Spracherkennung für: .ts vermutete Sprache: Unknown {[0] [0] [0]} [Methode: Schwerpunktbildung, einfache Gewichte, sechs Dimensionen]
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import * as mediaStore from "../../media/store.js";
import * as webMedia from "../../media/web-media.js";
import * as videoGenerationRuntime from "../../video-generation/runtime.js";
import * as videoGenerateBackground from "./video-generate-background.js";
import { createVideoGenerateTool } from "./video-generate-tool.js";
const taskRuntimeInternalMocks = vi.hoisted(() => ({
listTasksForOwnerKey: vi.fn(),
}));
const taskExecutorMocks = vi.hoisted(() => ({
recordTaskRunProgressByRunId: vi.fn(),
failTaskRunByRunId: vi.fn(),
completeTaskRunByRunId: vi.fn(),
createRunningTaskRun: vi.fn(),
}));
vi.mock("../../tasks/runtime-internal.js", () => taskRuntimeInternalMocks);
vi.mock("../../tasks/detached-task-runtime.js", () => taskExecutorMocks);
function asConfig(value: unknown): OpenClawConfig {
return value as OpenClawConfig;
}
function mockVideoPluginProvider(capabilities: Record<string, unknown> = {}) {
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockRetur nValue([
{
id: "video-plugin",
defaultModel: "vid-v1",
models: ["vid-v1"],
capabilities,
generateVideo: vi.fn(async () => ({
videos: [{ buffer: Buffer.from("x"), mimeType: "video/mp4" }],
})),
},
]);
}
function createVideoPluginTool() {
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "video-plugin/vid-v1" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
return tool;
}
function mockSavedVideoResult(fileName = "out.mp4") {
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "video-plugin",
model: "vid-v1",
attempts: [],
ignoredOverrides: [],
videos: [{ buffer: Buffer.from("video-bytes"), mimeType: "video/mp4", fileName }],
});
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: `/tmp/${fileName}`,
id: fileName,
size: 11,
contentType: "video/mp4",
});
return generateSpy;
}
function resetVideoGenerateMocks() {
vi.restoreAllMocks();
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([]);
taskRuntimeInternalMocks.listTasksForOwnerKey.mockReset();
taskRuntimeInternalMocks.listTasksForOwnerKey.mockReturnValue([]);
taskExecutorMocks.createRunningTaskRun.mockReset();
taskExecutorMocks.completeTaskRunByRunId.mockReset();
taskExecutorMocks.failTaskRunByRunId.mockReset();
taskExecutorMocks.recordTaskRunProgressByRunId.mockReset();
}
describe("createVideoGenerateTool", () => {
beforeEach(resetVideoGenerateMocks);
afterEach(() => {
vi.unstubAllEnvs();
});
it("returns null when no video-generation config or auth-backed provider is available", () => {
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([]);
expect(createVideoGenerateTool({ config: asConfig({}) })).toBeNull();
});
it("registers when video-generation config is present", () => {
expect(
createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "qwen/wan2.6-t2v" },
},
},
}),
}),
).not.toBeNull();
});
it("generates videos, saves them, and emits MEDIA paths without a session-backed detach", async () => {
taskExecutorMocks.createRunningTaskRun.mockReturnValue({
taskId: "task-123",
runtime: "cli",
requesterSessionKey: "agent:main:discord:direct:123",
ownerKey: "agent:main:discord:direct:123",
scopeKind: "session",
task: "friendly lobster surfing",
status: "running",
deliveryStatus: "not_applicable",
notifyPolicy: "silent",
createdAt: Date.now(),
});
taskExecutorMocks.completeTaskRunByRunId.mockReturnValue(undefined);
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "qwen",
model: "wan2.6-t2v",
attempts: [],
ignoredOverrides: [],
videos: [
{
buffer: Buffer.from("video-bytes"),
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
metadata: { taskId: "task-1" },
});
const saveSpy = vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: "/tmp/generated-lobster.mp4",
id: "generated-lobster.mp4",
size: 11,
contentType: "video/mp4",
});
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
mediaMaxMb: 8,
videoGenerationModel: { primary: "qwen/wan2.6-t2v" },
},
},
}),
});
expect(tool).not.toBeNull();
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-1", { prompt: "friendly lobster surfing" });
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(saveSpy).toHaveBeenCalledWith(
Buffer.from("video-bytes"),
"video/mp4",
"tool-video-generation",
8 * 1024 * 1024,
"lobster.mp4",
);
expect(text).toContain("Generated 1 video with qwen/wan2.6-t2v.");
expect(text).toContain("MEDIA:/tmp/generated-lobster.mp4");
expect(result.details).toMatchObject({
provider: "qwen",
model: "wan2.6-t2v",
count: 1,
media: {
mediaUrls: ["/tmp/generated-lobster.mp4"],
},
paths: ["/tmp/generated-lobster.mp4"],
metadata: { taskId: "task-1" },
});
expect(taskExecutorMocks.createRunningTaskRun).not.toHaveBeenCalled();
expect(taskExecutorMocks.completeTaskRunByRunId).not.toHaveBeenCalled();
});
it("surfaces url-only generated videos without saving local files", async () => {
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "vydra",
model: "veo3",
attempts: [],
ignoredOverrides: [],
videos: [
{
url: "https://example.com/generated-lobster.mp4",
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
metadata: { taskId: "task-1" },
});
const saveSpy = vi.spyOn(mediaStore, "saveMediaBuffer");
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "vydra/veo3" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-url", { prompt: "friendly lobster surfing" });
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(saveSpy).not.toHaveBeenCalled();
expect(text).toContain("Generated 1 video with vydra/veo3.");
expect(text).toContain("MEDIA:https://example.com/generated-lobster.mp4");
expect(result.details).toMatchObject({
provider: "vydra",
model: "veo3",
count: 1,
media: {
mediaUrls: ["https://example.com/generated-lobster.mp4"],
},
paths: ["https://example.com/generated-lobster.mp4"],
metadata: { taskId: "task-1" },
});
});
it("starts background generation and wakes the session with url-only MEDIA lines", async () => {
taskExecutorMocks.createRunningTaskRun.mockReturnValue({
taskId: "task-123",
runtime: "cli",
requesterSessionKey: "agent:main:discord:direct:123",
ownerKey: "agent:main:discord:direct:123",
scopeKind: "session",
task: "friendly lobster surfing",
status: "running",
deliveryStatus: "not_applicable",
notifyPolicy: "silent",
createdAt: Date.now(),
});
const wakeSpy = vi
.spyOn(videoGenerateBackground, "wakeVideoGenerationTaskCompletion")
.mockResolvedValue(undefined);
const saveSpy = vi.spyOn(mediaStore, "saveMediaBuffer");
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "vydra",
model: "veo3",
attempts: [],
ignoredOverrides: [],
videos: [
{
url: "https://example.com/generated-lobster.mp4",
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
metadata: { taskId: "task-1" },
});
let scheduledWork: (() => Promise<void>) | undefined;
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "vydra/veo3" },
},
},
}),
agentSessionKey: "agent:main:discord:direct:123",
requesterOrigin: {
channel: "discord",
to: "channel:1",
},
scheduleBackgroundWork: (work) => {
scheduledWork = work;
},
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-1", { prompt: "friendly lobster surfing" });
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).toContain("Background task started for video generation (task-123).");
expect(text).toContain("Do not call video_generate again for this request.");
expect(result.details).toMatchObject({
async: true,
status: "started",
task: {
taskId: "task-123",
},
});
expect(typeof scheduledWork).toBe("function");
await scheduledWork?.();
expect(saveSpy).not.toHaveBeenCalled();
expect(taskExecutorMocks.recordTaskRunProgressByRunId).toHaveBeenCalledWith(
expect.objectContaining({
runId: expect.stringMatching(/^tool:video_generate:/),
progressSummary: "Generating video",
}),
);
expect(taskExecutorMocks.completeTaskRunByRunId).toHaveBeenCalledWith(
expect.objectContaining({
runId: expect.stringMatching(/^tool:video_generate:/),
}),
);
expect(wakeSpy).toHaveBeenCalledWith(
expect.objectContaining({
handle: expect.objectContaining({
taskId: "task-123",
}),
status: "ok",
mediaUrls: ["https://example.com/generated-lobster.mp4"],
result: expect.stringContaining("MEDIA:https://example.com/generated-lobster.mp4"),
}),
);
});
it("surfaces provider generation failures inline when there is no detached session", async () => {
vi.spyOn(videoGenerationRuntime, "generateVideo").mockRejectedValue(new Error("queue boom"));
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "qwen/wan2.6-t2v" },
},
},
}),
});
expect(tool).not.toBeNull();
if (!tool) {
throw new Error("expected video_generate tool");
}
await expect(tool.execute("call-2", { prompt: "broken lobster" })).rejects.toThrow(
"queue boom",
);
expect(taskExecutorMocks.failTaskRunByRunId).not.toHaveBeenCalled();
});
it("shows duration normalization details from runtime metadata", async () => {
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "google",
model: "veo-3.1-fast-generate-preview",
attempts: [],
ignoredOverrides: [],
videos: [
{
buffer: Buffer.from("video-bytes"),
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
normalization: {
durationSeconds: {
requested: 5,
applied: 6,
supportedValues: [4, 6, 8],
},
},
metadata: {
requestedDurationSeconds: 5,
normalizedDurationSeconds: 6,
supportedDurationSeconds: [4, 6, 8],
},
});
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: "/tmp/generated-lobster.mp4",
id: "generated-lobster.mp4",
size: 11,
contentType: "video/mp4",
});
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "google/veo-3.1-fast-generate-preview" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-1", {
prompt: "friendly lobster surfing",
durationSeconds: 5,
});
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).toContain("Duration normalized: requested 5s; used 6s.");
expect(result.details).toMatchObject({
durationSeconds: 6,
requestedDurationSeconds: 5,
supportedDurationSeconds: [4, 6, 8],
normalization: {
durationSeconds: {
requested: 5,
applied: 6,
supportedValues: [4, 6, 8],
},
},
});
});
it("surfaces normalized video geometry from runtime metadata", async () => {
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "runway",
model: "gen4.5",
attempts: [],
ignoredOverrides: [],
videos: [
{
buffer: Buffer.from("video-bytes"),
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
normalization: {
aspectRatio: {
applied: "16:9",
derivedFrom: "size",
},
},
metadata: {
requestedSize: "1280x720",
normalizedAspectRatio: "16:9",
},
});
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: "/tmp/generated-lobster.mp4",
id: "generated-lobster.mp4",
size: 11,
contentType: "video/mp4",
});
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "runway/gen4.5" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-1", {
prompt: "friendly lobster surfing",
size: "1280x720",
});
expect(result.details).toMatchObject({
aspectRatio: "16:9",
normalization: {
aspectRatio: {
applied: "16:9",
derivedFrom: "size",
},
},
metadata: {
requestedSize: "1280x720",
normalizedAspectRatio: "16:9",
},
});
expect(result.details).not.toHaveProperty("size");
});
it("lists supported provider durations when advertised", async () => {
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
{
id: "google",
defaultModel: "veo-3.1-fast-generate-preview",
models: ["veo-3.1-fast-generate-preview"],
capabilities: {
generate: {
maxDurationSeconds: 8,
supportedDurationSeconds: [4, 6, 8],
},
imageToVideo: {
enabled: true,
maxInputImages: 1,
maxDurationSeconds: 8,
supportedDurationSeconds: [4, 6, 8],
},
},
generateVideo: vi.fn(async () => {
throw new Error("not used");
}),
},
]);
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "google/veo-3.1-fast-generate-preview" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-1", { action: "list" });
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).toContain("modes=generate/imageToVideo");
expect(text).toContain("supportedDurationSeconds=4/6/8");
expect(result.details).toMatchObject({
providers: [
expect.objectContaining({
id: "google",
modes: ["generate", "imageToVideo"],
}),
],
});
});
it("rejects image-to-video when the provider disables that mode", async () => {
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
{
id: "video-plugin",
defaultModel: "vid-v1",
models: ["vid-v1"],
capabilities: {
imageToVideo: {
enabled: false,
},
},
generateVideo: vi.fn(async () => {
throw new Error("not used");
}),
},
]);
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo");
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "video-plugin/vid-v1" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
await expect(
tool.execute("call-1", {
prompt: "lobster timelapse",
image: "data:image/png;base64,cG5n",
}),
).rejects.toThrow("video-plugin does not support image-to-video reference inputs.");
expect(generateSpy).not.toHaveBeenCalled();
});
it("warns when optional provider overrides are ignored", async () => {
vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
{
id: "openai",
defaultModel: "sora-2",
models: ["sora-2"],
capabilities: {
generate: {
supportsSize: true,
},
},
generateVideo: vi.fn(async () => {
throw new Error("not used");
}),
},
]);
vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
provider: "openai",
model: "sora-2",
attempts: [],
ignoredOverrides: [
{ key: "resolution", value: "720P" },
{ key: "audio", value: false },
{ key: "watermark", value: false },
],
videos: [
{
buffer: Buffer.from("video-bytes"),
mimeType: "video/mp4",
fileName: "lobster.mp4",
},
],
});
vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
path: "/tmp/generated-lobster.mp4",
id: "generated-lobster.mp4",
size: 11,
contentType: "video/mp4",
});
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "openai/sora-2" },
},
},
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
const result = await tool.execute("call-openai-generate", {
prompt: "A lobster on a neon bridge",
size: "1280x720",
resolution: "720P",
audio: false,
watermark: false,
});
const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
expect(text).toContain("Generated 1 video with openai/sora-2.");
expect(text).toContain(
"Warning: Ignored unsupported overrides for openai/sora-2: resolution=720P, audio=false, watermark=false.",
);
expect(result).toMatchObject({
details: {
size: "1280x720",
warning:
"Ignored unsupported overrides for openai/sora-2: resolution=720P, audio=false, watermark=false.",
ignoredOverrides: [
{ key: "resolution", value: "720P" },
{ key: "audio", value: false },
{ key: "watermark", value: false },
],
},
});
expect(result.details).not.toHaveProperty("resolution");
expect(result.details).not.toHaveProperty("audio");
expect(result.details).not.toHaveProperty("watermark");
});
it("rejects providerOptions that is not a plain JSON object", async () => {
mockVideoPluginProvider();
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo");
const tool = createVideoPluginTool();
// Array-shaped providerOptions should be rejected up front, not cast to a
// Record with numeric-string keys and silently forwarded.
await expect(
tool.execute("call-1", {
prompt: "lobster",
providerOptions: ["seed", 42] as unknown as Record<string, unknown>,
}),
).rejects.toThrow(
"providerOptions must be a JSON object keyed by provider-specific option name.",
);
// String providerOptions should also be rejected.
await expect(
tool.execute("call-2", {
prompt: "lobster",
providerOptions: "seed=42" as unknown as Record<string, unknown>,
}),
).rejects.toThrow(
"providerOptions must be a JSON object keyed by provider-specific option name.",
);
expect(generateSpy).not.toHaveBeenCalled();
});
it("forwards providerOptions to the runtime for valid JSON-object payloads", async () => {
mockVideoPluginProvider({
providerOptions: { seed: "number", draft: "boolean" },
});
const generateSpy = mockSavedVideoResult();
const tool = createVideoPluginTool();
await tool.execute("call-1", {
prompt: "lobster",
providerOptions: { seed: 42, draft: true },
});
expect(generateSpy).toHaveBeenCalledWith(
expect.objectContaining({
providerOptions: { seed: 42, draft: true },
}),
);
});
it("rejects *Roles arrays that are longer than the asset list", async () => {
mockVideoPluginProvider({
imageToVideo: { enabled: true, maxInputImages: 2 },
});
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo");
const tool = createVideoPluginTool();
await expect(
tool.execute("call-1", {
prompt: "lobster",
image: "data:image/png;base64,cG5n",
// Only one image is provided, so passing two roles is an off-by-one bug.
imageRoles: ["first_frame", "last_frame"],
}),
).rejects.toThrow(/imageRoles has 2 entries but only 1 reference image/);
expect(generateSpy).not.toHaveBeenCalled();
});
it("rejects *Roles that are not arrays", async () => {
mockVideoPluginProvider();
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo");
const tool = createVideoPluginTool();
await expect(
tool.execute("call-1", {
prompt: "lobster",
imageRoles: "first_frame" as unknown as string[],
}),
).rejects.toThrow(
"imageRoles must be a JSON array of role strings, parallel to the reference list.",
);
expect(generateSpy).not.toHaveBeenCalled();
});
it("attaches positional role hints to loaded reference assets", async () => {
mockVideoPluginProvider({
imageToVideo: { enabled: true, maxInputImages: 2 },
});
const generateSpy = mockSavedVideoResult();
const tool = createVideoPluginTool();
await tool.execute("call-1", {
prompt: "lobster",
images: ["data:image/png;base64,Zmlyc3Q=", "data:image/png;base64,bGFzdA=="],
imageRoles: ["first_frame", "last_frame"],
});
expect(generateSpy).toHaveBeenCalledTimes(1);
const call = generateSpy.mock.calls[0]?.[0] as {
inputImages?: Array<{ role?: string }>;
};
expect(call.inputImages).toHaveLength(2);
expect(call.inputImages?.[0]?.role).toBe("first_frame");
expect(call.inputImages?.[1]?.role).toBe("last_frame");
});
it("passes web_fetch SSRF policy when loading reference assets", async () => {
mockVideoPluginProvider({
imageToVideo: { enabled: true, maxInputImages: 1 },
});
vi.spyOn(webMedia, "loadWebMedia").mockResolvedValue({
kind: "image",
buffer: Buffer.from("image"),
contentType: "image/png",
});
mockSavedVideoResult();
const tool = createVideoGenerateTool({
config: asConfig({
agents: {
defaults: {
videoGenerationModel: { primary: "video-plugin/vid-v1" },
},
},
tools: { web: { fetch: { ssrfPolicy: { allowRfc2544BenchmarkRange: true } } } },
}),
});
if (!tool) {
throw new Error("expected video_generate tool");
}
await tool.execute("call-1", {
prompt: "lobster",
image: "/tmp/reference.png",
});
expect(webMedia.loadWebMedia).toHaveBeenCalledWith(
"/tmp/reference.png",
expect.objectContaining({
ssrfPolicy: { allowRfc2544BenchmarkRange: true },
}),
);
});
it("rejects audio data: URLs via the templated rejection branch", async () => {
mockVideoPluginProvider({
maxInputAudios: 1,
});
const generateSpy = vi.spyOn(videoGenerationRuntime, "generateVideo");
const tool = createVideoPluginTool();
await expect(
tool.execute("call-1", {
prompt: "lobster",
audioRef: "data:audio/mpeg;base64,bXAz",
}),
).rejects.toThrow("audio data: URLs are not supported for video_generate.");
expect(generateSpy).not.toHaveBeenCalled();
});
it("accepts aspectRatio=adaptive and forwards it to the runtime", async () => {
mockVideoPluginProvider();
const generateSpy = mockSavedVideoResult();
const tool = createVideoPluginTool();
await tool.execute("call-1", {
prompt: "lobster",
aspectRatio: "adaptive",
});
expect(generateSpy).toHaveBeenCalledWith(expect.objectContaining({ aspectRatio: "adaptive" }));
});
it("rejects unsupported aspectRatio values", async () => {
mockVideoPluginProvider();
const tool = createVideoPluginTool();
await expect(
tool.execute("call-1", {
prompt: "lobster",
aspectRatio: "17:9",
}),
).rejects.toThrow(
"aspectRatio must be one of 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9, or adaptive",
);
});
});
¤ Dauer der Verarbeitung: 0.3 Sekunden
(vorverarbeitet am 2026-04-27)
¤
*© Formatika GbR, Deutschland
|
|