From d4f09cea778e2017cc5dfba965c01b551ffa4774 Mon Sep 17 00:00:00 2001 From: n1sp Date: Fri, 22 May 2026 14:29:54 +0800 Subject: [PATCH] feat: add bytedance/seedance-2.0[-fast] (T2V/I2V/R2V, 6 endpoints) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register ByteDance Seedance 2.0 video generation models under a new `bytedance/` vendor namespace: bytedance/seedance-2.0/text-to-video bytedance/seedance-2.0/image-to-video bytedance/seedance-2.0/reference-to-video bytedance/seedance-2.0-fast/text-to-video bytedance/seedance-2.0-fast/image-to-video bytedance/seedance-2.0-fast/reference-to-video URL convention: `/vendors/bytedance/v1///generation` (mule-router canonical after the muledance→seedance migration). Polling reuses the existing `${apiPath}/${taskId}` pattern. Parameter schemas extracted directly from mule-router/tasks/handlers/mule/models/payloads/seedance.py: - T2V: prompt (req), resolution, aspect_ratio, duration, generate_audio, seed, web_search, camera_fixed (std only), watermark (std only) - I2V: image (req), last_frame_image, prompt, resolution, ... - R2V: prompt, images[], videos[] (https-only), audios[], resolution, ... Std vs fast differs in: resolution (std accepts 1080p, fast does not); T2V fast omits camera_fixed / watermark (server rejects non-false). The `model` request-body field is auto-injected by mule-router's payload_transformer and is intentionally NOT exposed as a CLI flag. Also: - Add `last_frame_image` to IMAGE_PARAM_NAMES so seedance I2V local file paths get auto-converted to base64. - Bump @mulerouter/core and mulerouter to 0.4.0 (publish workflow verifies the two versions match). --- packages/cli/package.json | 2 +- packages/cli/tests/run.test.ts | 23 +++ packages/core/package.json | 2 +- packages/core/src/image.ts | 1 + .../core/src/models/bytedance/_builders.ts | 159 ++++++++++++++++++ packages/core/src/models/bytedance/index.ts | 6 + .../seedance-2.0-fast-image-to-video.ts | 21 +++ .../seedance-2.0-fast-reference-to-video.ts | 22 +++ .../seedance-2.0-fast-text-to-video.ts | 22 +++ .../bytedance/seedance-2.0-image-to-video.ts | 23 +++ .../seedance-2.0-reference-to-video.ts | 23 +++ .../bytedance/seedance-2.0-text-to-video.ts | 23 +++ packages/core/src/models/index.ts | 1 + packages/core/tests/models.test.ts | 101 ++++++++++- 14 files changed, 423 insertions(+), 6 deletions(-) create mode 100644 packages/core/src/models/bytedance/_builders.ts create mode 100644 packages/core/src/models/bytedance/index.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-fast-image-to-video.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-fast-reference-to-video.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-fast-text-to-video.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-image-to-video.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-reference-to-video.ts create mode 100644 packages/core/src/models/bytedance/seedance-2.0-text-to-video.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 1f7b586..83467bb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "mulerouter", - "version": "0.3.2", + "version": "0.4.0", "description": "CLI for MuleRouter/MuleRun multimodal AI APIs — generate images, videos, speech, and music", "type": "module", "license": "MIT", diff --git a/packages/cli/tests/run.test.ts b/packages/cli/tests/run.test.ts index e2c4374..45552f2 100644 --- a/packages/cli/tests/run.test.ts +++ b/packages/cli/tests/run.test.ts @@ -100,4 +100,27 @@ describe("run command internals", () => { }); }); }); + + describe("seedance endpoint resolution", () => { + it("should resolve 3-part seedance identifiers", () => { + const t2v = resolveEndpoint("bytedance/seedance-2.0/text-to-video"); + expect(t2v.provider).toBe("bytedance"); + expect(t2v.action).toBe("text-to-video"); + expect(t2v.apiPath).toBe("/vendors/bytedance/v1/seedance-2.0/text-to-video/generation"); + expect(t2v.resultKey).toBe("videos"); + }); + + it("should throw for 2-part seedance identifier (ambiguous: 3 actions)", () => { + expect(() => resolveEndpoint("bytedance/seedance-2.0")).toThrow("Multiple actions"); + }); + + it("should distinguish std vs fast variants", () => { + const std = resolveEndpoint("bytedance/seedance-2.0/text-to-video"); + const fast = resolveEndpoint("bytedance/seedance-2.0-fast/text-to-video"); + expect(std.modelName).toBe("seedance-2.0"); + expect(fast.modelName).toBe("seedance-2.0-fast"); + expect(std.parameters.find((p) => p.name === "camera_fixed")).toBeDefined(); + expect(fast.parameters.find((p) => p.name === "camera_fixed")).toBeUndefined(); + }); + }); }); diff --git a/packages/core/package.json b/packages/core/package.json index ab70c96..dd365f5 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@mulerouter/core", - "version": "0.3.2", + "version": "0.4.0", "description": "Core library for MuleRouter/MuleRun multimodal AI APIs", "type": "module", "license": "MIT", diff --git a/packages/core/src/image.ts b/packages/core/src/image.ts index da4b5b3..9955a0e 100644 --- a/packages/core/src/image.ts +++ b/packages/core/src/image.ts @@ -8,6 +8,7 @@ export const IMAGE_PARAM_NAMES = new Set([ "images", "first_frame", "last_frame", + "last_frame_image", "first_frame_url", "last_frame_url", "ref_images_url", diff --git a/packages/core/src/models/bytedance/_builders.ts b/packages/core/src/models/bytedance/_builders.ts new file mode 100644 index 0000000..ed5b77e --- /dev/null +++ b/packages/core/src/models/bytedance/_builders.ts @@ -0,0 +1,159 @@ +import type { ModelParameter } from "../../types.js"; + +/** + * Shared parameter definitions for ByteDance Seedance 2.0 series. + * Source: extracted from mule-router payloads/seedance.py + docs/api/bytedance/. + * + * Two model variants (seedance-2.0 vs seedance-2.0-fast) differ only in: + * - `resolution` enum: std accepts 1080p, fast does not. + * - T2V fast does NOT accept `camera_fixed` / `watermark` non-false values + * (so the fast T2V parameter list omits those two flags entirely). + * + * `model` is auto-injected by mule-router from URL segments and MUST NOT be + * exposed as a CLI parameter. + */ + +const RESOLUTIONS_STD = ["480p", "720p", "1080p"] as const; +const RESOLUTIONS_FAST = ["480p", "720p"] as const; +const ASPECT_RATIOS = ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9", "adaptive"] as const; + +/** Parameters common to all 6 seedance endpoints. */ +const commonParameters: ModelParameter[] = [ + { + name: "aspect_ratio", + type: "string", + description: "Output video aspect ratio", + default: "adaptive", + enum: [...ASPECT_RATIOS], + }, + { + name: "duration", + type: "integer", + description: "Video duration in seconds. Discrete: -1 (model decides) or 4..15", + default: 5, + }, + { + name: "generate_audio", + type: "boolean", + description: "Generate audio track alongside video", + default: true, + }, + { + name: "seed", + type: "integer", + description: "Random seed for reproducibility (-1 to 4294967295)", + }, +]; + +function resolutionParam(forFast: boolean): ModelParameter { + return { + name: "resolution", + type: "string", + description: "Output video resolution", + default: "720p", + enum: forFast ? [...RESOLUTIONS_FAST] : [...RESOLUTIONS_STD], + }; +} + +/** Build T2V parameters for std (seedance-2.0) or fast variant. */ +export function seedanceT2vParameters(forFast: boolean): ModelParameter[] { + const base: ModelParameter[] = [ + { + name: "prompt", + type: "string", + description: "Text prompt describing the video to generate", + required: true, + }, + resolutionParam(forFast), + ...commonParameters, + { + name: "web_search", + type: "boolean", + description: "Enable web search grounding for the prompt", + default: false, + }, + ]; + + if (!forFast) { + // camera_fixed / watermark only accepted on std variant + base.push( + { + name: "camera_fixed", + type: "boolean", + description: "Lock camera position (no panning/zooming)", + default: false, + }, + { + name: "watermark", + type: "boolean", + description: "Add watermark to the output video", + default: false, + }, + ); + } + + return base; +} + +/** Build I2V parameters for std or fast variant. */ +export function seedanceI2vParameters(forFast: boolean): ModelParameter[] { + return [ + { + name: "image", + type: "string", + description: + "First frame image. HTTP(S) URL, data: URI, or local file path (auto-converted to base64)", + required: true, + }, + { + name: "last_frame_image", + type: "string", + description: + "Optional last frame image. HTTP(S) URL, data: URI, or local file path (auto-converted to base64)", + }, + { + name: "prompt", + type: "string", + description: "Optional text prompt for video motion guidance", + }, + resolutionParam(forFast), + ...commonParameters, + ]; +} + +/** Build R2V parameters for std or fast variant. */ +export function seedanceR2vParameters(forFast: boolean): ModelParameter[] { + return [ + { + name: "prompt", + type: "string", + description: "Optional text prompt for video generation", + }, + { + name: "images", + type: "array", + description: + 'Reference images (1-9). JSON array of HTTP(S) URLs / data: URIs / local file paths. At least one of --images / --videos must be provided. Example: --images \'["/tmp/a.png","https://example.com/b.jpg"]\'', + }, + { + name: "videos", + type: "array", + description: + "Reference videos (1-3). JSON array of HTTPS URLs ONLY (no http, no base64). At least one of --images / --videos must be provided. Example: --videos '[\"https://example.com/clip.mp4\"]'", + }, + { + name: "audios", + type: "array", + description: + "Reference audios (1-3). JSON array of URLs / data: URIs. Cannot be used alone — must be combined with --images or --videos. Example: --audios '[\"https://example.com/voice.mp3\"]'", + }, + resolutionParam(forFast), + ...commonParameters, + { + name: "web_search", + type: "boolean", + description: "Enable web search grounding for the prompt", + default: false, + }, + ]; +} diff --git a/packages/core/src/models/bytedance/index.ts b/packages/core/src/models/bytedance/index.ts new file mode 100644 index 0000000..4cf1e0c --- /dev/null +++ b/packages/core/src/models/bytedance/index.ts @@ -0,0 +1,6 @@ +import "./seedance-2.0-text-to-video.js"; +import "./seedance-2.0-image-to-video.js"; +import "./seedance-2.0-reference-to-video.js"; +import "./seedance-2.0-fast-text-to-video.js"; +import "./seedance-2.0-fast-image-to-video.js"; +import "./seedance-2.0-fast-reference-to-video.js"; diff --git a/packages/core/src/models/bytedance/seedance-2.0-fast-image-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-fast-image-to-video.ts new file mode 100644 index 0000000..bdda5a2 --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-fast-image-to-video.ts @@ -0,0 +1,21 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceI2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0-fast", + action: "image-to-video", + provider: "bytedance", + modelName: "seedance-2.0-fast", + description: "ByteDance Seedance 2.0-fast image-to-video: faster variant (max 720p)", + inputTypes: ["text", "image"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0-fast/image-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + parameters: seedanceI2vParameters(true), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/bytedance/seedance-2.0-fast-reference-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-fast-reference-to-video.ts new file mode 100644 index 0000000..8eff007 --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-fast-reference-to-video.ts @@ -0,0 +1,22 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceR2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0-fast", + action: "reference-to-video", + provider: "bytedance", + modelName: "seedance-2.0-fast", + description: + "ByteDance Seedance 2.0-fast reference-to-video: faster multi-modal references variant (max 720p)", + inputTypes: ["text", "image", "video", "audio"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0-fast/reference-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + parameters: seedanceR2vParameters(true), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/bytedance/seedance-2.0-fast-text-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-fast-text-to-video.ts new file mode 100644 index 0000000..7eb5835 --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-fast-text-to-video.ts @@ -0,0 +1,22 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceT2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0-fast", + action: "text-to-video", + provider: "bytedance", + modelName: "seedance-2.0-fast", + description: + "ByteDance Seedance 2.0-fast text-to-video: faster variant (max 720p, no camera_fixed/watermark)", + inputTypes: ["text"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0-fast/text-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + parameters: seedanceT2vParameters(true), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/bytedance/seedance-2.0-image-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-image-to-video.ts new file mode 100644 index 0000000..f5fd816 --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-image-to-video.ts @@ -0,0 +1,23 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceI2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0", + action: "image-to-video", + provider: "bytedance", + modelName: "seedance-2.0", + description: + "ByteDance Seedance 2.0 image-to-video: animate an image (optional last-frame target), up to 1080p", + inputTypes: ["text", "image"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0/image-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + tags: ["SOTA"], + parameters: seedanceI2vParameters(false), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/bytedance/seedance-2.0-reference-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-reference-to-video.ts new file mode 100644 index 0000000..8f20559 --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-reference-to-video.ts @@ -0,0 +1,23 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceR2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0", + action: "reference-to-video", + provider: "bytedance", + modelName: "seedance-2.0", + description: + "ByteDance Seedance 2.0 reference-to-video: multi-modal references (images/videos/audios), up to 1080p", + inputTypes: ["text", "image", "video", "audio"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0/reference-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + tags: ["SOTA"], + parameters: seedanceR2vParameters(false), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/bytedance/seedance-2.0-text-to-video.ts b/packages/core/src/models/bytedance/seedance-2.0-text-to-video.ts new file mode 100644 index 0000000..9d637cd --- /dev/null +++ b/packages/core/src/models/bytedance/seedance-2.0-text-to-video.ts @@ -0,0 +1,23 @@ +import { registerEndpoint } from "../../registry.js"; +import type { ModelEndpoint } from "../../types.js"; +import { seedanceT2vParameters } from "./_builders.js"; + +const endpoint: ModelEndpoint = { + modelId: "bytedance/seedance-2.0", + action: "text-to-video", + provider: "bytedance", + modelName: "seedance-2.0", + description: + "ByteDance Seedance 2.0 text-to-video: generate videos from text prompts (up to 1080p, 4-15s)", + inputTypes: ["text"], + outputType: "video", + apiPath: "/vendors/bytedance/v1/seedance-2.0/text-to-video/generation", + availableOn: ["mulerouter", "mulerun"], + resultKey: "videos", + tags: ["SOTA"], + parameters: seedanceT2vParameters(false), +}; + +registerEndpoint(endpoint); + +export { endpoint }; diff --git a/packages/core/src/models/index.ts b/packages/core/src/models/index.ts index c66a40d..19e07d7 100644 --- a/packages/core/src/models/index.ts +++ b/packages/core/src/models/index.ts @@ -1,5 +1,6 @@ // Import all vendor modules to register their endpoints with the global registry. import "./alibaba/index.js"; +import "./bytedance/index.js"; import "./google/index.js"; import "./klingai/index.js"; import "./midjourney/index.js"; diff --git a/packages/core/tests/models.test.ts b/packages/core/tests/models.test.ts index 10e44ac..f606fb9 100644 --- a/packages/core/tests/models.test.ts +++ b/packages/core/tests/models.test.ts @@ -9,19 +9,112 @@ describe("models", () => { it("should register all expected endpoints", () => { const all = registry.listAll(); - // Sum of per-provider counts below: 15 + 7 + 7 + 2 + 4 + 2 = 37 - expect(all.length).toBeGreaterThanOrEqual(37); + // Sum of per-provider counts below: 15 + 6 + 7 + 7 + 2 + 4 + 2 = 43 + expect(all.length).toBeGreaterThanOrEqual(43); }); it("should register all expected providers", () => { const providers = registry.getProviders(); expect(providers).toContain("alibaba"); + expect(providers).toContain("bytedance"); expect(providers).toContain("google"); expect(providers).toContain("klingai"); expect(providers).toContain("midjourney"); expect(providers).toContain("minimax"); expect(providers).toContain("openai"); - expect(providers.length).toBeGreaterThanOrEqual(6); + expect(providers.length).toBeGreaterThanOrEqual(7); + }); + + describe("bytedance", () => { + it("should have 6 endpoints", () => { + const endpoints = registry.listByProvider("bytedance"); + expect(endpoints).toHaveLength(6); + }); + + it("should register all seedance-2.0 actions", () => { + const t2v = registry.get("bytedance/seedance-2.0", "text-to-video"); + const i2v = registry.get("bytedance/seedance-2.0", "image-to-video"); + const r2v = registry.get("bytedance/seedance-2.0", "reference-to-video"); + expect(t2v).toBeDefined(); + expect(i2v).toBeDefined(); + expect(r2v).toBeDefined(); + expect(t2v?.apiPath).toBe("/vendors/bytedance/v1/seedance-2.0/text-to-video/generation"); + expect(i2v?.apiPath).toBe("/vendors/bytedance/v1/seedance-2.0/image-to-video/generation"); + expect(r2v?.apiPath).toBe("/vendors/bytedance/v1/seedance-2.0/reference-to-video/generation"); + }); + + it("should register all seedance-2.0-fast actions", () => { + expect(registry.get("bytedance/seedance-2.0-fast", "text-to-video")).toBeDefined(); + expect(registry.get("bytedance/seedance-2.0-fast", "image-to-video")).toBeDefined(); + expect(registry.get("bytedance/seedance-2.0-fast", "reference-to-video")).toBeDefined(); + }); + + it("should use 'videos' as resultKey for all 6 endpoints", () => { + const endpoints = registry.listByProvider("bytedance"); + expect(endpoints.every((e) => e.resultKey === "videos")).toBe(true); + expect(endpoints.every((e) => e.outputType === "video")).toBe(true); + }); + + it("should be available on both mulerouter and mulerun", () => { + const endpoints = registry.listByProvider("bytedance"); + for (const e of endpoints) { + expect(e.availableOn).toContain("mulerouter"); + expect(e.availableOn).toContain("mulerun"); + } + }); + + it("seedance-2.0 T2V should require prompt and accept camera_fixed/watermark", () => { + const t2v = registry.get("bytedance/seedance-2.0", "text-to-video"); + const prompt = t2v?.parameters.find((p) => p.name === "prompt"); + expect(prompt?.required).toBe(true); + const names = t2v?.parameters.map((p) => p.name) ?? []; + expect(names).toContain("camera_fixed"); + expect(names).toContain("watermark"); + const resolution = t2v?.parameters.find((p) => p.name === "resolution"); + expect(resolution?.enum).toContain("1080p"); + }); + + it("seedance-2.0-fast T2V should omit camera_fixed/watermark and reject 1080p", () => { + const t2v = registry.get("bytedance/seedance-2.0-fast", "text-to-video"); + const names = t2v?.parameters.map((p) => p.name) ?? []; + expect(names).not.toContain("camera_fixed"); + expect(names).not.toContain("watermark"); + const resolution = t2v?.parameters.find((p) => p.name === "resolution"); + expect(resolution?.enum).not.toContain("1080p"); + expect(resolution?.enum).toContain("720p"); + }); + + it("seedance-2.0 I2V should require image and accept 1080p", () => { + const i2v = registry.get("bytedance/seedance-2.0", "image-to-video"); + const image = i2v?.parameters.find((p) => p.name === "image"); + expect(image?.required).toBe(true); + const lastFrame = i2v?.parameters.find((p) => p.name === "last_frame_image"); + expect(lastFrame).toBeDefined(); + expect(lastFrame?.required).not.toBe(true); + const resolution = i2v?.parameters.find((p) => p.name === "resolution"); + expect(resolution?.enum).toContain("1080p"); + }); + + it("seedance-2.0 R2V should expose images/videos/audios as arrays (no single required)", () => { + const r2v = registry.get("bytedance/seedance-2.0", "reference-to-video"); + const images = r2v?.parameters.find((p) => p.name === "images"); + const videos = r2v?.parameters.find((p) => p.name === "videos"); + const audios = r2v?.parameters.find((p) => p.name === "audios"); + expect(images?.type).toBe("array"); + expect(videos?.type).toBe("array"); + expect(audios?.type).toBe("array"); + // conditional required (images OR videos) is enforced server-side, not by CLI + expect(images?.required).not.toBe(true); + expect(videos?.required).not.toBe(true); + }); + + it("should NOT define `model` as a CLI parameter (auto-injected by mule-router)", () => { + const endpoints = registry.listByProvider("bytedance"); + for (const e of endpoints) { + const names = e.parameters.map((p) => p.name); + expect(names).not.toContain("model"); + } + }); }); describe("alibaba", () => { @@ -185,7 +278,7 @@ describe("models", () => { expect(videos.length).toBeGreaterThan(0); expect(audios.length).toBe(4); // minimax only - expect(images.length + videos.length + audios.length).toBeGreaterThanOrEqual(37); + expect(images.length + videos.length + audios.length).toBeGreaterThanOrEqual(43); }); it("should filter SOTA models", () => {