From 11926b1348da48ea863ba32abdd74d99c39d8c65 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 14:08:47 -0800 Subject: [PATCH 01/35] fix(timeline): refuse colors that are not colors before they reach FFmpeg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A color clip's color and the canvas background went into the filter graph unquoted (`color=c=…`, `pad=…:color=…`), so a value like `black[p];movie=/path[m];[p][m]overlay` added filters: it read a file outside the workspace into the export, and `metadata=…:file=` created one. Validate colors where they enter (the add_color verb for MCP and the UI, and `clip timeline new --background` / `add-color --color`), and have the compiler refuse a stored value that is not a color with a CompileError. Accepted: FFmpeg color names in any case, #rrggbb[aa], 0xrrggbb[aa], with an optional @alpha. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/timeline-color-validation.md | 3 - src/cli-timeline.ts | 10 +- src/timeline/color.ts | 58 ++++++ src/timeline/compile.ts | 16 +- src/timeline/verbs.ts | 5 +- tests/integration/timeline-color.test.ts | 117 ++++++++++++ tests/timeline-color.test.ts | 211 ++++++++++++++++++++++ 7 files changed, 409 insertions(+), 11 deletions(-) delete mode 100644 docs/backlog/timeline-color-validation.md create mode 100644 src/timeline/color.ts create mode 100644 tests/integration/timeline-color.test.ts create mode 100644 tests/timeline-color.test.ts diff --git a/docs/backlog/timeline-color-validation.md b/docs/backlog/timeline-color-validation.md deleted file mode 100644 index 643dac7..0000000 --- a/docs/backlog/timeline-color-validation.md +++ /dev/null @@ -1,3 +0,0 @@ -# Timeline color validation - -Timeline `color` / `background` fields are bare `z.string()` (src/timeline/composition.ts, ops.ts) and `quoteColor` in src/timeline/compile.ts passes non-`#` values into the filtergraph unescaped, so a value like `red,drawtext=...` adds a filter; validate them as hex or a named-color allowlist in the schema. diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e623c1e..1141d94 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -1,5 +1,6 @@ import { appendOp } from './session/store.js'; import { CaptionError, DEFAULT_CAPTION_WORDS, planCaptions } from './timeline/captions.js'; +import { invalidColorMessage, isColor } from './timeline/color.js'; import { buildFrameAtPlan, checkExportable, @@ -134,6 +135,11 @@ function optionalNum(name: string, value: string | undefined): number | undefine return n; } +function colorFlag(name: string, value: string | undefined): string | undefined { + if (value === undefined || isColor(value)) return value; + throw new Error(`--${name} ${invalidColorMessage(value)}`); +} + function fitFlag(value: string | undefined): Fit | undefined { if (value === undefined) return undefined; const parsed = FitSchema.safeParse(value); @@ -177,7 +183,7 @@ export async function runTimeline(args: string[]): Promise { width: flags.width ? num(flags.width, 1920) : undefined, height: flags.height ? num(flags.height, 1080) : undefined, fps: flags.fps ? num(flags.fps, 30) : undefined, - background: flags.background, + background: colorFlag('background', flags.background), fit: fitFlag(flags.fit), }), ); @@ -244,7 +250,7 @@ export async function runTimeline(args: string[]): Promise { const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = colorClip({ id: makeClipId(), - color: flags.color, + color: colorFlag('color', flags.color), durationSec: num(duration, 2), startSec: start, }); diff --git a/src/timeline/color.ts b/src/timeline/color.ts new file mode 100644 index 0000000..9e9c214 --- /dev/null +++ b/src/timeline/color.ts @@ -0,0 +1,58 @@ +import { z } from 'zod'; + +/** + * Timeline colors reach FFmpeg as bare option values (`color=c=…`, + * `pad=…:color=…`), where a comma, semicolon or bracket starts a new filter. So a + * color must be one FFmpeg reads as a color and nothing more: anything else + * could add filters such as `movie=`, which reads any file on disk. + */ + +/** FFmpeg's color names (`ffmpeg -colors`), which it matches in any case. */ +const COLOR_NAMES = new Set( + `aliceblue antiquewhite aqua aquamarine azure beige bisque black blanchedalmond blue + blueviolet brown burlywood cadetblue chartreuse chocolate coral cornflowerblue cornsilk + crimson cyan darkblue darkcyan darkgoldenrod darkgray darkgreen darkkhaki darkmagenta + darkolivegreen darkorange darkorchid darkred darksalmon darkseagreen darkslateblue + darkslategray darkturquoise darkviolet deeppink deepskyblue dimgray dodgerblue firebrick + floralwhite forestgreen fuchsia gainsboro ghostwhite gold goldenrod gray green greenyellow + honeydew hotpink indianred indigo ivory khaki lavender lavenderblush lawngreen lemonchiffon + lightblue lightcoral lightcyan lightgoldenrodyellow lightgreen lightgrey lightpink + lightsalmon lightseagreen lightskyblue lightslategray lightsteelblue lightyellow lime + limegreen linen magenta maroon mediumaquamarine mediumblue mediumorchid mediumpurple + mediumseagreen mediumslateblue mediumspringgreen mediumturquoise mediumvioletred + midnightblue mintcream mistyrose moccasin navajowhite navy oldlace olive olivedrab orange + orangered orchid palegoldenrod palegreen paleturquoise palevioletred papayawhip peachpuff + peru pink plum powderblue purple red rosybrown royalblue saddlebrown salmon sandybrown + seagreen seashell sienna silver skyblue slateblue slategray snow springgreen steelblue tan + teal thistle tomato turquoise violet wheat white whitesmoke yellow yellowgreen`.split(/\s+/), +); + +const HEX = /^(?:#|0x)?[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?$/; +const HEX_ALPHA = /^0x[0-9a-fA-F]{1,2}$/; +const DECIMAL_ALPHA = /^(?:\d+(?:\.\d*)?|\.\d+)$/; + +/** Whether `value` is a color FFmpeg reads: a color name or hex (`#rrggbb`, + * `0xrrggbbaa`), with an optional `@alpha` from 0 to 1 (or 0x00 to 0xff). */ +export function isColor(value: string): boolean { + const [base = '', alpha, ...rest] = value.split('@'); + if (rest.length > 0) return false; + if (!HEX.test(base) && !COLOR_NAMES.has(base.toLowerCase())) return false; + return ( + alpha === undefined || + HEX_ALPHA.test(alpha) || + (DECIMAL_ALPHA.test(alpha) && Number(alpha) <= 1) + ); +} + +/** Why `value` was refused, with the forms that work. */ +export function invalidColorMessage(value: string): string { + return ( + `${JSON.stringify(value)} is not a color. Use a color name like black or white, ` + + 'or hex like #ffcc00, optionally with @0.5 for opacity.' + ); +} + +/** A color from an MCP client, the UI or the CLI (see `isColor`). */ +export const ColorSchema = z + .string() + .refine(isColor, { error: (issue) => invalidColorMessage(String(issue.input)) }); diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 45dc785..869aa1c 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -3,6 +3,7 @@ import { buildPreviewFrameArgs } from '../ffmpeg/args/preview.js'; import { buildTransitionArgs } from '../ffmpeg/args/transition.js'; import { quoteFilterArg } from '../ffmpeg/escape.js'; import { fontFileFor, MissingFontError } from '../ffmpeg/font.js'; +import { invalidColorMessage, isColor } from './color.js'; import { type Clip, type Composition, @@ -449,7 +450,7 @@ function buildSegmentStep( '-f', 'lavfi', '-i', - `color=c=${quoteColor(clip.color)}:s=${width}x${height}:r=${fps}`, + `color=c=${quoteColor(clip.color, `Color clip "${clip.id}" color`)}:s=${width}x${height}:r=${fps}`, ); inputArgs.push( '-f', @@ -465,7 +466,7 @@ function buildSegmentStep( '-f', 'lavfi', '-i', - `color=c=${quoteColor(background)}:s=${width}x${height}:r=${fps}`, + `color=c=${quoteColor(background, CANVAS_BACKGROUND)}:s=${width}x${height}:r=${fps}`, ); inputArgs.push( '-f', @@ -555,7 +556,7 @@ function framingFilters( height: number, background: string, ): string[] { - const pad = `color=${quoteColor(background)}`; + const pad = `color=${quoteColor(background, CANVAS_BACKGROUND)}`; if (isDefaultFraming(framing)) { return [ `scale=${width}:${height}:force_original_aspect_ratio=decrease`, @@ -635,8 +636,13 @@ const ANCHOR_EXPR: Record = { 'bottom-right': { x: `w-text_w-${EDGE}`, y: `h-text_h-${EDGE}` }, }; -/** color filter `c=` rejects '#rrggbb'; normalize to 0x form, pass names through. */ -function quoteColor(color: string): string { +const CANVAS_BACKGROUND = 'Canvas background'; + +/** color filter `c=` rejects '#rrggbb'; normalize to 0x form, pass names through. + * The value lands unquoted in the graph, so anything but a color is refused: a + * stored document is only as safe as whatever wrote it. */ +function quoteColor(color: string, what: string): string { + if (!isColor(color)) throw new CompileError(`${what} ${invalidColorMessage(color)}`); return color.startsWith('#') ? `0x${color.slice(1)}` : color; } diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index b72e9ec..8558382 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { DEFAULT_CAPTION_TRACK, DEFAULT_CAPTION_WORDS, planCaptions } from './captions.js'; +import { ColorSchema } from './color.js'; import type { MediaInfo } from './compile.js'; import { AnchorSchema, @@ -109,7 +110,9 @@ const EDITABLE_VERB_SCHEMAS = [ 'Optional clip id — supply one to reference this clip in a later verb of the SAME batch.', ), durationSec: z.number().positive(), - color: z.string().optional().describe('Card color (default black).'), + color: ColorSchema.optional().describe( + 'Card color: a name like black or white, or hex like #ffcc00 (default black).', + ), track: z.string().optional(), startSec: z.number().nonnegative().optional(), }), diff --git a/tests/integration/timeline-color.test.ts b/tests/integration/timeline-color.test.ts new file mode 100644 index 0000000..946f302 --- /dev/null +++ b/tests/integration/timeline-color.test.ts @@ -0,0 +1,117 @@ +import { access, mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { execa } from 'execa'; +import ffmpegStatic from 'ffmpeg-static'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { runFfmpeg } from '../../src/ffmpeg/run.js'; +import { buildMcpServer } from '../../src/mcp/server.js'; +import { mutateComposition } from '../../src/timeline/document-store.js'; + +const ffmpegPath = (): string => process.env.MAKEMYCLIP_FFMPEG_PATH ?? ffmpegStatic ?? 'ffmpeg'; + +// Unit tests assert the compiler refuses a color that is filter syntax; this +// checks that such a value is live in real FFmpeg (it writes a file outside the +// workspace) and that the MCP edit and export paths never hand it over. +const IS_CI = process.env.CI === 'true'; + +let workspace: string; +let outside: string; +let saved: string | undefined; + +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-color-')); + outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; +}); + +afterEach(async () => { + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); + await rm(outside, { recursive: true, force: true }); +}); + +const exists = (path: string): Promise => + access(path).then( + () => true, + () => false, + ); + +/** The average RGB of a video's first frame. */ +async function firstFrameRgb(video: string): Promise { + const { stdout } = await execa( + ffmpegPath(), + [ + ...['-hide_banner', '-loglevel', 'error', '-i', video, '-frames:v', '1'], + ...['-vf', 'scale=1:1:flags=area', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-'], + ], + { encoding: 'buffer' }, + ); + return [...stdout]; +} + +function payload(result: unknown): Record { + const content = (result as { content: { type: string; text?: string }[] }).content; + return JSON.parse(content.find((c) => c.type === 'text')?.text ?? '{}'); +} + +describe.skipIf(!IS_CI)('timeline colors against real FFmpeg', () => { + it('refuses a color that adds a filter, which FFmpeg would otherwise run', async () => { + const written = join(outside, 'written.txt'); + const hostile = `black,metadata=mode=print:file=${written}`; + + // Control: placed in pad= as the old compiler placed a background, it creates a file. + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error', '-f', 'lavfi', '-i', 'color=c=red:s=16x16'], + ...['-vf', `pad=16:16:0:0:color=${hostile},setsar=1`, '-frames:v', '1', '-f', 'null', '-'], + ]); + expect(await exists(written)).toBe(true); + await rm(written); + + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = buildMcpServer(); + await server.connect(serverTransport); + const client = new Client({ name: 'test', version: '1.0.0' }); + await client.connect(clientTransport); + try { + const refused = await client.callTool({ + name: 'timeline_edit', + arguments: { verbs: [{ verb: 'add_color', durationSec: 1, color: hostile }] }, + }); + expect((refused as { isError?: boolean }).isError).toBe(true); + + await client.callTool({ + name: 'timeline_edit', + arguments: { verbs: [{ verb: 'add_color', durationSec: 1, color: '#12ab34' }] }, + }); + // A stored document can still carry one (a hand edit, an older version). + await mutateComposition([{ op: 'setCanvas', width: 320, height: 240, background: hostile }]); + + const out = join(workspace, 'out.mp4'); + const blocked = payload( + await client.callTool({ name: 'timeline_export', arguments: { output: out } }), + ); + expect(String(blocked.error)).toMatch(/^Canvas background ".*" is not a color\./); + expect(await exists(out)).toBe(false); + expect(await exists(written)).toBe(false); + + // The valid colors still render: '#12ab34' reaches FFmpeg as 0x12ab34. + await mutateComposition([{ op: 'setCanvas', background: 'black' }]); + const exported = payload( + await client.callTool({ name: 'timeline_export', arguments: { output: out } }), + ); + expect(exported.error).toBeUndefined(); + const [r = 0, g = 0, b = 0] = await firstFrameRgb(out); + expect(Math.abs(r - 0x12)).toBeLessThan(12); + expect(Math.abs(g - 0xab)).toBeLessThan(12); + expect(Math.abs(b - 0x34)).toBeLessThan(12); + } finally { + await client.close(); + await server.close(); + } + }, 60_000); +}); diff --git a/tests/timeline-color.test.ts b/tests/timeline-color.test.ts new file mode 100644 index 0000000..785b410 --- /dev/null +++ b/tests/timeline-color.test.ts @@ -0,0 +1,211 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runTimeline } from '../src/cli-timeline.js'; +import { ColorSchema, isColor } from '../src/timeline/color.js'; +import { + type CompileContext, + CompileError, + checkExportable, + compileTimeline, +} from '../src/timeline/compile.js'; +import { type Composition, emptyComposition } from '../src/timeline/composition.js'; +import { readComposition } from '../src/timeline/document-store.js'; +import { applyOps, colorClip, mediaClip, videoTrack } from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; +import { CompositionVerbSchema, EditableVerbSchema } from '../src/timeline/verbs.js'; + +const M1 = 'm_aaaaaaaaaaaa' as MediaId; + +/** Values that would add filters to the graph if they reached FFmpeg unquoted. */ +const INJECTIONS = [ + 'red,drawtext=text=x', + 'black[p];movie=/etc/passwd[m];[p][m]overlay', + "black,metadata=mode=print:file=/tmp/x.txt:key='", + 'red:s=2x2', + 'red@0.5,null', +]; + +function ctx(): CompileContext { + return { + media: new Map([[M1, { path: '/ws/a.mp4', hasAudio: true }]]), + dir: '/ws', + output: '/ws/out.mp4', + }; +} + +/** A stored document holding one clip, as a hand edit or an older version could write it. */ +function docWith( + clip: ReturnType | ReturnType, + background = 'black', +): Composition { + return applyOps(emptyComposition({ background }), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { op: 'addClip', trackId: 'v0', clip }, + ]); +} + +const card = (color: string) => colorClip({ id: 'c1', color, durationSec: 2, startSec: 0 }); +const media = () => + mediaClip({ id: 'm1', mediaId: M1, sourceInSec: 0, sourceOutSec: 2, startSec: 0 }); + +function stepArgs(comp: Composition): string[] { + return compileTimeline(comp, ctx()).steps[0]?.args ?? []; +} + +describe('isColor', () => { + it('accepts FFmpeg color names in any case, hex, and an @alpha', () => { + for (const value of [ + 'black', + 'Red', + 'AliceBlue', + '#ffcc00', + '#FFCC0080', + '0xffcc00', + 'ffcc00', + 'black@0.6', + 'white@1', + 'red@.5', + '#ffcc00@0x80', + ]) { + expect(isColor(value), value).toBe(true); + } + }); + + it('refuses filter syntax, unknown names and malformed hex or alpha', () => { + for (const value of [ + ...INJECTIONS, + '', + ' red', + 'grey', // FFmpeg knows gray and lightgrey, not grey + 'notacolor', + '#fff', + '0XFFCC00', + 'red@1.5', + 'red@0x100', + 'red@', + 'red@@0.5', + 'rgb(255,0,0)', + ]) { + expect(isColor(value), value).toBe(false); + } + }); + + it('refuses a long malformed alpha in linear time', () => { + const start = performance.now(); + expect(isColor(`red@${'1'.repeat(100_000)}x`)).toBe(false); + expect(performance.now() - start).toBeLessThan(500); + }); + + it('gives the schema a message naming the value and the forms that work', () => { + const result = ColorSchema.safeParse('red,drawtext=text=x'); + expect(result.success).toBe(false); + expect(result.error?.issues[0]?.message).toBe( + '"red,drawtext=text=x" is not a color. Use a color name like black or white, or hex like #ffcc00, optionally with @0.5 for opacity.', + ); + }); +}); + +describe('add_color verb', () => { + it('refuses a color that is not a color, for MCP clients and the UI alike', () => { + for (const schema of [EditableVerbSchema, CompositionVerbSchema]) { + for (const color of INJECTIONS) { + const result = schema.safeParse({ verb: 'add_color', durationSec: 1, color }); + expect(result.success, color).toBe(false); + expect(result.error?.issues[0]?.path).toEqual(['color']); + } + } + }); + + it('keeps hex, names and the default', () => { + expect( + EditableVerbSchema.parse({ verb: 'add_color', durationSec: 1, color: '#12ab34' }), + ).toEqual({ verb: 'add_color', durationSec: 1, color: '#12ab34' }); + expect( + EditableVerbSchema.parse({ verb: 'add_color', durationSec: 1, color: 'navy@0.5' }), + ).toEqual({ verb: 'add_color', durationSec: 1, color: 'navy@0.5' }); + expect(EditableVerbSchema.parse({ verb: 'add_color', durationSec: 1 })).toEqual({ + verb: 'add_color', + durationSec: 1, + }); + }); +}); + +describe('compileTimeline colors', () => { + it('passes a valid card color and canvas background into the args, # as 0x', () => { + const args = stepArgs(docWith(card('#12ab34'), 'Navy@0.5')); + expect(args.slice(0, 5)).toEqual([ + '-y', + '-f', + 'lavfi', + '-i', + 'color=c=0x12ab34:s=1920x1080:r=30', + ]); + const graph = args[args.indexOf('-filter_complex') + 1] ?? ''; + expect(graph).toContain('pad=1920:1080:(ow-iw)/2:(oh-ih)/2:color=Navy@0.5,setsar=1'); + }); + + it('refuses a stored card color that is not a color', () => { + for (const color of INJECTIONS) { + expect(() => stepArgs(docWith(card(color))), color).toThrow(CompileError); + } + expect(() => stepArgs(docWith(card('red,drawtext=text=x')))).toThrow( + 'Color clip "c1" color "red,drawtext=text=x" is not a color.', + ); + }); + + it('refuses a stored canvas background that is not a color', () => { + for (const background of INJECTIONS) { + expect(() => stepArgs(docWith(media(), background)), background).toThrow(CompileError); + } + expect(() => stepArgs(docWith(media(), 'red,drawtext=text=x'))).toThrow( + 'Canvas background "red,drawtext=text=x" is not a color.', + ); + }); + + it('reports the bad color as an export blocker', () => { + const report = checkExportable(docWith(card('red,null')), ctx().media, '/ws'); + expect(report.exportable).toBe(false); + expect(report.blockers[0]).toMatch(/^Color clip "c1" color "red,null" is not a color\./); + }); +}); + +describe('clip timeline color flags', () => { + let workspace: string; + let saved: string | undefined; + + beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-color-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; + vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); + }); + + it('new --background takes a color and refuses anything else before replacing the doc', async () => { + await runTimeline(['new', '--background', '#102030']); + expect((await readComposition()).background).toBe('#102030'); + + await expect( + runTimeline(['new', '--background', 'black[p];movie=/etc/passwd[m];[p][m]overlay']), + ).rejects.toThrow('--background "black[p];movie=/etc/passwd[m];[p][m]overlay" is not a color.'); + expect((await readComposition()).background).toBe('#102030'); + }); + + it('add-color --color takes a color and refuses anything else without adding a clip', async () => { + await runTimeline(['new']); + await runTimeline(['add-color', '1', '--color', 'papayawhip']); + await expect(runTimeline(['add-color', '1', '--color', 'red,drawtext=text=x'])).rejects.toThrow( + '--color "red,drawtext=text=x" is not a color.', + ); + const clips = (await readComposition()).tracks.flatMap((t) => t.clips); + expect(clips.map((c) => (c.kind === 'color' ? c.color : c.kind))).toEqual(['papayawhip']); + }); +}); From df882f2b2667992be50a6de720a4b57769201134 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 14:58:41 -0800 Subject: [PATCH 02/35] feat(timeline): add named timelines and make_short A workspace now holds several timelines: main stays at composition.json, others live in timelines// with their own rev, lock and undo history. The make_short verb builds a vertical short on its own timeline from transcript word ranges of a source timeline: padded edges, cut from the source's clips so its silence and retake cuts carry in, a 9:16 cover canvas, -14 LUFS and captions, as one undoable edit. The CLI gains --timeline, `short`, `list` and `delete`; every MCP timeline tool takes an optional timeline id and timeline_show lists them. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/segment-duration-rounding.md | 3 + src/cli-timeline.ts | 294 ++++++++++--- src/index.ts | 20 + src/mcp/server.ts | 189 +++++---- src/timeline/document-store.ts | 321 ++++++++++---- src/timeline/schema.ts | 14 + src/timeline/shorts.ts | 341 +++++++++++++++ src/timeline/verbs.ts | 92 +++- src/ui/server.ts | 6 +- src/ui/timeline-tools.ts | 2 + tests/document-store.test.ts | 134 +++++- tests/integration/timeline-shorts.test.ts | 159 +++++++ tests/mcp-server.test.ts | 133 +++++- tests/timeline-captions.test.ts | 3 + tests/timeline-shorts.test.ts | 492 ++++++++++++++++++++++ tests/timeline-verbs.test.ts | 6 + 16 files changed, 2000 insertions(+), 209 deletions(-) create mode 100644 docs/backlog/segment-duration-rounding.md create mode 100644 src/timeline/shorts.ts create mode 100644 tests/integration/timeline-shorts.test.ts create mode 100644 tests/timeline-shorts.test.ts diff --git a/docs/backlog/segment-duration-rounding.md b/docs/backlog/segment-duration-rounding.md new file mode 100644 index 0000000..46e8c92 --- /dev/null +++ b/docs/backlog/segment-duration-rounding.md @@ -0,0 +1,3 @@ +# Segment duration rounding + +Export renders each clip as its own segment and joins them, and each segment comes out a little longer than its clip: the video rounds up to a whole frame and the AAC audio to a whole 1024-sample frame. A three-piece short planned at 16.200 s exported as 488 video frames (16.267 s at 30 fps) and 16.25 s of audio. Check whether the audio-video offset accumulates over a timeline with hundreds of cuts (a silence-cut hour), and if it does, trim each segment to a frame-aligned length or carry the remainder into the next segment. diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e623c1e..ceafd6e 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -9,6 +9,7 @@ import { import { AnchorSchema, type Clip, + type Composition, clipDuration, clipEndSec, clipsAtTime, @@ -24,11 +25,15 @@ import { type TextStyle, } from './timeline/composition.js'; import { + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, undoLastDocOp, } from './timeline/document-store.js'; import { @@ -49,6 +54,8 @@ import { } from './timeline/reframe.js'; import { reviewCuts } from './timeline/review.js'; import { runPlan } from './timeline/run-plan.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './timeline/schema.js'; +import { checkShortTimelines, planShort, type ShortRange } from './timeline/shorts.js'; import { analyzeSilence, DEFAULT_SENSITIVITY, SensitivitySchema } from './timeline/silence.js'; import { clipWords, @@ -63,6 +70,9 @@ import { getWorkspace, newOutputPath, resolveInput } from './workspace.js'; const TIMELINE_HELP = `clip timeline — build and export a non-destructive composition +Every command works on the main timeline unless --timeline names another +(a short made with \`short\` is a timeline of its own). + clip timeline new [--width W --height H --fps F --background C --fit contain|cover] clip timeline add-media [ ] [--start S] [--track T] clip timeline add-text [--start S] [--track T] [--font N --color C --anchor A --background C] @@ -81,6 +91,9 @@ const TIMELINE_HELP = `clip timeline — build and export a non-destructive comp clip timeline reframe <9:16|1:1|4:5|16:9|WxH> [--fit cover|contain] [--x 0..1] [--y 0..1] [--dry-run] clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset] clip timeline loudness <-14|-16|LUFS|off> + clip timeline short [ …] [--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] [--loudness LUFS|off] [--replace] [--dry-run] + clip timeline list + clip timeline delete clip timeline show clip timeline at clip timeline frame [] @@ -96,7 +109,7 @@ interface Parsed { } /** Flags that take no value, so they don't swallow the next argument. */ -const BOOLEAN_FLAGS = new Set(['dry-run', 'reset']); +const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace']); function parseArgs(args: string[]): Parsed { const positional: string[] = []; @@ -148,6 +161,36 @@ function framingFitFlag(value: string | undefined): FramingFit | undefined { return parsed.data; } +function timelineId(value: string): string { + const parsed = TimelineIdSchema.safeParse(value); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${value}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; +} + +function wordIndex(value: string | undefined): number { + const n = Number(value); + if (value === undefined || value.trim() === '' || !Number.isInteger(n) || n < 0) { + throw new Error(`A word index is a whole number from 0, got "${value}".`); + } + return n; +} + +/** `off`, or a loudness target within the export's range. */ +function loudnessValue(value: string, usage: string): number | null { + if (value === 'off') return null; + const parsed = TargetLufsSchema.safeParse(Number(value)); + if (!parsed.success) { + throw new Error( + `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, + ); + } + return parsed.data; +} + /** A short, human-readable label for a clip (for `show` / `at`). */ function clipLabel(clip: Clip): string { switch (clip.kind) { @@ -161,8 +204,10 @@ function clipLabel(clip: Clip): string { } export async function runTimeline(args: string[]): Promise { - const [sub, ...rest] = args; - const { positional, flags } = parseArgs(rest); + const parsed = parseArgs(args); + const [sub, ...positional] = parsed.positional; + const { flags } = parsed; + const timeline = flags.timeline !== undefined ? timelineId(flags.timeline) : MAIN_TIMELINE; switch (sub) { case undefined: @@ -180,8 +225,16 @@ export async function runTimeline(args: string[]): Promise { background: flags.background, fit: fitFlag(flags.fit), }), + timeline, ); - out({ created: true, width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }); + out({ + created: true, + timeline, + width: comp.width, + height: comp.height, + fps: comp.fps, + fit: comp.fit, + }); return; } @@ -197,7 +250,7 @@ export async function runTimeline(args: string[]): Promise { }); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = mediaClip({ id: makeClipId(), @@ -206,7 +259,9 @@ export async function runTimeline(args: string[]): Promise { sourceOutSec: num(outSec, result.ref.durationSec), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'media', clipId: clip.id, mediaId: result.mediaId, trackId, startSec: start }); return; } @@ -217,7 +272,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error('Usage: clip timeline add-text '); } const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const style: Record = {}; if (flags.font) style.fontSize = num(flags.font, 48); @@ -231,7 +286,9 @@ export async function runTimeline(args: string[]): Promise { startSec: start, style, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'text', clipId: clip.id, trackId, startSec: start }); return; } @@ -240,7 +297,7 @@ export async function runTimeline(args: string[]): Promise { const [duration] = positional; if (duration === undefined) throw new Error('Usage: clip timeline add-color '); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = colorClip({ id: makeClipId(), @@ -248,7 +305,9 @@ export async function runTimeline(args: string[]): Promise { durationSec: num(duration, 2), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'color', clipId: clip.id, trackId, startSec: start }); return; } @@ -258,17 +317,20 @@ export async function runTimeline(args: string[]): Promise { if (!afterClipId) throw new Error('Usage: clip timeline transition [] []'); const trackId = flags.track || DEFAULT_TRACK; - await mutateComposition([ - { - op: 'addTransition', - trackId, - transition: { - afterClipId, - kind: (kind as never) ?? 'fade', - durationSec: num(duration, 1), + await mutateComposition( + [ + { + op: 'addTransition', + trackId, + transition: { + afterClipId, + kind: (kind as never) ?? 'fade', + durationSec: num(duration, 1), + }, }, - }, - ]); + ], + { timeline }, + ); out({ added: 'transition', afterClipId, kind: kind ?? 'fade', trackId }); return; } @@ -278,9 +340,10 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || inSec === undefined || outSec === undefined) { throw new Error('Usage: clip timeline trim '); } - await mutateComposition([ - { op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }, - ]); + await mutateComposition( + [{ op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }], + { timeline }, + ); out({ trimmed: clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }); return; } @@ -289,9 +352,17 @@ export async function runTimeline(args: string[]): Promise { const [clipId, startSec] = positional; if (!clipId || startSec === undefined) throw new Error('Usage: clip timeline move '); - await mutateComposition([ - { op: 'moveClip', clipId, startSec: num(startSec, 0), toTrackId: flags.track || undefined }, - ]); + await mutateComposition( + [ + { + op: 'moveClip', + clipId, + startSec: num(startSec, 0), + toTrackId: flags.track || undefined, + }, + ], + { timeline }, + ); out({ moved: clipId, startSec: num(startSec, 0), trackId: flags.track || undefined }); return; } @@ -301,7 +372,9 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || atSec === undefined) throw new Error('Usage: clip timeline split '); const newClipId = makeClipId(); - await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }]); + await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }], { + timeline, + }); out({ split: clipId, into: [clipId, newClipId], atSec: num(atSec, 0) }); return; } @@ -309,7 +382,7 @@ export async function runTimeline(args: string[]): Promise { case 'remove': { const [clipId] = positional; if (!clipId) throw new Error('Usage: clip timeline remove '); - await mutateComposition([{ op: 'removeClip', clipId }]); + await mutateComposition([{ op: 'removeClip', clipId }], { timeline }); out({ removed: clipId }); return; } @@ -324,7 +397,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error(`--sensitivity must be between 0 and 1, got "${flags.sensitivity}".`); } const sensitivity = parsed.data; - const comp = await readComposition(); + const comp = await readComposition(timeline); const cut = await analyzeSilence(comp, clipId, { media: await buildMediaMap(), sensitivity }); const report = { clipId, @@ -342,7 +415,7 @@ export async function runTimeline(args: string[]): Promise { return; } // Analysis read `comp` outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, clipIds: [clipId, ...cut.op.newClipIds], rev: doc.rev }); return; } @@ -359,7 +432,7 @@ export async function runTimeline(args: string[]): Promise { if (!aspect.success && !size) { throw new Error(`Unknown shape "${shape}": use 9:16, 1:1, 4:5, 16:9 or WIDTHxHEIGHT.`); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const plan = planReframe(comp, { ...(aspect.success ? { aspect: aspect.data } @@ -373,7 +446,7 @@ export async function runTimeline(args: string[]): Promise { out({ ...report, applied: false, ...(flags['dry-run'] ? { dryRun: true } : {}) }); return; } - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } @@ -385,7 +458,7 @@ export async function runTimeline(args: string[]): Promise { 'Usage: clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const ops = planFraming(comp, { clipId, fit: framingFitFlag(flags.fit), @@ -398,7 +471,7 @@ export async function runTimeline(args: string[]): Promise { out({ clipId, applied: false, message: 'Nothing to change.' }); return; } - const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev, timeline }); const framed = findClip(doc, clipId); out({ clipId, @@ -413,30 +486,122 @@ export async function runTimeline(args: string[]): Promise { const [value] = positional; const usage = 'Usage: clip timeline loudness '; if (!value) throw new Error(usage); - let targetLufs: number | null = null; - if (value !== 'off') { - const parsed = TargetLufsSchema.safeParse(Number(value)); - if (!parsed.success) { - throw new Error( - `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, - ); - } - targetLufs = parsed.data; - } - const comp = await readComposition(); + const targetLufs = loudnessValue(value, usage); + const comp = await readComposition(timeline); if ((comp.loudness?.targetLufs ?? null) === targetLufs) { out({ loudness: comp.loudness, applied: false }); return; } const doc = await mutateComposition([{ op: 'setLoudness', targetLufs }], { expectedBaseRev: comp.rev, + timeline, }); out({ loudness: doc.loudness, applied: true, rev: doc.rev }); return; } + case 'short': { + const [id, ...rangeArgs] = positional; + const usage = + 'Usage: clip timeline short [ …] ' + + '[--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] ' + + '[--loudness LUFS|off] [--replace] [--dry-run]'; + if (!id || rangeArgs.length === 0 || rangeArgs.length % 3 !== 0) throw new Error(usage); + const target = timelineId(id); + if (flags.timeline !== undefined && timeline !== target) { + throw new Error( + `\`short\` builds the timeline named by its first argument ("${target}"), not --timeline. Use --source for the timeline to cut from.`, + ); + } + const sourceId = flags.source !== undefined ? timelineId(flags.source) : MAIN_TIMELINE; + checkShortTimelines(target, sourceId); + const ranges: ShortRange[] = []; + for (let i = 0; i < rangeArgs.length; i += 3) { + ranges.push({ + clipId: rangeArgs[i] ?? '', + fromIndex: wordIndex(rangeArgs[i + 1]), + toIndex: wordIndex(rangeArgs[i + 2]), + }); + } + const aspect = flags.aspect === undefined ? undefined : AspectSchema.safeParse(flags.aspect); + if (aspect && !aspect.success) { + throw new Error(`--aspect must be 9:16, 1:1, 4:5 or 16:9, got "${flags.aspect}".`); + } + const maxWords = num(flags.words, DEFAULT_CAPTION_WORDS); + if (!Number.isInteger(maxWords) || maxWords < 1 || maxWords > 12) { + throw new CaptionError( + `--words must be a whole number from 1 to 12, got "${flags.words}".`, + ); + } + + let current: Composition; + try { + current = await readComposition(target); + } catch (err) { + if (!(err instanceof NoTimelineError)) throw err; + current = emptyComposition(); + } + const media = await buildMediaMap(); + const plan = await planShort( + current, + await readComposition(sourceId), + { + ranges, + aspect: aspect?.data, + x: optionalNum('x', flags.x), + y: optionalNum('y', flags.y), + captions: flags['no-captions'] !== 'true', + maxWords, + targetLufs: + flags.loudness === undefined ? undefined : loudnessValue(flags.loudness, usage), + replace: flags.replace === 'true', + }, + (comp, clipId, transcriptId) => storedTranscriptForClip(comp, clipId, media, transcriptId), + ); + const report = { + timeline: target, + source: sourceId, + durationSec: plan.durationSec, + pieces: plan.pieces, + captions: plan.captions, + warnings: plan.warnings, + }; + if (flags['dry-run']) { + out({ ...report, applied: false, dryRun: true }); + return; + } + // The source and transcripts were read outside the write lock; reject if + // the short's timeline moved since. + const doc = await mutateComposition(plan.ops, { + expectedBaseRev: current.rev, + timeline: target, + create: true, + }); + out({ + ...report, + applied: true, + canvas: { width: doc.width, height: doc.height, fit: doc.fit }, + loudness: doc.loudness, + rev: doc.rev, + }); + return; + } + + case 'list': + out({ timelines: await listTimelines() }); + return; + + case 'delete': { + const [id] = positional; + if (!id) throw new Error('Usage: clip timeline delete '); + const target = timelineId(id); + await deleteTimeline(target); + out({ deleted: target }); + return; + } + case 'check': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const sheetPath = flags.sheet ? resolveInput(flags.sheet) : newOutputPath('cut-sheet', 'jpg'); out(await reviewCuts(comp, media, sheetPath, getWorkspace())); @@ -450,7 +615,7 @@ export async function runTimeline(args: string[]): Promise { 'Usage: clip timeline transcribe [--model ] [--language ]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); out( await transcribeClip(comp, clipId, await buildMediaMap(), { model: flags.model ? resolveInput(flags.model) : undefined, @@ -471,7 +636,7 @@ export async function runTimeline(args: string[]): Promise { : 'Usage: clip timeline remove-retakes [--dry-run]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const { clip, transcript } = await storedTranscriptForClip( comp, @@ -492,7 +657,7 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcript was read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([op], { expectedBaseRev: comp.rev, timeline }); out({ clipId, applied: true, ranges, rev: doc.rev }); return; } @@ -511,7 +676,7 @@ export async function runTimeline(args: string[]): Promise { if (flags.background) style.background = flags.background === 'none' ? null : flags.background; if (flags.anchor) style.anchor = AnchorSchema.parse(flags.anchor); - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const plan = await planCaptions( comp, @@ -530,15 +695,16 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcripts were read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } case 'show': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const { exportable, blockers } = checkExportable(comp, await buildMediaMap(), getWorkspace()); out({ + timeline, rev: comp.rev, durationSec: compositionDuration(comp), canvas: { width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }, @@ -575,7 +741,7 @@ export async function runTimeline(args: string[]): Promise { const [at] = positional; if (!at) throw new Error('Usage: clip timeline at '); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); out({ atSec, clips: clipsAtTime(comp, atSec).map((h) => ({ @@ -593,7 +759,7 @@ export async function runTimeline(args: string[]): Promise { const [at, output] = positional; if (!at) throw new Error('Usage: clip timeline frame []'); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-frame', 'jpg'); const plan = buildFrameAtPlan( @@ -607,7 +773,7 @@ export async function runTimeline(args: string[]): Promise { } case 'undo': { - const { undone, doc, label } = await undoLastDocOp(); + const { undone, doc, label } = await undoLastDocOp(timeline); out( undone ? { undone: true, label, rev: doc.rev } @@ -617,7 +783,7 @@ export async function runTimeline(args: string[]): Promise { } case 'redo': { - const { redone, doc, label } = await redoDocOp(); + const { redone, doc, label } = await redoDocOp(timeline); out( redone ? { redone: true, label, rev: doc.rev } @@ -627,7 +793,7 @@ export async function runTimeline(args: string[]): Promise { } case 'log': { - const log = await readDocOpLog(); + const log = await readDocOpLog(timeline); out({ rev: log.rev, cursor: log.cursor, @@ -653,10 +819,14 @@ export async function runTimeline(args: string[]): Promise { const target = output ? resolveInput(output) : newOutputPath('timeline-export', interchangeExtension(parsed.data)); - const result = await exportInterchange(await readComposition(), await buildMediaRefs(), { - format: parsed.data, - output: target, - }); + const result = await exportInterchange( + await readComposition(timeline), + await buildMediaRefs(), + { + format: parsed.data, + output: target, + }, + ); out({ exported: target, format: result.format, @@ -666,7 +836,7 @@ export async function runTimeline(args: string[]): Promise { }); return; } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-export', 'mp4'); const plan = compileTimeline(comp, { media, dir: getWorkspace(), output: finalOutput }); diff --git a/src/index.ts b/src/index.ts index 76eb806..30de026 100644 --- a/src/index.ts +++ b/src/index.ts @@ -165,12 +165,17 @@ export { CompositionCorruptError, compositionOpsPath, compositionPath, + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, overwriteComposition, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, + type TimelineListing, undoLastDocOp, writeComposition, writeCompositionIfUnchanged, @@ -225,6 +230,7 @@ export { export { type LoudnessResult, type RunPlanResult, runPlan } from './timeline/run-plan.js'; export { AudioStreamSchema, + MAIN_TIMELINE, type MediaId, MediaIdSchema, type MediaRef, @@ -232,8 +238,22 @@ export { makeMediaId, type Timecode, TimecodeSchema, + TimelineIdSchema, VideoStreamSchema, } from './timeline/schema.js'; +export { + checkShortTimelines, + DEFAULT_SHORT_LUFS, + planShort, + SHORT_LEAD_SEC, + SHORT_TAIL_SEC, + ShortError, + type ShortPiece, + type ShortPlan, + type ShortRange, + type ShortRequest, + type ShortTranscriptLookup, +} from './timeline/shorts.js'; export { analyzeSilence, DEFAULT_SENSITIVITY, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 7416f5d..db573d5 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -15,6 +15,8 @@ import { import type { Composition } from '../timeline/composition.js'; import { applyVerbs, + listTimelines, + NoTimelineError, readComposition, redoDocOp, undoLastDocOp, @@ -28,6 +30,8 @@ import { import { buildMediaMap } from '../timeline/media-registry.js'; import { reviewCuts } from '../timeline/review.js'; import { runPlan } from '../timeline/run-plan.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; import { transcribeClip } from '../timeline/transcript.js'; import { EditableVerbSchema } from '../timeline/verbs.js'; import { doctor } from '../tools/doctor.js'; @@ -98,12 +102,30 @@ async function jpegContent( */ async function describeComposition( comp: Composition, -): Promise { + timeline: string, +): Promise { const media = await buildMediaMap(); const { exportable, blockers } = checkExportable(comp, media, getWorkspace()); - return { ...summarizeComposition(comp), exportable, blockers }; + return { timeline, ...summarizeComposition(comp), exportable, blockers }; } +/** Answer a missing timeline or a short that can't be made with `{ error }`: + * both are the agent's to fix, like an export it can't render. */ +async function answeringTimelineErrors( + run: () => Promise, +): Promise> { + try { + return await run(); + } catch (err) { + if (err instanceof NoTimelineError || err instanceof ShortError) { + return jsonResult({ error: err.message }); + } + throw err; + } +} + +const timelineInput = TimelineIdSchema.optional(); + /** * Build the MakeMyClip Editor MCP server — a second front door onto the SAME * op-aware, undoable CompositionDoc the CLI, the `clip ui`, and the Claude Code @@ -119,8 +141,9 @@ export function buildMcpServer(): McpServer { { title: 'Show the timeline', description: - 'Read the current timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', + 'Read a timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers) — plus `timelines`, the ids of every timeline in the workspace (main and each short). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', inputSchema: { + timeline: timelineInput, atSec: z .number() .nonnegative() @@ -130,35 +153,37 @@ export function buildMcpServer(): McpServer { ), }, }, - async ({ atSec }) => { - const comp = await readComposition(); - const summary = jsonResult(await describeComposition(comp)); - if (atSec === undefined) return summary; - const output = newOutputPath('mcp-frame', 'jpg'); - try { - const plan = buildFrameAtPlan( - comp, - { - media: await confinedMediaMap(comp), - dir: getWorkspace(), - output, - tag: `mcp${randomBytes(4).toString('hex')}`, - }, - atSec, - ); - await runPlan(plan); - return { content: [...summary.content, await jpegContent(output)] }; - } catch (err) { - if (!(err instanceof CompileError)) throw err; - const frameError = { - type: 'text' as const, - text: JSON.stringify({ frameError: err.message }), - }; - return { content: [...summary.content, frameError] }; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE, atSec }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const timelines = (await listTimelines()).map((t) => t.id); + const summary = jsonResult({ ...(await describeComposition(comp, timeline)), timelines }); + if (atSec === undefined) return summary; + const output = newOutputPath('mcp-frame', 'jpg'); + try { + const plan = buildFrameAtPlan( + comp, + { + media: await confinedMediaMap(comp), + dir: getWorkspace(), + output, + tag: `mcp${randomBytes(4).toString('hex')}`, + }, + atSec, + ); + await runPlan(plan); + return { content: [...summary.content, await jpegContent(output)] }; + } catch (err) { + if (!(err instanceof CompileError)) throw err; + const frameError = { + type: 'text' as const, + text: JSON.stringify({ frameError: err.message }), + }; + return { content: [...summary.content, frameError] }; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -167,22 +192,24 @@ export function buildMcpServer(): McpServer { title: 'Check the cuts', description: "Review every hard cut (abutting clips with no transition). Reports cuts whose audio is loud right at the edge, which usually means a clipped word or a click, and returns a contact sheet image: one row per cut, the outgoing clip's last frame then the incoming clip's first. Run it after cutting (e.g. remove_silence) and before exporting.", + inputSchema: { timeline: timelineInput }, }, - async () => { - const comp = await readComposition(); - const media = await confinedMediaMap(comp); - const output = newOutputPath('cut-sheet', 'jpg'); - try { - const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); - const summary = jsonResult({ - ...review, - sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, - }); - return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const media = await confinedMediaMap(comp); + const output = newOutputPath('cut-sheet', 'jpg'); + try { + const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); + const summary = jsonResult({ + ...review, + sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, + }); + return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -190,13 +217,19 @@ export function buildMcpServer(): McpServer { { title: 'Edit the timeline', description: - 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', - inputSchema: { verbs: z.array(EditableVerbSchema).min(1) }, - }, - async ({ verbs }) => { - const { doc, ops } = await applyVerbs(verbs, makeVerbContext()); - return jsonResult({ applied: ops.length, document: await describeComposition(doc) }); + 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). ' + + 'make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as "best-1" (never main). It plays the given word ranges (indices from timeline_transcript on the source timeline, default main) with a little padding, cut from the source\'s clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical), look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. ' + + 'Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', + inputSchema: { timeline: timelineInput, verbs: z.array(EditableVerbSchema).min(1) }, }, + async ({ timeline = MAIN_TIMELINE, verbs }) => + answeringTimelineErrors(async () => { + const { doc, ops } = await applyVerbs(verbs, makeVerbContext(), timeline); + return jsonResult({ + applied: ops.length, + document: await describeComposition(doc, timeline), + }); + }), ); server.registerTool( @@ -206,34 +239,46 @@ export function buildMcpServer(): McpServer { description: "Transcribe a media clip's speech with the local whisper.cpp and return its words with timeline times. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript. A word's `index` is stable for its source file.", inputSchema: { + timeline: timelineInput, clipId: z.string().describe('Media clip to transcribe.'), language: WhisperLanguageSchema.optional(), }, }, - async ({ clipId, language }) => { - const comp = await readComposition(); - return jsonResult( - await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), - ); - }, + async ({ timeline = MAIN_TIMELINE, clipId, language }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + return jsonResult( + await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), + ); + }), ); server.registerTool( 'timeline_undo', - { title: 'Undo', description: 'Undo the most recent timeline edit.' }, - async () => { - const { undone, label } = await undoLastDocOp(); - return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + { + title: 'Undo', + description: 'Undo the most recent edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { undone, label } = await undoLastDocOp(timeline); + return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + }), ); server.registerTool( 'timeline_redo', - { title: 'Redo', description: 'Redo the most recently undone timeline edit.' }, - async () => { - const { redone, label } = await redoDocOp(); - return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + { + title: 'Redo', + description: 'Redo the most recently undone edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { redone, label } = await redoDocOp(timeline); + return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + }), ); server.registerTool( @@ -241,10 +286,11 @@ export function buildMcpServer(): McpServer { { title: 'Export the timeline', description: - 'Export the current timeline document to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + + 'Export a timeline (default main; pass `timeline` for a short) to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + 'fcpxml writes an FCPXML project for Final Cut Pro and DaVinci Resolve; xmeml writes Final Cut Pro 7 XML for Premiere Pro (Resolve reads it too), with text as .srt caption files beside it. ' + 'Both reference the source media by absolute path, refuse what the editor file cannot hold with the fix to make, and return notes on what is carried only in part.', inputSchema: { + timeline: timelineInput, format: z .enum(['mp4', 'fcpxml', 'xmeml']) .optional() @@ -255,9 +301,9 @@ export function buildMcpServer(): McpServer { .describe('Output file inside the workspace (relative paths resolve against it).'), }, }, - async ({ format = 'mp4', output }) => { - const comp = await readComposition(); + async ({ timeline = MAIN_TIMELINE, format = 'mp4', output }) => { try { + const comp = await readComposition(timeline); if (format !== 'mp4') { const target = output ? resolveInWorkspace(output) @@ -290,7 +336,8 @@ export function buildMcpServer(): McpServer { if ( err instanceof CompileError || err instanceof InterchangeError || - err instanceof WorkspaceBoundaryError + err instanceof WorkspaceBoundaryError || + err instanceof NoTimelineError ) { return jsonResult({ error: err.message }); } diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index d3f71c1..2a16dc1 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -1,10 +1,20 @@ import { randomBytes } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { readdir, readFile, rm } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; import { atomicWriteFile } from '../session/store.js'; -import { createRevisionedStore } from '../storage/revisioned-store.js'; -import { ensureWorkspace, getWorkspace } from '../workspace.js'; -import { type Composition, CompositionSchema, emptyComposition } from './composition.js'; +import { createRevisionedStore, type RevisionedStore } from '../storage/revisioned-store.js'; +import { + ensureWorkspace, + getWorkspace, + resolveInWorkspace, + WorkspaceBoundaryError, +} from '../workspace.js'; +import { + type Composition, + CompositionSchema, + compositionDuration, + emptyComposition, +} from './composition.js'; import { type DocOpLog, type DocOpLogEntry, @@ -17,17 +27,67 @@ import { undo as undoLog, } from './doc-op-log.js'; import { applyOps, applyOpsTracked, type CompositionOp } from './ops.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './schema.js'; import { type CompositionVerb, lowerVerbs, type VerbContext } from './verbs.js'; +/* + * A workspace holds several named timelines. `main` stays at the workspace root + * (`composition.json` + `composition-ops.json`), so existing workspaces and older + * `clip` versions keep working; every other timeline keeps the same two files in + * `timelines//`. Each timeline has its own rev, write lock and undo history. + */ const COMPOSITION_FILE = 'composition.json'; const COMPOSITION_OPS_FILE = 'composition-ops.json'; +const TIMELINES_DIR = 'timelines'; + +/** Thrown for a malformed timeline id, or for an operation `main` can't take. */ +export class TimelineIdError extends Error { + constructor(message: string) { + super(message); + this.name = 'TimelineIdError'; + } +} + +/** Thrown when a timeline other than `main` doesn't exist. Only `make_short` and + * `clip timeline new --timeline` create one, so a typo can't edit an empty document. */ +export class NoTimelineError extends Error { + readonly timeline: string; + constructor(timeline: string) { + super( + `No timeline "${timeline}" in the workspace. List them with \`clip timeline list\` ` + + `(timeline_show reports them too). make_short creates a short's timeline.`, + ); + this.name = 'NoTimelineError'; + this.timeline = timeline; + } +} -export function compositionPath(): string { - return resolve(getWorkspace(), COMPOSITION_FILE); +function checkedTimelineId(timeline: string): string { + const parsed = TimelineIdSchema.safeParse(timeline); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${timeline}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; } -export function compositionOpsPath(): string { - return resolve(getWorkspace(), COMPOSITION_OPS_FILE); +function timelineFile(timeline: string, file: string): string { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) return resolve(getWorkspace(), file); + // Checked against the workspace: `timelines/` could be a symlink out of it. + return resolveInWorkspace(join(TIMELINES_DIR, timeline, file)); +} + +export function compositionPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_FILE); +} + +export function compositionOpsPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_OPS_FILE); +} + +function isMissingFile(err: unknown): boolean { + return err instanceof Error && 'code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'); } /** Thrown when composition.json exists but cannot be parsed/validated — surfaced, @@ -65,31 +125,52 @@ export class CompositionConflictError extends Error { } } -export async function readComposition(): Promise { +/** The stored document, or null when the timeline has none yet. */ +async function readStoredComposition(timeline: string): Promise { await ensureWorkspace(); + const path = compositionPath(timeline); let raw: string; try { - raw = await readFile(compositionPath(), 'utf-8'); + raw = await readFile(path, 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyComposition(); + if (isMissingFile(err)) return null; throw err; } try { return CompositionSchema.parse(JSON.parse(raw)); } catch (err) { - throw new CompositionCorruptError(compositionPath(), { cause: err }); + throw new CompositionCorruptError(path, { cause: err }); } } -export async function writeComposition(comp: Composition): Promise { - await atomicWriteFile(compositionPath(), `${JSON.stringify(comp, null, 2)}\n`); +/** The document an edit starts from. A missing `main` is an empty document (a + * fresh workspace); any other missing timeline is an error unless the edit + * `create`s it. */ +async function readForEdit(timeline: string, create: boolean): Promise { + const comp = await readStoredComposition(timeline); + if (comp) return comp; + if (create || timeline === MAIN_TIMELINE) return emptyComposition(); + throw new NoTimelineError(timeline); +} + +/** Read a timeline's document (default `main`). Throws `NoTimelineError` for a + * timeline other than `main` that doesn't exist. */ +export async function readComposition(timeline: string = MAIN_TIMELINE): Promise { + return readForEdit(timeline, false); } -/** On-disk rev, treating a corrupt file as rev 0 — a deliberate overwrite is - * about to discard it anyway. Non-corruption read errors still propagate. */ -async function readCompositionRevTolerant(): Promise { +export async function writeComposition( + comp: Composition, + timeline: string = MAIN_TIMELINE, +): Promise { + await atomicWriteFile(compositionPath(timeline), `${JSON.stringify(comp, null, 2)}\n`); +} + +/** On-disk rev, treating a corrupt or missing file as rev 0 — a deliberate + * overwrite is about to replace it anyway. Other read errors still propagate. */ +async function readCompositionRevTolerant(timeline: string): Promise { try { - return (await readComposition()).rev; + return (await readStoredComposition(timeline))?.rev ?? 0; } catch (err) { if (err instanceof CompositionCorruptError) return 0; throw err; @@ -97,16 +178,26 @@ async function readCompositionRevTolerant(): Promise { } // Serialization + compare-and-swap, shared with the session store (see -// `storage/revisioned-store.ts`) so co-editing the doc (the agent + `clip ui`) -// can't silently lose updates. -const docStore = createRevisionedStore({ - read: readComposition, - write: writeComposition, - getRev: (comp) => comp.rev, - withRev: (comp, rev) => ({ ...comp, rev }), - readRevTolerant: readCompositionRevTolerant, - onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), -}); +// `storage/revisioned-store.ts`) so co-editing a timeline (the agent + `clip ui`) +// can't silently lose updates. One store per timeline, so each has its own lock. +const stores = new Map>(); + +function storeFor(timeline: string): RevisionedStore { + const id = checkedTimelineId(timeline); + let store = stores.get(id); + if (!store) { + store = createRevisionedStore({ + read: () => readComposition(id), + write: (comp) => writeComposition(comp, id), + getRev: (comp) => comp.rev, + withRev: (comp, rev) => ({ ...comp, rev }), + readRevTolerant: () => readCompositionRevTolerant(id), + onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), + }); + stores.set(id, store); + } + return store; +} // ─── op-log (undo/redo) ────────────────────────────────────────────────────── // @@ -130,13 +221,13 @@ function labelForOps(ops: CompositionOp[]): string { /** Read the raw op-log file. Missing OR corrupt → an empty log: undo history is a * convenience, never worth failing a read for. The caller reconciles it against * the authoritative document rev. */ -async function readDocOpLogFile(): Promise { +async function readDocOpLogFile(timeline: string): Promise { await ensureWorkspace(); let raw: string; try { - raw = await readFile(compositionOpsPath(), 'utf-8'); + raw = await readFile(compositionOpsPath(timeline), 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyDocOpLog(0); + if (isMissingFile(err)) return emptyDocOpLog(0); throw err; } try { @@ -146,8 +237,8 @@ async function readDocOpLogFile(): Promise { } } -async function writeDocOpLog(log: DocOpLog): Promise { - await atomicWriteFile(compositionOpsPath(), serializeDocOpLog(log)); +async function writeDocOpLog(log: DocOpLog, timeline: string): Promise { + await atomicWriteFile(compositionOpsPath(timeline), serializeDocOpLog(log)); } /** The op-log reconciled against the current document rev — the trustworthy view @@ -155,10 +246,10 @@ async function writeDocOpLog(log: DocOpLog): Promise { * inside the store's exclusive section so an interleaved IN-PROCESS commit can't * make a consistent log momentarily read as empty (cross-process reads stay * best-effort, as everywhere in this store). */ -export async function readDocOpLog(): Promise { - return docStore.runExclusive(async () => { - const docRev = (await readComposition()).rev; - return reconcile(await readDocOpLogFile(), docRev); +export async function readDocOpLog(timeline: string = MAIN_TIMELINE): Promise { + return storeFor(timeline).runExclusive(async () => { + const docRev = (await readComposition(timeline)).rev; + return reconcile(await readDocOpLogFile(timeline), docRev); }); } @@ -176,19 +267,21 @@ export async function readDocOpLog(): Promise { * shipping surface). */ async function commitDocAndLog( + timeline: string, + create: boolean, derive: ( doc: Composition, log: DocOpLog, ) => { doc: Composition; log: DocOpLog; result: T } | null, ): Promise<{ doc: Composition; result: T } | null> { - return docStore.runExclusive(async () => { + return storeFor(timeline).runExclusive(async () => { for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const log = reconcile(await readDocOpLogFile(), current.rev); + const current = await readForEdit(timeline, create); + const log = reconcile(await readDocOpLogFile(timeline), current.rev); const derived = derive(current, log); if (derived === null) return null; - const observed = (await readComposition()).rev; + const observed = (await readForEdit(timeline, create)).rev; if (observed !== current.rev) { if (attempt === MAX_COMMIT_ATTEMPTS) { throw new CompositionConflictError(current.rev, observed); @@ -199,8 +292,8 @@ async function commitDocAndLog( const nextRev = current.rev + 1; const committedDoc = { ...derived.doc, rev: nextRev }; const committedLog = { ...derived.log, rev: nextRev }; - await writeDocOpLog(committedLog); - await writeComposition(committedDoc); + await writeDocOpLog(committedLog, timeline); + await writeComposition(committedDoc, timeline); return { doc: committedDoc, result: derived.result }; } throw new CompositionConflictError(-1, -1); @@ -211,14 +304,18 @@ async function commitDocAndLog( * Read → apply ops → write, returning the new composition, and record the batch * (with its computed inverse) to the op-log so the edit is undoable. Serialized + * rev compare-and-swap via the shared revisioned store, so the agent and the UI - * can co-edit without lost updates. An empty batch is a true no-op. + * can co-edit without lost updates. An empty batch is a true no-op. `timeline` + * defaults to `main`; `create` lets the edit start a timeline that doesn't exist + * yet (from an empty document) instead of throwing `NoTimelineError`. */ export async function mutateComposition( ops: CompositionOp[], - opts?: { expectedBaseRev?: number }, + opts?: { expectedBaseRev?: number; timeline?: string; create?: boolean }, ): Promise { - if (ops.length === 0) return readComposition(); - const committed = await commitDocAndLog((current, log) => { + const timeline = opts?.timeline ?? MAIN_TIMELINE; + const create = opts?.create ?? false; + if (ops.length === 0) return readForEdit(timeline, create); + const committed = await commitDocAndLog(timeline, create, (current, log) => { // Optimistic-concurrency guard for callers whose ops were lowered against a // specific revision (e.g. `applyVerbs`, where a default append point is baked // from a snapshot read): if the doc moved under us, reject so the caller can @@ -235,7 +332,7 @@ export async function mutateComposition( }; return { doc, log: recordOps(log, entry), result: null }; }); - return committed ? committed.doc : readComposition(); + return committed ? committed.doc : readForEdit(timeline, create); } /** @@ -243,12 +340,16 @@ export async function mutateComposition( * op-aware mutation path the agent and `clip ui` share. Lowering (which ingests * files and mints ids) runs outside the write lock; `mutateComposition` then * validates, applies, and records the ops (so the edit lands in the undo stack). - * Returns the new document and the ops that were applied. + * Returns the new document and the ops that were applied. Edits `timeline` + * (default `main`), which must exist unless the batch starts with `make_short`. */ export async function applyVerbs( verbs: CompositionVerb[], ctx: VerbContext, + timeline: string = MAIN_TIMELINE, ): Promise<{ doc: Composition; ops: CompositionOp[] }> { + const create = verbs[0]?.verb === 'make_short'; + const editCtx: VerbContext = { ...ctx, timeline }; // The default append point (`trackEnd`) is baked from the snapshot we lower // against, so if another in-process writer commits between the read and the // apply, re-lower against the fresh doc rather than land overlapping clips. A @@ -257,10 +358,14 @@ export async function applyVerbs( // a burst of N concurrent edits each re-lower and land instead of the (N-cap) // tail silently losing to a `CompositionConflictError`. for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const ops = await lowerVerbs(current, verbs, ctx); + const current = await readForEdit(timeline, create); + const ops = await lowerVerbs(current, verbs, editCtx); try { - const doc = await mutateComposition(ops, { expectedBaseRev: current.rev }); + const doc = await mutateComposition(ops, { + expectedBaseRev: current.rev, + timeline, + create, + }); return { doc, ops }; } catch (err) { if (err instanceof CompositionConflictError && attempt < MAX_COMMIT_ATTEMPTS) continue; @@ -273,33 +378,33 @@ export async function applyVerbs( /** Undo the most recent recorded edit (apply its inverse), moving the op-log * cursor back. `undone: false` when there is nothing to undo. */ -export async function undoLastDocOp(): Promise<{ +export async function undoLastDocOp(timeline: string = MAIN_TIMELINE): Promise<{ undone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = undoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.inverse), log: step.log, result: step.entry.label }; }); - if (!committed) return { undone: false, doc: await readComposition(), label: null }; + if (!committed) return { undone: false, doc: await readComposition(timeline), label: null }; return { undone: true, doc: committed.doc, label: committed.result }; } /** Redo the most recently undone edit (re-apply its forward ops), moving the * cursor forward. `redone: false` when there is nothing to redo. */ -export async function redoDocOp(): Promise<{ +export async function redoDocOp(timeline: string = MAIN_TIMELINE): Promise<{ redone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = redoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.forward), log: step.log, result: step.entry.label }; }); - if (!committed) return { redone: false, doc: await readComposition(), label: null }; + if (!committed) return { redone: false, doc: await readComposition(timeline), label: null }; return { redone: true, doc: committed.doc, label: committed.result }; } @@ -318,9 +423,10 @@ export async function redoDocOp(): Promise<{ export async function writeCompositionIfUnchanged( comp: Composition, expectedRev: number, + timeline: string = MAIN_TIMELINE, ): Promise { - const committed = await docStore.writeIfUnchanged(comp, expectedRev); - await writeDocOpLog(emptyDocOpLog(committed.rev)); + const committed = await storeFor(timeline).writeIfUnchanged(comp, expectedRev); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); return committed; } @@ -332,12 +438,12 @@ export async function writeCompositionIfUnchanged( * replay an inverse against the wrong document. Log-first ordering (the seeded * log is empty anyway, so a crash resolves to the same empty history). */ -async function overwriteDocAndResetLog(comp: Composition): Promise { - return docStore.runExclusive(async () => { - const baseRev = await readCompositionRevTolerant(); +async function overwriteDocAndResetLog(comp: Composition, timeline: string): Promise { + return storeFor(timeline).runExclusive(async () => { + const baseRev = await readCompositionRevTolerant(timeline); const committed: Composition = { ...comp, rev: baseRev + 1 }; - await writeDocOpLog(emptyDocOpLog(committed.rev)); - await writeComposition(committed); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); + await writeComposition(committed, timeline); return committed; }); } @@ -348,12 +454,85 @@ async function overwriteDocAndResetLog(comp: Composition): Promise * recovered). Resets undo history — a recovered document has none — and advances * `rev` so a stale log can never collide with the new revision. */ -export function overwriteComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); +export function overwriteComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); } /** Start a fresh timeline (`clip timeline new`): replace the document, clear undo - * history, and advance `rev` past the previous one (never reuse a rev). */ -export function resetComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); + * history, and advance `rev` past the previous one (never reuse a rev). Creates + * `timeline` when it doesn't exist yet. */ +export function resetComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); +} + +// ─── the workspace's timelines ─────────────────────────────────────────────── + +/** A timeline in the workspace, or one whose document can't be read. */ +export type TimelineListing = + | { id: string; width: number; height: number; durationSec: number; rev: number } + | { id: string; error: string }; + +/** `main` first, then every `timelines//` holding a document, by id. A + * document that can't be read is listed with its error instead of failing the + * list; a `timelines/` folder that leads out of the workspace is refused. */ +export async function listTimelines(): Promise { + await ensureWorkspace(); + const listed: TimelineListing[] = []; + for (const id of [MAIN_TIMELINE, ...(await timelineFolders())]) { + try { + const comp = await readStoredComposition(id); + if (!comp && id !== MAIN_TIMELINE) continue; + const doc = comp ?? emptyComposition(); + listed.push({ + id, + width: doc.width, + height: doc.height, + durationSec: compositionDuration(doc), + rev: doc.rev, + }); + } catch (err) { + if (!(err instanceof CompositionCorruptError || err instanceof WorkspaceBoundaryError)) { + throw err; + } + listed.push({ id, error: err.message }); + } + } + return listed; +} + +async function timelineFolders(): Promise { + let names: string[]; + try { + names = await readdir(resolveInWorkspace(TIMELINES_DIR)); + } catch (err) { + if (isMissingFile(err)) return []; + throw err; + } + return names + .filter((name) => name !== MAIN_TIMELINE && TimelineIdSchema.safeParse(name).success) + .sort(); +} + +/** Remove a timeline and its undo history. `main` can't be deleted. */ +export async function deleteTimeline(timeline: string): Promise { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) { + throw new TimelineIdError( + "The main timeline can't be deleted. Start it over with `clip timeline new`.", + ); + } + await storeFor(timeline).runExclusive(async () => { + try { + if (!(await readStoredComposition(timeline))) throw new NoTimelineError(timeline); + } catch (err) { + // A corrupt document is deleted like any other. + if (!(err instanceof CompositionCorruptError)) throw err; + } + await rm(dirname(compositionPath(timeline)), { recursive: true, force: true }); + }); } diff --git a/src/timeline/schema.ts b/src/timeline/schema.ts index 88b07c5..76881ae 100644 --- a/src/timeline/schema.ts +++ b/src/timeline/schema.ts @@ -9,6 +9,20 @@ export const MediaIdSchema = z.string().regex(/^m_[a-f0-9]{12}$/, { message: 'MediaId must be of the form m_<12 hex chars>', }); +/** The timeline every workspace starts with, kept at `composition.json`. */ +export const MAIN_TIMELINE = 'main'; + +/** A named timeline in the workspace: `main`, or a short such as `best-1`. */ +export const TimelineIdSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9-]{0,47}$/, { + message: + 'A timeline id is 1–48 lowercase letters, digits or hyphens, starting with a letter or digit (e.g. "best-1").', + }) + .describe( + 'Timeline to work on (default "main"). A short made with make_short is a timeline of its own.', + ); + export const VideoStreamSchema = z.object({ codec: z.string(), width: z.number().int().positive(), diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts new file mode 100644 index 0000000..5fb7637 --- /dev/null +++ b/src/timeline/shorts.ts @@ -0,0 +1,341 @@ +import { planCaptions } from './captions.js'; +import { speedFactor, videoLayers } from './compile.js'; +import { + type Clip, + type Composition, + type Effect, + type MediaClip, + makeClipId, +} from './composition.js'; +import { applyOps, type CompositionOp, mediaClip, videoTrack } from './ops.js'; +import { type Aspect, canvasFor, planReframe } from './reframe.js'; +import { MAIN_TIMELINE, type MediaId } from './schema.js'; +import type { Transcript } from './transcript.js'; + +/** + * Shorts: a vertical clip cut out of a longer timeline by transcript word ranges, + * built on a timeline of its own. It is cut from the source timeline's clips, not + * from the raw file, so silence and retake cuts already made there carry in. It + * is a snapshot: later edits to the source don't flow into it. The agent or the + * person picks the words; nothing here scores highlights. + */ + +/** Kept before a range's first word, so its attack isn't clipped. */ +export const SHORT_LEAD_SEC = 0.1; +/** Kept after a range's last word, so its release and the breath after it aren't clipped. */ +export const SHORT_TAIL_SEC = 0.25; +/** Loudness a short is normalized to unless asked otherwise: the vertical platforms' -14 LUFS. */ +export const DEFAULT_SHORT_LUFS = -14; + +const SHORT_TRACK = 'v0'; +/** A piece shorter than this is dropped: too short to hear, just a flash. */ +const MIN_PIECE_SEC = 0.05; +const SAME_SEC = 1e-6; + +/** Words `fromIndex..toIndex` (inclusive) of the transcript behind a media clip + * on the source timeline. */ +export interface ShortRange { + clipId: string; + fromIndex: number; + toIndex: number; + /** The `transcriptId` the indices were read from; refused if it changed. */ + transcriptId?: string; +} + +export interface ShortRequest { + /** Played in this order, so a later moment can open the short as its hook. */ + ranges: ShortRange[]; + aspect?: Aspect; + /** Horizontal position of the picture in the frame: 0 left, 0.5 center, 1 right. */ + x?: number; + y?: number; + /** Burn in captions (default true). */ + captions?: boolean; + maxWords?: number; + /** Default -14; null leaves loudness off. */ + targetLufs?: number | null; + /** Rebuild a timeline that already holds a short. */ + replace?: boolean; +} + +/** A window of a source file the short plays, in source seconds. */ +export interface ShortPiece { + mediaId: MediaId; + sourceInSec: number; + sourceOutSec: number; +} + +export interface ShortPlan { + ops: CompositionOp[]; + pieces: ShortPiece[]; + durationSec: number; + /** Captions written (0 with captions off). */ + captions: number; + warnings: string[]; +} + +/** Thrown when a short can't be made from the request. */ +export class ShortError extends Error { + constructor(message: string) { + super(message); + this.name = 'ShortError'; + } +} + +/** A media clip of `comp` and its source's stored transcript (never transcribed here). */ +export type ShortTranscriptLookup = ( + comp: Composition, + clipId: string, + transcriptId?: string, +) => Promise<{ clip: MediaClip; transcript: Transcript }>; + +/** Refuse a short on `main` or on the timeline it is cut from. */ +export function checkShortTimelines(target: string, source: string = MAIN_TIMELINE): void { + if (target === MAIN_TIMELINE) { + throw new ShortError( + 'A short is built on a timeline of its own: pick a new timeline id such as "best-1", not main.', + ); + } + if (source === target) { + throw new ShortError( + `A short can't be cut from its own timeline "${target}". Cut it from main or another timeline.`, + ); + } +} + +interface WordWindow { + /** Padded edges, source seconds. */ + start: number; + end: number; + /** First word's start and last word's end. */ + speechStart: number; + speechEnd: number; +} + +interface Piece extends ShortPiece { + effects: Effect[]; +} + +/** + * Plan the edit that builds a short on `target` from `source`: for each range, + * the source timeline's clips under its words (padded), laid end to end on a + * fresh picture track, on a 9:16 canvas filled edge to edge, normalized to + * -14 LUFS and captioned. One batch, so one undo takes the whole short away. + */ +export async function planShort( + target: Composition, + source: Composition, + request: ShortRequest, + lookup: ShortTranscriptLookup, + mintId: () => string = makeClipId, +): Promise { + if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); + if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { + throw new ShortError( + 'That timeline already holds a short. Pass replace to rebuild it, or pick a new timeline id.', + ); + } + const picture = videoLayers(source).picture; + if (!picture) { + throw new ShortError( + 'The source timeline has no picture track to cut from: it is empty or has more than one.', + ); + } + + const pieces: Piece[] = []; + for (const range of request.ranges) { + const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); + const window = wordWindow(transcript, range, clip.id); + const found = piecesUnder(picture.clips, clip.mediaId, window); + if (found.length === 0) { + throw new ShortError( + `Words ${range.fromIndex}..${range.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, + ); + } + pieces.push(...found); + } + const joined = joinContinuous(pieces); + + const ops: CompositionOp[] = target.tracks.map( + (t): CompositionOp => ({ op: 'removeTrack', trackId: t.id }), + ); + const canvas = canvasFor(source, request.aspect ?? '9:16'); + ops.push( + { + op: 'setCanvas', + width: canvas.width, + height: canvas.height, + fps: source.fps, + background: source.background, + fit: 'cover', + }, + { op: 'addTrack', track: videoTrack({ id: SHORT_TRACK }) }, + ); + let at = 0; + for (const piece of joined) { + ops.push({ + op: 'addClip', + trackId: SHORT_TRACK, + clip: mediaClip({ + id: mintId(), + mediaId: piece.mediaId, + sourceInSec: piece.sourceInSec, + sourceOutSec: piece.sourceOutSec, + // Rounded to the microsecond: clean numbers, far inside the export's + // abutting tolerance. + startSec: Math.round(at * 1e6) / 1e6, + effects: piece.effects, + }), + }); + at += piece.sourceOutSec - piece.sourceInSec; + } + + if (request.x !== undefined || request.y !== undefined) { + const framed = planReframe(applyOps(target, ops), { + width: canvas.width, + height: canvas.height, + fit: 'cover', + x: request.x, + y: request.y, + }); + ops.push(...framed.ops); + } + + const targetLufs = request.targetLufs === undefined ? DEFAULT_SHORT_LUFS : request.targetLufs; + if ((target.loudness?.targetLufs ?? null) !== targetLufs) { + ops.push({ op: 'setLoudness', targetLufs }); + } + + let captions = 0; + if (request.captions !== false) { + const built = applyOps(target, ops); + const plan = await planCaptions( + built, + { maxWords: request.maxWords }, + (clipId, transcriptId) => lookup(built, clipId, transcriptId), + mintId, + ); + ops.push(...plan.ops); + captions = plan.captions; + } + + return { + ops, + pieces: joined.map(({ mediaId, sourceInSec, sourceOutSec }) => ({ + mediaId, + sourceInSec, + sourceOutSec, + })), + durationSec: roundMs(at), + captions, + warnings: [], + }; +} + +/** + * The source window a range plays: its first word's start minus the lead to its + * last word's end plus the tail. Each side stops at the neighboring word's edge, + * so the padding never takes in part of a word outside the range, and at the + * start of the file. + */ +function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): WordWindow { + const first = transcript.words[range.fromIndex]; + const last = transcript.words[range.toIndex]; + if (!first || !last || range.toIndex < range.fromIndex) { + throw new ShortError( + `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clipId}" ` + + `(it has ${transcript.words.length} words, numbered from 0).`, + ); + } + const before = transcript.words[range.fromIndex - 1]; + const after = transcript.words[range.toIndex + 1]; + const start = Math.min( + first.startSec, + Math.max(first.startSec - SHORT_LEAD_SEC, before?.endSec ?? 0, 0), + ); + const end = Math.max( + last.endSec, + Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), + ); + return { + start: roundMs(start), + end: roundMs(end), + speechStart: first.startSec, + speechEnd: last.endSec, + }; +} + +/** + * The parts of a window that the source timeline's picture clips of one file + * play, in timeline order. A moment the timeline plays twice is taken once, and + * a part holding only padding (no word of the range) is left out, since it would + * only add a jump cut. + */ +function piecesUnder(clips: readonly Clip[], mediaId: MediaId, window: WordWindow): Piece[] { + const taken: [number, number][] = []; + const pieces: Piece[] = []; + for (const clip of [...clips].sort((a, b) => a.startSec - b.startSec)) { + if (clip.kind !== 'media' || clip.mediaId !== mediaId) continue; + const lo = Math.max(window.start, clip.sourceInSec); + const hi = Math.min(window.end, clip.sourceOutSec); + if (hi <= lo) continue; + for (const [a, b] of subtract([lo, hi], taken)) { + if (b - a < MIN_PIECE_SEC || b < window.speechStart || a > window.speechEnd) continue; + const speed = speedFactor(clip.effects); + if (speed !== 1) { + throw new ShortError( + `Clip "${clip.id}" on the source timeline plays at ${speed}x, and a short can't include a speed change yet. ` + + 'Remove its speed effect or pick words outside it.', + ); + } + pieces.push({ + mediaId, + sourceInSec: a, + sourceOutSec: b, + // Fades belonged to the source clip's edges, not the short's. + effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), + }); + taken.push([a, b]); + } + } + return pieces; +} + +/** `[lo, hi]` minus the `taken` intervals, in order. */ +function subtract([lo, hi]: [number, number], taken: [number, number][]): [number, number][] { + let parts: [number, number][] = [[lo, hi]]; + for (const [a, b] of taken) { + parts = parts.flatMap(([p, q]): [number, number][] => { + if (b <= p || a >= q) return [[p, q]]; + const rest: [number, number][] = []; + if (a > p) rest.push([p, a]); + if (b < q) rest.push([b, q]); + return rest; + }); + } + return parts; +} + +/** Merge pieces that continue one another in the same file with the same + * effects (a split, not a cut, on the source), so the short has no cut there. */ +function joinContinuous(pieces: Piece[]): Piece[] { + const joined: Piece[] = []; + for (const piece of pieces) { + const prev = joined[joined.length - 1]; + if ( + prev && + prev.mediaId === piece.mediaId && + Math.abs(prev.sourceOutSec - piece.sourceInSec) < SAME_SEC && + JSON.stringify(prev.effects) === JSON.stringify(piece.effects) + ) { + joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; + } else { + joined.push(piece); + } + } + return joined; +} + +function roundMs(sec: number): number { + return Math.round(sec * 1000) / 1000; +} diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index b72e9ec..f9167c0 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -21,7 +21,8 @@ import { planFraming, planReframe, } from './reframe.js'; -import type { MediaId } from './schema.js'; +import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from './schema.js'; +import { checkShortTimelines, planShort } from './shorts.js'; import { SensitivitySchema, type SilenceCut } from './silence.js'; import { clipWords, @@ -277,6 +278,71 @@ const EDITABLE_VERB_SCHEMAS = [ 'Normalize the exported mp4 to this integrated loudness in LUFS: -14 for YouTube, Shorts, Reels and TikTok, -16 for podcasts. null turns it off. Applies to mp4 export only.', ), }), + z.object({ + verb: z.literal('make_short'), + ranges: z + .array( + z.object({ + clipId: z + .string() + .describe( + 'A media clip on the source timeline whose transcript the indices come from.', + ), + fromIndex: z + .number() + .int() + .nonnegative() + .describe('First word of the range: its `index` from timeline_transcript.'), + toIndex: z.number().int().nonnegative().describe('Last word of the range (inclusive).'), + transcriptId: z + .string() + .optional() + .describe( + 'The `transcriptId` the indices were read from; refused if the clip was transcribed again since.', + ), + }), + ) + .min(1) + .max(20) + .describe( + 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences; 15–60 s in all is typical.', + ), + source: TimelineIdSchema.optional().describe( + 'Timeline to cut from (default "main"). Its silence and retake cuts carry into the short.', + ), + aspect: AspectSchema.optional().describe('Output shape (default 9:16).'), + x: z + .number() + .min(0) + .max(1) + .optional() + .describe( + 'Horizontal position of the picture: 0 shows the left edge of the source, 0.5 the center, 1 the right edge. Put the speaker in frame.', + ), + y: z.number().min(0).max(1).optional().describe('Vertical position: 0 top, 1 bottom.'), + captions: z + .boolean() + .optional() + .describe('Burn in captions from the transcript (default true).'), + maxWords: z + .number() + .int() + .min(1) + .max(12) + .optional() + .describe( + `Most words per caption (default ${DEFAULT_CAPTION_WORDS}); 1 shows one word at a time.`, + ), + targetLufs: TargetLufsSchema.nullable() + .optional() + .describe( + 'Export loudness in LUFS (default -14, for Shorts, Reels and TikTok); null for off.', + ), + replace: z + .boolean() + .optional() + .describe('Rebuild this timeline when it already holds a short (default: refuse).'), + }), z.object({ verb: z.literal('transition'), afterClipId: z @@ -331,8 +397,12 @@ export interface VerbContext { clipId: string, transcriptId?: string, ) => Promise<{ clip: MediaClip; transcript: Transcript; media: Map }>; + /** Read another timeline of the workspace (make_short's source). */ + readTimeline: (timeline: string) => Promise; /** Track a clip is placed on when the verb doesn't say. */ defaultTrack: string; + /** The timeline being edited (default `main`); `applyVerbs` sets it. */ + timeline?: string; } const DEFAULT_TRACK = 'v0'; @@ -493,6 +563,26 @@ export async function lowerVerb( return (comp.loudness?.targetLufs ?? null) === verb.targetLufs ? [] : [{ op: 'setLoudness', targetLufs: verb.targetLufs }]; + case 'make_short': { + const sourceId = verb.source ?? MAIN_TIMELINE; + checkShortTimelines(ctx.timeline ?? MAIN_TIMELINE, sourceId); + const plan = await planShort( + comp, + await ctx.readTimeline(sourceId), + { + ranges: verb.ranges, + aspect: verb.aspect, + x: verb.x, + y: verb.y, + captions: verb.captions, + maxWords: verb.maxWords, + targetLufs: verb.targetLufs, + replace: verb.replace, + }, + ctx.transcript, + ); + return plan.ops; + } case 'transition': return [ { diff --git a/src/ui/server.ts b/src/ui/server.ts index 735a0d4..26b2859 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -42,6 +42,7 @@ import { CompositionOpError } from '../timeline/ops.js'; import { ReframeError } from '../timeline/reframe.js'; import { runPlan } from '../timeline/run-plan.js'; import type { MediaId } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; import { SilenceAnalysisError } from '../timeline/silence.js'; import { TranscriptError, transcribeClip } from '../timeline/transcript.js'; import { CompositionVerbSchema } from '../timeline/verbs.js'; @@ -60,11 +61,12 @@ import { import { clipConfinedMedia, confinedMediaRefs, makeVerbContext } from './timeline-tools.js'; /** Errors about the clip or its setup, not the server: silence removal or - * transcription can't work on this clip, whisper.cpp isn't set up, or a - * reframe asks for a size or position that can't be. */ + * transcription can't work on this clip, whisper.cpp isn't set up, a reframe + * asks for a size or position that can't be, or a short can't be made. */ function isClipConditionError(err: unknown): boolean { return ( err instanceof ReframeError || + err instanceof ShortError || err instanceof SilenceAnalysisError || err instanceof TranscriptError || err instanceof TranscriptionUnavailableError diff --git a/src/ui/timeline-tools.ts b/src/ui/timeline-tools.ts index 15235b6..f1ac55d 100644 --- a/src/ui/timeline-tools.ts +++ b/src/ui/timeline-tools.ts @@ -7,6 +7,7 @@ import { type Fit, findClip, } from '../timeline/composition.js'; +import { readComposition } from '../timeline/document-store.js'; import { buildMediaMap, buildMediaRefs } from '../timeline/media-registry.js'; import { ownFraming } from '../timeline/reframe.js'; import type { MediaId, MediaRef } from '../timeline/schema.js'; @@ -33,6 +34,7 @@ export function makeVerbContext(): VerbContext { }); return { mediaId: result.mediaId, durationSec: result.ref.durationSec }; }, + readTimeline: (timeline) => readComposition(timeline), analyzeSilence: async (comp, clipId, sensitivity) => analyzeSilence(comp, clipId, { media: await clipConfinedMedia(comp, clipId), sensitivity }), transcript: async (comp, clipId, transcriptId) => { diff --git a/tests/document-store.test.ts b/tests/document-store.test.ts index 1a6c647..f8430bc 100644 --- a/tests/document-store.test.ts +++ b/tests/document-store.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -6,16 +6,25 @@ import { emptyComposition } from '../src/timeline/composition.js'; import { CompositionConflictError, CompositionCorruptError, + compositionOpsPath, compositionPath, + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, overwriteComposition, readComposition, + readDocOpLog, + redoDocOp, resetComposition, + TimelineIdError, + undoLastDocOp, writeComposition, writeCompositionIfUnchanged, } from '../src/timeline/document-store.js'; import { mediaClip, videoTrack } from '../src/timeline/ops.js'; import type { MediaId } from '../src/timeline/schema.js'; +import { WorkspaceBoundaryError } from '../src/workspace.js'; const M1 = 'm_aaaaaaaaaaaa' as MediaId; @@ -141,3 +150,126 @@ describe('overwriteComposition (recovery primitive)', () => { expect(reread.rev).toBe(1); }); }); + +describe('named timelines', () => { + it('keeps main at the workspace root and every other timeline in timelines//', () => { + expect(compositionPath()).toBe(join(workspace, 'composition.json')); + expect(compositionPath('main')).toBe(join(workspace, 'composition.json')); + expect(compositionOpsPath('main')).toBe(join(workspace, 'composition-ops.json')); + expect(compositionPath('best-1')).toBe( + join(workspace, 'timelines', 'best-1', 'composition.json'), + ); + expect(compositionOpsPath('best-1')).toBe( + join(workspace, 'timelines', 'best-1', 'composition-ops.json'), + ); + }); + + it('refuses a malformed timeline id before touching a path', () => { + for (const bad of ['../main', 'Best', '', '-x', 'a/b', 'x'.repeat(49)]) { + expect(() => compositionPath(bad)).toThrow(TimelineIdError); + } + }); + + it('throws NoTimelineError for a missing timeline other than main', async () => { + await expect(readComposition('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 1280 }], { timeline: 'nope' }), + ).rejects.toBeInstanceOf(NoTimelineError); + await expect(undoLastDocOp('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(redoDocOp('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(readDocOpLog('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(writeCompositionIfUnchanged(emptyComposition(), 0, 'nope')).rejects.toBeInstanceOf( + NoTimelineError, + ); + expect((await readComposition('main')).tracks).toEqual([]); + }); + + it('creates a timeline with create or reset, then keeps its own rev and undo history', async () => { + await mutateComposition([{ op: 'setCanvas', width: 1280 }]); // main rev 1 + await mutateComposition([{ op: 'setCanvas', height: 720 }]); // main rev 2 + + const short = await mutateComposition([{ op: 'setCanvas', width: 1080, height: 1920 }], { + timeline: 'best-1', + create: true, + }); + expect(short.rev).toBe(1); + await resetComposition(emptyComposition({ fps: 24 }), 'best-2'); + expect((await readComposition('best-2')).fps).toBe(24); + + await mutateComposition([{ op: 'setCanvas', fps: 60 }], { timeline: 'best-1' }); + expect((await readComposition('best-1')).rev).toBe(2); + expect((await readComposition()).rev).toBe(2); + expect((await readDocOpLog('best-1')).entries).toHaveLength(2); + + await undoLastDocOp('best-1'); + expect((await readComposition('best-1')).fps).toBe(30); + expect((await readComposition()).height).toBe(720); + expect((await readDocOpLog()).cursor).toBe(2); + }); + + it('serializes writes per timeline, so concurrent edits of two timelines both land', async () => { + await resetComposition(emptyComposition(), 'best-1'); + await Promise.all([ + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'a' }) }], { + timeline: 'best-1', + }), + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'b' }) }], { + timeline: 'best-1', + }), + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'm' }) }]), + ]); + expect((await readComposition('best-1')).tracks.map((t) => t.id).sort()).toEqual(['a', 'b']); + expect((await readComposition()).tracks.map((t) => t.id)).toEqual(['m']); + }); + + it('lists main first, then each timeline by id, and a corrupt one with its error', async () => { + await resetComposition(emptyComposition({ width: 1080, height: 1920 }), 'zeta'); + await resetComposition(emptyComposition(), 'alpha'); + await resetComposition(emptyComposition(), 'broken'); + await writeFile(compositionPath('broken'), 'not json'); + await mkdir(join(workspace, 'timelines', 'empty-folder'), { recursive: true }); + await writeFile(join(workspace, 'timelines', 'stray.txt'), 'x'); + + const listed = await listTimelines(); + expect(listed.map((t) => t.id)).toEqual(['main', 'alpha', 'broken', 'zeta']); + expect(listed[0]).toEqual({ id: 'main', width: 1920, height: 1080, durationSec: 0, rev: 0 }); + expect(listed[3]).toMatchObject({ id: 'zeta', width: 1080, height: 1920, rev: 1 }); + expect(listed[2]).toMatchObject({ id: 'broken', error: expect.stringMatching(/corrupt/) }); + }); + + it('deletes a timeline, refuses main, and reports a missing one', async () => { + await resetComposition(emptyComposition(), 'best-1'); + await deleteTimeline('best-1'); + await expect(readComposition('best-1')).rejects.toBeInstanceOf(NoTimelineError); + expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); + + await expect(deleteTimeline('best-1')).rejects.toBeInstanceOf(NoTimelineError); + await expect(deleteTimeline('main')).rejects.toBeInstanceOf(TimelineIdError); + }); + + it('deletes a corrupt timeline', async () => { + await resetComposition(emptyComposition(), 'broken'); + await writeFile(compositionPath('broken'), 'not json'); + await deleteTimeline('broken'); + expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); + }); + + it('refuses a timelines/ folder that is a symlink out of the workspace', async () => { + const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); + try { + await mkdir(join(outside, 'best-1')); + await writeFile( + join(outside, 'best-1', 'composition.json'), + JSON.stringify(emptyComposition()), + ); + await symlink(outside, join(workspace, 'timelines')); + await expect(readComposition('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 2 }], { timeline: 'best-2', create: true }), + ).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect(listTimelines()).rejects.toBeInstanceOf(WorkspaceBoundaryError); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration/timeline-shorts.test.ts b/tests/integration/timeline-shorts.test.ts new file mode 100644 index 0000000..a5476af --- /dev/null +++ b/tests/integration/timeline-shorts.test.ts @@ -0,0 +1,159 @@ +import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runTimeline } from '../../src/cli-timeline.js'; +import { probe } from '../../src/ffmpeg/probe.js'; +import { listFilters, runFfmpeg } from '../../src/ffmpeg/run.js'; +import { readComposition } from '../../src/timeline/document-store.js'; +import type { Transcript } from '../../src/timeline/transcript.js'; + +// A short cut from a 10-minute recording, end to end through the CLI: the +// source's cuts carry in, the export is 9:16 at -14 LUFS, main is untouched and +// one undo takes the short away. The transcript is a fixture, so no whisper. +const IS_CI = process.env.CI === 'true'; +// CI's FFmpeg build has no drawtext, so captions are burned in only where it does. +const HAS_DRAWTEXT = IS_CI && (await listFilters()).has('drawtext'); + +let workspace: string; +let saved: string | undefined; + +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-shorts-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); +}); + +/** Run a `clip timeline` subcommand and parse the JSON it prints. */ +async function cli>(args: string[]): Promise { + let printed = ''; + const spy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + printed += String(chunk); + return true; + }); + try { + await runTimeline(args); + } finally { + spy.mockRestore(); + } + return JSON.parse(printed) as T; +} + +/** Integrated loudness of a file, LUFS, measured by ebur128. */ +async function integratedLufs(path: string): Promise { + const { stderr } = await runFfmpeg([ + ...['-hide_banner', '-nostats', '-i', path, '-map', '0:a:0'], + ...['-af', 'ebur128', '-f', 'null', '-'], + ]); + const summary = stderr.slice(stderr.lastIndexOf('Summary:')); + const match = /I:\s+(-?[\d.]+) LUFS/.exec(summary); + if (!match?.[1]) throw new Error(`No integrated loudness in:\n${summary}`); + return Number(match[1]); +} + +/** 10 minutes of 720p30 test pattern over a quiet tone stepping between two + * levels (about -26 LUFS), with a 2 s GOP: a 30 s piece looped by stream copy, + * which takes seconds instead of minutes to make. */ +async function tenMinuteRecording(path: string): Promise { + const piece = join(workspace, 'piece.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'testsrc=size=1280x720:rate=30:duration=30'], + ...['-f', 'lavfi', '-i', "aevalsrc='(0.05+0.05*lt(mod(t,4),2))*sin(2*PI*220*t)':s=48000:d=30"], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', piece], + ]); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-stream_loop', '19', '-i', piece, '-c', 'copy', path], + ]); +} + +/** Word i spoken from i+0.2 s to i+0.7 s, as whisper would store it. */ +async function storeTranscript(mediaId: string, source: string): Promise { + const file = await stat(source); + const transcript: Transcript = { + version: 1, + mediaId: mediaId as Transcript['mediaId'], + language: 'en', + source: 'whisper-cpp', + model: 'fixture', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: file.size, + sourceMtimeMs: file.mtimeMs, + words: Array.from({ length: 599 }, (_, i) => ({ + text: i % 8 === 7 ? `word${i}.` : `word${i}`, + startSec: i + 0.2, + endSec: i + 0.7, + })), + }; + await mkdir(join(workspace, 'transcripts'), { recursive: true }); + await writeFile(join(workspace, 'transcripts', `${mediaId}.json`), JSON.stringify(transcript)); +} + +describe.skipIf(!IS_CI)('shorts against real FFmpeg', () => { + it('cuts a 9:16 short at -14 LUFS from a 10-minute recording, leaving main alone', async () => { + const source = join(workspace, 'podcast.mp4'); + await tenMinuteRecording(source); + await cli(['new', '--width', '1280', '--height', '720']); + const added = await cli<{ clipId: string; mediaId: string }>(['add-media', source]); + await storeTranscript(added.mediaId, source); + // A retake cut on main (words 545..547) carries into the short. + await cli(['cut-words', added.clipId, '545', '547']); + const main = await readComposition(); + + const short = await cli<{ + durationSec: number; + pieces: { sourceInSec: number; sourceOutSec: number }[]; + canvas: { width: number; height: number; fit: string }; + captions: number; + rev: number; + }>([ + 'short', + 'best-1', + ...[added.clipId, '540', '552'], + ...[added.clipId, '100', '105'], + ...['--x', '0.3'], + ...(HAS_DRAWTEXT ? [] : ['--no-captions']), + ]); + expect(short.pieces).toEqual([ + expect.objectContaining({ sourceInSec: 540.1, sourceOutSec: 545.2 }), + expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.95 }), + expect.objectContaining({ sourceInSec: 100.1, sourceOutSec: 105.95 }), + ]); + expect(short.durationSec).toBe(16.2); + expect(short.canvas).toEqual({ width: 720, height: 1280, fit: 'cover' }); + if (HAS_DRAWTEXT) expect(short.captions).toBeGreaterThan(0); + + const output = join(workspace, 'best-1.mp4'); + const exported = await cli<{ loudness: { mode: string } }>([ + ...['--timeline', 'best-1', 'export', output], + ]); + expect(exported.loudness.mode).not.toBe('skipped'); + const probed = await probe(output); + expect(probed.video).toMatchObject({ width: 720, height: 1280 }); + // Each piece is its own segment, which export rounds up to a whole video + // frame (and AAC frame): up to one frame per piece, plus one AAC frame. + const slack = short.pieces.length / 30 + 1024 / 48000; + expect(Math.abs(probed.durationSec - short.durationSec)).toBeLessThanOrEqual(slack); + expect(Math.abs((await integratedLufs(output)) + 14)).toBeLessThan(1); + + expect(await readComposition()).toEqual(main); + const listed = await cli<{ timelines: { id: string }[] }>(['list']); + expect(listed.timelines.map((t) => t.id)).toEqual(['main', 'best-1']); + + const undo = await cli<{ undone: boolean }>(['--timeline', 'best-1', 'undo']); + expect(undo.undone).toBe(true); + const emptied = await cli<{ tracks: unknown[] }>(['--timeline', 'best-1', 'show']); + expect(emptied.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(main.rev); + }, 240_000); +}); diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index 3b69f1c..7fc7271 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -6,6 +6,11 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildMcpServer } from '../src/mcp/server.js'; +import { appendOp } from '../src/session/store.js'; +import { mutateComposition, readComposition } from '../src/timeline/document-store.js'; +import { mediaClip, videoTrack } from '../src/timeline/ops.js'; +import { makeMediaId } from '../src/timeline/schema.js'; +import type { Transcript } from '../src/timeline/transcript.js'; const pkg = createRequire(import.meta.url)('../package.json') as { version: string }; @@ -220,3 +225,129 @@ describe('MCP server', () => { } }); }); + +describe('MCP shorts (named timelines)', () => { + /** Main holds a 30 s registered recording with a stored transcript: nothing renders. */ + async function recordingOnMain(): Promise { + const source = join(workspace, 'talk.mp4'); + await writeFile(source, 'video bytes'); + const mediaId = makeMediaId(source); + await appendOp({ + tool: 'ingest', + args: { path: source }, + result: { + mediaId, + ref: { path: source, durationSec: 30, audio: { codec: 'aac' }, video: { codec: 'h264' } }, + }, + }); + const file = await stat(source); + const transcript: Transcript = { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-tiny.en.bin', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: file.size, + sourceMtimeMs: file.mtimeMs, + words: Array.from({ length: 20 }, (_, i) => ({ + text: `word${i}`, + startSec: i + 1, + endSec: i + 1.5, + })), + }; + await mkdir(join(workspace, 'transcripts'), { recursive: true }); + await writeFile(join(workspace, 'transcripts', `${mediaId}.json`), JSON.stringify(transcript)); + await mutateComposition([ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId, sourceOutSec: 30, startSec: 0 }), + }, + ]); + } + + it('makes a short on its own timeline with timeline_edit, leaving main alone', async () => { + await recordingOnMain(); + const mainBefore = await readComposition(); + const { client, close } = await connect(); + try { + const edit = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { + timeline: 'clip-1', + verbs: [ + { + verb: 'make_short', + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 6 }], + x: 0.4, + }, + ], + }, + }), + ); + const doc = edit.document as Record; + expect(doc.timeline).toBe('clip-1'); + expect(doc.canvas).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); + expect(doc.loudness).toEqual({ targetLufs: -14 }); + expect(doc.exportable).toBe(true); + expect((doc.tracks as { id: string }[]).map((t) => t.id)).toEqual(['v0', 'captions']); + expect(await readComposition()).toEqual(mainBefore); + + const show = payload(await client.callTool({ name: 'timeline_show', arguments: {} })); + expect(show.timeline).toBe('main'); + expect(show.timelines).toEqual(['main', 'clip-1']); + + const undo = payload( + await client.callTool({ name: 'timeline_undo', arguments: { timeline: 'clip-1' } }), + ); + expect(undo.undone).toBe(true); + const emptied = payload( + await client.callTool({ name: 'timeline_show', arguments: { timeline: 'clip-1' } }), + ); + expect(emptied.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(mainBefore.rev); + } finally { + await close(); + } + }); + + it('answers a missing timeline or a short on main with an error', async () => { + await recordingOnMain(); + const { client, close } = await connect(); + try { + for (const name of ['timeline_show', 'timeline_check', 'timeline_undo', 'timeline_export']) { + const result = payload(await client.callTool({ name, arguments: { timeline: 'nope' } })); + expect(result.error).toMatch(/No timeline "nope"/); + } + const typo = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { timeline: 'nope', verbs: [{ verb: 'remove', clipId: 'c1' }] }, + }), + ); + expect(typo.error).toMatch(/No timeline "nope"/); + + const onMain = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { + verbs: [{ verb: 'make_short', ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 6 }] }], + }, + }), + ); + expect(onMain.error).toMatch(/not main/); + + const badId = await client.callTool({ + name: 'timeline_show', + arguments: { timeline: '../main' }, + }); + expect(badId.isError).toBe(true); + } finally { + await close(); + } + }); +}); diff --git a/tests/timeline-captions.test.ts b/tests/timeline-captions.test.ts index 1b944b3..4e47a19 100644 --- a/tests/timeline-captions.test.ts +++ b/tests/timeline-captions.test.ts @@ -322,6 +322,9 @@ describe('caption_speech verb', () => { const { clip, transcript } = await lookup(new Map([[M, RECORDING]]))(clipId); return { clip, transcript, media: new Map() }; }, + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', }); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts new file mode 100644 index 0000000..c2d1267 --- /dev/null +++ b/tests/timeline-shorts.test.ts @@ -0,0 +1,492 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + type Composition, + emptyComposition, + findClip, + type MediaClip, +} from '../src/timeline/composition.js'; +import { + applyVerbs, + NoTimelineError, + readComposition, + readDocOpLog, + resetComposition, + undoLastDocOp, +} from '../src/timeline/document-store.js'; +import { + applyOps, + type CompositionOp, + mediaClip, + textClip, + videoTrack, +} from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; +import { + checkShortTimelines, + planShort, + ShortError, + type ShortRequest, + type ShortTranscriptLookup, +} from '../src/timeline/shorts.js'; +import { NoTranscriptError, type Transcript, TranscriptError } from '../src/timeline/transcript.js'; +import { EditableVerbSchema, lowerVerb, type VerbContext } from '../src/timeline/verbs.js'; + +const M = 'm_aaaaaaaaaaaa' as MediaId; +const N = 'm_bbbbbbbbbbbb' as MediaId; + +function transcriptOf(mediaId: MediaId, spans: [number, number][]): Transcript { + return { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-tiny.en.bin', + modelKey: '', + requestedLanguage: 'auto', + sourceSize: 0, + sourceMtimeMs: 0, + words: spans.map(([startSec, endSec], i) => ({ text: `w${i}`, startSec, endSec })), + }; +} + +/** Word i spoken from i+1 s to i+1.5 s: half-second pauses between words. */ +const SPACED = transcriptOf( + M, + Array.from({ length: 20 }, (_, i): [number, number] => [i + 1, i + 1.5]), +); + +/** A recording on the source timeline's picture track. */ +function sourceOf(clips: MediaClip[], canvas: Partial = {}): Composition { + return applyOps(emptyComposition({ width: 1920, height: 1080, ...canvas }), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ...clips.map((clip): CompositionOp => ({ op: 'addClip', trackId: 'v0', clip })), + ]); +} + +const clipOf = ( + id: string, + sourceInSec: number, + sourceOutSec: number, + startSec: number, + extra: Partial = {}, +): MediaClip => + mediaClip({ id, mediaId: M, sourceInSec, sourceOutSec, startSec, effects: extra.effects }); + +function lookupFor(transcripts: Transcript[]): ShortTranscriptLookup { + return async (comp, clipId) => { + const found = findClip(comp, clipId); + if (found?.clip.kind !== 'media') throw new TranscriptError(`No clip "${clipId}"`); + const transcript = transcripts.find((t) => t.mediaId === found.clip.mediaId); + if (!transcript) throw new NoTranscriptError(`Clip "${clipId}" has no transcript yet.`); + return { clip: found.clip as MediaClip, transcript }; + }; +} + +function counter(prefix = 's'): () => string { + let n = 0; + return () => `${prefix}${++n}`; +} + +async function plan( + source: Composition, + request: ShortRequest, + opts: { target?: Composition; transcripts?: Transcript[] } = {}, +) { + return planShort( + opts.target ?? emptyComposition(), + source, + { captions: false, ...request }, + lookupFor(opts.transcripts ?? [SPACED]), + counter(), + ); +} + +type AddClipOp = Extract; +const addedClips = (ops: CompositionOp[]): AddClipOp['clip'][] => + ops.filter((o): o is AddClipOp => o.op === 'addClip').map((o) => o.clip); + +describe('planShort: edges', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + + it('pads a range 0.10 s before its first word and 0.25 s after its last', async () => { + const short = await plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.durationSec).toBe(2.85); + }); + + it('stops the padding at the neighboring words and at the start of the file', async () => { + const tight = transcriptOf(M, [ + [0.03, 0.3], + [0.32, 0.6], + [0.65, 0.9], + [0.95, 1.2], + ]); + const middle = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, + { transcripts: [tight] }, + ); + expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.3, sourceOutSec: 0.95 }]); + + const first = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 0 }] }, + { transcripts: [tight] }, + ); + expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.32 }]); + }); + + it('stops at the end of the source clip', async () => { + const short = await plan(sourceOf([clipOf('c1', 0, 5.6, 0)]), { + ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }], + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + }); + + it('refuses word indices outside the transcript', async () => { + await expect( + plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 18, toIndex: 25 }] }), + ).rejects.toThrow(/aren't in the transcript.*20 words/); + await expect( + plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 4, toIndex: 2 }] }), + ).rejects.toBeInstanceOf(ShortError); + }); +}); + +describe('planShort: cut from the source timeline', () => { + it('keeps the silence cuts made on the source: three sub-clips give three pieces', async () => { + const cut = sourceOf([ + clipOf('c1', 0, 3.7, 0), + clipOf('c2', 3.9, 5.7, 3.7), + clipOf('c3', 5.9, 12, 5.5), + ]); + const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 6 }] }); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.7 }, + { mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.7 }, + { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.75 }, + ]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.8, 2.6]); + expect(short.durationSec).toBe(4.45); + }); + + it('refuses words that were cut out of the source timeline', async () => { + const retakeCut = sourceOf([clipOf('c1', 0, 3.7, 0), clipOf('c3', 5.9, 12, 3.7)]); + await expect( + plan(retakeCut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }), + ).rejects.toThrow(/cut out of the source timeline/); + }); + + it('leaves out a sliver that holds only padding', async () => { + // The tail padding of word 4 (ends 5.5) reaches 5.75, just into c2. + const cut = sourceOf([clipOf('c1', 0, 5.6, 0), clipOf('c2', 5.7, 12, 5.6)]); + const short = await plan(cut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + }); + + it('plays the ranges in the order given', async () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const short = await plan(whole, { + ranges: [ + { clipId: 'c1', fromIndex: 6, toIndex: 7 }, + { clipId: 'c1', fromIndex: 1, toIndex: 2 }, + ], + }); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 6.9, sourceOutSec: 8.75 }, + { mediaId: M, sourceInSec: 1.9, sourceOutSec: 3.75 }, + ]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.85]); + }); + + it('cuts ranges from two recordings', async () => { + const other = transcriptOf(N, [[0.5, 1]]); + const two = sourceOf([ + clipOf('c1', 0, 30, 0), + mediaClip({ id: 'd1', mediaId: N, sourceOutSec: 2, startSec: 30 }), + ]); + const short = await plan( + two, + { + ranges: [ + { clipId: 'd1', fromIndex: 0, toIndex: 0 }, + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + ], + }, + { transcripts: [SPACED, other] }, + ); + expect(short.pieces.map((p) => p.mediaId)).toEqual([N, M]); + }); + + it('takes a moment the source plays twice only once', async () => { + const repeated = sourceOf([clipOf('c1', 0, 10, 0), clipOf('c2', 2, 6, 10)]); + const short = await plan(repeated, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + }); + + it('joins pieces that a split (not a cut) left continuous', async () => { + const split = sourceOf([clipOf('c1', 0, 4, 0), clipOf('c2', 4, 10, 4)]); + const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + }); + + it('keeps color and volume, and drops the fades of the source clip', async () => { + const source = sourceOf([ + clipOf('c1', 0, 30, 0, { + effects: [ + { type: 'fadeIn', durationSec: 1 }, + { type: 'adjust', brightness: 0.1, contrast: 1, saturation: 1 }, + { type: 'volume', gain: 1.5 }, + ], + }), + ]); + const short = await plan(source, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(addedClips(short.ops)[0]?.effects.map((e) => e.type)).toEqual(['adjust', 'volume']); + }); + + it('refuses a range over a clip with a speed change', async () => { + const fast = sourceOf([clipOf('c1', 0, 30, 0, { effects: [{ type: 'speed', factor: 2 }] })]); + await expect( + plan(fast, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }), + ).rejects.toThrow(/2x.*speed/); + }); + + it('refuses a source timeline with no picture track', async () => { + await expect( + plan(emptyComposition(), { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }), + ).rejects.toThrow(/no picture track/); + }); +}); + +describe('planShort: the edit', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + + it('builds a 9:16 cover canvas, the pieces, the framing, -14 LUFS and captions', async () => { + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], x: 0.3 }, + lookupFor([SPACED]), + counter(), + ); + expect(short.ops).toEqual([ + { op: 'setCanvas', width: 1080, height: 1920, fps: 30, background: 'black', fit: 'cover' }, + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ + id: 's1', + mediaId: M, + sourceInSec: 2.9, + sourceOutSec: 5.75, + startSec: 0, + }), + }, + { op: 'setTransform', clipId: 's1', transform: { x: 0.3 } }, + { op: 'setLoudness', targetLufs: -14 }, + { op: 'addTrack', track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip', + trackId: 'captions', + clip: textClip({ + id: 's2', + text: 'w2 w3 w4', + startSec: 0.1, + durationSec: 2.75, + style: { fontSize: 54, color: 'white', background: 'black@0.6', anchor: 'bottom-center' }, + }), + }, + ]); + expect(short.captions).toBe(1); + expect(short.warnings).toEqual([]); + }); + + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { + const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); + const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; + const vertical = await plan(hd, { ranges: [range] }); + expect(vertical.ops[0]).toMatchObject({ width: 720, height: 1280, fps: 25 }); + const square = await plan(hd, { ranges: [range], aspect: '1:1' }); + expect(square.ops[0]).toMatchObject({ width: 720, height: 720 }); + }); + + it('leaves loudness off with null, and captions out when asked', async () => { + const short = await plan(whole, { + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + targetLufs: null, + }); + expect(short.ops.map((o) => o.op)).toEqual(['setCanvas', 'addTrack', 'addClip']); + expect(short.captions).toBe(0); + }); + + it('refuses a timeline that already holds a short, and rebuilds it with replace', async () => { + const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; + const first = await plan(whole, { ranges: [range] }); + const made = applyOps(emptyComposition(), first.ops); + await expect(plan(whole, { ranges: [range] }, { target: made })).rejects.toThrow(/replace/); + + const rebuilt = await plan(whole, { ranges: [range], replace: true }, { target: made }); + // Loudness is already -14, so only the tracks and clips are rebuilt. + expect(rebuilt.ops.map((o) => o.op)).toEqual([ + 'removeTrack', + 'setCanvas', + 'addTrack', + 'addClip', + ]); + }); + + it('reuses a timeline left empty by an undo without replace', async () => { + const emptied = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ]); + const short = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }, + { target: emptied }, + ); + expect(short.ops[0]).toEqual({ op: 'removeTrack', trackId: 'v0' }); + expect(() => applyOps(emptied, short.ops)).not.toThrow(); + }); +}); + +describe('checkShortTimelines', () => { + it('refuses main as the short, and the short as its own source', () => { + expect(() => checkShortTimelines('main')).toThrow(/not main/); + expect(() => checkShortTimelines('best-1', 'best-1')).toThrow(/its own timeline/); + expect(() => checkShortTimelines('best-1')).not.toThrow(); + expect(() => checkShortTimelines('best-2', 'best-1')).not.toThrow(); + }); +}); + +describe('make_short verb', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const ctxFor = (timeline?: string, sources: Record = {}): VerbContext => ({ + ingest: async () => { + throw new Error('not used'); + }, + analyzeSilence: async () => { + throw new Error('not used'); + }, + transcript: async (comp, clipId, transcriptId) => ({ + ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), + media: new Map(), + }), + readTimeline: async (id) => { + const doc = sources[id]; + if (!doc) throw new NoTimelineError(id); + return doc; + }, + defaultTrack: 'v0', + timeline, + }); + const verb = { + verb: 'make_short' as const, + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + }; + + it('lowers to the planned short, cut from main by default', async () => { + const ops = await lowerVerb(emptyComposition(), verb, ctxFor('best-1', { main: whole })); + const doc = applyOps(emptyComposition(), ops); + expect(doc).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); + expect(doc.loudness).toEqual({ targetLufs: -14 }); + expect(doc.tracks.map((t) => t.id)).toEqual(['v0', 'captions']); + }); + + it('cuts from another timeline with source', async () => { + const ops = await lowerVerb( + emptyComposition(), + { ...verb, source: 'long-cut' }, + ctxFor('best-1', { 'long-cut': whole }), + ); + expect(ops.length).toBeGreaterThan(0); + }); + + it('refuses main as its target', async () => { + await expect( + lowerVerb(emptyComposition(), verb, ctxFor(undefined, { main: whole })), + ).rejects.toBeInstanceOf(ShortError); + }); + + it('is an editable verb with bounded ranges and settings', () => { + const ok = (extra: Record): boolean => + EditableVerbSchema.safeParse({ ...verb, ...extra }).success; + expect(ok({})).toBe(true); + expect(ok({ ranges: [] })).toBe(false); + expect(ok({ ranges: Array.from({ length: 21 }, () => verb.ranges[0]) })).toBe(false); + expect(ok({ ranges: [{ clipId: 'c1', fromIndex: -1, toIndex: 2 }] })).toBe(false); + expect(ok({ x: 1.5 })).toBe(false); + expect(ok({ maxWords: 13 })).toBe(false); + expect(ok({ targetLufs: -40 })).toBe(false); + expect(ok({ targetLufs: null, captions: false, replace: true, aspect: '4:5' })).toBe(true); + expect(ok({ source: 'Main Cut' })).toBe(false); + }); +}); + +describe('make_short on the store', () => { + let workspace: string; + let saved: string | undefined; + beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-shorts-test-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; + }); + afterEach(async () => { + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); + }); + + const ctx: VerbContext = { + ingest: async () => { + throw new Error('not used'); + }, + analyzeSilence: async () => { + throw new Error('not used'); + }, + transcript: async (comp, clipId, transcriptId) => ({ + ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), + media: new Map(), + }), + readTimeline: (id) => readComposition(id), + defaultTrack: 'v0', + }; + const makeShort = { + verb: 'make_short' as const, + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + }; + + it('creates the short as one undoable edit and leaves main alone', async () => { + const main = await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + + const { doc } = await applyVerbs([makeShort], ctx, 'best-1'); + expect(doc).toMatchObject({ width: 1080, height: 1920, rev: 1 }); + expect(await readComposition()).toEqual(main); + expect((await readDocOpLog('best-1')).entries).toHaveLength(1); + + const undo = await undoLastDocOp('best-1'); + expect(undo.undone).toBe(true); + expect(undo.doc.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(main.rev); + }); + + it('creates a timeline only when the batch starts with make_short', async () => { + await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + const addText = { verb: 'add_text' as const, text: 'Hook', durationSec: 1 }; + await expect(applyVerbs([addText], ctx, 'best-1')).rejects.toBeInstanceOf(NoTimelineError); + await expect(applyVerbs([addText, makeShort], ctx, 'best-1')).rejects.toBeInstanceOf( + NoTimelineError, + ); + + const { doc } = await applyVerbs([makeShort, { ...addText, track: 'hook' }], ctx, 'best-1'); + expect(doc.tracks.map((t) => t.id)).toEqual(['v0', 'captions', 'hook']); + }); + + it('refuses to make a short on main', async () => { + await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + await expect(applyVerbs([makeShort], ctx)).rejects.toBeInstanceOf(ShortError); + }); +}); diff --git a/tests/timeline-verbs.test.ts b/tests/timeline-verbs.test.ts index 88a4a3c..afd7f96 100644 --- a/tests/timeline-verbs.test.ts +++ b/tests/timeline-verbs.test.ts @@ -30,6 +30,9 @@ const ctx = (durationSec = 10): VerbContext => ({ transcript: async () => { throw new Error('not used'); }, + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', }); @@ -413,6 +416,9 @@ describe('applyVerbs (op-aware + undoable)', () => { openGate = resolve; }); const barrierCtx: VerbContext = { + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', analyzeSilence: async () => { throw new Error('not used'); From 13adac9b9d0a3a18e73073872322e1ab5d3f3505 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:03:26 -0800 Subject: [PATCH 03/35] fix(timeline): keep padding-only and replayed padding out of shorts A piece that only touches a range's words (they were cut out exactly) is dropped, so make_short refuses instead of playing padding alone. A range that continues the one before it in the same file starts where that one's padding ended, so neighboring words play straight through. Co-Authored-By: Claude Opus 5.5 --- src/timeline/shorts.ts | 88 +++++++++++++++++++++++++---------- tests/timeline-shorts.test.ts | 49 +++++++++++++++++++ 2 files changed, 113 insertions(+), 24 deletions(-) diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 5fb7637..0bfef72 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -104,6 +104,7 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI } interface WordWindow { + mediaId: MediaId; /** Padded edges, source seconds. */ start: number; end: number; @@ -114,6 +115,9 @@ interface WordWindow { interface Piece extends ShortPiece { effects: Effect[]; + /** The source clip's speed, refused only if the piece is played. */ + speed: number; + sourceClipId: string; } /** @@ -142,17 +146,28 @@ export async function planShort( ); } - const pieces: Piece[] = []; + const windows: WordWindow[] = []; for (const range of request.ranges) { const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); - const window = wordWindow(transcript, range, clip.id); - const found = piecesUnder(picture.clips, clip.mediaId, window); + windows.push(wordWindow(transcript, range, clip)); + } + const pieces: Piece[] = []; + windows.forEach((window, i) => { + const range = request.ranges[i]; + const found = piecesUnder(picture.clips, withoutReplay(window, windows[i - 1])); if (found.length === 0) { throw new ShortError( - `Words ${range.fromIndex}..${range.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, + `Words ${range?.fromIndex}..${range?.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, ); } pieces.push(...found); + }); + const sped = pieces.find((p) => p.speed !== 1); + if (sped) { + throw new ShortError( + `Clip "${sped.sourceClipId}" on the source timeline plays at ${sped.speed}x, and a short can't include a speed change yet. ` + + 'Remove its speed effect or pick words outside it.', + ); } const joined = joinContinuous(pieces); @@ -238,12 +253,12 @@ export async function planShort( * so the padding never takes in part of a word outside the range, and at the * start of the file. */ -function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): WordWindow { +function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): WordWindow { const first = transcript.words[range.fromIndex]; const last = transcript.words[range.toIndex]; if (!first || !last || range.toIndex < range.fromIndex) { throw new ShortError( - `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clipId}" ` + + `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clip.id}" ` + `(it has ${transcript.words.length} words, numbered from 0).`, ); } @@ -258,6 +273,7 @@ function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), ); return { + mediaId: clip.mediaId, start: roundMs(start), end: roundMs(end), speechStart: first.startSec, @@ -266,39 +282,62 @@ function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): } /** - * The parts of a window that the source timeline's picture clips of one file - * play, in timeline order. A moment the timeline plays twice is taken once, and - * a part holding only padding (no word of the range) is left out, since it would - * only add a jump cut. + * A window that picks up later in the same file than the one before it starts + * where that one's padding ended, so two ranges of neighboring words play + * straight through instead of replaying the padding they share. A range that + * repeats or goes back to earlier words is left as asked. + */ +function withoutReplay(window: WordWindow, before: WordWindow | undefined): WordWindow { + if ( + !before || + before.mediaId !== window.mediaId || + window.speechStart < before.speechEnd || + window.start >= before.end + ) { + return window; + } + return { ...window, start: Math.min(before.end, window.speechStart) }; +} + +/** + * The parts of a window that the source timeline's picture clips of its file + * play, in timeline order. A moment the timeline plays twice is taken once. + * Parts a split left continuous are joined first; then a part holding none of + * the range's words (only padding) is left out, since it would only add a jump + * cut, as is a sliver too short to see. */ -function piecesUnder(clips: readonly Clip[], mediaId: MediaId, window: WordWindow): Piece[] { +function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { const taken: [number, number][] = []; - const pieces: Piece[] = []; + const parts: Piece[] = []; for (const clip of [...clips].sort((a, b) => a.startSec - b.startSec)) { - if (clip.kind !== 'media' || clip.mediaId !== mediaId) continue; + if (clip.kind !== 'media' || clip.mediaId !== window.mediaId) continue; const lo = Math.max(window.start, clip.sourceInSec); const hi = Math.min(window.end, clip.sourceOutSec); if (hi <= lo) continue; for (const [a, b] of subtract([lo, hi], taken)) { - if (b - a < MIN_PIECE_SEC || b < window.speechStart || a > window.speechEnd) continue; - const speed = speedFactor(clip.effects); - if (speed !== 1) { - throw new ShortError( - `Clip "${clip.id}" on the source timeline plays at ${speed}x, and a short can't include a speed change yet. ` + - 'Remove its speed effect or pick words outside it.', - ); - } - pieces.push({ - mediaId, + parts.push({ + mediaId: window.mediaId, sourceInSec: a, sourceOutSec: b, // Fades belonged to the source clip's edges, not the short's. effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), + speed: speedFactor(clip.effects), + sourceClipId: clip.id, }); taken.push([a, b]); } } - return pieces; + return joinContinuous(parts).filter( + (p) => p.sourceOutSec - p.sourceInSec >= MIN_PIECE_SEC && holdsSpeech(p, window), + ); +} + +/** Whether a piece plays any of the range's words, not just touches their edge. */ +function holdsSpeech(piece: Piece, window: WordWindow): boolean { + if (window.speechEnd > window.speechStart) { + return piece.sourceOutSec > window.speechStart && piece.sourceInSec < window.speechEnd; + } + return piece.sourceInSec <= window.speechStart && piece.sourceOutSec >= window.speechEnd; } /** `[lo, hi]` minus the `taken` intervals, in order. */ @@ -326,6 +365,7 @@ function joinContinuous(pieces: Piece[]): Piece[] { prev && prev.mediaId === piece.mediaId && Math.abs(prev.sourceOutSec - piece.sourceInSec) < SAME_SEC && + prev.speed === piece.speed && JSON.stringify(prev.effects) === JSON.stringify(piece.effects) ) { joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index c2d1267..49006cf 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -187,6 +187,55 @@ describe('planShort: cut from the source timeline', () => { expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); }); + it('refuses words cut out exactly, where only padding touches them', async () => { + // Word 2 (3.0–3.5) was cut out exactly: c1 ends at its start, c2 starts at its end. + const exact = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3.5, 30, 3)]); + await expect( + plan(exact, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }), + ).rejects.toThrow(/cut out of the source timeline/); + }); + + it('keeps the lead padding across a split at the first word', async () => { + const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); + const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.75 }]); + }); + + it('plays two ranges of neighboring words straight through, without replaying padding', async () => { + const close = transcriptOf(M, [ + [1, 1.5], + [1.7, 2.2], + [2.9, 3.4], + ]); + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const short = await plan( + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + ], + }, + { transcripts: [close] }, + ); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }]); + + const repeated = await plan( + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + { clipId: 'c1', fromIndex: 0, toIndex: 1 }, + ], + }, + { transcripts: [close] }, + ); + expect(repeated.pieces).toEqual([ + { mediaId: M, sourceInSec: 1.6, sourceOutSec: 2.45 }, + { mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }, + ]); + }); + it('plays the ranges in the order given', async () => { const whole = sourceOf([clipOf('c1', 0, 30, 0)]); const short = await plan(whole, { From b6688cba6914d83cd10e276d7b89372b00d616af Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:05:05 -0800 Subject: [PATCH 04/35] docs(docs): describe shorts in the README, SKILL.md and llms.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the long recording → shorts quick start, the `short` command, the make_short verb in the MCP lists and an FAQ entry, and polishes the missing-timeline and non-empty-short errors for CLI users. Co-Authored-By: Claude Opus 5.5 --- README.md | 32 ++++++++++++++++++++++++++++++-- SKILL.md | 5 ++++- llms.txt | 3 ++- mcpb/manifest.json | 2 +- src/timeline/document-store.ts | 4 ++-- src/timeline/shorts.ts | 2 +- 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 29bfeef..73d4bae 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,13 @@ Version 0.5.0 turns a raw talking-head recording into a clean, captioned cut: ve | **Captions** | Burned-in captions timed to the words, as short phrases or one word at a time | `clip timeline captions` | | **Reframe** | Turns a landscape cut into 9:16, 1:1 or 4:5, filling the frame and choosing the part that shows; zoom and position per clip | `clip timeline reframe` | | **Loudness** | Evens out the exported mp4's volume to a target (-14 LUFS for YouTube, Shorts, Reels and TikTok; -16 for podcasts), measured in two passes | `clip timeline loudness` | +| **Shorts from a long recording** | Turns ranges of words from a long recording into a vertical short on a timeline of its own: captioned, at -14 LUFS, with the cuts already made on the recording; the long cut stays as it is | `clip timeline short` | | **Cut check** | Flags cuts where a word may be clipped, with a contact sheet of the frames on each side | `clip timeline check` | | **Export to Final Cut Pro and DaVinci Resolve** | FCPXML 1.10 project | `clip timeline export --format fcpxml` | | **Export to Premiere Pro** | Final Cut Pro 7 XML project, captions as `.srt` files | `clip timeline export --format xmeml` | | **Render** | mp4 of the finished cut | `clip timeline export` | -All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check isn't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. +All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check and shorts aren't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. --- @@ -149,6 +150,29 @@ A cut splits the clip into pieces: the first keeps its id, and `clip timeline sh --- +## Quick start: a long recording → vertical shorts + +Each short is a timeline of its own, cut from the main timeline's clips, so the pauses and retakes you cut there stay cut. In Claude Code or Claude Desktop, ask: + +> *Transcribe podcast.mp4 and cut the pauses. Then find the three strongest moments, 20 to 60 seconds each, and make each one a short.* + +The same steps from the CLI: + +```bash +clip timeline new +clip timeline add-media podcast.mp4 # prints the clip id, e.g. clip_ec2f047d +clip timeline transcribe clip_ec2f047d # each word with its index and time +clip timeline silence clip_ec2f047d # cut the pauses on main +clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 as a 9:16 short +clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off +clip timeline --timeline best-1 export best-1.mp4 +clip timeline list # main and every short +``` + +`short` keeps 0.10 s before the first word and 0.25 s after the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. + +--- + ## Another example: a product demo from screen recordings Five screen recordings → one polished product demo. Type this into Claude Code (or Claude Desktop, via the MCP server): @@ -374,7 +398,7 @@ MakeMyClip Editor is built for that job: an open-source video editor that Claude ### Is there an MCP server for video editing? -Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. +Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, make shorts, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. ### How do I remove silence from a video automatically? @@ -388,6 +412,10 @@ Yes. MakeMyClip Editor transcribes with whisper.cpp on your machine; install it Yes. After transcription, MakeMyClip Editor's `clip timeline captions` (or the UI's Captions button) puts captions timed to the words on a track above the video, and export burns them in. Captions come as short phrases or one word at a time, with your choice of size, color, position, and background. Non-Latin scripts use an installed font that has the glyphs. In the Premiere export, captions become `.srt` files. On Linux, the bundled FFmpeg can't draw text, so set `MAKEMYCLIP_FFMPEG_PATH` to an FFmpeg built with libfreetype and libharfbuzz. +### Can it cut Shorts or Reels out of a podcast? + +Yes. Transcribe the recording, then `clip timeline short best-1 ` (or the `make_short` verb from chat) turns that range of words into a vertical short on a timeline of its own: 9:16, filling the frame, captioned, normalized to -14 LUFS, with the silence and retake cuts already made on the recording. Make as many as you like; the long cut stays untouched, and each short exports on its own. You or the agent pick the moments: nothing scores highlights for you, and nothing is uploaded. + ### Can I finish an AI edit in DaVinci Resolve, Premiere Pro or Final Cut Pro? Yes. MakeMyClip Editor's `clip timeline export cut.fcpxml` writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve, and `clip timeline export cut.xml` writes Final Cut Pro 7 XML for Premiere Pro. Both keep the cuts, gaps, fades, volume, and cross dissolves, and point at your original files by path. Anything the format can't hold (speed, color adjustments, chroma key, a reframe's fill, zoom or position) is refused with the fix to make. diff --git a/SKILL.md b/SKILL.md index f28502d..e3bb63b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: editor -description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. +description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; cut clips or shorts from a long video or podcast; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. license: MIT compatibility: Requires Node 24+. FFmpeg is auto-downloaded via ffmpeg-static on first use; override with MAKEMYCLIP_FFMPEG_PATH. Transcription (transcript cuts, retakes, captions) needs a user-installed whisper.cpp and a model file set in MAKEMYCLIP_WHISPER_MODEL. metadata: @@ -22,6 +22,7 @@ Trigger this skill when the user asks to: - Cut the silence, pauses, or dead air out of a recording ("tighten it up", "remove the gaps") - Remove repeated takes or bad takes, or cut a sentence or words by the transcript - Clean up a talking-head video, tutorial, lecture, or video podcast +- Cut clips or vertical shorts (Shorts, Reels, TikTok) out of a long video, webinar or podcast - Add captions or subtitles timed to the speech, or titles and lower-thirds - Zoom, pan, or apply Ken Burns to a region of a clip - Assemble multiple clips into one (concat, montage) @@ -332,6 +333,8 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it keeps 0.10 s before the first word and 0.25 s after the last, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. + To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. ### Open the local UI (implemented) diff --git a/llms.txt b/llms.txt index e245b0a..aa8b461 100644 --- a/llms.txt +++ b/llms.txt @@ -21,6 +21,7 @@ - **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time) - **Reframe:** `clip timeline reframe 9:16` turns a landscape cut vertical (also 1:1, 4:5); `--x` picks the part that shows, `clip timeline framing ` sets one clip's zoom and position - **Loudness:** `clip timeline loudness -14` normalizes the exported mp4 to -14 LUFS (or `-16` for podcasts) in two passes +- **Shorts from a long recording:** `clip timeline short best-1 ` (MCP: the `make_short` verb) turns a range of transcript words into a 9:16 short on a timeline of its own, captioned and at -14 LUFS, keeping the cuts made on the main timeline; `--timeline ` points any command at a short - **Cut check:** `clip timeline check --sheet cuts.jpg` flags cuts that may clip a word and writes a contact sheet - **Export to pro editors:** `clip timeline export --format fcpxml` (Final Cut Pro, DaVinci Resolve) or `--format xmeml` (Premiere Pro; captions as .srt) - Every timeline edit is one undoable step (`clip timeline undo` / `redo`), from the CLI, the MCP server or the browser UI @@ -52,7 +53,7 @@ Plus session meta-ops not in the tool registry: snapshot, undo, inspect, delete. - Different category from Runway/Pika (generative AI), Premiere/DaVinci (heavy NLEs) — but it exports its cut to Final Cut Pro, Resolve and Premiere for finishing - Overlaps with auto-editor (open-source CLI: silence cuts, transcript edits, NLE export); MakeMyClip Editor adds a shared timeline document that an agent and a person (browser UI) both edit, with undo per step - Transcription is local (whisper.cpp); nothing is uploaded -- Single workspace folder holds composition.json (the timeline), session.json (the op log), output files, snapshots/ — fully version-controllable +- Single workspace folder holds composition.json (the main timeline), timelines/ (shorts and other named timelines), session.json (the op log), output files, snapshots/ — fully version-controllable - Stream-copy for lossless trim/split/concat; re-encodes only when filter graphs require it ## Architecture diff --git a/mcpb/manifest.json b/mcpb/manifest.json index 2841441..dc6e9c5 100644 --- a/mcpb/manifest.json +++ b/mcpb/manifest.json @@ -40,7 +40,7 @@ }, { "name": "timeline_edit", - "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, transition." + "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, make a vertical short from transcript words, transition." }, { "name": "timeline_check", diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index 2a16dc1..b5ff3ed 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -55,7 +55,7 @@ export class NoTimelineError extends Error { constructor(timeline: string) { super( `No timeline "${timeline}" in the workspace. List them with \`clip timeline list\` ` + - `(timeline_show reports them too). make_short creates a short's timeline.`, + '(timeline_show reports them too). A short gets its timeline from make_short (`clip timeline short`).', ); this.name = 'NoTimelineError'; this.timeline = timeline; @@ -493,7 +493,7 @@ export async function listTimelines(): Promise { id, width: doc.width, height: doc.height, - durationSec: compositionDuration(doc), + durationSec: Math.round(compositionDuration(doc) * 1000) / 1000, rev: doc.rev, }); } catch (err) { diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 0bfef72..0fbb9ab 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -136,7 +136,7 @@ export async function planShort( if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { throw new ShortError( - 'That timeline already holds a short. Pass replace to rebuild it, or pick a new timeline id.', + 'That timeline already holds a short. Pass replace (`--replace`) to rebuild it, or pick a new timeline id.', ); } const picture = videoLayers(source).picture; From bc8e1d9fee05127dcf6e0b6d220f1c1d8e239c51 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:14:53 -0800 Subject: [PATCH 05/35] fix(timeline): tune short edges to whisper.cpp's word timing whisper.cpp words abut, each pause going to the word before it, so the neighbor-word clamp removed all padding. Measured on a 30-sentence say recording with base.en and on jfk.wav: a range now starts 0.25 s before its first word (stopping halfway into the word before) and ends with its last word, which already runs through the pause after it. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- SKILL.md | 2 +- docs/backlog/short-edges-snap-to-pause.md | 3 ++ src/index.ts | 1 - src/timeline/shorts.ts | 32 +++++++------ tests/integration/timeline-shorts.test.ts | 8 ++-- tests/timeline-shorts.test.ts | 58 +++++++++++------------ 7 files changed, 55 insertions(+), 51 deletions(-) create mode 100644 docs/backlog/short-edges-snap-to-pause.md diff --git a/README.md b/README.md index 73d4bae..0dd305d 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ clip timeline --timeline best-1 export best-1.mp4 clip timeline list # main and every short ``` -`short` keeps 0.10 s before the first word and 0.25 s after the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. +`short` starts 0.25 s before the first word and ends with the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. --- diff --git a/SKILL.md b/SKILL.md index e3bb63b..a11612f 100644 --- a/SKILL.md +++ b/SKILL.md @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it keeps 0.10 s before the first word and 0.25 s after the last, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/docs/backlog/short-edges-snap-to-pause.md b/docs/backlog/short-edges-snap-to-pause.md new file mode 100644 index 0000000..3bcb401 --- /dev/null +++ b/docs/backlog/short-edges-snap-to-pause.md @@ -0,0 +1,3 @@ +# Snap short edges to the pause + +whisper.cpp (`-ml 1 -sow`) gives each pause to the word before it, so at a sentence edge its words run late. On a 30-sentence test recording (macOS `say`, model base.en), a short ending on a sentence's last word ran past the pause into the next sentence in 18 of 29 cases, by up to 0.35 s. Even with the 0.25 s lead, 3 of 29 starts lost up to 0.1 s of the first word. No fixed padding fixes both. Move each edge of a range to the quietest point of the audio near it: run a silencedetect pass over a second or so around the edge and keep the edge inside the pause between the range and its neighbors. `-dtw base.en` didn't change the per-word offsets in a quick check. diff --git a/src/index.ts b/src/index.ts index 30de026..bea7419 100644 --- a/src/index.ts +++ b/src/index.ts @@ -246,7 +246,6 @@ export { DEFAULT_SHORT_LUFS, planShort, SHORT_LEAD_SEC, - SHORT_TAIL_SEC, ShortError, type ShortPiece, type ShortPlan, diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 0fbb9ab..3cc598c 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -20,10 +20,14 @@ import type { Transcript } from './transcript.js'; * person picks the words; nothing here scores highlights. */ -/** Kept before a range's first word, so its attack isn't clipped. */ -export const SHORT_LEAD_SEC = 0.1; -/** Kept after a range's last word, so its release and the breath after it aren't clipped. */ -export const SHORT_TAIL_SEC = 0.25; +/** + * Kept before a range's first word. whisper.cpp hands each pause to the word + * before it, so the next word tends to start late: on test speech its first + * words started up to 0.35 s after the voice did. 0.25 s left 3 of 29 sentence + * starts short by at most 0.1 s. A range ends where its last word does, with no + * padding: that word already runs on through the pause after it. + */ +export const SHORT_LEAD_SEC = 0.25; /** Loudness a short is normalized to unless asked otherwise: the vertical platforms' -14 LUFS. */ export const DEFAULT_SHORT_LUFS = -14; @@ -248,10 +252,9 @@ export async function planShort( } /** - * The source window a range plays: its first word's start minus the lead to its - * last word's end plus the tail. Each side stops at the neighboring word's edge, - * so the padding never takes in part of a word outside the range, and at the - * start of the file. + * The source window a range plays: its first word's start minus the lead, to + * its last word's end. The lead stops halfway into the word before (whisper.cpp + * words abut, so that word also holds the pause) and at the start of the file. */ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): WordWindow { const first = transcript.words[range.fromIndex]; @@ -263,19 +266,18 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); } const before = transcript.words[range.fromIndex - 1]; - const after = transcript.words[range.toIndex + 1]; const start = Math.min( first.startSec, - Math.max(first.startSec - SHORT_LEAD_SEC, before?.endSec ?? 0, 0), - ); - const end = Math.max( - last.endSec, - Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), + Math.max( + first.startSec - SHORT_LEAD_SEC, + before ? (before.startSec + before.endSec) / 2 : 0, + 0, + ), ); return { mediaId: clip.mediaId, start: roundMs(start), - end: roundMs(end), + end: last.endSec, speechStart: first.startSec, speechEnd: last.endSec, }; diff --git a/tests/integration/timeline-shorts.test.ts b/tests/integration/timeline-shorts.test.ts index a5476af..753c7d3 100644 --- a/tests/integration/timeline-shorts.test.ts +++ b/tests/integration/timeline-shorts.test.ts @@ -125,11 +125,11 @@ describe.skipIf(!IS_CI)('shorts against real FFmpeg', () => { ...(HAS_DRAWTEXT ? [] : ['--no-captions']), ]); expect(short.pieces).toEqual([ - expect.objectContaining({ sourceInSec: 540.1, sourceOutSec: 545.2 }), - expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.95 }), - expect.objectContaining({ sourceInSec: 100.1, sourceOutSec: 105.95 }), + expect.objectContaining({ sourceInSec: 539.95, sourceOutSec: 545.2 }), + expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.7 }), + expect.objectContaining({ sourceInSec: 99.95, sourceOutSec: 105.7 }), ]); - expect(short.durationSec).toBe(16.2); + expect(short.durationSec).toBe(16); expect(short.canvas).toEqual({ width: 720, height: 1280, fit: 'cover' }); if (HAS_DRAWTEXT) expect(short.captions).toBeGreaterThan(0); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 49006cf..e67e3f5 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -111,13 +111,13 @@ const addedClips = (ops: CompositionOp[]): AddClipOp['clip'][] => describe('planShort: edges', () => { const whole = sourceOf([clipOf('c1', 0, 30, 0)]); - it('pads a range 0.10 s before its first word and 0.25 s after its last', async () => { + it('starts a range 0.25 s before its first word and ends it with its last', async () => { const short = await plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); - expect(short.durationSec).toBe(2.85); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); + expect(short.durationSec).toBe(2.75); }); - it('stops the padding at the neighboring words and at the start of the file', async () => { + it('stops the lead halfway into the word before, and at the start of the file', async () => { const tight = transcriptOf(M, [ [0.03, 0.3], [0.32, 0.6], @@ -129,21 +129,21 @@ describe('planShort: edges', () => { { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, { transcripts: [tight] }, ); - expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.3, sourceOutSec: 0.95 }]); + expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.165, sourceOutSec: 0.9 }]); const first = await plan( whole, { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 0 }] }, { transcripts: [tight] }, ); - expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.32 }]); + expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.3 }]); }); it('stops at the end of the source clip', async () => { - const short = await plan(sourceOf([clipOf('c1', 0, 5.6, 0)]), { + const short = await plan(sourceOf([clipOf('c1', 0, 5.3, 0)]), { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }], }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.75, sourceOutSec: 5.3 }]); }); it('refuses word indices outside the transcript', async () => { @@ -165,12 +165,12 @@ describe('planShort: cut from the source timeline', () => { ]); const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 6 }] }); expect(short.pieces).toEqual([ - { mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.7 }, + { mediaId: M, sourceInSec: 2.75, sourceOutSec: 3.7 }, { mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.7 }, - { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.75 }, + { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.5 }, ]); - expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.8, 2.6]); - expect(short.durationSec).toBe(4.45); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.95, 2.75]); + expect(short.durationSec).toBe(4.35); }); it('refuses words that were cut out of the source timeline', async () => { @@ -181,10 +181,10 @@ describe('planShort: cut from the source timeline', () => { }); it('leaves out a sliver that holds only padding', async () => { - // The tail padding of word 4 (ends 5.5) reaches 5.75, just into c2. - const cut = sourceOf([clipOf('c1', 0, 5.6, 0), clipOf('c2', 5.7, 12, 5.6)]); - const short = await plan(cut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + // The lead of word 3 (starts 4.0) reaches back to 3.75, just into the end of c1. + const cut = sourceOf([clipOf('c1', 0, 3.8, 0), clipOf('c2', 3.85, 12, 3.8)]); + const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 3, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.85, sourceOutSec: 5.5 }]); }); it('refuses words cut out exactly, where only padding touches them', async () => { @@ -198,7 +198,7 @@ describe('planShort: cut from the source timeline', () => { it('keeps the lead padding across a split at the first word', async () => { const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 3.5 }]); }); it('plays two ranges of neighboring words straight through, without replaying padding', async () => { @@ -218,7 +218,7 @@ describe('planShort: cut from the source timeline', () => { }, { transcripts: [close] }, ); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2.2 }]); const repeated = await plan( whole, @@ -231,8 +231,8 @@ describe('planShort: cut from the source timeline', () => { { transcripts: [close] }, ); expect(repeated.pieces).toEqual([ - { mediaId: M, sourceInSec: 1.6, sourceOutSec: 2.45 }, - { mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }, + { mediaId: M, sourceInSec: 1.45, sourceOutSec: 2.2 }, + { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2.2 }, ]); }); @@ -245,10 +245,10 @@ describe('planShort: cut from the source timeline', () => { ], }); expect(short.pieces).toEqual([ - { mediaId: M, sourceInSec: 6.9, sourceOutSec: 8.75 }, - { mediaId: M, sourceInSec: 1.9, sourceOutSec: 3.75 }, + { mediaId: M, sourceInSec: 6.75, sourceOutSec: 8.5 }, + { mediaId: M, sourceInSec: 1.75, sourceOutSec: 3.5 }, ]); - expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.85]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.75]); }); it('cuts ranges from two recordings', async () => { @@ -273,13 +273,13 @@ describe('planShort: cut from the source timeline', () => { it('takes a moment the source plays twice only once', async () => { const repeated = sourceOf([clipOf('c1', 0, 10, 0), clipOf('c2', 2, 6, 10)]); const short = await plan(repeated, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); }); it('joins pieces that a split (not a cut) left continuous', async () => { const split = sourceOf([clipOf('c1', 0, 4, 0), clipOf('c2', 4, 10, 4)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); }); it('keeps color and volume, and drops the fades of the source clip', async () => { @@ -330,8 +330,8 @@ describe('planShort: the edit', () => { clip: mediaClip({ id: 's1', mediaId: M, - sourceInSec: 2.9, - sourceOutSec: 5.75, + sourceInSec: 2.75, + sourceOutSec: 5.5, startSec: 0, }), }, @@ -344,8 +344,8 @@ describe('planShort: the edit', () => { clip: textClip({ id: 's2', text: 'w2 w3 w4', - startSec: 0.1, - durationSec: 2.75, + startSec: 0.25, + durationSec: 2.5, style: { fontSize: 54, color: 'white', background: 'black@0.6', anchor: 'bottom-center' }, }), }, From 7dec1b1bcfdeb17b632b590280612805037276bc Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:23:42 -0800 Subject: [PATCH 06/35] fix(timeline): confine timeline folders and caption only the chosen words A timeline's folder is now checked against the workspace too, not only its files, so a timelines/ symlink out of the workspace can't receive temp files or a recursive delete through files that link back in. Captions on a short come only from the chosen words, not the word the lead padding reaches into, and a range whose words were all cut is refused even when a clip still covers the gap between them. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/review-test-timeout.md | 3 +++ src/timeline/document-store.ts | 13 ++++++++--- src/timeline/shorts.ts | 30 ++++++++++++++++++++----- tests/document-store.test.ts | 29 ++++++++++++++++++++++++ tests/timeline-shorts.test.ts | 35 +++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 docs/backlog/review-test-timeout.md diff --git a/docs/backlog/review-test-timeout.md b/docs/backlog/review-test-timeout.md new file mode 100644 index 0000000..955a01f --- /dev/null +++ b/docs/backlog/review-test-timeout.md @@ -0,0 +1,3 @@ +# Review integration test timeout + +`tests/integration/timeline-review.test.ts` ("flags a cut planted mid-tone…") runs under vitest's default 5 s timeout but renders several FFmpeg steps, so it sometimes times out on a busy machine (seen once in two runs on `feat/shorts` at 18511f1). Give it an explicit timeout like the other integration tests (`120_000`). diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index b5ff3ed..d149aa2 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -1,6 +1,6 @@ import { randomBytes } from 'node:crypto'; import { readdir, readFile, rm } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { join, resolve } from 'node:path'; import { atomicWriteFile } from '../session/store.js'; import { createRevisionedStore, type RevisionedStore } from '../storage/revisioned-store.js'; import { @@ -72,9 +72,16 @@ function checkedTimelineId(timeline: string): string { return parsed.data; } +/** A non-main timeline's folder, checked against the workspace: `timelines/` + * or the folder could be a symlink out of it, and writes (temp files) and + * deletes act on the folder itself, not only on the files in it. */ +function timelineDir(timeline: string): string { + return resolveInWorkspace(join(TIMELINES_DIR, checkedTimelineId(timeline))); +} + function timelineFile(timeline: string, file: string): string { if (checkedTimelineId(timeline) === MAIN_TIMELINE) return resolve(getWorkspace(), file); - // Checked against the workspace: `timelines/` could be a symlink out of it. + timelineDir(timeline); return resolveInWorkspace(join(TIMELINES_DIR, timeline, file)); } @@ -533,6 +540,6 @@ export async function deleteTimeline(timeline: string): Promise { // A corrupt document is deleted like any other. if (!(err instanceof CompositionCorruptError)) throw err; } - await rm(dirname(compositionPath(timeline)), { recursive: true, force: true }); + await rm(timelineDir(timeline), { recursive: true, force: true }); }); } diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 3cc598c..bd5a43f 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -109,6 +109,10 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI interface WordWindow { mediaId: MediaId; + /** The range's words, by index into its transcript. */ + fromIndex: number; + toIndex: number; + words: { startSec: number; endSec: number }[]; /** Padded edges, source seconds. */ start: number; end: number; @@ -228,10 +232,19 @@ export async function planShort( let captions = 0; if (request.captions !== false) { const built = applyOps(target, ops); + // Only the asked-for words: the lead reaches into the word before, which + // whisper.cpp stretches over the pause, and it isn't part of the short. const plan = await planCaptions( built, { maxWords: request.maxWords }, - (clipId, transcriptId) => lookup(built, clipId, transcriptId), + async (clipId, transcriptId) => { + const found = await lookup(built, clipId, transcriptId); + const asked = windows.filter((w) => w.mediaId === found.transcript.mediaId); + const words = found.transcript.words.filter((_, i) => + asked.some((w) => i >= w.fromIndex && i <= w.toIndex), + ); + return { ...found, transcript: { ...found.transcript, words } }; + }, mintId, ); ops.push(...plan.ops); @@ -276,6 +289,9 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); return { mediaId: clip.mediaId, + fromIndex: range.fromIndex, + toIndex: range.toIndex, + words: transcript.words.slice(range.fromIndex, range.toIndex + 1), start: roundMs(start), end: last.endSec, speechStart: first.startSec, @@ -334,12 +350,14 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { ); } -/** Whether a piece plays any of the range's words, not just touches their edge. */ +/** Whether a piece plays any of the range's words, not just touches one's edge + * or the gap between two that were cut. */ function holdsSpeech(piece: Piece, window: WordWindow): boolean { - if (window.speechEnd > window.speechStart) { - return piece.sourceOutSec > window.speechStart && piece.sourceInSec < window.speechEnd; - } - return piece.sourceInSec <= window.speechStart && piece.sourceOutSec >= window.speechEnd; + return window.words.some((w) => + w.endSec > w.startSec + ? piece.sourceOutSec > w.startSec && piece.sourceInSec < w.endSec + : piece.sourceInSec <= w.startSec && piece.sourceOutSec >= w.endSec, + ); } /** `[lo, hi]` minus the `taken` intervals, in order. */ diff --git a/tests/document-store.test.ts b/tests/document-store.test.ts index f8430bc..d3226f8 100644 --- a/tests/document-store.test.ts +++ b/tests/document-store.test.ts @@ -254,6 +254,35 @@ describe('named timelines', () => { expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); }); + it('refuses a timeline folder out of the workspace even when its files link back in', async () => { + const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); + try { + // timelines/ leads out; the files in it link back to documents inside. + const kept = join(workspace, 'kept'); + await mkdir(kept); + await writeFile(join(kept, 'composition.json'), JSON.stringify(emptyComposition())); + await writeFile(join(kept, 'composition-ops.json'), '{}'); + await mkdir(join(outside, 'best-1')); + for (const file of ['composition.json', 'composition-ops.json']) { + await symlink(join(kept, file), join(outside, 'best-1', file)); + } + await symlink(outside, join(workspace, 'timelines')); + + await expect(readComposition('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 2 }], { timeline: 'best-1' }), + ).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect(deleteTimeline('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + const { readdir } = await import('node:fs/promises'); + expect((await readdir(join(outside, 'best-1'))).sort()).toEqual([ + 'composition-ops.json', + 'composition.json', + ]); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + it('refuses a timelines/ folder that is a symlink out of the workspace', async () => { const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); try { diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index e67e3f5..8ac8798 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -195,6 +195,21 @@ describe('planShort: cut from the source timeline', () => { ).rejects.toThrow(/cut out of the source timeline/); }); + it('refuses a range whose words were all cut, leaving only the gap between them', async () => { + const apart = transcriptOf(M, [ + [1, 1.5], + [4, 4.5], + ]); + const gapOnly = sourceOf([clipOf('c1', 1.5, 4, 0)]); + await expect( + plan( + gapOnly, + { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 1 }] }, + { transcripts: [apart] }, + ), + ).rejects.toThrow(/cut out of the source timeline/); + }); + it('keeps the lead padding across a split at the first word', async () => { const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); @@ -354,6 +369,26 @@ describe('planShort: the edit', () => { expect(short.warnings).toEqual([]); }); + it('captions only the asked-for words, not the word the lead reaches into', async () => { + // whisper.cpp words abut: w0 runs through the pause before w1. + const abutting = transcriptOf(M, [ + [0, 1], + [1, 1.5], + [1.5, 2], + [2, 3], + ]); + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, + lookupFor([abutting]), + counter(), + ); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }]); + const captions = addedClips(short.ops).filter((c) => c.kind === 'text'); + expect(captions.map((c) => (c.kind === 'text' ? c.text : ''))).toEqual(['w1 w2']); + }); + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; From 638e0ff2488bf9050a136ae3c0753524177f4b59 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:30:54 -0800 Subject: [PATCH 07/35] fix(timeline): caption each piece of a short from its own range Captions came from the union of every range's words, so with reordered neighboring ranges one range's word could show in another's lead. Each piece now captions only the ranges it plays, from the transcripts the ranges were already read with. Co-Authored-By: Claude Opus 5.5 --- src/timeline/shorts.ts | 102 ++++++++++++++++++++++------------ tests/timeline-shorts.test.ts | 31 +++++++++++ 2 files changed, 96 insertions(+), 37 deletions(-) diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index bd5a43f..febf983 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -1,16 +1,22 @@ -import { planCaptions } from './captions.js'; +import { + DEFAULT_CAPTION_TRACK, + DEFAULT_CAPTION_WORDS, + defaultCaptionStyle, + groupCaptions, +} from './captions.js'; import { speedFactor, videoLayers } from './compile.js'; import { type Clip, type Composition, + clipEndSec, type Effect, type MediaClip, makeClipId, } from './composition.js'; -import { applyOps, type CompositionOp, mediaClip, videoTrack } from './ops.js'; +import { applyOps, type CompositionOp, mediaClip, textClip, videoTrack } from './ops.js'; import { type Aspect, canvasFor, planReframe } from './reframe.js'; import { MAIN_TIMELINE, type MediaId } from './schema.js'; -import type { Transcript } from './transcript.js'; +import { clipWords, type Transcript } from './transcript.js'; /** * Shorts: a vertical clip cut out of a longer timeline by transcript word ranges, @@ -109,6 +115,7 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI interface WordWindow { mediaId: MediaId; + transcript: Transcript; /** The range's words, by index into its transcript. */ fromIndex: number; toIndex: number; @@ -126,6 +133,9 @@ interface Piece extends ShortPiece { /** The source clip's speed, refused only if the piece is played. */ speed: number; sourceClipId: string; + transcript: Transcript; + /** The word ranges the piece plays, which are what it captions. */ + asked: { fromIndex: number; toIndex: number }[]; } /** @@ -195,23 +205,21 @@ export async function planShort( { op: 'addTrack', track: videoTrack({ id: SHORT_TRACK }) }, ); let at = 0; - for (const piece of joined) { - ops.push({ - op: 'addClip', - trackId: SHORT_TRACK, - clip: mediaClip({ - id: mintId(), - mediaId: piece.mediaId, - sourceInSec: piece.sourceInSec, - sourceOutSec: piece.sourceOutSec, - // Rounded to the microsecond: clean numbers, far inside the export's - // abutting tolerance. - startSec: Math.round(at * 1e6) / 1e6, - effects: piece.effects, - }), + const placed = joined.map((piece) => { + const clip = mediaClip({ + id: mintId(), + mediaId: piece.mediaId, + sourceInSec: piece.sourceInSec, + sourceOutSec: piece.sourceOutSec, + // Rounded to the microsecond: clean numbers, far inside the export's + // abutting tolerance. + startSec: Math.round(at * 1e6) / 1e6, + effects: piece.effects, }); + ops.push({ op: 'addClip', trackId: SHORT_TRACK, clip }); at += piece.sourceOutSec - piece.sourceInSec; - } + return { piece, clip }; + }); if (request.x !== undefined || request.y !== undefined) { const framed = planReframe(applyOps(target, ops), { @@ -231,24 +239,37 @@ export async function planShort( let captions = 0; if (request.captions !== false) { - const built = applyOps(target, ops); - // Only the asked-for words: the lead reaches into the word before, which - // whisper.cpp stretches over the pause, and it isn't part of the short. - const plan = await planCaptions( - built, - { maxWords: request.maxWords }, - async (clipId, transcriptId) => { - const found = await lookup(built, clipId, transcriptId); - const asked = windows.filter((w) => w.mediaId === found.transcript.mediaId); - const words = found.transcript.words.filter((_, i) => - asked.some((w) => i >= w.fromIndex && i <= w.toIndex), - ); - return { ...found, transcript: { ...found.transcript, words } }; - }, - mintId, - ); - ops.push(...plan.ops); - captions = plan.captions; + // Each piece captions only its own range's words: the lead reaches into the + // word before, which whisper.cpp stretches over the pause, and that word + // isn't part of the short even when another range asks for it. + const style = defaultCaptionStyle(applyOps(target, ops)); + const cues = placed.flatMap(({ piece, clip }) => { + const words = piece.transcript.words.filter((_, i) => + piece.asked.some((r) => i >= r.fromIndex && i <= r.toIndex), + ); + return groupCaptions( + clipWords({ ...piece.transcript, words }, clip), + clipEndSec(clip), + request.maxWords ?? DEFAULT_CAPTION_WORDS, + ); + }); + if (cues.length > 0) { + ops.push({ op: 'addTrack', track: videoTrack({ id: DEFAULT_CAPTION_TRACK }) }); + for (const cue of cues) { + ops.push({ + op: 'addClip', + trackId: DEFAULT_CAPTION_TRACK, + clip: textClip({ + id: mintId(), + text: cue.text, + startSec: cue.startSec, + durationSec: roundMs(cue.endSec - cue.startSec), + style, + }), + }); + } + } + captions = cues.length; } return { @@ -289,6 +310,7 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); return { mediaId: clip.mediaId, + transcript, fromIndex: range.fromIndex, toIndex: range.toIndex, words: transcript.words.slice(range.fromIndex, range.toIndex + 1), @@ -341,6 +363,8 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), speed: speedFactor(clip.effects), sourceClipId: clip.id, + transcript: window.transcript, + asked: [{ fromIndex: window.fromIndex, toIndex: window.toIndex }], }); taken.push([a, b]); } @@ -388,7 +412,11 @@ function joinContinuous(pieces: Piece[]): Piece[] { prev.speed === piece.speed && JSON.stringify(prev.effects) === JSON.stringify(piece.effects) ) { - joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; + joined[joined.length - 1] = { + ...prev, + sourceOutSec: piece.sourceOutSec, + asked: [...prev.asked, ...piece.asked], + }; } else { joined.push(piece); } diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 8ac8798..0a110ae 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -389,6 +389,37 @@ describe('planShort: the edit', () => { expect(captions.map((c) => (c.kind === 'text' ? c.text : ''))).toEqual(['w1 w2']); }); + it("captions each piece from its own range, even when another range asks for the lead's word", async () => { + const abutting = transcriptOf(M, [ + [0, 1], + [1, 2], + [2, 3], + ]); + const short = await planShort( + emptyComposition(), + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + ], + }, + lookupFor([abutting]), + counter(), + ); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }, + { mediaId: M, sourceInSec: 0, sourceOutSec: 1 }, + ]); + const captions = addedClips(short.ops).flatMap((c) => + c.kind === 'text' ? [{ text: c.text, startSec: c.startSec }] : [], + ); + expect(captions).toEqual([ + { text: 'w1', startSec: 0.25 }, + { text: 'w0', startSec: 1.25 }, + ]); + }); + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; From 18c8cf09c73c7e323aa0fb5b05dd5f1d832f476c Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:41:29 -0800 Subject: [PATCH 08/35] perf(timeline): render a media frame from a short window of its source The frame preview encoded the clip's whole segment to grab one frame, which took minutes on a long recording. A media clip's frame is now one step: seek the source to the frame and run the segment's own video chain on timestamps offset to the frame's time, so fades, captions and speed read the same time they do in the segment. Text and color cards keep the two-step path. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/frame-preview-short-window.md | 3 - src/timeline/compile.ts | 151 ++++++++++++++------- tests/timeline-introspect.test.ts | 89 +++++++++--- 3 files changed, 173 insertions(+), 70 deletions(-) delete mode 100644 docs/backlog/frame-preview-short-window.md diff --git a/docs/backlog/frame-preview-short-window.md b/docs/backlog/frame-preview-short-window.md deleted file mode 100644 index 177b62f..0000000 --- a/docs/backlog/frame-preview-short-window.md +++ /dev/null @@ -1,3 +0,0 @@ -# Frame preview encodes a short window - -`buildFrameAtPlan` (`src/timeline/compile.ts`) encodes the whole clip's segment to grab one frame, so every framing change in the browser UI re-encodes the full clip before the viewer updates. On a long uncut recording that takes seconds. Encode only a short window around the requested time (`-ss` before the input, `-t` of a few frames) and adjust the frame offset, keeping the same filter chain so the preview still matches export. diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 45dc785..dcf1bb3 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -399,6 +399,59 @@ function segPath(dir: string, index: number, clipId: string, tag?: string): stri return resolve(dir, `${SEG_PREFIX}-${tagInfix(tag)}${index}-${clipId}.mp4`); } +/** + * A clip's segment filters after its input: the video chain (framing onto the + * canvas, a text clip's own text, effects, then the text over it) and the audio + * effects. Shared by export and the frame preview, so a preview can't drift + * from what export draws. + */ +function clipChains( + comp: Composition, + clip: Clip, + index: number, + ctx: CompileContext, + overlays: TextClip[], + fadeSuppress: FadeSuppression = { in: false, out: false }, +): { chain: string; audio: string[]; textFiles: StepSideFile[] } { + const outDur = outputDuration(clip); + const { width, height, fps, background } = comp; + const { video: fx, audio } = effectFilters(clip, outDur, fadeSuppress); + const textFiles: StepSideFile[] = []; + + if (clip.kind === 'text') { + const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); + textFiles.push({ path: textfile, content: clip.text }); + fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); + } + + // Overlay times are timeline seconds; the segment runs in output time, which + // a speed effect scales. + const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; + const end = clipEndSec(clip); + for (const overlay of overlays) { + const from = Math.max(overlay.startSec, clip.startSec); + const to = Math.min(clipEndSec(overlay), end); + if (to - from <= ABUT_EPSILON) continue; + const textfile = resolve( + ctx.dir, + `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, + ); + textFiles.push({ path: textfile, content: overlay.text }); + const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; + fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); + } + + // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish + // for generated sources that are already canvas-sized, but keeps the chain uniform.) + const geometry = [ + ...framingFilters(clipFraming(comp, clip), width, height, background), + 'setsar=1', + `fps=${fps}`, + 'format=yuv420p', + ]; + return { chain: [...geometry, ...fx].join(','), audio, textFiles }; +} + /** * Normalize one clip to a self-contained segment: canvas-sized video at the * composition fps, effects baked in, and a guaranteed stereo AAC audio track @@ -419,8 +472,14 @@ function buildSegmentStep( const outDur = outputDuration(clip); const { width, height, fps, background } = comp; const out = segPath(ctx.dir, index, clip.id, ctx.tag); - const { video: fx, audio: afx } = effectFilters(clip, outDur, fadeSuppress); - const textFiles: StepSideFile[] = []; + const { chain: videoChain, audio: afx, textFiles } = clipChains( + comp, + clip, + index, + ctx, + overlays, + fadeSuppress, + ); const inputArgs: string[] = []; let videoSource: string; // filtergraph input label feeding the video chain, e.g. [0:v] @@ -475,38 +534,8 @@ function buildSegmentStep( ); videoSource = '[0:v]'; audioSource = '1:a'; - const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); - textFiles.push({ path: textfile, content: clip.text }); - fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); - } - - // Overlay times are timeline seconds; the segment runs in output time, which - // a speed effect scales. - const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; - const end = clipEndSec(clip); - for (const overlay of overlays) { - const from = Math.max(overlay.startSec, clip.startSec); - const to = Math.min(clipEndSec(overlay), end); - if (to - from <= ABUT_EPSILON) continue; - const textfile = resolve( - ctx.dir, - `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, - ); - textFiles.push({ path: textfile, content: overlay.text }); - const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; - fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); } - // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish - // for generated sources that are already canvas-sized, but keeps the chain uniform.) - const geometry = [ - ...framingFilters(clipFraming(comp, clip), width, height, background), - 'setsar=1', - `fps=${fps}`, - 'format=yuv420p', - ]; - - const videoChain = [...geometry, ...fx].join(','); // Always normalize audio to a uniform stereo/48k layout BEFORE any effects, so // the fold's concat/acrossfade can't silently downmix when a source clip is // mono (and so every segment exposes a real [a] output). aformat is a harmless @@ -692,18 +721,18 @@ function assertAbutting(clips: Clip[]): void { /** * Render ONE frame at timeline time `atSec` — the agent's "eyes" on the doc. - * Goes through the REAL segment path (`buildSegmentStep`) so the preview can't - * diverge from what export would produce, then extracts a still from that - * normalized segment. Two steps: encode the active clip's segment, then grab the - * frame at the mapped offset. Runs the SAME abutting check as export, so it fails - * (rather than guessing a clip) on an unexportable doc; throws `CompileError` past - * the end, in a gap, or on an overlap, and inherits the v1 single-video-track / - * no-compositing guards. + * Uses the segment's own video chain (`clipChains`) so the preview can't diverge + * from what export would produce. A media clip is one step: seek the source to + * the frame and run the chain on a window starting there, with the timestamps + * offset so fades and text windows read the time they would in the segment, so + * the cost doesn't grow with the clip's length. A text or color card (generated, + * and short) is two: encode its segment, then grab the frame from it. Runs the + * SAME abutting check as export, so it fails (rather than guessing a clip) on an + * unexportable doc; throws `CompileError` past the end, in a gap, or on an + * overlap, and inherits the v1 single-video-track / no-compositing guards. * - * Known v1 limits (flag, don't fix here): a frame inside a transition window - * shows the underlying clip, not the xfade blend; `-ss` is keyframe-accurate - * (off by up to a GOP) — fine for a thumbnail; and a long clip re-encodes a whole - * segment to grab one frame. + * Known v1 limit (flag, don't fix here): a frame inside a transition window + * shows the underlying clip, not the xfade blend. */ export function buildFrameAtPlan( comp: Composition, @@ -728,9 +757,6 @@ export function buildFrameAtPlan( } assertCompilable(clip); - // Encode the clip's segment exactly as export would, then extract the frame. - const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); - // Map doc-local time to the post-speed SEGMENT timebase. The ratio is 1 unless // the clip carries a speed effect, which shortens the segment relative to the // document extent — so a frame still lands on the source content the document @@ -741,9 +767,40 @@ export function buildFrameAtPlan( const segTime = clipDur > 0 ? (localDoc * outDur) / clipDur : 0; const lastFrame = Math.max(0, outDur - 1 / comp.fps); const clamped = Math.max(0, Math.min(segTime, lastFrame)); + const label = `frame:${clip.id}@${atSec}`; + if (clip.kind === 'media') { + const info = ctx.media.get(clip.mediaId); + if (!info) { + throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + // Seconds into the clip's source. The chain's speed `setpts` divides the + // offset back into segment time, which is what its fades and text read. + const local = Math.round(clamped * speedFactor(clip.effects) * 1000) / 1000; + const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); + const frameStep: FfmpegStep = { + label, + args: [ + '-y', + '-ss', + seconds(clip.sourceInSec + local), + '-i', + info.path, + '-filter_complex', + `[0:v]setpts=PTS-STARTPTS+${local}/TB,${chain}[v]`, + ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], + ctx.output, + ], + output: ctx.output, + textFiles, + }; + return { steps: [frameStep], output: ctx.output, durationSec: 0 }; + } + + // Encode the card's segment exactly as export would, then extract the frame. + const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { - label: `frame:${clip.id}@${atSec}`, + label, args: buildPreviewFrameArgs({ input: segStep.output, output: ctx.output, atSec: clamped }), output: ctx.output, textFiles: [], diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index e514515..c89df21 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { buildFrameAtPlan, CompileError, type MediaInfo } from '../src/timeline/compile.js'; +import { + buildFrameAtPlan, + CompileError, + compileTimeline, + type MediaInfo, +} from '../src/timeline/compile.js'; import { clipsAtTime, emptyComposition } from '../src/timeline/composition.js'; import { applyOps, mediaClip, textClip, videoTrack } from '../src/timeline/ops.js'; import type { MediaId } from '../src/timeline/schema.js'; @@ -65,18 +70,48 @@ describe('buildFrameAtPlan', () => { ]); } - it('encodes the clip segment then extracts the frame at the doc-local offset', () => { - const plan = buildFrameAtPlan(oneClip(), CTX('/ws/frame.jpg'), 3); - expect(plan.steps).toHaveLength(2); - const [seg, frame] = plan.steps; - expect(seg?.label).toBe('segment:c1'); - expect(frame?.label).toContain('frame:c1'); - expect(frame?.output).toBe('/ws/frame.jpg'); - // -ss -i … - expect(frame?.args.slice(0, 2)).toEqual(['-y', '-ss']); - expect(Number(frame?.args[2])).toBeCloseTo(3); - expect(frame?.args[4]).toBe(seg?.output); // reads the encoded segment - expect(frame?.args.at(-1)).toBe('/ws/frame.jpg'); + /** The `-filter_complex` graph of a step. */ + const graphOf = (args: string[] | undefined): string => + args?.[(args?.indexOf('-filter_complex') ?? -2) + 1] ?? ''; + + it('renders a media frame in one step from the source, through the segment chain', () => { + const doc = applyOps(oneClip(), [ + { op: 'addEffect', clipId: 'c1', effect: { type: 'fadeIn', durationSec: 1 } }, + ]); + const plan = buildFrameAtPlan(doc, CTX('/ws/frame.jpg'), 3); + expect(plan.steps).toHaveLength(1); + const [frame] = plan.steps; + expect(frame?.label).toBe('frame:c1@3'); + expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '3', '-i', '/in.mp4']); + expect(frame?.args.slice(-8)).toEqual([ + '-map', + '[v]', + '-an', + '-frames:v', + '1', + '-q:v', + '2', + '/ws/frame.jpg', + ]); + // The same video chain export runs, on timestamps offset to the frame's time. + const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); + const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); + expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS-STARTPTS+3/TB,${segmentChain}[v]`); + }); + + it('seeks into the source from the clip in-point and stops at its last frame', () => { + const doc = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId: M, sourceInSec: 100, sourceOutSec: 108, startSec: 0 }), + }, + ]); + const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; + // 30 fps: the last frame starts at 8 - 1/30 s. + expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.967']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.967\/TB,/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -90,9 +125,17 @@ describe('buildFrameAtPlan', () => { expect(graph).toContain("crop=w='min(iw,1080)':h='min(ih,1920)':x='(iw-ow)*0'"); }); - it('a tagged context gives the segment a collision-free path; untagged stays stable', () => { + it('a tagged context gives a card segment a collision-free path; untagged stays stable', () => { + const card = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: textClip({ id: 'c1', text: 'hi', durationSec: 8, startSec: 0 }), + }, + ]); const segOut = (tag?: string) => - buildFrameAtPlan(oneClip(), { ...CTX('/ws/frame.jpg'), tag }, 3).steps[0]?.output; + buildFrameAtPlan(card, { ...CTX('/ws/frame.jpg'), tag }, 3).steps[0]?.output; // Untagged keeps the deterministic name export also uses. expect(segOut()).toBe('/ws/tl-seg-0-c1.mp4'); // A tag is mixed in, so a preview can't overwrite/unlink an export's segment… @@ -106,12 +149,13 @@ describe('buildFrameAtPlan', () => { { op: 'addClip', trackId: 'v0', - clip: mediaClip({ id: 'c2', mediaId: M, sourceOutSec: 4, startSec: 8 }), + clip: mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1, sourceOutSec: 5, startSec: 8 }), }, ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) - expect(plan.steps[0]?.label).toBe('segment:c2'); - expect(Number(plan.steps[1]?.args[2])).toBeCloseTo(2); + expect(plan.steps[0]?.label).toBe('frame:c2@10'); + expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '3']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+2\/TB,/); }); it('maps doc-local time to the post-speed segment timebase for a speed clip', () => { @@ -130,8 +174,13 @@ describe('buildFrameAtPlan', () => { }), }, ]); - const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4); - expect(Number(plan.steps[1]?.args[2])).toBeCloseTo(2); + // The frame shows source second 4; the chain's setpts=PTS/2 turns the +4 s + // offset into segment time 2, which the fades and text windows read. + const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; + expect(frame?.args.slice(1, 3)).toEqual(['-ss', '4']); + const graph = graphOf(frame?.args); + expect(graph).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+4\/TB,/); + expect(graph).toContain('setpts=PTS/2'); }); it('works on a text clip (no media required)', () => { From 28529ff1acd40f11af4a40632cc0123586f924c4 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:44:21 -0800 Subject: [PATCH 09/35] fix(timeline): keep a sped-up clip on the canvas frame grid After setpts, a sped-up clip's frames reached the encoder off the canvas frame grid, and the encoder's frame dropping repeated some frames and shifted others by up to two frames. An fps filter right after the speed setpts picks the nearest frame for each slot, so export is steady and the frame preview shows the frame export plays. Adds the CI-gated check: the window frame matches the frame of the fully encoded clip at PSNR >= 40 dB mid-clip, inside a fade and at 2x, and a frame 9 minutes into a 10-minute clip renders in under a second. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 21 +-- .../integration/timeline-frame-window.test.ts | 161 ++++++++++++++++++ 2 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 tests/integration/timeline-frame-window.test.ts diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index dcf1bb3..d3c4d1b 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -322,6 +322,7 @@ interface FadeSuppression { function effectFilters( clip: Clip, outDur: number, + fps: number, suppress: FadeSuppression = { in: false, out: false }, ): FilterChains { const video: string[] = []; @@ -343,7 +344,10 @@ function effectFilters( if (clip.kind === 'media') { const f = speedFactor(clip.effects); if (f !== 1) { - video.push(`setpts=PTS/${f}`); + // Back to the canvas rate right away: retimed frames otherwise reach the + // encoder off its frame grid, and which ones it drops or repeats varies, + // so a frame preview couldn't show the frame export plays. + video.push(`setpts=PTS/${f}`, `fps=${fps}`); audio.push(atempoChain(f)); } } @@ -415,7 +419,7 @@ function clipChains( ): { chain: string; audio: string[]; textFiles: StepSideFile[] } { const outDur = outputDuration(clip); const { width, height, fps, background } = comp; - const { video: fx, audio } = effectFilters(clip, outDur, fadeSuppress); + const { video: fx, audio } = effectFilters(clip, outDur, fps, fadeSuppress); const textFiles: StepSideFile[] = []; if (clip.kind === 'text') { @@ -472,14 +476,11 @@ function buildSegmentStep( const outDur = outputDuration(clip); const { width, height, fps, background } = comp; const out = segPath(ctx.dir, index, clip.id, ctx.tag); - const { chain: videoChain, audio: afx, textFiles } = clipChains( - comp, - clip, - index, - ctx, - overlays, - fadeSuppress, - ); + const { + chain: videoChain, + audio: afx, + textFiles, + } = clipChains(comp, clip, index, ctx, overlays, fadeSuppress); const inputArgs: string[] = []; let videoSource: string; // filtergraph input label feeding the video chain, e.g. [0:v] diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts new file mode 100644 index 0000000..7c54764 --- /dev/null +++ b/tests/integration/timeline-frame-window.test.ts @@ -0,0 +1,161 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { buildPreviewFrameArgs } from '../../src/ffmpeg/args/preview.js'; +import { listFilters, runFfmpeg } from '../../src/ffmpeg/run.js'; +import { buildFrameAtPlan, compileTimeline, type MediaInfo } from '../../src/timeline/compile.js'; +import { + type Composition, + emptyComposition, + type MediaClip, +} from '../../src/timeline/composition.js'; +import { applyOps, mediaClip, textClip, videoTrack } from '../../src/timeline/ops.js'; +import { runPlan } from '../../src/timeline/run-plan.js'; +import type { MediaId } from '../../src/timeline/schema.js'; + +// The frame preview runs the segment's video chain on a window of the source +// starting at the frame. This checks it against the frame cut out of the fully +// encoded clip, which is what export plays, and that it doesn't grow with the +// clip's length. +const IS_CI = process.env.CI === 'true'; +const HAS_DRAWTEXT = IS_CI && (await listFilters()).has('drawtext'); +const M = 'm_aaaaaaaaaaaa' as MediaId; + +let workspace: string; +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-frame-window-')); +}); +afterEach(async () => { + await rm(workspace, { recursive: true, force: true }); +}); + +/** 720p30 test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +async function testPattern(path: string, seconds: number): Promise { + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=30:duration=${seconds}`], + ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', path], + ]); +} + +/** A vertical timeline holding one clip of the source, framed off-center. */ +function vertical(clip: MediaClip, withText = false): Composition { + return applyOps(emptyComposition({ width: 1080, height: 1920, fit: 'cover' }), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { op: 'addClip', trackId: 'v0', clip: { ...clip, transform: { ...clip.transform, x: 0.3 } } }, + ...(withText + ? [ + { op: 'addTrack' as const, track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip' as const, + trackId: 'captions', + clip: textClip({ + id: 't1', + text: 'Caption over the picture', + startSec: 2, + durationSec: 2, + style: { fontSize: 54, background: 'black@0.6', anchor: 'bottom-center' }, + }), + }, + ] + : []), + ]); +} + +/** PSNR between two images, dB. */ +async function psnr(a: string, b: string): Promise { + const { stderr } = await runFfmpeg([ + ...['-hide_banner', '-i', a, '-i', b, '-lavfi', 'psnr', '-f', 'null', '-'], + ]); + const match = /average:([\d.]+|inf)/.exec(stderr); + if (!match?.[1]) throw new Error(`No PSNR in:\n${stderr}`); + return match[1] === 'inf' ? Number.POSITIVE_INFINITY : Number(match[1]); +} + +describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg', () => { + it('matches the frame of the fully encoded clip: mid-clip, in a fade, at 2x', async () => { + const source = join(workspace, 'pattern.mp4'); + await testPattern(source, 20); + const media = new Map([[M, { path: source, hasAudio: true }]]); + const cases: { name: string; doc: Composition; atSec: number; exportSec: number }[] = [ + { + name: 'mid-clip', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 15, startSec: 0 }), + HAS_DRAWTEXT, + ), + atSec: 3, + exportSec: 3, + }, + { + name: 'inside a fade', + doc: vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 5, + sourceOutSec: 15, + startSec: 0, + effects: [{ type: 'fadeIn', durationSec: 2 }], + }), + ), + atSec: 1, + exportSec: 1, + }, + { + // 8 s of source at 2x plays in 4 s: timeline 4 s is export 2 s. + name: 'at 2x speed', + doc: vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 4, + sourceOutSec: 12, + startSec: 0, + effects: [ + { type: 'speed', factor: 2 }, + { type: 'fadeIn', durationSec: 3 }, + ], + }), + ), + atSec: 4, + exportSec: 2, + }, + ]; + + for (const { name, doc, atSec, exportSec } of cases) { + const tag = name.replace(/\W+/g, '-'); + const exported = join(workspace, `${tag}.mp4`); + await runPlan(compileTimeline(doc, { media, dir: workspace, output: exported })); + const full = join(workspace, `${tag}-full.jpg`); + await runFfmpeg(buildPreviewFrameArgs({ input: exported, output: full, atSec: exportSec })); + + const windowFrame = join(workspace, `${tag}-window.jpg`); + const plan = buildFrameAtPlan(doc, { media, dir: workspace, output: windowFrame }, atSec); + expect(plan.steps, name).toHaveLength(1); + await runPlan(plan); + + expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); + } + }, 180_000); + + it('renders a frame near the end of a 10-minute clip in under a second', async () => { + const piece = join(workspace, 'piece.mp4'); + const source = join(workspace, 'ten-minutes.mp4'); + await testPattern(piece, 30); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-stream_loop', '19', '-i', piece, '-c', 'copy', source], + ]); + const media = new Map([[M, { path: source, hasAudio: true }]]); + const doc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 600, startSec: 0 })); + const output = join(workspace, 'late.jpg'); + + const started = performance.now(); + await runPlan(buildFrameAtPlan(doc, { media, dir: workspace, output }, 540)); + expect(performance.now() - started).toBeLessThan(1000); + }, 120_000); +}); From 6488d7ad67635427ed5a152d65753684c7446b56 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:50:51 -0800 Subject: [PATCH 10/35] feat(timeline): outline a long recording as sentences timelineOutline lists the timeline's speech as numbered sentences, across silence-cut sub-clips, one 40 KB page at a time with nextFromSec. MCP timeline_transcript returns it when no clipId is given (view, fromSec and toSec added), and `clip timeline transcribe --sentences [--from --to]` prints one line per sentence. make_short now warns when a range starts or ends mid-sentence, and timeline_edit passes the warnings on. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- SKILL.md | 4 +- llms.txt | 2 +- src/cli-timeline.ts | 46 +++++-- src/mcp/server.ts | 50 +++++-- src/timeline/captions.ts | 8 +- src/timeline/document-store.ts | 12 +- src/timeline/shorts.ts | 38 +++++- src/timeline/transcript.ts | 167 ++++++++++++++++++++++- src/timeline/verbs.ts | 6 +- tests/mcp-server.test.ts | 61 ++++++++- tests/timeline-outline.test.ts | 240 +++++++++++++++++++++++++++++++++ tests/timeline-shorts.test.ts | 41 +++++- 13 files changed, 640 insertions(+), 38 deletions(-) create mode 100644 tests/timeline-outline.test.ts diff --git a/README.md b/README.md index 0dd305d..ff87911 100644 --- a/README.md +++ b/README.md @@ -161,8 +161,9 @@ The same steps from the CLI: ```bash clip timeline new clip timeline add-media podcast.mp4 # prints the clip id, e.g. clip_ec2f047d -clip timeline transcribe clip_ec2f047d # each word with its index and time +clip timeline transcribe clip_ec2f047d # transcribe it (once per file) clip timeline silence clip_ec2f047d # cut the pauses on main +clip timeline transcribe --sentences | head -80 # one line per sentence: time, word indices, text clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 as a 9:16 short clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off clip timeline --timeline best-1 export best-1.mp4 diff --git a/SKILL.md b/SKILL.md index a11612f..7553f49 100644 --- a/SKILL.md +++ b/SKILL.md @@ -323,7 +323,7 @@ To cut the pauses out of a talking clip already on the timeline, use `timeline s After cutting, run `timeline check` before exporting: it lists every hard cut whose audio is loud right at the join (usually a clipped word or a click) and writes a contact sheet (`--sheet out.jpg`) with the outgoing clip's last frame and the incoming clip's first for each cut. Open the sheet to see the jumps. A flagged cut is a reason to trim or move that boundary, not a verdict. Export fades the audio over 8 ms on each side of every hard cut, so a cut through steady sound doesn't click; the check still flags cuts in sound, since those can clip a word. -To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. +To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. For a long recording, read `timeline transcribe --sentences` instead: one line per sentence of the whole timeline (`12:34 #1400–1413 clip_ab12 text`, the time, the word indices, the clip it starts in), and `--from S --to S` for a stretch; pipe it through `head` to read a page at a time. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. To cut by words, use the `index` values from `timeline transcribe`: `timeline cut-words ` removes those words (inclusive) and moves later clips on the track up. Add `--transcript ` from the same `timeline transcribe` output so the cut is refused, not misplaced, if the clip was transcribed again since. The transcript also lists `retakes`, phrases said twice in a row; `timeline remove-retakes ` cuts the earlier take of each (`--dry-run` lists them first). Each is one edit that `timeline undo` reverses. A cut splits the clip, so transcribe the resulting clips again before the next cut, and run `timeline check` afterwards. @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence; widen it to the sentence's edge unless the cut is meant. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/llms.txt b/llms.txt index aa8b461..fd5e3a9 100644 --- a/llms.txt +++ b/llms.txt @@ -15,7 +15,7 @@ ## What v0.5.0 does - **Remove silence:** `clip timeline silence ` cuts the pauses out of a recording (`--sensitivity`, `--dry-run`) -- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine +- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine; `clip timeline transcribe --sentences` lists a long recording as numbered sentences - **Edit by transcript:** `clip timeline cut-words ` cuts a range of words - **Remove repeated takes:** `clip timeline remove-retakes ` cuts the earlier take of each phrase said twice in a row - **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index ceafd6e..e9f1b91 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -62,6 +62,7 @@ import { findRetakes, planWordCut, storedTranscriptForClip, + timelineOutline, transcribeClip, } from './timeline/transcript.js'; import { DEFAULT_VERB_TRACK as DEFAULT_TRACK, ensureTrack, trackEnd } from './timeline/verbs.js'; @@ -84,7 +85,7 @@ Every command works on the main timeline unless --timeline names another clip timeline remove clip timeline silence [--sensitivity 0..1] [--dry-run] clip timeline check [--sheet ] - clip timeline transcribe [--model ] [--language ] + clip timeline transcribe [] [--sentences] [--from S] [--to S] [--model ] [--language ] clip timeline cut-words [--transcript ] clip timeline remove-retakes [--dry-run] [--transcript ] clip timeline captions [] [--words N] [--track T] [--font N --color C --anchor A --background C|none] [--dry-run] @@ -109,7 +110,7 @@ interface Parsed { } /** Flags that take no value, so they don't swallow the next argument. */ -const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace']); +const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace', 'sentences']); function parseArgs(args: string[]): Parsed { const positional: string[] = []; @@ -171,6 +172,15 @@ function timelineId(value: string): string { return parsed.data; } +/** Timeline time as m:ss, or h:mm:ss from an hour. */ +function clock(sec: number): string { + const whole = Math.floor(sec); + const s = String(whole % 60).padStart(2, '0'); + const m = Math.floor(whole / 60) % 60; + const h = Math.floor(whole / 3600); + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${s}` : `${m}:${s}`; +} + function wordIndex(value: string | undefined): number { const n = Number(value); if (value === undefined || value.trim() === '' || !Number.isInteger(n) || n < 0) { @@ -610,18 +620,30 @@ export async function runTimeline(args: string[]): Promise { case 'transcribe': { const [clipId] = positional; - if (!clipId) { - throw new Error( - 'Usage: clip timeline transcribe [--model ] [--language ]', - ); - } const comp = await readComposition(timeline); - out( - await transcribeClip(comp, clipId, await buildMediaMap(), { - model: flags.model ? resolveInput(flags.model) : undefined, - language: flags.language || undefined, - }), + const opts = { + model: flags.model ? resolveInput(flags.model) : undefined, + language: flags.language || undefined, + }; + if (clipId && flags.sentences !== 'true') { + out(await transcribeClip(comp, clipId, await buildMediaMap(), opts)); + return; + } + // One line per sentence, cheap to read and to `head`: no page limit here. + const outline = await timelineOutline(comp, await buildMediaMap(), { + ...opts, + clipId, + fromSec: optionalNum('from', flags.from), + toSec: optionalNum('to', flags.to), + maxBytes: Number.POSITIVE_INFINITY, + }); + const lines = Object.entries(outline.transcriptIds).map( + ([mediaId, id]) => `transcript ${mediaId}: ${id}`, ); + for (const s of outline.sentences) { + lines.push(`${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId} ${s.text}`); + } + process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; } diff --git a/src/mcp/server.ts b/src/mcp/server.ts index db573d5..5d95c6a 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -32,7 +32,7 @@ import { reviewCuts } from '../timeline/review.js'; import { runPlan } from '../timeline/run-plan.js'; import { MAIN_TIMELINE, TimelineIdSchema } from '../timeline/schema.js'; import { ShortError } from '../timeline/shorts.js'; -import { transcribeClip } from '../timeline/transcript.js'; +import { timelineOutline, transcribeClip } from '../timeline/transcript.js'; import { EditableVerbSchema } from '../timeline/verbs.js'; import { doctor } from '../tools/doctor.js'; import { WhisperLanguageSchema } from '../transcribe/whisper.js'; @@ -218,15 +218,16 @@ export function buildMcpServer(): McpServer { title: 'Edit the timeline', description: 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). ' + - 'make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as "best-1" (never main). It plays the given word ranges (indices from timeline_transcript on the source timeline, default main) with a little padding, cut from the source\'s clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical), look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. ' + + "make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as \"best-1\" (never main). It plays the given word ranges (read the source timeline's sentences with timeline_transcript and no clipId; the source defaults to main) with a little padding, cut from the source's clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical); the result's warnings name a range that starts or ends mid-sentence. Look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. " + 'Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', inputSchema: { timeline: timelineInput, verbs: z.array(EditableVerbSchema).min(1) }, }, async ({ timeline = MAIN_TIMELINE, verbs }) => answeringTimelineErrors(async () => { - const { doc, ops } = await applyVerbs(verbs, makeVerbContext(), timeline); + const { doc, ops, warnings } = await applyVerbs(verbs, makeVerbContext(), timeline); return jsonResult({ applied: ops.length, + ...(warnings.length > 0 ? { warnings } : {}), document: await describeComposition(doc, timeline), }); }), @@ -235,21 +236,50 @@ export function buildMcpServer(): McpServer { server.registerTool( 'timeline_transcript', { - title: 'Transcribe a clip', + title: 'Transcribe the speech', description: - "Transcribe a media clip's speech with the local whisper.cpp and return its words with timeline times. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript. A word's `index` is stable for its source file.", + "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short (or cut_words). " + + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + + "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { timeline: timelineInput, - clipId: z.string().describe('Media clip to transcribe.'), + clipId: z + .string() + .optional() + .describe("Media clip to transcribe. Leave out for the whole timeline's sentences."), + view: z + .enum(['words', 'sentences']) + .optional() + .describe('words (default with a clipId) or sentences (default without one).'), + fromSec: z + .number() + .nonnegative() + .optional() + .describe("Sentences starting at or after this timeline time (a page's nextFromSec)."), + toSec: z + .number() + .positive() + .optional() + .describe('Sentences starting before this timeline time.'), language: WhisperLanguageSchema.optional(), }, }, - async ({ timeline = MAIN_TIMELINE, clipId, language }) => + async ({ timeline = MAIN_TIMELINE, clipId, view, fromSec, toSec, language }) => answeringTimelineErrors(async () => { const comp = await readComposition(timeline); - return jsonResult( - await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), - ); + const media = await confinedMediaMap(comp); + if ((view ?? (clipId ? 'words' : 'sentences')) === 'words') { + if (!clipId) { + return jsonResult({ + error: "The words view needs a clipId. Leave view out for the timeline's sentences.", + }); + } + return jsonResult(await transcribeClip(comp, clipId, media, { language })); + } + const outline = await timelineOutline(comp, media, { clipId, fromSec, toSec, language }); + // Compact, as the page size is counted: a long recording's outline is + // what has to fit in one result. + return { content: [{ type: 'text' as const, text: JSON.stringify(outline) }] }; }), ); diff --git a/src/timeline/captions.ts b/src/timeline/captions.ts index b16c45b..7c59c62 100644 --- a/src/timeline/captions.ts +++ b/src/timeline/captions.ts @@ -24,8 +24,8 @@ export const DEFAULT_CAPTION_WORDS = 7; /** Widest caption, in Latin characters (a CJK character counts as two). */ const MAX_CAPTION_WIDTH = 32; -/** A pause this long between words starts a new caption. */ -const PAUSE_SEC = 0.7; +/** A pause this long between words starts a new caption, or a new sentence. */ +export const PAUSE_SEC = 0.7; /** How long a caption stays up after its last word, unless the next one starts. */ const HOLD_SEC = 0.4; /** Shortest caption shown. */ @@ -270,7 +270,9 @@ export function joinWords(texts: string[]): string { return out; } -function endsSentence(text: string): boolean { +/** Whether a word ends a sentence: it ends in . ! ? … (or 。!?), maybe + * followed by a closing quote or bracket. */ +export function endsSentence(text: string): boolean { return /[.!?。!?…]["')\]」』]?$/u.test(text); } diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index d149aa2..2eb18b4 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -354,9 +354,14 @@ export async function applyVerbs( verbs: CompositionVerb[], ctx: VerbContext, timeline: string = MAIN_TIMELINE, -): Promise<{ doc: Composition; ops: CompositionOp[] }> { +): Promise<{ doc: Composition; ops: CompositionOp[]; warnings: string[] }> { const create = verbs[0]?.verb === 'make_short'; - const editCtx: VerbContext = { ...ctx, timeline }; + let warnings: string[] = []; + const editCtx: VerbContext = { + ...ctx, + timeline, + warn: (message) => warnings.push(message), + }; // The default append point (`trackEnd`) is baked from the snapshot we lower // against, so if another in-process writer commits between the read and the // apply, re-lower against the fresh doc rather than land overlapping clips. A @@ -366,6 +371,7 @@ export async function applyVerbs( // tail silently losing to a `CompositionConflictError`. for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { const current = await readForEdit(timeline, create); + warnings = []; const ops = await lowerVerbs(current, verbs, editCtx); try { const doc = await mutateComposition(ops, { @@ -373,7 +379,7 @@ export async function applyVerbs( timeline, create, }); - return { doc, ops }; + return { doc, ops, warnings }; } catch (err) { if (err instanceof CompositionConflictError && attempt < MAX_COMMIT_ATTEMPTS) continue; throw err; diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index febf983..e6d3c16 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -2,7 +2,9 @@ import { DEFAULT_CAPTION_TRACK, DEFAULT_CAPTION_WORDS, defaultCaptionStyle, + endsSentence, groupCaptions, + PAUSE_SEC, } from './captions.js'; import { speedFactor, videoLayers } from './compile.js'; import { @@ -281,7 +283,7 @@ export async function planShort( })), durationSec: roundMs(at), captions, - warnings: [], + warnings: windows.flatMap((window, i) => sentenceEdgeWarnings(window, i + 1)), }; } @@ -321,6 +323,40 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): }; } +/** + * Where a range starts or ends inside a sentence: right after a word that + * doesn't end one (with no real pause between), or right before the next word + * of the same sentence. Named by the words on either side; the short is still + * made, since a cut mid-sentence can be meant. + */ +function sentenceEdgeWarnings(window: WordWindow, rangeNumber: number): string[] { + const words = window.transcript.words; + const warnings: string[] = []; + const midSentence = ( + a: (typeof words)[number] | undefined, + b: (typeof words)[number] | undefined, + ) => + a !== undefined && + b !== undefined && + !endsSentence(a.text) && + b.startSec - a.endSec < PAUSE_SEC; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + if (midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber} starts mid-sentence, between "${before?.text}" and "${first?.text}" (word ${window.fromIndex}).`, + ); + } + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber} ends mid-sentence, between "${last?.text}" and "${after?.text}" (word ${window.toIndex}).`, + ); + } + return warnings; +} + /** * A window that picks up later in the same file than the one before it starts * where that one's padding ended, so two ranges of neighboring words play diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 5452d55..b2dba11 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -13,7 +13,8 @@ import { WhisperLanguageSchema, } from '../transcribe/whisper.js'; import { getWorkspace, resolveInWorkspace } from '../workspace.js'; -import { checkExportable, type MediaInfo, withoutTextOverlays } from './compile.js'; +import { endsSentence, joinWords, PAUSE_SEC } from './captions.js'; +import { checkExportable, type MediaInfo, videoLayers, withoutTextOverlays } from './compile.js'; import { type Composition, findClip, type MediaClip, makeClipId } from './composition.js'; import { applyOp, type CompositionOp, type SourceRange } from './ops.js'; import { type MediaId, MediaIdSchema } from './schema.js'; @@ -305,6 +306,170 @@ export async function transcribeClip( }; } +/** Longest sentence an outline line holds; whisper sometimes leaves out the + * punctuation, and a line has to end somewhere. */ +const MAX_SENTENCE_WORDS = 40; +/** Default size of one outline page, in bytes of compact JSON: about 10k + * tokens, which an MCP client takes in one result. */ +export const OUTLINE_PAGE_BYTES = 40_000; + +/** A word in timeline order, with the media clip it plays in. */ +export interface OutlineWord extends ClipWord { + clipId: string; +} + +/** One sentence of the timeline's speech. `fromIndex..toIndex` are word indices + * into its source's transcript, as `cut_words` and `make_short` take them, and + * `clipId` is the clip its first word plays in. */ +export interface OutlineSentence { + startSec: number; + endSec: number; + clipId: string; + fromIndex: number; + toIndex: number; + text: string; +} + +export interface TimelineOutline { + sentences: OutlineSentence[]; + /** The `transcriptId` of each source the sentences come from. */ + transcriptIds: Record; + /** Where the next page starts (pass it as `fromSec`); absent on the last page. */ + nextFromSec?: number; +} + +/** + * Group words (one source, timeline order) into sentences: a sentence ends at + * sentence-ending punctuation, before a pause longer than `PAUSE_SEC`, after + * `MAX_SENTENCE_WORDS` words, or where the timeline goes back in the source. + */ +export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { + const groups: OutlineWord[][] = []; + let current: OutlineWord[] = []; + for (const word of words) { + const prev = current[current.length - 1]; + if ( + prev && + (endsSentence(prev.text) || + word.startSec - prev.endSec > PAUSE_SEC || + word.index < prev.index || + current.length >= MAX_SENTENCE_WORDS) + ) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + return groups.flatMap((group) => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return []; + return [ + { + startSec: first.startSec, + endSec: last.endSec, + clipId: first.clipId, + fromIndex: first.index, + toIndex: last.index, + text: joinWords(group.map((w) => w.text)), + }, + ]; + }); +} + +export interface OutlineOptions extends TranscribeOptions { + /** Only this media clip's words. */ + clipId?: string; + /** Sentences starting at or after this timeline time. */ + fromSec?: number; + /** Sentences starting before this timeline time. */ + toSec?: number; + /** Page size in bytes of compact JSON (default `OUTLINE_PAGE_BYTES`). */ + maxBytes?: number; + /** How a source's transcript is had (default: `transcribeMedia`, cached). */ + transcribe?: (mediaId: MediaId, info: MediaInfo) => Promise; +} + +/** + * The timeline's speech as sentences, in timeline order: every media clip on + * the picture track, each source transcribed once (or read from its stored + * transcript). A sentence runs on across cuts between clips of the same source + * (a silence cut splits a clip mid-sentence), and a word a cut splits in two is + * listed once. A page holds at most `maxBytes` of compact JSON; `nextFromSec` + * says where the next one starts. + */ +export async function timelineOutline( + comp: Composition, + media: Map, + opts: OutlineOptions = {}, +): Promise { + const transcribe = opts.transcribe ?? ((mediaId, info) => transcribeMedia(mediaId, info, opts)); + const picture = videoLayers(comp).picture; + const clips = (picture?.clips ?? []) + .filter((c): c is MediaClip => c.kind === 'media') + .filter((c) => opts.clipId === undefined || c.id === opts.clipId) + .sort((a, b) => a.startSec - b.startSec); + if (opts.clipId !== undefined && clips.length === 0) { + // The clip isn't a media clip on the picture track: say why, as a lookup would. + locateMediaClip(comp, opts.clipId, media); + } + + const transcripts = new Map(); + const sentences: OutlineSentence[] = []; + let run: OutlineWord[] = []; + let runMedia: MediaId | undefined; + for (const clip of clips) { + let transcript = transcripts.get(clip.mediaId); + if (transcript === undefined) { + const info = media.get(clip.mediaId); + if (!info) { + throw new TranscriptError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + transcript = info.hasAudio ? await transcribe(clip.mediaId, info) : null; + transcripts.set(clip.mediaId, transcript); + } + if (!transcript) continue; + if (clip.mediaId !== runMedia) { + sentences.push(...outlineSentences(run)); + run = []; + runMedia = clip.mediaId; + } + for (const word of clipWords(transcript, clip)) { + // A word a cut runs through shows in the clips on both sides of it: one + // word, from its start in the first to its end in the last. + const last = run[run.length - 1]; + if (last?.index === word.index) run[run.length - 1] = { ...last, endSec: word.endSec }; + else run.push({ ...word, clipId: clip.id }); + } + } + sentences.push(...outlineSentences(run)); + + const transcriptIds: Record = {}; + for (const [mediaId, transcript] of transcripts) { + if (transcript) transcriptIds[mediaId] = transcriptIdOf(transcript); + } + const fromSec = opts.fromSec ?? 0; + const toSec = opts.toSec ?? Number.POSITIVE_INFINITY; + const inRange = sentences.filter((s) => s.startSec >= fromSec && s.startSec < toSec); + + const maxBytes = opts.maxBytes ?? OUTLINE_PAGE_BYTES; + // The page's own fields, with room for the largest nextFromSec. + let bytes = Buffer.byteLength( + JSON.stringify({ sentences: [], transcriptIds, nextFromSec: 99999.999 }), + ); + const page: OutlineSentence[] = []; + for (const sentence of inRange) { + const size = Buffer.byteLength(JSON.stringify(sentence)) + 1; + if (bytes + size > maxBytes && page.length > 0) { + return { sentences: page, transcriptIds, nextFromSec: sentence.startSec }; + } + page.push(sentence); + bytes += size; + } + return { sentences: page, transcriptIds }; +} + /** An inclusive range of transcript word indices. */ export interface WordRange { fromIndex: number; diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index f9167c0..4e8a308 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -305,7 +305,7 @@ const EDITABLE_VERB_SCHEMAS = [ .min(1) .max(20) .describe( - 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences; 15–60 s in all is typical.', + 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences from the timeline_transcript sentence outline; 15–60 s in all is typical. A range that starts or ends mid-sentence is made anyway, with a warning.', ), source: TimelineIdSchema.optional().describe( 'Timeline to cut from (default "main"). Its silence and retake cuts carry into the short.', @@ -403,6 +403,9 @@ export interface VerbContext { defaultTrack: string; /** The timeline being edited (default `main`); `applyVerbs` sets it. */ timeline?: string; + /** Hears what a verb did that the caller may not have meant (e.g. a short + * cut mid-sentence); `applyVerbs` collects them. */ + warn?: (message: string) => void; } const DEFAULT_TRACK = 'v0'; @@ -581,6 +584,7 @@ export async function lowerVerb( }, ctx.transcript, ); + for (const warning of plan.warnings) ctx.warn?.(warning); return plan.ops; } case 'transition': diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index 7fc7271..e9665fa 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -228,7 +228,7 @@ describe('MCP server', () => { describe('MCP shorts (named timelines)', () => { /** Main holds a 30 s registered recording with a stored transcript: nothing renders. */ - async function recordingOnMain(): Promise { + async function recordingOnMain(): Promise<{ mediaId: string }> { const source = join(workspace, 'talk.mp4'); await writeFile(source, 'video bytes'); const mediaId = makeMediaId(source); @@ -267,6 +267,7 @@ describe('MCP shorts (named timelines)', () => { clip: mediaClip({ id: 'c1', mediaId, sourceOutSec: 30, startSec: 0 }), }, ]); + return { mediaId }; } it('makes a short on its own timeline with timeline_edit, leaving main alone', async () => { @@ -289,6 +290,10 @@ describe('MCP shorts (named timelines)', () => { }, }), ); + expect(edit.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "word1" and "word2" (word 2).', + 'Range 1 ends mid-sentence, between "word6" and "word7" (word 6).', + ]); const doc = edit.document as Record; expect(doc.timeline).toBe('clip-1'); expect(doc.canvas).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); @@ -315,6 +320,58 @@ describe('MCP shorts (named timelines)', () => { } }); + it("outlines the timeline's sentences without a clipId, compactly", async () => { + const { mediaId } = await recordingOnMain(); + // A configured model the stored transcript matches, so nothing is transcribed. + const model = join(workspace, 'ggml-base.en.bin'); + await writeFile(model, 'model bytes'); + const m = await stat(model); + const transcriptFile = join(workspace, 'transcripts', `${mediaId}.json`); + const stored = JSON.parse(await readFile(transcriptFile, 'utf8')) as Transcript; + await writeFile( + transcriptFile, + JSON.stringify({ ...stored, modelKey: `${model}:${m.size}:${m.mtimeMs}` }), + ); + const env = { + model: process.env.MAKEMYCLIP_WHISPER_MODEL, + path: process.env.MAKEMYCLIP_WHISPER_PATH, + }; + process.env.MAKEMYCLIP_WHISPER_MODEL = model; + process.env.MAKEMYCLIP_WHISPER_PATH = join(workspace, 'no-whisper'); + const { client, close } = await connect(); + try { + const result = (await client.callTool({ name: 'timeline_transcript', arguments: {} })) as { + content: { type: string; text: string }[]; + }; + const text = result.content[0]?.text ?? ''; + expect(text).not.toContain('\n'); + const outline = JSON.parse(text) as { + sentences: { clipId: string; fromIndex: number; toIndex: number; text: string }[]; + transcriptIds: Record; + nextFromSec?: number; + }; + expect(outline.sentences).toEqual([ + expect.objectContaining({ clipId: 'c1', fromIndex: 0, toIndex: 19 }), + ]); + expect(Object.values(outline.transcriptIds)).toHaveLength(1); + expect(outline.nextFromSec).toBeUndefined(); + + const noClip = payload( + await client.callTool({ name: 'timeline_transcript', arguments: { view: 'words' } }), + ); + expect(noClip.error).toMatch(/needs a clipId/); + } finally { + await close(); + for (const [key, value] of [ + ['MAKEMYCLIP_WHISPER_MODEL', env.model], + ['MAKEMYCLIP_WHISPER_PATH', env.path], + ] as const) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); + it('answers a missing timeline or a short on main with an error', async () => { await recordingOnMain(); const { client, close } = await connect(); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts new file mode 100644 index 0000000..81a99e9 --- /dev/null +++ b/tests/timeline-outline.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from 'vitest'; +import type { MediaInfo } from '../src/timeline/compile.js'; +import { type Composition, emptyComposition, type MediaClip } from '../src/timeline/composition.js'; +import { + applyOps, + type CompositionOp, + mediaClip, + textClip, + videoTrack, +} from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; +import { + clipWords, + type OutlineWord, + outlineSentences, + type Transcript, + TranscriptError, + timelineOutline, +} from '../src/timeline/transcript.js'; + +const M = 'm_aaaaaaaaaaaa' as MediaId; +const N = 'm_bbbbbbbbbbbb' as MediaId; +const O = 'm_cccccccccccc' as MediaId; +const MEDIA = new Map([ + [M, { path: '/talk.mp4', hasAudio: true }], + [N, { path: '/broll.mp4', hasAudio: false }], + [O, { path: '/guest.mp4', hasAudio: true }], +]); + +function transcriptOf(mediaId: MediaId, words: Transcript['words']): Transcript { + return { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-base.en.bin', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: 0, + sourceMtimeMs: 0, + words, + }; +} + +/** Abutting words, as whisper.cpp gives them: 0.4 s each from 0. */ +function spoken(texts: string[]): Transcript['words'] { + return texts.map((text, i) => ({ + text, + startSec: Math.round(i * 0.4 * 1000) / 1000, + endSec: Math.round((i + 1) * 0.4 * 1000) / 1000, + })); +} + +function timelineOf(clips: MediaClip[]): Composition { + return applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ...clips.map((clip): CompositionOp => ({ op: 'addClip', trackId: 'v0', clip })), + ]); +} + +const transcribeFrom = + (...transcripts: Transcript[]) => + async (mediaId: MediaId): Promise => { + const found = transcripts.find((t) => t.mediaId === mediaId); + if (!found) throw new TranscriptError(`no transcript for ${mediaId}`); + return found; + }; + +const ms = (sec: number): number => Math.round(sec * 1000) / 1000; +const word = (index: number, text: string, startSec: number, endSec: number): OutlineWord => ({ + index, + text, + startSec: ms(startSec), + endSec: ms(endSec), + clipId: 'c1', +}); + +describe('outlineSentences', () => { + it('ends a sentence at its punctuation', () => { + const words = ['Hello', 'there.', 'How', 'are', 'you?', 'Fine'].map((t, i) => + word(i, t, i * 0.4, (i + 1) * 0.4), + ); + expect(outlineSentences(words)).toEqual([ + { startSec: 0, endSec: 0.8, clipId: 'c1', fromIndex: 0, toIndex: 1, text: 'Hello there.' }, + { startSec: 0.8, endSec: 2, clipId: 'c1', fromIndex: 2, toIndex: 4, text: 'How are you?' }, + { startSec: 2, endSec: 2.4, clipId: 'c1', fromIndex: 5, toIndex: 5, text: 'Fine' }, + ]); + }); + + it('ends one at a long pause, after 40 words, and where the source goes back', () => { + const paused = [word(0, 'so', 0, 0.3), word(1, 'anyway', 1.2, 1.5)]; + expect(outlineSentences(paused).map((s) => s.text)).toEqual(['so', 'anyway']); + + const run = Array.from({ length: 45 }, (_, i) => word(i, 'and', i * 0.3, (i + 1) * 0.3)); + expect(outlineSentences(run).map((s) => [s.fromIndex, s.toIndex])).toEqual([ + [0, 39], + [40, 44], + ]); + + const reordered = [word(10, 'later', 0, 0.4), word(2, 'earlier', 0.4, 0.8)]; + expect(outlineSentences(reordered)).toHaveLength(2); + }); +}); + +describe('timelineOutline', () => { + const talk = transcriptOf( + M, + spoken(['First', 'point', 'here.', 'Second', 'point', 'runs', 'on', 'long.', 'Third', 'one.']), + ); + + it('keeps word indices across silence-cut sub-clips, listing a split word once', async () => { + // Cuts at 1.0–1.1 (through "here.", 0.8–1.2) and 2.7–3.0 (before "long.", 2.8–3.2). + const comp = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 1, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.1, sourceOutSec: 2.7, startSec: 1 }), + mediaClip({ id: 'c3', mediaId: M, sourceInSec: 3, sourceOutSec: 4, startSec: 2.6 }), + ]); + const outline = await timelineOutline(comp, MEDIA, { transcribe: transcribeFrom(talk) }); + expect(outline.sentences).toEqual([ + { + startSec: 0, + endSec: 1.1, + clipId: 'c1', + fromIndex: 0, + toIndex: 2, + text: 'First point here.', + }, + { + startSec: 1.1, + endSec: 2.8, + clipId: 'c2', + fromIndex: 3, + toIndex: 7, + text: 'Second point runs on long.', + }, + { startSec: 2.8, endSec: 3.6, clipId: 'c3', fromIndex: 8, toIndex: 9, text: 'Third one.' }, + ]); + // The same indices and times clipWords gives each clip. + const placed = comp.tracks[0]?.clips.flatMap((c) => + c.kind === 'media' ? clipWords(talk, c) : [], + ); + for (const s of outline.sentences) { + expect(placed?.find((w) => w.index === s.fromIndex)?.startSec).toBe(s.startSec); + expect(placed?.findLast((w) => w.index === s.toIndex)?.endSec).toBe(s.endSec); + } + expect(Object.keys(outline.transcriptIds)).toEqual([M]); + expect(outline.nextFromSec).toBeUndefined(); + }); + + it('reads each source once, skips one without sound, and breaks sentences between sources', async () => { + const guest = transcriptOf(O, spoken(['Hello', 'world'])); + const calls: MediaId[] = []; + const comp = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'o1', mediaId: O, sourceOutSec: 0.8, startSec: 0.8 }), + mediaClip({ id: 'b1', mediaId: N, sourceOutSec: 2, startSec: 1.6 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.2, sourceOutSec: 2, startSec: 3.6 }), + ]); + const outline = await timelineOutline(comp, MEDIA, { + transcribe: async (mediaId) => { + calls.push(mediaId); + return mediaId === O ? guest : talk; + }, + }); + expect(calls).toEqual([M, O]); + expect(outline.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex, s.text])).toEqual([ + ['c1', 0, 1, 'First point'], + ['o1', 0, 1, 'Hello world'], + ['c2', 3, 4, 'Second point'], + ]); + expect(Object.keys(outline.transcriptIds)).toEqual([M, O]); + }); + + it('pages by size and range, so pages put back together give every sentence once', async () => { + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 4, startSec: 0 })]); + const transcribe = transcribeFrom(talk); + const all = (await timelineOutline(comp, MEDIA, { transcribe })).sentences; + + const pages: string[] = []; + const seen = []; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec, maxBytes: 250 }); + pages.push(JSON.stringify(page)); + seen.push(...page.sentences); + fromSec = page.nextFromSec; + } + expect(pages.length).toBeGreaterThan(1); + for (const page of pages) expect(Buffer.byteLength(page)).toBeLessThanOrEqual(250); + expect(seen).toEqual(all); + + const middle = await timelineOutline(comp, MEDIA, { transcribe, fromSec: 1, toSec: 3 }); + expect(middle.sentences.map((s) => s.fromIndex)).toEqual([3]); + }); + + it('pages a 60-minute, 9,000-word recording at 40 KB', async () => { + const texts = Array.from({ length: 9000 }, (_, i) => + i % 14 === 13 ? `sentence${i}.` : `word${i}`, + ); + const hour = transcriptOf(M, spoken(texts)); + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 3600, startSec: 0 })]); + const transcribe = transcribeFrom(hour); + + let count = 0; + let pages = 0; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec }); + expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(40_000); + count += page.sentences.length; + pages++; + fromSec = page.nextFromSec; + } + expect(count).toBe(Math.ceil(9000 / 14)); + expect(pages).toBeGreaterThan(1); + }); + + it("outlines one clip, and says why a clip that isn't media can't be", async () => { + const comp = applyOps( + timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 1, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.2, sourceOutSec: 4, startSec: 1 }), + ]), + [ + { op: 'addTrack', track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip', + trackId: 'captions', + clip: textClip({ id: 't1', text: 'hi', startSec: 0, durationSec: 1 }), + }, + ], + ); + const transcribe = transcribeFrom(talk); + const one = await timelineOutline(comp, MEDIA, { transcribe, clipId: 'c1' }); + expect(one.sentences.map((s) => s.clipId)).toEqual(['c1']); + await expect(timelineOutline(comp, MEDIA, { transcribe, clipId: 't1' })).rejects.toThrow( + /text clip/, + ); + }); +}); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 0a110ae..b315fa8 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -366,7 +366,46 @@ describe('planShort: the edit', () => { }, ]); expect(short.captions).toBe(1); - expect(short.warnings).toEqual([]); + expect(short.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "w1" and "w2" (word 2).', + 'Range 1 ends mid-sentence, between "w4" and "w5" (word 4).', + ]); + }); + + it('warns only where a range cuts into a sentence', async () => { + // "One two. Three four five. Six" with a long pause before "Six". + const said = { + ...transcriptOf(M, [ + [1, 1.4], + [1.5, 1.9], + [2.2, 2.6], + [2.7, 3.1], + [3.2, 3.6], + [4.6, 5], + ]), + }; + said.words = said.words.map((w, i) => ({ + ...w, + text: ['One', 'two.', 'Three', 'four', 'five.', 'Six'][i] ?? w.text, + })); + const warnings = async (fromIndex: number, toIndex: number): Promise => + ( + await plan( + sourceOf([clipOf('c1', 0, 30, 0)]), + { ranges: [{ clipId: 'c1', fromIndex, toIndex }] }, + { transcripts: [said] }, + ) + ).warnings; + expect(await warnings(2, 4)).toEqual([]); // a whole sentence + expect(await warnings(0, 5)).toEqual([]); // the recording's edges + expect(await warnings(3, 4)).toEqual([ + 'Range 1 starts mid-sentence, between "Three" and "four" (word 3).', + ]); + expect(await warnings(2, 3)).toEqual([ + 'Range 1 ends mid-sentence, between "four" and "five." (word 3).', + ]); + // "five." ends the sentence; the 1 s pause before "Six" ends one too. + expect(await warnings(5, 5)).toEqual([]); }); it('captions only the asked-for words, not the word the lead reaches into', async () => { From c7866bfb7c175fcea0e9d20d47457ddb1aba4be0 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:59:44 -0800 Subject: [PATCH 11/35] fix(timeline): seek frames down to the microsecond, keep replayed words The window frame rounded its seek to the millisecond, which could land past a clip's last frame (no image) or its out-point; it now rounds down at microsecond precision. The outline merged a word repeated by consecutive clips as if a cut split it; it merges only when the next clip carries on later in the source, and a word played again starts a new sentence. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 10 +++++-- src/timeline/transcript.ts | 15 ++++++++--- .../integration/timeline-frame-window.test.ts | 9 ++++++- tests/timeline-introspect.test.ts | 5 ++-- tests/timeline-outline.test.ts | 27 +++++++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index d3c4d1b..8a732c3 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,10 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +function floorUs(sec: number): number { + return Math.floor(sec * 1e6 + 1e-6) / 1e6; +} + /** The font file for a text clip, with a missing font reported against the clip. * Text over the picture always gets a file: see `ffmpeg/font.ts`. */ function fontFor(ctx: CompileContext, clip: TextClip, always: boolean): string | undefined { @@ -777,14 +781,16 @@ export function buildFrameAtPlan( } // Seconds into the clip's source. The chain's speed `setpts` divides the // offset back into segment time, which is what its fades and text read. - const local = Math.round(clamped * speedFactor(clip.effects) * 1000) / 1000; + // Rounded down, to the microsecond: rounding up could seek past the clip's + // last frame (no image) or past its out-point. + const local = floorUs(clamped * speedFactor(clip.effects)); const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, args: [ '-y', '-ss', - seconds(clip.sourceInSec + local), + String(floorUs(clip.sourceInSec + local)), '-i', info.path, '-filter_complex', diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index b2dba11..f9c20bd 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -352,7 +352,7 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { prev && (endsSentence(prev.text) || word.startSec - prev.endSec > PAUSE_SEC || - word.index < prev.index || + word.index <= prev.index || current.length >= MAX_SENTENCE_WORDS) ) { groups.push(current); @@ -419,6 +419,7 @@ export async function timelineOutline( const sentences: OutlineSentence[] = []; let run: OutlineWord[] = []; let runMedia: MediaId | undefined; + let prevClip: MediaClip | undefined; for (const clip of clips) { let transcript = transcripts.get(clip.mediaId); if (transcript === undefined) { @@ -435,13 +436,21 @@ export async function timelineOutline( run = []; runMedia = clip.mediaId; } + // A clip that carries on later in the source than the one before it (a cut + // between them), rather than playing part of it again. + const continues = + prevClip?.mediaId === clip.mediaId && clip.sourceInSec >= prevClip.sourceOutSec - 1e-6; for (const word of clipWords(transcript, clip)) { // A word a cut runs through shows in the clips on both sides of it: one // word, from its start in the first to its end in the last. const last = run[run.length - 1]; - if (last?.index === word.index) run[run.length - 1] = { ...last, endSec: word.endSec }; - else run.push({ ...word, clipId: clip.id }); + if (continues && last?.index === word.index) { + run[run.length - 1] = { ...last, endSec: word.endSec }; + } else { + run.push({ ...word, clipId: clip.id }); + } } + prevClip = clip; } sentences.push(...outlineSentences(run)); diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 7c54764..e7d139a 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -140,6 +140,13 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); } + + // The very end of a clip that ends with its source: the seek must not pass + // the last frame, or there is no image. + const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); + const last = join(workspace, 'last.jpg'); + await runPlan(buildFrameAtPlan(whole, { media, dir: workspace, output: last }, 19.999)); + expect((await stat(last)).size).toBeGreaterThan(0); }, 180_000); it('renders a frame near the end of a 10-minute clip in under a second', async () => { diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index c89df21..6d6a872 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -110,8 +110,9 @@ describe('buildFrameAtPlan', () => { ]); const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.967']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.967\/TB,/); + // Rounded down, never past that frame or the clip's out-point. + expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.966666\/TB,/); }); it('frames the picture the way export does (fit, zoom, position)', () => { diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 81a99e9..9337355 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -147,6 +147,33 @@ describe('timelineOutline', () => { expect(outline.nextFromSec).toBeUndefined(); }); + it('keeps a word the timeline plays twice as two, where a clip replays the source', async () => { + const hello = transcriptOf(M, spoken(['Hello.', 'and', 'so'])); + const transcribe = transcribeFrom(hello); + const twice = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 0.4, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceOutSec: 0.4, startSec: 0.4 }), + ]); + expect( + (await timelineOutline(twice, MEDIA, { transcribe })).sentences.map((s) => [ + s.clipId, + s.startSec, + s.endSec, + ]), + ).toEqual([ + ['c1', 0, 0.4], + ['c2', 0.4, 0.8], + ]); + + const andAgain = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0.4, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 0.4, sourceOutSec: 1.2, startSec: 0.4 }), + ]); + expect( + (await timelineOutline(andAgain, MEDIA, { transcribe })).sentences.map((s) => s.text), + ).toEqual(['and', 'and so']); + }); + it('reads each source once, skips one without sound, and breaks sentences between sources', async () => { const guest = transcriptOf(O, spoken(['Hello', 'world'])); const calls: MediaId[] = []; From 390ad0cca19079d72471e09d8941fcce6c4df094 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:08:53 -0800 Subject: [PATCH 12/35] fix(timeline): read a frame preview from a second before the frame A source slower than the canvas shows a frame export repeats into later slots, and seeking straight to the frame skipped it (at a clip's end, no image at all). The window now reads from a second earlier and trims to the first frame at the frame's segment time, timestamped from the seek point. An outline of an explicit clip off the picture track lists its sentences, and the MCP outline no longer suggests cut_words for sentences that can span clips. Co-Authored-By: Claude Opus 5.5 --- src/mcp/server.ts | 2 +- src/timeline/compile.ts | 21 ++++++++++----- src/timeline/transcript.ts | 15 +++++------ .../integration/timeline-frame-window.test.ts | 27 ++++++++++++++++--- tests/timeline-introspect.test.ts | 27 ++++++++++++------- tests/timeline-outline.test.ts | 11 ++++++++ 6 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 5d95c6a..bd4f7f0 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -238,7 +238,7 @@ export function buildMcpServer(): McpServer { { title: 'Transcribe the speech', description: - "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short (or cut_words). " + + "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short. " + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 8a732c3..04e9233 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,9 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +/** How much of the source a frame preview reads before the frame. */ +const FRAME_PREROLL_SEC = 1; + function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -779,22 +782,28 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Seconds into the clip's source. The chain's speed `setpts` divides the - // offset back into segment time, which is what its fades and text read. - // Rounded down, to the microsecond: rounding up could seek past the clip's - // last frame (no image) or past its out-point. + // Where the window starts, in seconds into the clip's source: a little + // before the frame, so a source frame that export repeats into it (a source + // slower than the canvas) is still read. An input seek starts timestamps at + // the seek point, so adding `from` gives source time even when the first + // frame read comes after it. The chain's speed `setpts` divides + // the offset back into segment time, which its fades and text read, and + // `trim` then takes the first frame at the frame's segment time, as reading + // it out of the encoded segment would. Rounded down, to the microsecond, so + // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); + const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, args: [ '-y', '-ss', - String(floorUs(clip.sourceInSec + local)), + String(floorUs(clip.sourceInSec + from)), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS-STARTPTS+${local}/TB,${chain}[v]`, + `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${floorUs(clamped)}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index f9c20bd..3e22404 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -405,15 +405,12 @@ export async function timelineOutline( opts: OutlineOptions = {}, ): Promise { const transcribe = opts.transcribe ?? ((mediaId, info) => transcribeMedia(mediaId, info, opts)); - const picture = videoLayers(comp).picture; - const clips = (picture?.clips ?? []) - .filter((c): c is MediaClip => c.kind === 'media') - .filter((c) => opts.clipId === undefined || c.id === opts.clipId) - .sort((a, b) => a.startSec - b.startSec); - if (opts.clipId !== undefined && clips.length === 0) { - // The clip isn't a media clip on the picture track: say why, as a lookup would. - locateMediaClip(comp, opts.clipId, media); - } + const clips = + opts.clipId !== undefined + ? [locateMediaClip(comp, opts.clipId, media).clip] + : (videoLayers(comp).picture?.clips ?? []) + .filter((c): c is MediaClip => c.kind === 'media') + .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); const sentences: OutlineSentence[] = []; diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index e7d139a..2131c02 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -30,11 +30,11 @@ afterEach(async () => { await rm(workspace, { recursive: true, force: true }); }); -/** 720p30 test pattern (its counter and gradient move every frame) with a 2 s GOP. */ -async function testPattern(path: string, seconds: number): Promise { +/** 720p test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +async function testPattern(path: string, seconds: number, rate = 30): Promise { await runFfmpeg([ ...['-hide_banner', '-loglevel', 'error'], - ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=30:duration=${seconds}`], + ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=${rate}:duration=${seconds}`], ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], ...['-c:a', 'aac', '-shortest', path], @@ -141,6 +141,27 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); } + // A 24 fps source on the 30 fps canvas: export repeats some source frames, + // and the preview shows the one export plays, up to the clip's very end. + const slow = join(workspace, 'slow.mp4'); + await testPattern(slow, 4, 24); + const slowMedia = new Map([[M, { path: slow, hasAudio: true }]]); + const slowDoc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 4, startSec: 0 })); + const slowExport = join(workspace, 'slow-export.mp4'); + await runPlan( + compileTimeline(slowDoc, { media: slowMedia, dir: workspace, output: slowExport }), + ); + for (const atSec of [1.5, 3.999]) { + const full = join(workspace, `slow-${atSec}-full.jpg`); + const exportSec = Math.min(atSec, 4 - 1 / 30); + await runFfmpeg(buildPreviewFrameArgs({ input: slowExport, output: full, atSec: exportSec })); + const windowFrame = join(workspace, `slow-${atSec}-window.jpg`); + await runPlan( + buildFrameAtPlan(slowDoc, { media: slowMedia, dir: workspace, output: windowFrame }, atSec), + ); + expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); + } + // The very end of a clip that ends with its source: the seek must not pass // the last frame, or there is no image. const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 6d6a872..a0272cc 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,7 +82,8 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '3', '-i', '/in.mp4']); + // Read from 1 s before the frame, then take the first frame at 3 s. + expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '2', '-i', '/in.mp4']); expect(frame?.args.slice(-8)).toEqual([ '-map', '[v]', @@ -96,7 +97,9 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS-STARTPTS+3/TB,${segmentChain}[v]`); + expect(graphOf(frame?.args)).toBe( + `[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`, + ); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { @@ -111,8 +114,9 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.966666\/TB,/); + expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); + expect(graphOf(near?.args)).toMatch(/,trim=start=7\.966666\[v\]$/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -155,8 +159,9 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '3']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+2\/TB,/); + expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '2']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+1\/TB,/); + expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); it('maps doc-local time to the post-speed segment timebase for a speed clip', () => { @@ -175,13 +180,15 @@ describe('buildFrameAtPlan', () => { }), }, ]); - // The frame shows source second 4; the chain's setpts=PTS/2 turns the +4 s - // offset into segment time 2, which the fades and text windows read. + // The frame shows source second 4. Reading starts a second earlier, at 3; + // the chain's setpts=PTS/2 turns source time into segment time, which the + // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(1, 3)).toEqual(['-ss', '4']); + expect(frame?.args.slice(1, 3)).toEqual(['-ss', '3']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+4\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+3\/TB,/); expect(graph).toContain('setpts=PTS/2'); + expect(graph).toMatch(/,trim=start=2\[v\]$/); }); it('works on a text clip (no media required)', () => { diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 9337355..ed104fb 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -260,6 +260,17 @@ describe('timelineOutline', () => { const transcribe = transcribeFrom(talk); const one = await timelineOutline(comp, MEDIA, { transcribe, clipId: 'c1' }); expect(one.sentences.map((s) => s.clipId)).toEqual(['c1']); + // A clip off the picture track (here a second media track) is still outlined. + const twoTracks = applyOps(comp, [ + { op: 'addTrack', track: videoTrack({ id: 'v1' }) }, + { + op: 'addClip', + trackId: 'v1', + clip: mediaClip({ id: 'x1', mediaId: M, sourceInSec: 1.2, sourceOutSec: 2, startSec: 0 }), + }, + ]); + const off = await timelineOutline(twoTracks, MEDIA, { transcribe, clipId: 'x1' }); + expect(off.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex])).toEqual([['x1', 3, 4]]); await expect(timelineOutline(comp, MEDIA, { transcribe, clipId: 't1' })).rejects.toThrow( /text clip/, ); From d00188e5b5eaf26ed61591cee91c5d6fd2e939de Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:14:38 -0800 Subject: [PATCH 13/35] style(timeline): format the frame-plan test Co-Authored-By: Claude Opus 5.5 --- tests/timeline-introspect.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index a0272cc..b28a605 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -97,9 +97,7 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`, - ); + expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { From 8d889cc8f0ba56f49dc86073a55c7b7606400dbc Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:26:38 -0800 Subject: [PATCH 14/35] fix(timeline): take the canvas frame at or after an off-grid preview time trim rounds an off-grid start to the nearest frame, so a preview between two frames could show the earlier one where export shows the later. The boundary is now the next canvas frame. The CI-gated test's pattern turns its hue 24 degrees a frame, so a preview off by one frame fails it (21 dB with the old boundary), and it adds a case between two frames. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 5 ++++- .../integration/timeline-frame-window.test.ts | 18 ++++++++++++++++-- tests/timeline-introspect.test.ts | 8 +++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 04e9233..f316251 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -793,6 +793,9 @@ export function buildFrameAtPlan( // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); + // The first canvas frame at or after the frame's time: `trim` rounds an + // off-grid start to the nearest frame, which could be the one before. + const firstFrame = Math.ceil(clamped * comp.fps - 1e-6) / comp.fps; const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, @@ -803,7 +806,7 @@ export function buildFrameAtPlan( '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${floorUs(clamped)}[v]`, + `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 2131c02..e69bcd3 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -30,11 +30,17 @@ afterEach(async () => { await rm(workspace, { recursive: true, force: true }); }); -/** 720p test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +/** 720p test pattern with a 2 s GOP. Its hue turns 24 degrees a frame, so + * every frame differs all over and a frame off by one fails the PSNR check. */ async function testPattern(path: string, seconds: number, rate = 30): Promise { await runFfmpeg([ ...['-hide_banner', '-loglevel', 'error'], - ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=${rate}:duration=${seconds}`], + ...[ + '-f', + 'lavfi', + '-i', + `testsrc=size=1280x720:rate=${rate}:duration=${seconds},hue=h=t*${24 * rate}`, + ], ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], ...['-c:a', 'aac', '-shortest', path], @@ -90,6 +96,14 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' atSec: 3, exportSec: 3, }, + { + name: 'between two frames', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 15, startSec: 0 }), + ), + atSec: 3.01, + exportSec: 3.01, + }, { name: 'inside a fade', doc: vertical( diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index b28a605..e1642a4 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -114,7 +114,7 @@ describe('buildFrameAtPlan', () => { // Rounded down, never past that frame or the clip's out-point. expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); - expect(graphOf(near?.args)).toMatch(/,trim=start=7\.966666\[v\]$/); + expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -147,6 +147,12 @@ describe('buildFrameAtPlan', () => { expect(segOut('frame-aaa')).not.toBe(segOut('frame-bbb')); }); + it('takes the first frame at or after a time between frames, as export would', () => { + // 30 fps: 3.01 s falls between the frames at 3.0 and 3.0333. + const graph = graphOf(buildFrameAtPlan(oneClip(), CTX('/ws/f.jpg'), 3.01).steps[0]?.args); + expect(graph).toMatch(/,trim=start=3\.0333333\d*\[v\]$/); + }); + it('selects the second clip and offsets into it for a time in the second clip', () => { const doc = applyOps(oneClip(), [ { From 59c957a4250ac433409bfa5085f2fa9a35668125 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:29:52 -0800 Subject: [PATCH 15/35] fix(timeline): keep a preview within the frames a segment encodes Encoding a clip to its length keeps round(length x fps) frames, so on a clip whose length is off the frame grid the next-frame boundary could pick a frame export leaves out. The boundary now stops at the segment's last frame, with a unit test and a CI-gated case (21 dB without it). Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 9 ++++++--- tests/integration/timeline-frame-window.test.ts | 9 +++++++++ tests/timeline-introspect.test.ts | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index f316251..ac0ca4a 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -793,9 +793,12 @@ export function buildFrameAtPlan( // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); - // The first canvas frame at or after the frame's time: `trim` rounds an - // off-grid start to the nearest frame, which could be the one before. - const firstFrame = Math.ceil(clamped * comp.fps - 1e-6) / comp.fps; + // The first canvas frame at or after the frame's time (`trim` rounds an + // off-grid start to the nearest frame, which could be the one before), but + // no later than the segment's last frame: encoding to the clip's length + // keeps round(length x fps) frames. + const lastIndex = Math.max(0, Math.round(outDur * comp.fps) - 1); + const firstFrame = Math.min(Math.ceil(clamped * comp.fps - 1e-6), lastIndex) / comp.fps; const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index e69bcd3..162fd0b 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -104,6 +104,15 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' atSec: 3.01, exportSec: 3.01, }, + { + // 1.01 s encodes 30 frames at 30 fps; the last one is at 29/30 s. + name: 'the last frame of a clip off the frame grid', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 6.01, startSec: 0 }), + ), + atSec: 1.009, + exportSec: 29 / 30, + }, { name: 'inside a fade', doc: vertical( diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index e1642a4..1476d44 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -153,6 +153,20 @@ describe('buildFrameAtPlan', () => { expect(graph).toMatch(/,trim=start=3\.0333333\d*\[v\]$/); }); + it("stops at the segment's last frame on a clip whose length is off the frame grid", () => { + // 1.01 s at 30 fps encodes 30 frames, the last at 29/30 s. + const short = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 1.01, startSec: 0 }), + }, + ]); + const graph = graphOf(buildFrameAtPlan(short, CTX('/ws/f.jpg'), 1.009).steps[0]?.args); + expect(graph).toMatch(/,trim=start=0\.9666666\d*\[v\]$/); + }); + it('selects the second clip and offsets into it for a time in the second clip', () => { const doc = applyOps(oneClip(), [ { From 34b7f6704ac55c8a0b862fe87872aefcd8f81431 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:40:51 -0800 Subject: [PATCH 16/35] fix(timeline): keep held frames in previews, tie-safe outline pages A frame preview now keeps every frame from the keyframe before its seek (-noaccurate_seek) instead of a 1 s preroll, so a sparse source, such as a screen recording holding a still, previews the frame export holds (19.5 dB before, now matching). Frames before the clip's in-point are dropped as export's seek drops them. The outline names each sentence's source when the timeline has more than one, so its transcriptId can be found, and a page never splits sentences that start at the same time, so paging always moves on. Co-Authored-By: Claude Opus 5.5 --- src/mcp/server.ts | 2 +- src/timeline/compile.ts | 28 ++++++++--------- src/timeline/transcript.ts | 29 ++++++++++++++---- .../integration/timeline-frame-window.test.ts | 18 +++++++++++ tests/timeline-introspect.test.ts | 29 +++++++++++------- tests/timeline-outline.test.ts | 30 +++++++++++++++++++ 6 files changed, 105 insertions(+), 31 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index bd4f7f0..8ddc8b6 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -240,7 +240,7 @@ export function buildMcpServer(): McpServer { description: "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short. " + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + - "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", + "transcriptIds (by source; with more than one source each sentence names its mediaId) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { timeline: timelineInput, clipId: z diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index ac0ca4a..2657a68 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,9 +614,6 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } -/** How much of the source a frame preview reads before the frame. */ -const FRAME_PREROLL_SEC = 1; - function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -782,17 +779,17 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Where the window starts, in seconds into the clip's source: a little - // before the frame, so a source frame that export repeats into it (a source - // slower than the canvas) is still read. An input seek starts timestamps at - // the seek point, so adding `from` gives source time even when the first - // frame read comes after it. The chain's speed `setpts` divides - // the offset back into segment time, which its fades and text read, and - // `trim` then takes the first frame at the frame's segment time, as reading - // it out of the encoded segment would. Rounded down, to the microsecond, so - // the seek never passes the clip's last frame or its out-point. + // Seconds into the clip's source. The seek keeps every frame from the + // keyframe before it (`-noaccurate_seek`), so a frame that export repeats + // into this time (a source slower than the canvas, or a screen recording + // holding a still frame) is still read; its timestamps count from the seek + // point, so adding `local` gives source time from the clip's in-point. + // Frames before the in-point are dropped, as export's seek drops them. The + // chain's speed `setpts` turns source time into segment time, which its + // fades and text read, and `trim` takes the frame at the frame's segment + // time. Rounded down, to the microsecond, so the seek never passes the + // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); - const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length @@ -804,12 +801,13 @@ export function buildFrameAtPlan( label, args: [ '-y', + '-noaccurate_seek', '-ss', - String(floorUs(clip.sourceInSec + from)), + String(floorUs(clip.sourceInSec + local)), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${firstFrame}[v]`, + `[0:v]setpts=PTS+${local}/TB,trim=start=0,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 3e22404..686e83d 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -325,6 +325,9 @@ export interface OutlineSentence { startSec: number; endSec: number; clipId: string; + /** The sentence's source, named only when the timeline has more than one, + * so its `transcriptIds` entry can be found. */ + mediaId?: MediaId; fromIndex: number; toIndex: number; text: string; @@ -413,7 +416,7 @@ export async function timelineOutline( .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); - const sentences: OutlineSentence[] = []; + const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; let run: OutlineWord[] = []; let runMedia: MediaId | undefined; let prevClip: MediaClip | undefined; @@ -429,7 +432,7 @@ export async function timelineOutline( } if (!transcript) continue; if (clip.mediaId !== runMedia) { - sentences.push(...outlineSentences(run)); + if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); run = []; runMedia = clip.mediaId; } @@ -449,12 +452,16 @@ export async function timelineOutline( } prevClip = clip; } - sentences.push(...outlineSentences(run)); + if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); const transcriptIds: Record = {}; for (const [mediaId, transcript] of transcripts) { if (transcript) transcriptIds[mediaId] = transcriptIdOf(transcript); } + const named = Object.keys(transcriptIds).length > 1; + const sentences = bySource.flatMap(({ mediaId, sentences }) => + named ? sentences.map((s) => ({ ...s, mediaId })) : sentences, + ); const fromSec = opts.fromSec ?? 0; const toSec = opts.toSec ?? Number.POSITIVE_INFINITY; const inRange = sentences.filter((s) => s.startSec >= fromSec && s.startSec < toSec); @@ -465,10 +472,22 @@ export async function timelineOutline( JSON.stringify({ sentences: [], transcriptIds, nextFromSec: 99999.999 }), ); const page: OutlineSentence[] = []; - for (const sentence of inRange) { + for (const [i, sentence] of inRange.entries()) { const size = Buffer.byteLength(JSON.stringify(sentence)) + 1; if (bytes + size > maxBytes && page.length > 0) { - return { sentences: page, transcriptIds, nextFromSec: sentence.startSec }; + // The next page starts at a time, so sentences that start together stay + // on one page: split before them, or, when they fill this page from its + // first line, keep all of them here. + const tied = sentence.startSec; + const kept = page.filter((s) => s.startSec !== tied); + if (kept.length > 0) return { sentences: kept, transcriptIds, nextFromSec: tied }; + const rest = inRange.slice(i).findIndex((s) => s.startSec !== tied); + if (rest === -1) return { sentences: inRange, transcriptIds }; + return { + sentences: inRange.slice(0, i + rest), + transcriptIds, + nextFromSec: inRange[i + rest]?.startSec, + }; } page.push(sentence); bytes += size; diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 162fd0b..57d4a08 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -185,6 +185,24 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); } + // A sparse source (a frame every 5 s, like a screen recording holding a + // still): at 18 s export still shows the frame from 15 s. + const sparse = join(workspace, 'sparse.mp4'); + await testPattern(sparse, 25, 0.2); + const sparseMedia = new Map([[M, { path: sparse, hasAudio: true }]]); + const sparseDoc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 25, startSec: 0 })); + const sparseExport = join(workspace, 'sparse-export.mp4'); + await runPlan( + compileTimeline(sparseDoc, { media: sparseMedia, dir: workspace, output: sparseExport }), + ); + const sparseFull = join(workspace, 'sparse-full.jpg'); + await runFfmpeg(buildPreviewFrameArgs({ input: sparseExport, output: sparseFull, atSec: 18 })); + const sparseWindow = join(workspace, 'sparse-window.jpg'); + await runPlan( + buildFrameAtPlan(sparseDoc, { media: sparseMedia, dir: workspace, output: sparseWindow }, 18), + ); + expect(await psnr(sparseFull, sparseWindow), 'sparse source').toBeGreaterThanOrEqual(40); + // The very end of a clip that ends with its source: the seek must not pass // the last frame, or there is no image. const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 1476d44..9d08532 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,8 +82,15 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - // Read from 1 s before the frame, then take the first frame at 3 s. - expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '2', '-i', '/in.mp4']); + // Seek to 3 s, keeping the frames from the keyframe before it. + expect(frame?.args.slice(0, 6)).toEqual([ + '-y', + '-noaccurate_seek', + '-ss', + '3', + '-i', + '/in.mp4', + ]); expect(frame?.args.slice(-8)).toEqual([ '-map', '[v]', @@ -97,7 +104,9 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`); + expect(graphOf(frame?.args)).toBe( + `[0:v]setpts=PTS+3/TB,trim=start=0,${segmentChain},trim=start=3[v]`, + ); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { @@ -112,8 +121,8 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); + expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+7\.966666\/TB,trim=start=0,/); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -177,8 +186,8 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '2']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+1\/TB,/); + expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+2\/TB,/); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -198,13 +207,13 @@ describe('buildFrameAtPlan', () => { }), }, ]); - // The frame shows source second 4. Reading starts a second earlier, at 3; + // The frame shows source second 4; // the chain's setpts=PTS/2 turns source time into segment time, which the // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(1, 3)).toEqual(['-ss', '3']); + expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+3\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+4\/TB,/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index ed104fb..34c1fd3 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -144,6 +144,7 @@ describe('timelineOutline', () => { expect(placed?.findLast((w) => w.index === s.toIndex)?.endSec).toBe(s.endSec); } expect(Object.keys(outline.transcriptIds)).toEqual([M]); + expect(outline.sentences.some((s) => 'mediaId' in s)).toBe(false); expect(outline.nextFromSec).toBeUndefined(); }); @@ -196,6 +197,8 @@ describe('timelineOutline', () => { ['c2', 3, 4, 'Second point'], ]); expect(Object.keys(outline.transcriptIds)).toEqual([M, O]); + // Two sources: each sentence names its own, to find its transcriptId. + expect(outline.sentences.map((s) => s.mediaId)).toEqual([M, O, M]); }); it('pages by size and range, so pages put back together give every sentence once', async () => { @@ -220,6 +223,33 @@ describe('timelineOutline', () => { expect(middle.sentences.map((s) => s.fromIndex)).toEqual([3]); }); + it('keeps sentences that start together on one page, so paging always moves on', async () => { + // Four one-word sentences at the same instant (zero-length words), then two more. + const tied = transcriptOf(M, [ + { text: 'Lead.', startSec: 0, endSec: 0.4 }, + ...['A.', 'B.', 'C.', 'D.'].map((text) => ({ text, startSec: 1, endSec: 1 })), + { text: 'After.', startSec: 1.2, endSec: 1.6 }, + { text: 'End.', startSec: 1.6, endSec: 2 }, + ]); + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 2, startSec: 0 })]); + const transcribe = transcribeFrom(tied); + const all = (await timelineOutline(comp, MEDIA, { transcribe })).sentences; + for (const maxBytes of [300, 450]) { + const seen = []; + const starts: number[] = []; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + starts.push(fromSec); + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec, maxBytes }); + seen.push(...page.sentences); + fromSec = page.nextFromSec; + expect(starts.length).toBeLessThan(10); + } + expect(seen, `maxBytes ${maxBytes}`).toEqual(all); + expect(new Set(starts).size).toBe(starts.length); + } + }); + it('pages a 60-minute, 9,000-word recording at 40 KB', async () => { const texts = Array.from({ length: 9000 }, (_, i) => i % 14 === 13 ? `sentence${i}.` : `word${i}`, From 9b8029d2887c12a6e5ae61a3871ebe7e153dedff Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:48:46 -0800 Subject: [PATCH 17/35] fix(timeline): match export's frame timing tick for tick; outline ranges A frame preview restored its timestamps with a fractional offset while the seek rounds to whole stream ticks, so a 24 fps frame on a 30 fps canvas could land one canvas frame early (21 dB at 1.391 s). It now undoes the seek in whole ticks, leaving the timestamps export's own seek gives. An outline sentence ends where the timeline skips ahead over words it plays elsewhere (clips moved out of source order), so its index range never takes in words it doesn't show. The CLI sentence lines name the source when the timeline has more than one. Co-Authored-By: Claude Opus 5.5 --- src/cli-timeline.ts | 6 ++- src/timeline/compile.ts | 8 +++- src/timeline/transcript.ts | 45 +++++++++++++++---- .../integration/timeline-frame-window.test.ts | 4 +- tests/timeline-introspect.test.ts | 12 +++-- tests/timeline-outline.test.ts | 16 +++++++ 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e9f1b91..46130f8 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -641,7 +641,11 @@ export async function runTimeline(args: string[]): Promise { ([mediaId, id]) => `transcript ${mediaId}: ${id}`, ); for (const s of outline.sentences) { - lines.push(`${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId} ${s.text}`); + // With more than one source, its media id, to match a transcript line. + const source = s.mediaId ? ` ${s.mediaId}` : ''; + lines.push( + `${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId}${source} ${s.text}`, + ); } process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 2657a68..05cc08e 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -790,6 +790,7 @@ export function buildFrameAtPlan( // time. Rounded down, to the microsecond, so the seek never passes the // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); + const seek = floorUs(clip.sourceInSec + local); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length @@ -803,11 +804,14 @@ export function buildFrameAtPlan( '-y', '-noaccurate_seek', '-ss', - String(floorUs(clip.sourceInSec + local)), + String(seek), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${local}/TB,trim=start=0,${chain},trim=start=${firstFrame}[v]`, + // The seek shifted timestamps by (in-point + local) in whole stream + // ticks; shifting back by the same rounding leaves them as export's + // seek to the in-point does, tick for tick. + `[0:v]setpts=PTS+round(${seek}/TB)-round(${floorUs(clip.sourceInSec)}/TB),trim=start=0,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 686e83d..cd2d742 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -344,9 +344,14 @@ export interface TimelineOutline { /** * Group words (one source, timeline order) into sentences: a sentence ends at * sentence-ending punctuation, before a pause longer than `PAUSE_SEC`, after - * `MAX_SENTENCE_WORDS` words, or where the timeline goes back in the source. + * `MAX_SENTENCE_WORDS` words, where the timeline goes back in the source, or + * where it skips ahead over words `played` (by index) somewhere else on it, so + * a sentence's index range never takes in words it doesn't show. */ -export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { +export function outlineSentences( + words: OutlineWord[], + played: ReadonlySet = new Set(), +): OutlineSentence[] { const groups: OutlineWord[][] = []; let current: OutlineWord[] = []; for (const word of words) { @@ -356,6 +361,7 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { (endsSentence(prev.text) || word.startSec - prev.endSec > PAUSE_SEC || word.index <= prev.index || + skipsPlayed(prev.index, word.index, played) || current.length >= MAX_SENTENCE_WORDS) ) { groups.push(current); @@ -381,6 +387,11 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { }); } +function skipsPlayed(from: number, to: number, played: ReadonlySet): boolean { + for (let i = from + 1; i < to; i++) if (played.has(i)) return true; + return false; +} + export interface OutlineOptions extends TranscribeOptions { /** Only this media clip's words. */ clipId?: string; @@ -416,10 +427,9 @@ export async function timelineOutline( .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); - const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; - let run: OutlineWord[] = []; - let runMedia: MediaId | undefined; - let prevClip: MediaClip | undefined; + const placed: { clip: MediaClip; words: ClipWord[] }[] = []; + /** Word indices each source plays anywhere in the outlined clips. */ + const played = new Map>(); for (const clip of clips) { let transcript = transcripts.get(clip.mediaId); if (transcript === undefined) { @@ -431,8 +441,25 @@ export async function timelineOutline( transcripts.set(clip.mediaId, transcript); } if (!transcript) continue; + const words = clipWords(transcript, clip); + placed.push({ clip, words }); + const indices = played.get(clip.mediaId) ?? new Set(); + for (const w of words) indices.add(w.index); + played.set(clip.mediaId, indices); + } + + const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; + let run: OutlineWord[] = []; + let runMedia: MediaId | undefined; + let prevClip: MediaClip | undefined; + const flush = (): void => { + if (runMedia) { + bySource.push({ mediaId: runMedia, sentences: outlineSentences(run, played.get(runMedia)) }); + } + }; + for (const { clip, words } of placed) { if (clip.mediaId !== runMedia) { - if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); + flush(); run = []; runMedia = clip.mediaId; } @@ -440,7 +467,7 @@ export async function timelineOutline( // between them), rather than playing part of it again. const continues = prevClip?.mediaId === clip.mediaId && clip.sourceInSec >= prevClip.sourceOutSec - 1e-6; - for (const word of clipWords(transcript, clip)) { + for (const word of words) { // A word a cut runs through shows in the clips on both sides of it: one // word, from its start in the first to its end in the last. const last = run[run.length - 1]; @@ -452,7 +479,7 @@ export async function timelineOutline( } prevClip = clip; } - if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); + flush(); const transcriptIds: Record = {}; for (const [mediaId, transcript] of transcripts) { diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 57d4a08..ec4f692 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -174,7 +174,9 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' await runPlan( compileTimeline(slowDoc, { media: slowMedia, dir: workspace, output: slowExport }), ); - for (const atSec of [1.5, 3.999]) { + // 1.391 s: the seek's whole-tick rounding decides which side of a frame + // boundary the source frame at 1.4167 s lands on. + for (const atSec of [1.5, 1.391, 3.999]) { const full = join(workspace, `slow-${atSec}-full.jpg`); const exportSec = Math.min(atSec, 4 - 1 / 30); await runFfmpeg(buildPreviewFrameArgs({ input: slowExport, output: full, atSec: exportSec })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 9d08532..ba8c8fb 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -105,7 +105,7 @@ describe('buildFrameAtPlan', () => { const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+3/TB,trim=start=0,${segmentChain},trim=start=3[v]`, + `[0:v]setpts=PTS+round(3/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, ); }); @@ -122,7 +122,9 @@ describe('buildFrameAtPlan', () => { // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+7\.966666\/TB,trim=start=0,/); + expect(graphOf(near?.args)).toMatch( + /^\[0:v\]setpts=PTS\+round\(107\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, + ); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -187,7 +189,9 @@ describe('buildFrameAtPlan', () => { const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+2\/TB,/); + expect(graphOf(plan.steps[0]?.args)).toMatch( + /^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(1\/TB\),/, + ); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -213,7 +217,7 @@ describe('buildFrameAtPlan', () => { const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+4\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(4\/TB\)-round\(0\/TB\),/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 34c1fd3..dabf8d7 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -148,6 +148,22 @@ describe('timelineOutline', () => { expect(outline.nextFromSec).toBeUndefined(); }); + it('ends a sentence where the timeline skips words it plays elsewhere', async () => { + const run = transcriptOf(M, spoken('a b c d e f g h i j k l'.split(' '))); + // Words 0–1, then 10–11, then 2–9: clips moved out of source order. + const moved = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 4, sourceOutSec: 4.8, startSec: 0.8 }), + mediaClip({ id: 'c3', mediaId: M, sourceInSec: 0.8, sourceOutSec: 4, startSec: 1.6 }), + ]); + const outline = await timelineOutline(moved, MEDIA, { transcribe: transcribeFrom(run) }); + expect(outline.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex])).toEqual([ + ['c1', 0, 1], + ['c2', 10, 11], + ['c3', 2, 9], + ]); + }); + it('keeps a word the timeline plays twice as two, where a clip replays the source', async () => { const hello = transcriptOf(M, spoken(['Hello.', 'and', 'so'])); const transcribe = transcribeFrom(hello); From 19e6220ee6a1e0e041e4a1c8c3646ca12fd0f6af Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:55:45 -0800 Subject: [PATCH 18/35] fix(timeline): preroll slow-motion previews; end the sentence listing on a closed pipe A slow-motion frame can need the source frame before the keyframe its own time falls just after; the preview now seeks a second early as well as keeping the frames from the keyframe before (21 dB before at 0.25x). `clip timeline transcribe --sentences | head` ended with an unhandled EPIPE once head closed the pipe on a long listing; a closed pipe now just ends it. Co-Authored-By: Claude Opus 5.5 --- src/cli-timeline.ts | 4 +++ src/timeline/compile.ts | 15 +++++---- .../integration/timeline-frame-window.test.ts | 31 +++++++++++++++++++ tests/timeline-introspect.test.ts | 18 +++++------ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index 46130f8..8c3731d 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -647,6 +647,10 @@ export async function runTimeline(args: string[]): Promise { `${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId}${source} ${s.text}`, ); } + // `| head` closes the pipe early: that ends the listing, not with an error. + process.stdout.on('error', (err: NodeJS.ErrnoException) => { + if (err.code !== 'EPIPE') throw err; + }); process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; } diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 05cc08e..7f21fe1 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,9 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +/** How much of the source a frame preview reads before the frame, at least. */ +const FRAME_PREROLL_SEC = 1; + function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -779,18 +782,18 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Seconds into the clip's source. The seek keeps every frame from the - // keyframe before it (`-noaccurate_seek`), so a frame that export repeats - // into this time (a source slower than the canvas, or a screen recording - // holding a still frame) is still read; its timestamps count from the seek - // point, so adding `local` gives source time from the clip's in-point. + // Seconds into the clip's source. The seek lands a second before that and + // keeps every frame from the keyframe before it (`-noaccurate_seek`), so a + // frame that export repeats into this time (a source slower than the + // canvas, slow motion, or a screen recording holding a still frame) is + // still read; undoing the seek gives source time from the clip's in-point. // Frames before the in-point are dropped, as export's seek drops them. The // chain's speed `setpts` turns source time into segment time, which its // fades and text read, and `trim` takes the frame at the frame's segment // time. Rounded down, to the microsecond, so the seek never passes the // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); - const seek = floorUs(clip.sourceInSec + local); + const seek = floorUs(clip.sourceInSec + Math.max(0, local - FRAME_PREROLL_SEC)); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index ec4f692..258cc21 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -187,6 +187,37 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); } + // Slow motion: at 0.25x a canvas frame can need the source frame before the + // keyframe the frame's own time falls just after (2.5 s here, a 60-frame GOP). + const slowMotion = vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 0.01, + sourceOutSec: 4, + startSec: 0, + effects: [{ type: 'speed', factor: 0.25 }], + }), + ); + const slowMotionExport = join(workspace, 'slow-motion.mp4'); + await runPlan( + compileTimeline(slowMotion, { media: slowMedia, dir: workspace, output: slowMotionExport }), + ); + const slowMotionFull = join(workspace, 'slow-motion-full.jpg'); + // Timeline 2.491 s of a 0.25x clip plays at 9.964 s of its segment. + await runFfmpeg( + buildPreviewFrameArgs({ input: slowMotionExport, output: slowMotionFull, atSec: 9.964 }), + ); + const slowMotionWindow = join(workspace, 'slow-motion-window.jpg'); + await runPlan( + buildFrameAtPlan( + slowMotion, + { media: slowMedia, dir: workspace, output: slowMotionWindow }, + 2.491, + ), + ); + expect(await psnr(slowMotionFull, slowMotionWindow), 'slow motion').toBeGreaterThanOrEqual(40); + // A sparse source (a frame every 5 s, like a screen recording holding a // still): at 18 s export still shows the frame from 15 s. const sparse = join(workspace, 'sparse.mp4'); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index ba8c8fb..5049355 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,12 +82,12 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - // Seek to 3 s, keeping the frames from the keyframe before it. + // Seek a second before 3 s, keeping the frames from the keyframe before that. expect(frame?.args.slice(0, 6)).toEqual([ '-y', '-noaccurate_seek', '-ss', - '3', + '2', '-i', '/in.mp4', ]); @@ -105,7 +105,7 @@ describe('buildFrameAtPlan', () => { const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+round(3/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, + `[0:v]setpts=PTS+round(2/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, ); }); @@ -121,9 +121,9 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); + expect(near?.args.slice(2, 4)).toEqual(['-ss', '106.966666']); expect(graphOf(near?.args)).toMatch( - /^\[0:v\]setpts=PTS\+round\(107\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, + /^\[0:v\]setpts=PTS\+round\(106\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, ); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -188,9 +188,9 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); + expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '2']); expect(graphOf(plan.steps[0]?.args)).toMatch( - /^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(1\/TB\),/, + /^\[0:v\]setpts=PTS\+round\(2\/TB\)-round\(1\/TB\),/, ); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -215,9 +215,9 @@ describe('buildFrameAtPlan', () => { // the chain's setpts=PTS/2 turns source time into segment time, which the // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); + expect(frame?.args.slice(2, 4)).toEqual(['-ss', '3']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(4\/TB\)-round\(0\/TB\),/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(0\/TB\),/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); From 7665a7f34dc0290230d8ebdd6f417eda856c498f Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 17:52:08 -0800 Subject: [PATCH 19/35] feat(timeline): cut a short's edges in the pause the audio shows whisper.cpp gives each pause to the word before it and runs late at sentence ends, so a short cut by the transcript kept up to 0.35 s of the next sentence. make_short now measures the source's levels around each edge (one short FFmpeg read, 10 ms RMS frames), finds the pause the transcript boundary points at by a threshold set from those levels, and cuts 0.1 s into it. An edge with no clear pause keeps the transcript rule and warns; a word whisper places past the cut keeps its caption. On the 30-sentence say recording every sentence-short is clean: no edge clipped, none keeping a neighbor's speech (before: 18 ends kept the next sentence, 3 starts were clipped). On jfk.wav every cut lands in a pause. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- SKILL.md | 2 +- docs/backlog/cut-words-pause-edges.md | 3 + docs/backlog/short-edges-snap-to-pause.md | 3 - src/cli-timeline.ts | 11 +- src/ffmpeg/args/speech-levels.ts | 42 +++++++ src/index.ts | 13 ++ src/timeline/pauses.ts | 140 +++++++++++++++++++++ src/timeline/shorts.ts | 147 ++++++++++++++++++---- src/timeline/verbs.ts | 7 +- src/ui/timeline-tools.ts | 7 ++ tests/integration/timeline-pauses.test.ts | 57 +++++++++ tests/mcp-server.test.ts | 2 + tests/timeline-captions.test.ts | 1 + tests/timeline-pauses.test.ts | 137 ++++++++++++++++++++ tests/timeline-shorts.test.ts | 128 +++++++++++++++++-- tests/timeline-verbs.test.ts | 2 + 17 files changed, 664 insertions(+), 40 deletions(-) create mode 100644 docs/backlog/cut-words-pause-edges.md delete mode 100644 docs/backlog/short-edges-snap-to-pause.md create mode 100644 src/ffmpeg/args/speech-levels.ts create mode 100644 src/timeline/pauses.ts create mode 100644 tests/integration/timeline-pauses.test.ts create mode 100644 tests/timeline-pauses.test.ts diff --git a/README.md b/README.md index ff87911..0910a66 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ clip timeline --timeline best-1 export best-1.mp4 clip timeline list # main and every short ``` -`short` starts 0.25 s before the first word and ends with the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. +`short` cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone (where there's no clear pause it follows the transcript and warns), fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. --- diff --git a/SKILL.md b/SKILL.md index 7553f49..3e8389e 100644 --- a/SKILL.md +++ b/SKILL.md @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence; widen it to the sentence's edge unless the cut is meant. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone, so it never keeps the next sentence's first syllable, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence (widen it to the sentence's edge unless the cut is meant) and an edge with no clear pause in the audio (music under the speech, or no break), which follows the transcript instead: look at that edge or listen to it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/docs/backlog/cut-words-pause-edges.md b/docs/backlog/cut-words-pause-edges.md new file mode 100644 index 0000000..5c88d73 --- /dev/null +++ b/docs/backlog/cut-words-pause-edges.md @@ -0,0 +1,3 @@ +# Cut words at the pause + +`cut_words` and `remove_retakes` cut from the first word's start to the last word's end as whisper.cpp gives them (`planWordCut` in `src/timeline/transcript.ts`). whisper gives each pause to the word before it and runs late at sentence edges. On a 30-sentence test recording, cutting a sentence would also remove up to 0.35 s of the next sentence's first syllable, at 18 of 29 boundaries. Place those cut edges in the pause the audio shows, as `make_short` does (`src/timeline/pauses.ts`), through `VerbContext.speechLevels`. diff --git a/docs/backlog/short-edges-snap-to-pause.md b/docs/backlog/short-edges-snap-to-pause.md deleted file mode 100644 index 3bcb401..0000000 --- a/docs/backlog/short-edges-snap-to-pause.md +++ /dev/null @@ -1,3 +0,0 @@ -# Snap short edges to the pause - -whisper.cpp (`-ml 1 -sow`) gives each pause to the word before it, so at a sentence edge its words run late. On a 30-sentence test recording (macOS `say`, model base.en), a short ending on a sentence's last word ran past the pause into the next sentence in 18 of 29 cases, by up to 0.35 s. Even with the 0.25 s lead, 3 of 29 starts lost up to 0.1 s of the first word. No fixed padding fixes both. Move each edge of a range to the quietest point of the audio near it: run a silencedetect pass over a second or so around the edge and keep the edge inside the pause between the range and its neighbors. `-dtw base.en` didn't change the per-word offsets in a quick check. diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index 8773c7d..72633ac 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -45,6 +45,7 @@ import { } from './timeline/interchange.js'; import { buildMediaMap, buildMediaRefs } from './timeline/media-registry.js'; import { colorClip, mediaClip, textClip } from './timeline/ops.js'; +import { measureSpeechLevels } from './timeline/pauses.js'; import { AspectSchema, type FramingFit, @@ -572,7 +573,15 @@ export async function runTimeline(args: string[]): Promise { flags.loudness === undefined ? undefined : loudnessValue(flags.loudness, usage), replace: flags.replace === 'true', }, - (comp, clipId, transcriptId) => storedTranscriptForClip(comp, clipId, media, transcriptId), + { + transcript: (comp, clipId, transcriptId) => + storedTranscriptForClip(comp, clipId, media, transcriptId), + levels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + const info = clip?.kind === 'media' ? media.get(clip.mediaId) : undefined; + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, + }, ); const report = { timeline: target, diff --git a/src/ffmpeg/args/speech-levels.ts b/src/ffmpeg/args/speech-levels.ts new file mode 100644 index 0000000..6db6fa7 --- /dev/null +++ b/src/ffmpeg/args/speech-levels.ts @@ -0,0 +1,42 @@ +export interface SpeechLevelsArgs { + input: string; + /** Where to start measuring, in source seconds. */ + fromSec: number; + durationSec: number; +} + +/** Samples per measured frame at 16 kHz: 10 ms. */ +export const LEVEL_FRAME_SAMPLES = 160; +export const LEVEL_SAMPLE_RATE = 16000; + +/** + * The RMS level of every 10 ms of a source's first audio stream over a window, + * printed to stderr as `pts_time:…` / `lavfi.astats.Overall.RMS_level=…` lines. + * Mono at 16 kHz is plenty to tell speech from a pause, and the window is a few + * seconds, so it runs in well under a tenth of a second. + */ +export function buildSpeechLevelsArgs({ input, fromSec, durationSec }: SpeechLevelsArgs): string[] { + return [ + '-hide_banner', + '-nostats', + '-ss', + String(fromSec), + '-t', + String(durationSec), + '-i', + input, + '-map', + '0:a:0', + '-af', + [ + `aresample=${LEVEL_SAMPLE_RATE}`, + 'aformat=channel_layouts=mono', + `asetnsamples=n=${LEVEL_FRAME_SAMPLES}:p=0`, + 'astats=metadata=1:reset=1', + 'ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + ].join(','), + '-f', + 'null', + '-', + ]; +} diff --git a/src/index.ts b/src/index.ts index bea7419..7206d6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -199,6 +199,17 @@ export { textClip, videoTrack, } from './timeline/ops.js'; +export { + endInPause, + findPauses, + measureSpeechLevels, + PAUSE_KEEP_SEC, + type Pause, + parseSpeechLevels, + pauseAt, + type SpeechLevels, + startInPause, +} from './timeline/pauses.js'; export { type Aspect, AspectSchema, @@ -251,7 +262,9 @@ export { type ShortPlan, type ShortRange, type ShortRequest, + type ShortSources, type ShortTranscriptLookup, + type SpeechLevelsLookup, } from './timeline/shorts.js'; export { analyzeSilence, diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts new file mode 100644 index 0000000..0cfa147 --- /dev/null +++ b/src/timeline/pauses.ts @@ -0,0 +1,140 @@ +import { + buildSpeechLevelsArgs, + LEVEL_FRAME_SAMPLES, + LEVEL_SAMPLE_RATE, +} from '../ffmpeg/args/speech-levels.js'; +import { runFfmpeg } from '../ffmpeg/run.js'; + +/** + * Pauses in speech, found in the audio itself. whisper.cpp's word edges run + * late at sentence ends: it gives each pause to the word before it, and the + * next word can start a third of a second into its own speech. A cut placed by + * the transcript alone then keeps the start of the next sentence. Measured + * levels show where the voice actually stops, so an edge is moved into the + * pause the transcript points at. + */ + +/** RMS level of each 10 ms of a source's audio, dBFS, from `fromSec` on. */ +export interface SpeechLevels { + fromSec: number; + frameSec: number; + db: number[]; +} + +/** A stretch of a source quiet enough to be a pause, in source seconds. */ +export interface Pause { + startSec: number; + endSec: number; +} + +/** Room tone kept between a cut and the speech beside it. */ +export const PAUSE_KEEP_SEC = 0.1; +/** How far before the transcript's boundary the pause may lie: whisper runs + * up to about 0.75 s late there. */ +const SEARCH_BEFORE_SEC = 0.9; +/** …and after it, for a boundary whisper put early. */ +const SEARCH_AFTER_SEC = 0.05; +/** Measured around a boundary, for a noise floor and speech level of its own. */ +const MEASURE_BEFORE_SEC = 1.5; +const MEASURE_AFTER_SEC = 1; +/** Quiet means this far over the window's floor (its 5th percentile)… */ +const ABOVE_FLOOR_DB = 10; +/** …and this far under its speech (its 95th percentile). */ +const BELOW_SPEECH_DB = 20; +/** The shortest quiet stretch counted as a pause. */ +const MIN_PAUSE_SEC = 0.08; +/** Sound shorter than this inside a pause (a click, a lip noise) doesn't end it. */ +const MAX_BLIP_SEC = 0.04; +/** Two pauses at least this long in reach make the boundary ambiguous. */ +const COMPETING_PAUSE_SEC = 0.12; +const SILENT_DB = -120; + +/** Read the per-frame levels `buildSpeechLevelsArgs` prints. */ +export function parseSpeechLevels(stderr: string, fromSec: number): SpeechLevels { + const db: number[] = []; + for (const match of stderr.matchAll(/lavfi\.astats\.Overall\.RMS_level=(\S+)/g)) { + const value = Number(match[1]); + db.push(Number.isFinite(value) ? Math.max(value, SILENT_DB) : SILENT_DB); + } + return { fromSec, frameSec: LEVEL_FRAME_SAMPLES / LEVEL_SAMPLE_RATE, db }; +} + +/** Measure a source's levels around a boundary (runs FFmpeg). */ +export async function measureSpeechLevels( + path: string, + boundarySec: number, +): Promise { + const fromSec = Math.max(0, boundarySec - MEASURE_BEFORE_SEC); + const durationSec = boundarySec + MEASURE_AFTER_SEC - fromSec; + const { stderr } = await runFfmpeg(buildSpeechLevelsArgs({ input: path, fromSec, durationSec })); + return parseSpeechLevels(stderr, fromSec); +} + +/** Every pause in the levels, by a threshold set from the levels themselves. */ +export function findPauses(levels: SpeechLevels): Pause[] { + const sorted = [...levels.db].sort((a, b) => a - b); + const at = (q: number): number => sorted[Math.floor(q * (sorted.length - 1))] ?? SILENT_DB; + const threshold = Math.min(at(0.05) + ABOVE_FLOOR_DB, at(0.95) - BELOW_SPEECH_DB); + const quiet = levels.db.map((level) => level <= threshold); + const maxBlip = Math.round(MAX_BLIP_SEC / levels.frameSec); + // A short burst of sound between two quiet stretches is part of the pause. + for (let i = 0; i < quiet.length; ) { + if (quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && !quiet[j]) j++; + if (i > 0 && j < quiet.length && j - i <= maxBlip) quiet.fill(true, i, j); + i = j; + } + const pauses: Pause[] = []; + for (let i = 0; i < quiet.length; ) { + if (!quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && quiet[j]) j++; + if ((j - i) * levels.frameSec >= MIN_PAUSE_SEC - 1e-9) { + pauses.push({ + startSec: levels.fromSec + i * levels.frameSec, + endSec: levels.fromSec + j * levels.frameSec, + }); + } + i = j; + } + return pauses; +} + +/** + * The pause at a transcript boundary: the longest pause reaching into + * [boundary − 0.9 s, boundary + 0.05 s], but not before the start of the word + * the boundary ends. `none` when there is none (speech runs on, or music under + * it); `ambiguous` when two pauses of 0.12 s or more are in reach, since the + * transcript can't say which one it means. + */ +export function pauseAt( + levels: SpeechLevels, + boundarySec: number, + wordBeforeStartSec: number, +): Pause | 'none' | 'ambiguous' { + const from = Math.max(boundarySec - SEARCH_BEFORE_SEC, wordBeforeStartSec - SEARCH_AFTER_SEC); + const to = boundarySec + SEARCH_AFTER_SEC; + const inReach = findPauses(levels).filter((p) => p.endSec > from && p.startSec < to); + if (inReach.length === 0) return 'none'; + if (inReach.filter((p) => p.endSec - p.startSec >= COMPETING_PAUSE_SEC - 1e-9).length > 1) { + return 'ambiguous'; + } + return inReach.reduce((a, b) => (b.endSec - b.startSec > a.endSec - a.startSec ? b : a)); +} + +/** Where a short ends in a pause: a little room tone after the voice stops. */ +export function endInPause(pause: Pause): number { + return pause.startSec + Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} + +/** Where a short starts in a pause: a little room tone before the voice. */ +export function startInPause(pause: Pause): number { + return pause.endSec - Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index e6d3c16..1e5b216 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -16,6 +16,7 @@ import { makeClipId, } from './composition.js'; import { applyOps, type CompositionOp, mediaClip, textClip, videoTrack } from './ops.js'; +import { endInPause, pauseAt, type SpeechLevels, startInPause } from './pauses.js'; import { type Aspect, canvasFor, planReframe } from './reframe.js'; import { MAIN_TIMELINE, type MediaId } from './schema.js'; import { clipWords, type Transcript } from './transcript.js'; @@ -40,6 +41,8 @@ export const SHORT_LEAD_SEC = 0.25; export const DEFAULT_SHORT_LUFS = -14; const SHORT_TRACK = 'v0'; +/** A range whose snapped edges leave less than this keeps the transcript's. */ +const MIN_SNAPPED_SEC = 0.2; /** A piece shorter than this is dropped: too short to hear, just a flash. */ const MIN_PIECE_SEC = 0.05; const SAME_SEC = 1e-6; @@ -101,6 +104,23 @@ export type ShortTranscriptLookup = ( transcriptId?: string, ) => Promise<{ clip: MediaClip; transcript: Transcript }>; +/** The audio levels of a media clip's source around a time, in source seconds + * (runs FFmpeg); null when the source has no sound. */ +export type SpeechLevelsLookup = ( + comp: Composition, + clipId: string, + aroundSec: number, +) => Promise; + +/** What `planShort` reads beyond the documents. */ +export interface ShortSources { + transcript: ShortTranscriptLookup; + /** Moves each edge into the pause the audio shows there. Without it, edges + * follow the transcript. */ + levels?: SpeechLevelsLookup; + mintId?: () => string; +} + /** Refuse a short on `main` or on the timeline it is cut from. */ export function checkShortTimelines(target: string, source: string = MAIN_TIMELINE): void { if (target === MAIN_TIMELINE) { @@ -136,8 +156,9 @@ interface Piece extends ShortPiece { speed: number; sourceClipId: string; transcript: Transcript; - /** The word ranges the piece plays, which are what it captions. */ - asked: { fromIndex: number; toIndex: number }[]; + /** The word ranges the piece plays, which are what it captions, with the + * source window each was cut to. */ + asked: { fromIndex: number; toIndex: number; start: number; end: number }[]; } /** @@ -150,9 +171,9 @@ export async function planShort( target: Composition, source: Composition, request: ShortRequest, - lookup: ShortTranscriptLookup, - mintId: () => string = makeClipId, + sources: ShortSources, ): Promise { + const mintId = sources.mintId ?? makeClipId; if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { throw new ShortError( @@ -167,9 +188,16 @@ export async function planShort( } const windows: WordWindow[] = []; - for (const range of request.ranges) { - const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); - windows.push(wordWindow(transcript, range, clip)); + const warnings: string[] = []; + for (const [i, range] of request.ranges.entries()) { + const { clip, transcript } = await sources.transcript(source, range.clipId, range.transcriptId); + const window = wordWindow(transcript, range, clip); + warnings.push(...sentenceEdgeWarnings(window, i + 1)); + windows.push( + sources.levels + ? await snapToPauses(window, source, clip.id, sources.levels, i + 1, warnings) + : window, + ); } const pieces: Piece[] = []; windows.forEach((window, i) => { @@ -246,9 +274,19 @@ export async function planShort( // isn't part of the short even when another range asks for it. const style = defaultCaptionStyle(applyOps(target, ops)); const cues = placed.flatMap(({ piece, clip }) => { - const words = piece.transcript.words.filter((_, i) => - piece.asked.some((r) => i >= r.fromIndex && i <= r.toIndex), - ); + // A word whisper places past the cut (it runs late at a sentence end) is + // pulled inside it, so its caption isn't lost. + const words = piece.transcript.words.flatMap((w, i) => { + const range = piece.asked.find((r) => i >= r.fromIndex && i <= r.toIndex); + if (!range) return []; + if (w.startSec >= range.end) { + return [{ ...w, startSec: Math.max(range.start, range.end - 0.25), endSec: range.end }]; + } + if (w.endSec <= range.start) { + return [{ ...w, startSec: range.start, endSec: Math.min(range.end, range.start + 0.25) }]; + } + return [w]; + }); return groupCaptions( clipWords({ ...piece.transcript, words }, clip), clipEndSec(clip), @@ -283,7 +321,7 @@ export async function planShort( })), durationSec: roundMs(at), captions, - warnings: windows.flatMap((window, i) => sentenceEdgeWarnings(window, i + 1)), + warnings, }; } @@ -323,6 +361,68 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): }; } +/** + * Move a range's edges into the pauses its source's audio shows at its + * transcript boundaries: a little room tone before the first word and after + * the last, never the next sentence's first syllable. An edge with no clear + * pause (speech runs on, music under it, or two pauses the transcript can't + * tell apart) keeps the transcript rule, with a warning unless the range was + * cut mid-sentence there anyway. + */ +async function snapToPauses( + window: WordWindow, + source: Composition, + clipId: string, + levelsFor: SpeechLevelsLookup, + rangeNumber: number, + warnings: string[], +): Promise { + const words = window.transcript.words; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (!first || !last) return window; + + let { start, end } = window; + let startLevels: SpeechLevels | null; + let endLevels: SpeechLevels | null; + try { + startLevels = await levelsFor(source, clipId, first.startSec); + endLevels = startLevels && (await levelsFor(source, clipId, last.endSec)); + } catch { + // FFmpeg couldn't read the audio; the short is still made, by the transcript. + warnings.push( + `Range ${rangeNumber}: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.`, + ); + return window; + } + if (!startLevels) return window; // no sound to measure + const startPause = pauseAt(startLevels, first.startSec, before?.startSec ?? 0); + if (typeof startPause === 'object') start = startInPause(startPause); + else if (before && !midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber}: no clear pause before "${first.text}", so its start follows the transcript and may clip that word or keep the end of "${before.text}". Look at it or listen.`, + ); + } + const endPause = endLevels ? pauseAt(endLevels, last.endSec, last.startSec) : 'none'; + if (typeof endPause === 'object') end = endInPause(endPause); + else if (after && !midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber}: no clear pause after "${last.text}", so its end follows the transcript and may keep the start of "${after.text}". Look at it or listen.`, + ); + } + // A pause found on the wrong side of a short range would leave nothing. + if (end - start < MIN_SNAPPED_SEC) return window; + return { ...window, start: roundMs(start), end: roundMs(end) }; +} + +/** Whether two neighboring words belong to one sentence: no sentence end and + * no real pause between them. */ +function midSentence(a: { text: string; endSec: number }, b: { startSec: number }): boolean { + return !endsSentence(a.text) && b.startSec - a.endSec < PAUSE_SEC; +} + /** * Where a range starts or ends inside a sentence: right after a word that * doesn't end one (with no real pause between), or right before the next word @@ -332,26 +432,18 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): function sentenceEdgeWarnings(window: WordWindow, rangeNumber: number): string[] { const words = window.transcript.words; const warnings: string[] = []; - const midSentence = ( - a: (typeof words)[number] | undefined, - b: (typeof words)[number] | undefined, - ) => - a !== undefined && - b !== undefined && - !endsSentence(a.text) && - b.startSec - a.endSec < PAUSE_SEC; const before = words[window.fromIndex - 1]; const first = words[window.fromIndex]; - if (midSentence(before, first)) { + if (before && first && midSentence(before, first)) { warnings.push( - `Range ${rangeNumber} starts mid-sentence, between "${before?.text}" and "${first?.text}" (word ${window.fromIndex}).`, + `Range ${rangeNumber} starts mid-sentence, between "${before.text}" and "${first.text}" (word ${window.fromIndex}).`, ); } const last = words[window.toIndex]; const after = words[window.toIndex + 1]; - if (midSentence(last, after)) { + if (last && after && midSentence(last, after)) { warnings.push( - `Range ${rangeNumber} ends mid-sentence, between "${last?.text}" and "${after?.text}" (word ${window.toIndex}).`, + `Range ${rangeNumber} ends mid-sentence, between "${last.text}" and "${after.text}" (word ${window.toIndex}).`, ); } return warnings; @@ -400,7 +492,14 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { speed: speedFactor(clip.effects), sourceClipId: clip.id, transcript: window.transcript, - asked: [{ fromIndex: window.fromIndex, toIndex: window.toIndex }], + asked: [ + { + fromIndex: window.fromIndex, + toIndex: window.toIndex, + start: window.start, + end: window.end, + }, + ], }); taken.push([a, b]); } diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index 91e81b5..6e96d54 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -23,7 +23,7 @@ import { planReframe, } from './reframe.js'; import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from './schema.js'; -import { checkShortTimelines, planShort } from './shorts.js'; +import { checkShortTimelines, planShort, type SpeechLevelsLookup } from './shorts.js'; import { SensitivitySchema, type SilenceCut } from './silence.js'; import { clipWords, @@ -402,6 +402,9 @@ export interface VerbContext { ) => Promise<{ clip: MediaClip; transcript: Transcript; media: Map }>; /** Read another timeline of the workspace (make_short's source). */ readTimeline: (timeline: string) => Promise; + /** A media clip's source audio levels around a time (runs FFmpeg), for + * placing a short's edges in pauses; null when the source has no sound. */ + speechLevels: SpeechLevelsLookup; /** Track a clip is placed on when the verb doesn't say. */ defaultTrack: string; /** The timeline being edited (default `main`); `applyVerbs` sets it. */ @@ -585,7 +588,7 @@ export async function lowerVerb( targetLufs: verb.targetLufs, replace: verb.replace, }, - ctx.transcript, + { transcript: ctx.transcript, levels: ctx.speechLevels }, ); for (const warning of plan.warnings) ctx.warn?.(warning); return plan.ops; diff --git a/src/ui/timeline-tools.ts b/src/ui/timeline-tools.ts index f1ac55d..881628b 100644 --- a/src/ui/timeline-tools.ts +++ b/src/ui/timeline-tools.ts @@ -9,6 +9,7 @@ import { } from '../timeline/composition.js'; import { readComposition } from '../timeline/document-store.js'; import { buildMediaMap, buildMediaRefs } from '../timeline/media-registry.js'; +import { measureSpeechLevels } from '../timeline/pauses.js'; import { ownFraming } from '../timeline/reframe.js'; import type { MediaId, MediaRef } from '../timeline/schema.js'; import { analyzeSilence } from '../timeline/silence.js'; @@ -35,6 +36,12 @@ export function makeVerbContext(): VerbContext { return { mediaId: result.mediaId, durationSec: result.ref.durationSec }; }, readTimeline: (timeline) => readComposition(timeline), + speechLevels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + if (clip?.kind !== 'media') return null; + const info = (await clipConfinedMedia(comp, clipId)).get(clip.mediaId); + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, analyzeSilence: async (comp, clipId, sensitivity) => analyzeSilence(comp, clipId, { media: await clipConfinedMedia(comp, clipId), sensitivity }), transcript: async (comp, clipId, transcriptId) => { diff --git a/tests/integration/timeline-pauses.test.ts b/tests/integration/timeline-pauses.test.ts new file mode 100644 index 0000000..2488b31 --- /dev/null +++ b/tests/integration/timeline-pauses.test.ts @@ -0,0 +1,57 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { runFfmpeg } from '../../src/ffmpeg/run.js'; +import { measureSpeechLevels, pauseAt } from '../../src/timeline/pauses.js'; + +// The levels a short's edges are placed by, measured by real FFmpeg on an +// encoded file: a tone sounding 0.1–0.7 s of every second, over quiet noise. +const IS_CI = process.env.CI === 'true'; + +let workspace: string; +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-pauses-')); +}); +afterEach(async () => { + await rm(workspace, { recursive: true, force: true }); +}); + +describe.skipIf(!IS_CI)('pauses against real FFmpeg', () => { + it('finds the pause a late transcript boundary points at, inside the real one', async () => { + const source = join(workspace, 'talk.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'color=c=gray:s=320x240:r=30:d=8'], + ...[ + '-f', + 'lavfi', + '-i', + "aevalsrc='0.3*sin(2*PI*220*t)*between(mod(t,1),0.1,0.7)+0.001*(random(0)-0.5)':s=48000:d=8", + ], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', source], + ]); + + // "Speech" stops at 3.7 and starts again at 4.1; a transcript running + // 0.25 s late puts the boundary at 4.35, its last word from 3.35. + const levels = await measureSpeechLevels(source, 4.35); + const pause = pauseAt(levels, 4.35, 3.35); + if (typeof pause !== 'object') throw new Error(`no pause found: ${pause}`); + // Inside the real pause, by at most 40 ms (AAC smears a sharp ending by + // about 30 ms): a cut placed from it never moves toward the speech. + expect(pause.startSec).toBeGreaterThanOrEqual(3.7 - 1e-6); + expect(pause.startSec).toBeLessThanOrEqual(3.74); + expect(pause.endSec).toBeLessThanOrEqual(4.1 + 1e-6); + expect(pause.endSec).toBeGreaterThanOrEqual(4.06); + + // Where the tone never stops there is no pause to find. + const steady = join(workspace, 'steady.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'sine=frequency=220:sample_rate=48000:duration=4'], + ...['-c:a', 'aac', steady], + ]); + expect(pauseAt(await measureSpeechLevels(steady, 2), 2, 1.5)).toBe('none'); + }, 60_000); +}); diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index e9665fa..62a31d8 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -293,6 +293,8 @@ describe('MCP shorts (named timelines)', () => { expect(edit.warnings).toEqual([ 'Range 1 starts mid-sentence, between "word1" and "word2" (word 2).', 'Range 1 ends mid-sentence, between "word6" and "word7" (word 6).', + // The fixture's "video" is a few bytes, which FFmpeg can't measure. + "Range 1: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.", ]); const doc = edit.document as Record; expect(doc.timeline).toBe('clip-1'); diff --git a/tests/timeline-captions.test.ts b/tests/timeline-captions.test.ts index 4e47a19..79dfd99 100644 --- a/tests/timeline-captions.test.ts +++ b/tests/timeline-captions.test.ts @@ -322,6 +322,7 @@ describe('caption_speech verb', () => { const { clip, transcript } = await lookup(new Map([[M, RECORDING]]))(clipId); return { clip, transcript, media: new Map() }; }, + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, diff --git a/tests/timeline-pauses.test.ts b/tests/timeline-pauses.test.ts new file mode 100644 index 0000000..fbf6359 --- /dev/null +++ b/tests/timeline-pauses.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from 'vitest'; +import { buildSpeechLevelsArgs } from '../src/ffmpeg/args/speech-levels.js'; +import { + endInPause, + findPauses, + parseSpeechLevels, + pauseAt, + type SpeechLevels, + startInPause, +} from '../src/timeline/pauses.js'; + +/** Levels built from [seconds, dB] stretches, 10 ms a frame, from `fromSec`. */ +function levelsOf(stretches: [number, number][], fromSec = 10): SpeechLevels { + const db = stretches.flatMap(([sec, level]) => + Array.from({ length: Math.round(sec / 0.01) }, () => level), + ); + return { fromSec, frameSec: 0.01, db }; +} + +const near = (value: number): number => Math.round(value * 1000) / 1000; + +describe('buildSpeechLevelsArgs', () => { + it('measures 10 ms of mono 16 kHz audio a frame over the window', () => { + expect(buildSpeechLevelsArgs({ input: '/in.mp4', fromSec: 8.5, durationSec: 2.5 })).toEqual([ + '-hide_banner', + '-nostats', + '-ss', + '8.5', + '-t', + '2.5', + '-i', + '/in.mp4', + '-map', + '0:a:0', + '-af', + 'aresample=16000,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + '-f', + 'null', + '-', + ]); + }); +}); + +describe('parseSpeechLevels', () => { + it('reads one level a frame, digital silence as -120 dB', () => { + const stderr = [ + '[Parsed_ametadata_4 @ 0x1] frame:0 pts:0 pts_time:0', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-12.703566', + '[Parsed_ametadata_4 @ 0x1] frame:1 pts:160 pts_time:0.01', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-inf', + '[Parsed_ametadata_4 @ 0x1] frame:2 pts:320 pts_time:0.02', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-48.5', + ].join('\n'); + expect(parseSpeechLevels(stderr, 3)).toEqual({ + fromSec: 3, + frameSec: 0.01, + db: [-12.703566, -120, -48.5], + }); + }); +}); + +describe('findPauses', () => { + it('finds pauses by the window’s own floor and speech level', () => { + // A noisy room: speech at -18 dB, room tone at -45 dB. + const levels = levelsOf([ + [1, -18], + [0.3, -45], + [0.8, -18], + [0.05, -45], // too short to be a pause + [0.5, -18], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([[11, 11.3]]); + }); + + it('keeps a pause whole across a click shorter than 40 ms', () => { + const levels = levelsOf([ + [1, -20], + [0.15, -60], + [0.03, -25], + [0.15, -60], + [1, -20], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([ + [11, 11.33], + ]); + }); + + it('finds none in speech that never stops, or in a steady level', () => { + expect(findPauses(levelsOf([[3, -20]]))).toEqual([]); + const stepped = levelsOf([ + [1, -26], + [1, -20], + [1, -26], + ]); + expect(findPauses(stepped)).toEqual([]); + }); +}); + +describe('pauseAt', () => { + // Speech, a 0.4 s pause from 11.2, speech again: whisper puts the boundary at + // 11.75, inside the next sentence, and the last word started at 10.9. + const levels = levelsOf([ + [1.2, -20], + [0.4, -60], + [1, -20], + ]); + + it('finds the pause a late transcript boundary points at', () => { + const pause = pauseAt(levels, 11.75, 10.9); + expect(pause).toEqual({ startSec: expect.closeTo(11.2, 6), endSec: expect.closeTo(11.6, 6) }); + if (typeof pause !== 'object') throw new Error('expected a pause'); + expect(near(endInPause(pause))).toBe(11.3); + expect(near(startInPause(pause))).toBe(11.5); + }); + + it('keeps half a short pause on each side', () => { + expect(near(endInPause({ startSec: 5, endSec: 5.1 }))).toBe(5.05); + expect(near(startInPause({ startSec: 5, endSec: 5.1 }))).toBe(5.05); + }); + + it('looks no earlier than the start of the word the boundary ends', () => { + // The last word started at 11.65, after the pause: there's none for it. + expect(pauseAt(levels, 11.75, 11.65)).toBe('none'); + }); + + it('says none when speech runs on, and ambiguous when two pauses compete', () => { + expect(pauseAt(levelsOf([[3, -20]]), 11.5, 11)).toBe('none'); + const two = levelsOf([ + [1, -20], + [0.2, -60], + [0.3, -20], + [0.2, -60], + [1, -20], + ]); + expect(pauseAt(two, 11.7, 10.5)).toBe('ambiguous'); + }); +}); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index b315fa8..c2d3410 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -99,8 +99,7 @@ async function plan( opts.target ?? emptyComposition(), source, { captions: false, ...request }, - lookupFor(opts.transcripts ?? [SPACED]), - counter(), + { transcript: lookupFor(opts.transcripts ?? [SPACED]), mintId: counter() }, ); } @@ -333,8 +332,7 @@ describe('planShort: the edit', () => { emptyComposition(), whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], x: 0.3 }, - lookupFor([SPACED]), - counter(), + { transcript: lookupFor([SPACED]), mintId: counter() }, ); expect(short.ops).toEqual([ { op: 'setCanvas', width: 1080, height: 1920, fps: 30, background: 'black', fit: 'cover' }, @@ -420,8 +418,7 @@ describe('planShort: the edit', () => { emptyComposition(), whole, { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, - lookupFor([abutting]), - counter(), + { transcript: lookupFor([abutting]), mintId: counter() }, ); expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }]); const captions = addedClips(short.ops).filter((c) => c.kind === 'text'); @@ -443,8 +440,7 @@ describe('planShort: the edit', () => { { clipId: 'c1', fromIndex: 0, toIndex: 0 }, ], }, - lookupFor([abutting]), - counter(), + { transcript: lookupFor([abutting]), mintId: counter() }, ); expect(short.pieces).toEqual([ { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }, @@ -507,6 +503,120 @@ describe('planShort: the edit', () => { }); }); +/** Speech at [start, end] spans at -20 dB over -60 dB room tone, measured the way + * the levels capability measures it: 1.5 s before a time to 1 s after. */ +function levelsFrom(spans: [number, number][]): SpeechLevelsLookup { + return async (_comp, _clipId, aroundSec) => { + const fromSec = Math.max(0, aroundSec - 1.5); + const frames = Math.round((aroundSec + 1 - fromSec) / 0.01); + const db = Array.from({ length: frames }, (_, i) => { + const t = fromSec + (i + 0.5) * 0.01; + return spans.some(([a, b]) => t >= a && t < b) ? -20 : -60; + }); + return { fromSec, frameSec: 0.01, db }; + }; +} + +describe('planShort: edges in the pauses', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + // Word i is spoken from i+0.1 to i+0.9 s; whisper, 0.3 s late as at sentence + // ends, puts it at [i+0.3, i+1.3), each word running into the next. + const late = (texts: string[]): Transcript => { + const t = transcriptOf( + M, + texts.map((_, i): [number, number] => [i + 0.3, i + 1.3]), + ); + return { ...t, words: t.words.map((w, i) => ({ ...w, text: `${w.text}${texts[i]}` })) }; + }; + const spoken = Array.from({ length: 10 }, (_, i): [number, number] => [i + 0.1, i + 0.9]); + const sentences = (t: Transcript): Transcript => ({ + ...t, + words: t.words.map((w, i) => ({ ...w, text: `w${i}.` })), + }); + + async function snap( + transcript: Transcript, + levels: SpeechLevelsLookup | undefined, + range = { clipId: 'c1', fromIndex: 3, toIndex: 5 }, + captions = false, + ) { + return planShort( + emptyComposition(), + whole, + { ranges: [range], captions }, + { transcript: lookupFor([transcript]), levels, mintId: counter() }, + ); + } + + it('moves both edges into the pauses, off the next sentence', async () => { + const short = await snap(late(Array(10).fill('')), levelsFrom(spoken)); + // The pause before word 3 is 2.9–3.1 and the one after word 5 is 5.9–6.1: + // 0.1 s of room tone inside each, where the transcript alone gives 3.05–6.3. + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3, sourceOutSec: 6 }]); + expect(short.warnings.some((w) => w.includes('pause'))).toBe(false); + }); + + it('keeps the transcript edges with a warning where the audio shows no pause', async () => { + const talking = levelsFrom([[0, 30]]); + const short = await snap(sentences(late(Array(10).fill(''))), talking); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([ + 'Range 1: no clear pause before "w3.", so its start follows the transcript and may clip that word or keep the end of "w2.". Look at it or listen.', + 'Range 1: no clear pause after "w5.", so its end follows the transcript and may keep the start of "w6.". Look at it or listen.', + ]); + }); + + it('warns once where a range was cut mid-sentence anyway', async () => { + const short = await snap(late(Array(10).fill('')), levelsFrom([[0, 30]])); + expect(short.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "w2" and "w3" (word 3).', + 'Range 1 ends mid-sentence, between "w5" and "w6" (word 5).', + ]); + }); + + it('follows the transcript, quietly, for a source without sound', async () => { + const short = await snap(late(Array(10).fill('.')), async () => null); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([]); + }); + + it('still makes the short when the audio can’t be measured', async () => { + const short = await snap(late(Array(10).fill('.')), async () => { + throw new Error('FFmpeg failed'); + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([ + "Range 1: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.", + ]); + }); + + it('keeps the caption of a last word whisper places past the cut', async () => { + // Speech of word 5 ends at 5.6; whisper starts it at 5.8. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 4.3], + [4.3, 5.8], + [5.8, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 5.1], + [5.2, 5.6], + [6.2, 7], + ]; + const short = await snap(transcript, levelsFrom(speech), undefined, true); + expect(short.pieces.at(-1)?.sourceOutSec).toBe(5.7); + const texts = addedClips(short.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts.join(' ')).toContain('w5'); + }); +}); + describe('checkShortTimelines', () => { it('refuses main as the short, and the short as its own source', () => { expect(() => checkShortTimelines('main')).toThrow(/not main/); @@ -529,6 +639,7 @@ describe('make_short verb', () => { ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), media: new Map(), }), + speechLevels: async () => null, readTimeline: async (id) => { const doc = sources[id]; if (!doc) throw new NoTimelineError(id); @@ -606,6 +717,7 @@ describe('make_short on the store', () => { media: new Map(), }), readTimeline: (id) => readComposition(id), + speechLevels: async () => null, defaultTrack: 'v0', }; const makeShort = { diff --git a/tests/timeline-verbs.test.ts b/tests/timeline-verbs.test.ts index afd7f96..f2be506 100644 --- a/tests/timeline-verbs.test.ts +++ b/tests/timeline-verbs.test.ts @@ -30,6 +30,7 @@ const ctx = (durationSec = 10): VerbContext => ({ transcript: async () => { throw new Error('not used'); }, + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, @@ -416,6 +417,7 @@ describe('applyVerbs (op-aware + undoable)', () => { openGate = resolve; }); const barrierCtx: VerbContext = { + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, From dabc615f3c6f3ec55e782fd2c222ee2e506d9d09 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 17:59:36 -0800 Subject: [PATCH 20/35] fix(timeline): measure pauses on padded audio; keep pulled words in order Levels are measured with the transcription's aresample=async=1:first_pts=0, so a late audio start or a timestamp gap can't shift a pause; digital silence counts as quiet but is left out of the floor estimate, or room tone between sentences would read as sound. A word pulled inside a cut keeps its place after the word before it, and the speech check uses the pulled words, so a one-word range whisper places past its cut is kept. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/retake-test-model.md | 3 ++ src/ffmpeg/args/speech-levels.ts | 4 +- src/timeline/pauses.ts | 9 +++- src/timeline/shorts.ts | 53 ++++++++++++++------ tests/integration/timeline-pauses.test.ts | 23 +++++++++ tests/timeline-pauses.test.ts | 15 +++++- tests/timeline-shorts.test.ts | 60 +++++++++++++++++++++++ 7 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 docs/backlog/retake-test-model.md diff --git a/docs/backlog/retake-test-model.md b/docs/backlog/retake-test-model.md new file mode 100644 index 0000000..16fe1aa --- /dev/null +++ b/docs/backlog/retake-test-model.md @@ -0,0 +1,3 @@ +# Retake integration test depends on the model + +`tests/integration/timeline-transcript.test.ts` ("removes a repeated take as one undoable edit") runs whenever whisper.cpp and a model are set up. With `ggml-base.en.bin` it fails on `feat/shorts` too: the model's transcript of the test's spoken phrase has no immediate repeat, so `retakes` is empty. Make the test independent of the model (assert on a stored transcript with a known repeat, as the unit tests do) or pin the model it was written for. diff --git a/src/ffmpeg/args/speech-levels.ts b/src/ffmpeg/args/speech-levels.ts index 6db6fa7..cafb438 100644 --- a/src/ffmpeg/args/speech-levels.ts +++ b/src/ffmpeg/args/speech-levels.ts @@ -29,7 +29,9 @@ export function buildSpeechLevelsArgs({ input, fromSec, durationSec }: SpeechLev '0:a:0', '-af', [ - `aresample=${LEVEL_SAMPLE_RATE}`, + // A late start or a timestamp gap becomes silence, as for transcription, + // so frame n is always n x 10 ms after the seek point. + `aresample=${LEVEL_SAMPLE_RATE}:async=1:first_pts=0`, 'aformat=channel_layouts=mono', `asetnsamples=n=${LEVEL_FRAME_SAMPLES}:p=0`, 'astats=metadata=1:reset=1', diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts index 0cfa147..efc7170 100644 --- a/src/timeline/pauses.ts +++ b/src/timeline/pauses.ts @@ -48,6 +48,8 @@ const MAX_BLIP_SEC = 0.04; /** Two pauses at least this long in reach make the boundary ambiguous. */ const COMPETING_PAUSE_SEC = 0.12; const SILENT_DB = -120; +/** At or below this, a frame is digital silence, not a room. */ +const DIGITAL_SILENCE_DB = -100; /** Read the per-frame levels `buildSpeechLevelsArgs` prints. */ export function parseSpeechLevels(stderr: string, fromSec: number): SpeechLevels { @@ -72,10 +74,13 @@ export async function measureSpeechLevels( /** Every pause in the levels, by a threshold set from the levels themselves. */ export function findPauses(levels: SpeechLevels): Pause[] { - const sorted = [...levels.db].sort((a, b) => a - b); + // Digital silence (padding where the audio starts late, or a muted stretch) + // is quiet, but it isn't the room's floor: leave it out of the estimate, or + // the room tone between sentences would count as sound. + const sorted = levels.db.filter((level) => level > DIGITAL_SILENCE_DB).sort((a, b) => a - b); const at = (q: number): number => sorted[Math.floor(q * (sorted.length - 1))] ?? SILENT_DB; const threshold = Math.min(at(0.05) + ABOVE_FLOOR_DB, at(0.95) - BELOW_SPEECH_DB); - const quiet = levels.db.map((level) => level <= threshold); + const quiet = levels.db.map((level) => level <= DIGITAL_SILENCE_DB || level <= threshold); const maxBlip = Math.round(MAX_BLIP_SEC / levels.frameSec); // A short burst of sound between two quiet stretches is part of the pause. for (let i = 0; i < quiet.length; ) { diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 1e5b216..4671873 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -43,6 +43,10 @@ export const DEFAULT_SHORT_LUFS = -14; const SHORT_TRACK = 'v0'; /** A range whose snapped edges leave less than this keeps the transcript's. */ const MIN_SNAPPED_SEC = 0.2; +/** How long a word pulled inside a cut lasts. */ +const PULLED_WORD_SEC = 0.25; +/** The least a word starts after the one before it. */ +const WORD_ORDER_SEC = 0.01; /** A piece shorter than this is dropped: too short to hear, just a flash. */ const MIN_PIECE_SEC = 0.05; const SAME_SEC = 1e-6; @@ -274,19 +278,9 @@ export async function planShort( // isn't part of the short even when another range asks for it. const style = defaultCaptionStyle(applyOps(target, ops)); const cues = placed.flatMap(({ piece, clip }) => { - // A word whisper places past the cut (it runs late at a sentence end) is - // pulled inside it, so its caption isn't lost. - const words = piece.transcript.words.flatMap((w, i) => { - const range = piece.asked.find((r) => i >= r.fromIndex && i <= r.toIndex); - if (!range) return []; - if (w.startSec >= range.end) { - return [{ ...w, startSec: Math.max(range.start, range.end - 0.25), endSec: range.end }]; - } - if (w.endSec <= range.start) { - return [{ ...w, startSec: range.start, endSec: Math.min(range.end, range.start + 0.25) }]; - } - return [w]; - }); + const words = piece.asked.flatMap((r) => + wordsInside(piece.transcript.words.slice(r.fromIndex, r.toIndex + 1), r.start, r.end), + ); return groupCaptions( clipWords({ ...piece.transcript, words }, clip), clipEndSec(clip), @@ -414,7 +408,38 @@ async function snapToPauses( } // A pause found on the wrong side of a short range would leave nothing. if (end - start < MIN_SNAPPED_SEC) return window; - return { ...window, start: roundMs(start), end: roundMs(end) }; + start = roundMs(start); + end = roundMs(end); + return { ...window, start, end, words: wordsInside(window.words, start, end) }; +} + +/** + * A range's words kept inside the window it was cut to, in order. A word + * whisper places past an edge (it runs late at a sentence end) is pulled just + * inside, so the range still holds its speech and the word keeps its caption; + * no word starts before the one ahead of it, or a caption shown a word at a + * time would lose that one. + */ +function wordsInside( + words: W[], + start: number, + end: number, +): W[] { + let prevStart = Number.NEGATIVE_INFINITY; + return words.map((w) => { + let startSec = w.startSec; + let endSec = w.endSec; + if (startSec >= end) { + startSec = Math.max(start, end - PULLED_WORD_SEC); + endSec = end; + } else if (endSec <= start) { + startSec = start; + endSec = Math.min(end, start + PULLED_WORD_SEC); + } + startSec = Math.min(Math.max(startSec, prevStart + WORD_ORDER_SEC), endSec - WORD_ORDER_SEC); + prevStart = startSec; + return startSec === w.startSec && endSec === w.endSec ? w : { ...w, startSec, endSec }; + }); } /** Whether two neighboring words belong to one sentence: no sentence end and diff --git a/tests/integration/timeline-pauses.test.ts b/tests/integration/timeline-pauses.test.ts index 2488b31..e8641d3 100644 --- a/tests/integration/timeline-pauses.test.ts +++ b/tests/integration/timeline-pauses.test.ts @@ -45,6 +45,29 @@ describe.skipIf(!IS_CI)('pauses against real FFmpeg', () => { expect(pause.endSec).toBeLessThanOrEqual(4.1 + 1e-6); expect(pause.endSec).toBeGreaterThanOrEqual(4.06); + // Audio that starts a second into the file: the levels still count from + // the seek point, so the pause lands where it is (1.7–2.1), not 0.15 s early. + const late = join(workspace, 'late.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'color=c=gray:s=320x240:r=30:d=8'], + ...['-itsoffset', '1'], + ...[ + '-f', + 'lavfi', + '-i', + "aevalsrc='0.3*sin(2*PI*220*t)*between(mod(t,1),0.1,0.7)+0.001*(random(0)-0.5)':s=48000:d=7", + ], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', late], + ]); + const shifted = pauseAt(await measureSpeechLevels(late, 2.35), 2.35, 1.35); + if (typeof shifted !== 'object') throw new Error(`no pause found: ${shifted}`); + expect(shifted.startSec).toBeGreaterThanOrEqual(1.7 - 1e-6); + expect(shifted.startSec).toBeLessThanOrEqual(1.74); + expect(shifted.endSec).toBeLessThanOrEqual(2.1 + 1e-6); + expect(shifted.endSec).toBeGreaterThanOrEqual(2.06); + // Where the tone never stops there is no pause to find. const steady = join(workspace, 'steady.mp4'); await runFfmpeg([ diff --git a/tests/timeline-pauses.test.ts b/tests/timeline-pauses.test.ts index fbf6359..7819066 100644 --- a/tests/timeline-pauses.test.ts +++ b/tests/timeline-pauses.test.ts @@ -33,7 +33,7 @@ describe('buildSpeechLevelsArgs', () => { '-map', '0:a:0', '-af', - 'aresample=16000,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + 'aresample=16000:async=1:first_pts=0,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', '-f', 'null', '-', @@ -85,6 +85,19 @@ describe('findPauses', () => { ]); }); + it('sets the floor by the room, not by digital silence before the audio starts', () => { + const levels = levelsOf([ + [0.5, -120], // padding: the audio starts late + [1, -20], + [0.3, -62], // room tone between sentences + [1, -20], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([ + [10, 10.5], + [11.5, 11.8], + ]); + }); + it('finds none in speech that never stops, or in a steady level', () => { expect(findPauses(levelsOf([[3, -20]]))).toEqual([]); const stepped = levelsOf([ diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index c2d3410..cea617b 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -590,6 +590,66 @@ describe('planShort: edges in the pauses', () => { ]); }); + it('keeps every word’s caption, in order, one word at a time', async () => { + // Cut at 5.7: the word before the last starts at 5.6, the last at 5.8. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 5.6], + [5.6, 5.8], + [5.8, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 4.8], + [4.9, 5.6], + [6.2, 7], + ]; + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 5 }], maxWords: 1 }, + { transcript: lookupFor([transcript]), levels: levelsFrom(speech), mintId: counter() }, + ); + expect(short.pieces.at(-1)?.sourceOutSec).toBe(5.7); + const texts = addedClips(short.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts).toEqual(['w3', 'w4', 'w5']); + }); + + it('keeps a one-word range whose word whisper places past its cut', async () => { + // The word is spoken 5.2–5.6; whisper says 5.72–6.4. Its start finds two + // pauses (ambiguous, so the transcript's), its end the pause at 5.6. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 4.3], + [4.3, 5.72], + [5.72, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 4.9], + [5.2, 5.6], + [6.2, 7], + ]; + const short = await snap(transcript, levelsFrom(speech), { + clipId: 'c1', + fromIndex: 5, + toIndex: 5, + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 5.47, sourceOutSec: 5.7 }]); + }); + it('keeps the caption of a last word whisper places past the cut', async () => { // Speech of word 5 ends at 5.6; whisper starts it at 5.8. const transcript = transcriptOf(M, [ From 7d4da17dbc49cfe06e51459f8a07686915f4e239 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 18:04:08 -0800 Subject: [PATCH 21/35] fix(ui): label media clips with their file name Media clips on the browser timeline and in the clip inspector showed their media ID (m_821d98105d7c) instead of the file name. The session log's ingest entries already map each media ID to its path, so the UI now builds a name map from them and labels clips with the file's base name, falling back to the ID when the session doesn't name it. Closes #80 Co-Authored-By: Claude Opus 5.5 --- src/ui/web/src/App.tsx | 7 +- src/ui/web/src/components/ClipInspector.tsx | 5 +- src/ui/web/src/components/DocTimeline.tsx | 10 ++- src/ui/web/src/lib/composition.ts | 25 +++++-- src/ui/web/src/lib/playable-clips.ts | 2 +- tests/ui-clip-label.test.ts | 76 +++++++++++++++++++++ 6 files changed, 115 insertions(+), 10 deletions(-) create mode 100644 tests/ui-clip-label.test.ts diff --git a/src/ui/web/src/App.tsx b/src/ui/web/src/App.tsx index 8092dda..8f57d75 100644 --- a/src/ui/web/src/App.tsx +++ b/src/ui/web/src/App.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useMemo, useState } from 'react'; import { ClipInspector } from './components/ClipInspector.js'; import { DetailPane } from './components/DetailPane.js'; import { DocTimeline } from './components/DocTimeline.js'; @@ -23,7 +23,7 @@ import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts.js'; import { useSession } from './hooks/useSession.js'; import { useSessionSafety } from './hooks/useSessionSafety.js'; import { useTimeline } from './hooks/useTimeline.js'; -import { findClip, nextClipOnTrack } from './lib/composition.js'; +import { findClip, mediaNames, nextClipOnTrack } from './lib/composition.js'; import { applyTimelineVerbs, type Verb } from './lib/verbs.js'; export function App() { @@ -42,6 +42,7 @@ export function App() { const selectedEntry = selectedId === null ? null : (session.entries.find((e) => e.id === selectedId) ?? null); const selected = selectedClipId === null ? null : findClip(composition, selectedClipId); + const names = useMemo(() => mediaNames(session.entries), [session.entries]); // Selecting an op and selecting a doc-clip are mutually exclusive — each owns // the inspector pane, so picking one clears the other. @@ -224,6 +225,7 @@ export function App() { ; playheadSec: number; hasNext: boolean; /** The timeline's fit, which a clip without its own follows. */ @@ -84,7 +87,7 @@ export function ClipInspector({
-

{clipLabel(clip)}

+

{clipLabel(clip, mediaNames)}

{clip.kind} · {clip.id} · {start.toFixed(2)}–{end.toFixed(2)}s ( {clipDuration(clip).toFixed(2)}s) diff --git a/src/ui/web/src/components/DocTimeline.tsx b/src/ui/web/src/components/DocTimeline.tsx index fe1f4dd..44b8158 100644 --- a/src/ui/web/src/components/DocTimeline.tsx +++ b/src/ui/web/src/components/DocTimeline.tsx @@ -21,6 +21,7 @@ const MAX_THUMBS = 16; // cap thumbnails per clip to bound requests */ export function DocTimeline({ composition, + mediaNames, selectedClipId, onSelectClip, playheadSec, @@ -33,6 +34,8 @@ export function DocTimeline({ verbError, }: { composition: Composition; + /** File name of each media id, for labeling media clips. */ + mediaNames: ReadonlyMap; selectedClipId: string | null; onSelectClip: (clipId: string | null) => void; playheadSec: number; @@ -156,6 +159,7 @@ export function DocTimeline({ @@ -205,10 +209,12 @@ export function DocTimeline({ function ClipBlock({ clip, + label, selected, onSelect, }: { clip: Clip; + label: string; selected: boolean; onSelect: (clipId: string | null) => void; }) { @@ -220,10 +226,10 @@ function ClipBlock({ className={`doc-tl-clip doc-tl-clip-${clip.kind}${selected ? ' selected' : ''}`} style={{ left, width }} onClick={() => onSelect(selected ? null : clip.id)} - title={`${clipLabel(clip)} · ${clip.startSec.toFixed(2)}–${clipEndSec(clip).toFixed(2)}s`} + title={`${label} · ${clip.startSec.toFixed(2)}–${clipEndSec(clip).toFixed(2)}s`} > {clip.kind === 'media' ? : null} - {clipLabel(clip)} + {label} {clipDuration(clip).toFixed(1)}s ); diff --git a/src/ui/web/src/lib/composition.ts b/src/ui/web/src/lib/composition.ts index fe0f2f3..9576d7a 100644 --- a/src/ui/web/src/lib/composition.ts +++ b/src/ui/web/src/lib/composition.ts @@ -1,4 +1,5 @@ -import type { Clip, Composition, Track } from '../types.js'; +import type { Clip, Composition, SessionEntry, Track } from '../types.js'; +import { shortName } from './playable-clips.js'; // Frontend mirror of the doc geometry helpers in src/timeline/composition.ts. // The timeline view positions clips by their DOCUMENT extent (startSec + @@ -23,9 +24,25 @@ export function compositionDuration(comp: Composition): number { return end; } -/** Short, human-readable label for a clip on the timeline. */ -export function clipLabel(clip: Clip): string { - if (clip.kind === 'media') return clip.mediaId; +/** File name of each ingested media id, read from the session log's `ingest` + * entries (`{ mediaId, ref: { path } }`) — the same registry export resolves + * media through. */ +export function mediaNames(entries: SessionEntry[]): Map { + const names = new Map(); + for (const entry of entries) { + if (entry.tool !== 'ingest') continue; + const { mediaId, ref } = entry.result as { mediaId?: unknown; ref?: { path?: unknown } }; + if (typeof mediaId === 'string' && typeof ref?.path === 'string') { + names.set(mediaId, shortName(ref.path)); + } + } + return names; +} + +/** Short, human-readable label for a clip on the timeline. A media clip shows its + * file name, or its media id when the session log doesn't name it. */ +export function clipLabel(clip: Clip, names: ReadonlyMap): string { + if (clip.kind === 'media') return names.get(clip.mediaId) ?? clip.mediaId; if (clip.kind === 'text') return `“${clip.text.slice(0, 20)}”`; return `color · ${clip.color}`; } diff --git a/src/ui/web/src/lib/playable-clips.ts b/src/ui/web/src/lib/playable-clips.ts index e34955b..a9452f5 100644 --- a/src/ui/web/src/lib/playable-clips.ts +++ b/src/ui/web/src/lib/playable-clips.ts @@ -26,7 +26,7 @@ function isVideoPath(path: string): boolean { return VIDEO_EXT.has(ext); } -function shortName(path: string): string { +export function shortName(path: string): string { const base = path.split('/').pop() ?? path; // Strip the 8-char random prefix (`xxxxxxxx-`) that we add to import // filenames and tool outputs, so cards show "demo.mp4" instead of diff --git a/tests/ui-clip-label.test.ts b/tests/ui-clip-label.test.ts new file mode 100644 index 0000000..fa5adde --- /dev/null +++ b/tests/ui-clip-label.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { clipLabel, mediaNames } from '../src/ui/web/src/lib/composition.js'; +import type { Clip, SessionEntry } from '../src/ui/web/src/types.js'; + +function ingest(mediaId: string, path: string): SessionEntry { + return { + id: `op_${mediaId}`, + tool: 'ingest', + args: { path }, + result: { mediaId, ref: { path, audio: null, video: null } }, + timestamp: '2026-05-28T16:30:00.000Z', + }; +} + +function mediaClip(mediaId: string): Clip { + return { + kind: 'media', + id: 'clip_1', + mediaId, + startSec: 0, + sourceInSec: 0, + sourceOutSec: 5, + }; +} + +describe('mediaNames', () => { + it("maps each ingested media id to its file's base name", () => { + const names = mediaNames([ + ingest('m_aaa', '/work/media/talking.mp4'), + ingest('m_bbb', '/work/imports/2e0d99ef-broll.mov'), + ]); + expect(names.get('m_aaa')).toBe('talking.mp4'); + expect(names.get('m_bbb')).toBe('broll.mov'); // strips the import prefix + }); + + it('skips non-ingest entries and ingest results without a media id or path', () => { + const names = mediaNames([ + { + id: 'op_trim', + tool: 'trim', + args: {}, + result: { mediaId: 'm_x', path: '/work/out.mp4' }, + timestamp: '2026-05-28T16:30:00.000Z', + }, + { + id: 'op_bad', + tool: 'ingest', + args: {}, + result: { ref: { path: '/work/no-id.mp4' } }, + timestamp: '2026-05-28T16:30:00.000Z', + }, + ]); + expect(names.size).toBe(0); + }); +}); + +describe('clipLabel', () => { + it('labels a media clip with its file name', () => { + const names = mediaNames([ingest('m_aaa', '/work/media/talking.mp4')]); + expect(clipLabel(mediaClip('m_aaa'), names)).toBe('talking.mp4'); + }); + + it('falls back to the media id when the name is unknown', () => { + expect(clipLabel(mediaClip('m_821d98105d7c'), new Map())).toBe('m_821d98105d7c'); + }); + + it('labels text and color clips by their content', () => { + const names = new Map(); + expect( + clipLabel({ kind: 'text', id: 'c_t', startSec: 0, text: 'Hello', durationSec: 2 }, names), + ).toBe('“Hello”'); + expect( + clipLabel({ kind: 'color', id: 'c_c', startSec: 0, color: 'black', durationSec: 2 }, names), + ).toBe('color · black'); + }); +}); From e876187e1115e15b2498539cd16f46488e35abb2 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 18:08:35 -0800 Subject: [PATCH 22/35] fix(ui): draw the op list in the dark theme Each op in the list left of the monitor is a - +
{hasClips ? ( diff --git a/src/ui/web/src/components/ExportButton.tsx b/src/ui/web/src/components/ExportButton.tsx index 32d78c4..827aba6 100644 --- a/src/ui/web/src/components/ExportButton.tsx +++ b/src/ui/web/src/components/ExportButton.tsx @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; type ProjectFormat = 'fcpxml' | 'xmeml'; @@ -15,8 +16,8 @@ const PROJECTS: readonly { format: ProjectFormat; label: string; detail: string /** POST the export route for an editor project; throws with the server's * message (e.g. the 422 naming what the project can't hold). */ -async function exportProject(format: ProjectFormat): Promise { - const res = await fetch('/api/timeline/export', { +async function exportProject(format: ProjectFormat, timeline: string): Promise { + const res = await fetch(withTimeline('/api/timeline/export', timeline), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ format }), @@ -35,10 +36,12 @@ const fileName = (path: string): string => path.split(/[\\/]/).pop() ?? path; * carries only in part. */ export function ExportButton({ + timeline, busy, disabled, onExportVideo, }: { + timeline: string; busy: boolean; disabled: boolean; onExportVideo: () => void; @@ -72,7 +75,7 @@ export function ExportButton({ setError(null); setCopied(false); try { - setResult(await exportProject(format)); + setResult(await exportProject(format, timeline)); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { diff --git a/src/ui/web/src/components/FramingSection.tsx b/src/ui/web/src/components/FramingSection.tsx index 62d39d2..2576e34 100644 --- a/src/ui/web/src/components/FramingSection.tsx +++ b/src/ui/web/src/components/FramingSection.tsx @@ -110,7 +110,7 @@ export function FramingSection({ ); } -function Slider({ +export function Slider({ label, value, min, diff --git a/src/ui/web/src/components/TimelineTabs.tsx b/src/ui/web/src/components/TimelineTabs.tsx new file mode 100644 index 0000000..0d7057d --- /dev/null +++ b/src/ui/web/src/components/TimelineTabs.tsx @@ -0,0 +1,104 @@ +import { MAIN_TIMELINE, shapeLabel, type TimelineListing } from '../lib/timelines.js'; + +/** What the last short-making or export-all did, shown under the tabs. */ +export interface TimelineNotice { + kind: 'info' | 'warn'; + lines: string[]; +} + +/** + * One tab per timeline in the workspace, main and every short, each with its + * shape. A short's ✕ deletes it (the caller confirms), and "Export all shorts" + * renders each one. Shown once there is a short. + */ +export function TimelineTabs({ + timelines, + current, + exportingAll, + notice, + onSelect, + onDelete, + onExportAll, + onDismissNotice, +}: { + timelines: TimelineListing[]; + current: string; + exportingAll: boolean; + notice: TimelineNotice | null; + onSelect: (id: string) => void; + onDelete: (id: string) => void; + onExportAll: () => void; + onDismissNotice: () => void; +}) { + const shorts = timelines.filter((t) => t.id !== MAIN_TIMELINE); + if (shorts.length === 0 && notice === null) return null; + + return ( +
+
+
+ {timelines.map((t) => { + const isCurrent = t.id === current; + const shape = 'error' in t ? 'unreadable' : shapeLabel(t.width, t.height); + return ( +
+ + {t.id === MAIN_TIMELINE ? null : ( + + )} +
+ ); + })} +
+ +
+ {notice ? ( +
+
+ {notice.lines.map((line) => ( +
{line}
+ ))} +
+ +
+ ) : null} +
+ ); +} diff --git a/src/ui/web/src/components/TranscriptSection.tsx b/src/ui/web/src/components/TranscriptSection.tsx index e3f648e..307d3f9 100644 --- a/src/ui/web/src/components/TranscriptSection.tsx +++ b/src/ui/web/src/components/TranscriptSection.tsx @@ -1,6 +1,8 @@ import { useCallback, useEffect, useRef, useState } from 'react'; +import { isTimelineId, MAIN_TIMELINE, withTimeline } from '../lib/timelines.js'; import type { Verb } from '../lib/verbs.js'; import type { MediaClip } from '../types.js'; +import { Slider } from './FramingSection.js'; /** Mirrors ClipTranscript in src/timeline/transcript.ts. */ interface ClipWord { @@ -19,19 +21,28 @@ interface ClipTranscript { /** * Edit a media clip by its words: transcribe on request (the local whisper.cpp * is optional, so its setup message shows here when it's missing), click a word - * to move the playhead, shift-click to select a range, then cut it. Phrases - * said twice in a row are marked and can all be cut at once. Every cut is one - * undoable edit through the verb layer. + * to move the playhead, shift-click to select a range, then cut it or make a + * short of it. Phrases said twice in a row are marked and can all be cut at + * once. Every cut is one undoable edit through the verb layer. */ export function TranscriptSection({ + timeline, clip, busy, + nextShortId, onApply, + onMakeShort, onSeek, }: { + /** The timeline the clip is on, which a short is cut from. */ + timeline: string; clip: MediaClip; busy: boolean; + /** A name no timeline uses yet, offered for a new short. */ + nextShortId: string; onApply: (verbs: Verb[]) => Promise; + /** Make a short on timeline `id` and open it; resolves false when that failed. */ + onMakeShort: (id: string, verb: Verb) => Promise; onSeek: (sec: number) => void; }) { const [transcript, setTranscript] = useState(null); @@ -39,6 +50,10 @@ export function TranscriptSection({ const [error, setError] = useState(null); const [anchor, setAnchor] = useState(null); const [focus, setFocus] = useState(null); + // null until the name is edited, so the offered name follows the timelines. + const [shortName, setShortName] = useState(null); + const [shortX, setShortX] = useState(0.5); + const [nameError, setNameError] = useState(null); // Set once the transcript has been asked for; `latest` numbers requests so a // response for an older clip window is dropped instead of shown. const wanted = useRef(false); @@ -50,7 +65,9 @@ export function TranscriptSection({ setLoading(true); setError(null); try { - const res = await fetch(`/api/timeline/transcript/${encodeURIComponent(clip.id)}`); + const res = await fetch( + withTimeline(`/api/timeline/transcript/${encodeURIComponent(clip.id)}`, timeline), + ); const json = (await res.json()) as ClipTranscript & { error?: string }; if (request !== latest.current) return; if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`); @@ -60,7 +77,7 @@ export function TranscriptSection({ } finally { if (request === latest.current) setLoading(false); } - }, [clip.id]); + }, [clip.id, timeline]); // A cut or trim changes the clip's window: reload a transcript that was asked // for, even one still loading (its response is then dropped as outdated). The @@ -83,6 +100,33 @@ export function TranscriptSection({ from === null || to === null ? 0 : words.filter((w) => w.index >= from && w.index <= to).length; const inRetake = (index: number): boolean => retakes.some((r) => index >= r.fromIndex && index <= r.toIndex); + const name = shortName ?? nextShortId; + + const makeShort = async (): Promise => { + if (from === null || to === null || !transcript) return; + const id = name.trim(); + if (!isTimelineId(id)) { + setNameError('Name it with lowercase letters, digits and dashes, like best-1.'); + return; + } + if (id === MAIN_TIMELINE || id === timeline) { + setNameError(`A short needs a name of its own, not ${id}.`); + return; + } + const made = await onMakeShort(id, { + verb: 'make_short', + ranges: [ + { clipId: clip.id, fromIndex: from, toIndex: to, transcriptId: transcript.transcriptId }, + ], + source: timeline, + x: shortX, + }); + if (made) { + setAnchor(null); + setFocus(null); + setShortName(null); + } + }; return (
@@ -182,6 +226,45 @@ export function TranscriptSection({ ) : null}
+ {selectedCount > 0 ? ( +
+ + + {nameError ?
{nameError}
: null} + +

+ A 9:16 short of these words with captions, at -14 LUFS, its edges in the pauses. It + opens when made; Undo there takes it apart. +

+
+ ) : null}

Click a word to move the playhead; shift-click to select a range. Undo restores.

diff --git a/src/ui/web/src/components/Viewer.tsx b/src/ui/web/src/components/Viewer.tsx index 5782193..ddcbf55 100644 --- a/src/ui/web/src/components/Viewer.tsx +++ b/src/ui/web/src/components/Viewer.tsx @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; interface Exportable { exportable: boolean; @@ -16,7 +17,7 @@ type FrameState = * from the export. The playhead is debounced so scrubbing doesn't fire an FFmpeg * render per pixel, and a banner surfaces whether the document can render at all. */ -export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { +export function Viewer({ timeline, atSec, rev }: { timeline: string; atSec: number; rev: number }) { const [debouncedAt, setDebouncedAt] = useState(atSec); const [status, setStatus] = useState(null); const [frame, setFrame] = useState({ kind: 'loading' }); @@ -30,7 +31,7 @@ export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { // Re-check exportability whenever the document changes (rev cache-busts). useEffect(() => { let alive = true; - fetch(`/api/timeline/exportable?rev=${rev}`) + fetch(withTimeline(`/api/timeline/exportable?rev=${rev}`, timeline)) .then((r) => r.json() as Promise) .then((s) => { if (alive) setStatus(s); @@ -41,13 +42,13 @@ export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { return () => { alive = false; }; - }, [rev]); + }, [timeline, rev]); // Fetch the composited frame as a blob (not via ) so a superseded // scrub is ABORTED instead of left rendering, and a transient 503 reads as a // clean message rather than a broken image. The previous frame stays on screen // until the next one decodes — no blank flash while FFmpeg works. - const src = `/api/timeline/frame?at=${debouncedAt.toFixed(2)}&rev=${rev}`; + const src = withTimeline(`/api/timeline/frame?at=${debouncedAt.toFixed(2)}&rev=${rev}`, timeline); useEffect(() => { const controller = new AbortController(); fetch(src, { signal: controller.signal }) diff --git a/src/ui/web/src/hooks/useTimeline.ts b/src/ui/web/src/hooks/useTimeline.ts index e815df4..38a1460 100644 --- a/src/ui/web/src/hooks/useTimeline.ts +++ b/src/ui/web/src/hooks/useTimeline.ts @@ -1,4 +1,5 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; import type { Composition } from '../types.js'; const EMPTY: Composition = { @@ -14,34 +15,48 @@ const EMPTY: Composition = { }; /** - * Polls `/api/timeline` (the CompositionDoc — the source of truth for assembled - * edits) every `intervalMs`. Mirrors `useSession`: returns the latest doc plus - * loading/error state and a `refresh()` for callers that just mutated it (e.g. - * after a verb POST) and want the view to update without waiting for the poll. + * Polls `/api/timeline` for `timeline` (the CompositionDoc — the source of truth + * for assembled edits) every `intervalMs`. Mirrors `useSession`: returns the + * latest doc plus loading/error state and a `refresh()` for callers that just + * mutated it (e.g. after a verb POST) and want the view to update without + * waiting for the poll. `missing` is set when the timeline no longer exists. */ -export function useTimeline(intervalMs = 2000): { +export function useTimeline( + timeline: string, + intervalMs = 2000, +): { composition: Composition; loading: boolean; error: string | null; + missing: boolean; refresh: () => Promise; } { - const [composition, setComposition] = useState(EMPTY); + // Kept with the timeline it belongs to, so a switch shows nothing of the last one. + const [loaded, setLoaded] = useState<{ timeline: string; doc: Composition } | null>(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); + const [missing, setMissing] = useState(false); + // A response for a timeline switched away from is dropped, not shown. + const wanted = useRef(timeline); + wanted.current = timeline; const load = useCallback(async () => { try { - const res = await fetch('/api/timeline'); + const res = await fetch(withTimeline('/api/timeline', timeline)); + if (wanted.current !== timeline) return; + setMissing(res.status === 404); if (!res.ok) throw new Error(`HTTP ${res.status}`); const json = (await res.json()) as { document: Composition }; - setComposition(json.document); + if (wanted.current !== timeline) return; + setLoaded({ timeline, doc: json.document }); setError(null); setLoading(false); } catch (err) { + if (wanted.current !== timeline) return; setError(err instanceof Error ? err.message : String(err)); setLoading(false); } - }, []); + }, [timeline]); useEffect(() => { void load(); @@ -49,5 +64,6 @@ export function useTimeline(intervalMs = 2000): { return () => window.clearInterval(timer); }, [intervalMs, load]); - return { composition, loading, error, refresh: load }; + const composition = loaded?.timeline === timeline ? loaded.doc : EMPTY; + return { composition, loading, error, missing, refresh: load }; } diff --git a/src/ui/web/src/hooks/useTimelines.ts b/src/ui/web/src/hooks/useTimelines.ts new file mode 100644 index 0000000..179d8e8 --- /dev/null +++ b/src/ui/web/src/hooks/useTimelines.ts @@ -0,0 +1,48 @@ +import { useCallback, useEffect, useState } from 'react'; +import { isTimelineId, MAIN_TIMELINE, type TimelineListing } from '../lib/timelines.js'; + +/** The timeline the page's `?timeline=` names, so a reload keeps it. */ +function timelineInUrl(): string { + const id = new URLSearchParams(window.location.search).get('timeline'); + return id && isTimelineId(id) ? id : MAIN_TIMELINE; +} + +/** + * The workspace's timelines (main and every short), polled like the document, + * and the one being edited, kept in the page URL. + */ +export function useTimelines(intervalMs = 2000): { + timelines: TimelineListing[]; + current: string; + select: (id: string) => void; + refresh: () => Promise; +} { + const [timelines, setTimelines] = useState([]); + const [current, setCurrent] = useState(timelineInUrl); + + const load = useCallback(async () => { + try { + const res = await fetch('/api/timelines'); + if (!res.ok) return; + setTimelines(((await res.json()) as { timelines: TimelineListing[] }).timelines); + } catch { + // The next poll tries again; the tabs keep the last list meanwhile. + } + }, []); + + useEffect(() => { + void load(); + const timer = window.setInterval(load, intervalMs); + return () => window.clearInterval(timer); + }, [intervalMs, load]); + + const select = useCallback((id: string) => { + setCurrent(id); + const url = new URL(window.location.href); + if (id === MAIN_TIMELINE) url.searchParams.delete('timeline'); + else url.searchParams.set('timeline', id); + window.history.replaceState(null, '', url); + }, []); + + return { timelines, current, select, refresh: load }; +} diff --git a/src/ui/web/src/lib/timelines.ts b/src/ui/web/src/lib/timelines.ts new file mode 100644 index 0000000..683591a --- /dev/null +++ b/src/ui/web/src/lib/timelines.ts @@ -0,0 +1,34 @@ +/** The timeline at `composition.json`; every other timeline is a short. */ +export const MAIN_TIMELINE = 'main'; + +/** Mirrors TimelineIdSchema in src/timeline/schema.ts. */ +const TIMELINE_ID = /^[a-z0-9][a-z0-9-]{0,47}$/; + +/** Mirrors TimelineListing in src/timeline/document-store.ts. */ +export type TimelineListing = + | { id: string; width: number; height: number; durationSec: number; rev: number } + | { id: string; error: string }; + +export function isTimelineId(id: string): boolean { + return TIMELINE_ID.test(id); +} + +/** An API path for the given timeline: every /api/timeline route reads `?timeline=`. */ +export function withTimeline(path: string, timeline: string): string { + return `${path}${path.includes('?') ? '&' : '?'}timeline=${encodeURIComponent(timeline)}`; +} + +/** `16:9`, `9:16`, `1:1`, `4:5`… from the canvas size. */ +export function shapeLabel(width: number, height: number): string { + const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b)); + const d = gcd(width, height); + const [w, h] = [width / d, height / d]; + return w <= 32 && h <= 32 ? `${w}:${h}` : `${width}×${height}`; +} + +/** The first `short-N` no timeline uses yet. */ +export function nextShortId(ids: readonly string[]): string { + let n = 1; + while (ids.includes(`short-${n}`)) n++; + return `short-${n}`; +} diff --git a/src/ui/web/src/lib/verbs.ts b/src/ui/web/src/lib/verbs.ts index e528cf5..e3a0ecd 100644 --- a/src/ui/web/src/lib/verbs.ts +++ b/src/ui/web/src/lib/verbs.ts @@ -1,4 +1,5 @@ import type { Composition, Fit } from '../types.js'; +import { withTimeline } from './timelines.js'; /** A timeline editing verb (mirrors CompositionVerb in src/timeline/verbs.ts). * Loosely typed at the field level — the server validates with Zod, which is the @@ -44,26 +45,39 @@ export type Verb = kind?: string; durationSec?: number; track?: string; + } + | { + verb: 'make_short'; + ranges: { clipId: string; fromIndex: number; toIndex: number; transcriptId?: string }[]; + source?: string; + x?: number; }; /** - * POST editing verbs to /api/timeline/verbs — the same op-aware, undoable path - * the agent and CLI use. Throws with the server's message on a 4xx/5xx (e.g. a - * 422 for a bad clip reference or a 403 for a workspace-boundary violation) so - * the caller can surface it. `applied` is the op count; 0 means the verbs changed - * nothing (e.g. remove_silence found no pause). + * POST editing verbs to /api/timeline/verbs for `timeline` — the same op-aware, + * undoable path the agent and CLI use. Throws with the server's message on a + * 4xx/5xx (e.g. a 422 for a bad clip reference or a 403 for a workspace-boundary + * violation) so the caller can surface it. `applied` is the op count; 0 means the + * verbs changed nothing (e.g. remove_silence found no pause). `warnings` name what + * a verb did that may not have been meant (a short cut mid-sentence). */ export async function applyTimelineVerbs( verbs: Verb[], -): Promise<{ document: Composition; applied: number }> { - const res = await fetch('/api/timeline/verbs', { + timeline: string, +): Promise<{ document: Composition; applied: number; warnings: string[] }> { + const res = await fetch(withTimeline('/api/timeline/verbs', timeline), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ verbs }), }); - const json = (await res.json()) as { document?: Composition; applied?: number; error?: string }; + const json = (await res.json()) as { + document?: Composition; + applied?: number; + warnings?: string[]; + error?: string; + }; if (!res.ok || !json.document) { throw new Error(json.error ?? `HTTP ${res.status}`); } - return { document: json.document, applied: json.applied ?? 0 }; + return { document: json.document, applied: json.applied ?? 0, warnings: json.warnings ?? [] }; } diff --git a/src/ui/web/src/styles.css b/src/ui/web/src/styles.css index 94d3213..4bbb062 100644 --- a/src/ui/web/src/styles.css +++ b/src/ui/web/src/styles.css @@ -918,6 +918,113 @@ body, /* ─── Doc timeline — the CompositionDoc (the editor's centerpiece) ────── */ +/* ─── Timeline tabs: main and every short ─────────────────────────── */ + +.tl-tabs-bar { + flex: 0 0 auto; + background: var(--bg-1); + border-top: 1px solid var(--border-2); +} + +.tl-tabs { + display: flex; + align-items: flex-end; + gap: 0.75rem; + padding: 0.4rem 1.1rem 0; +} + +.tl-tabs-list { + display: flex; + gap: 0.25rem; + min-width: 0; + overflow-x: auto; +} + +.tl-tab { + display: flex; + align-items: center; + border: 1px solid var(--border); + border-bottom: none; + border-radius: var(--r-sm) var(--r-sm) 0 0; + background: var(--bg); + color: var(--text-2); + white-space: nowrap; +} + +.tl-tab.is-current { + background: #181b21; + border-color: var(--border-2); + color: var(--text); +} + +.tl-tab-select { + font: inherit; + font-size: 0.82rem; + display: flex; + align-items: center; + gap: 0.4rem; + padding: 0.35rem 0.4rem 0.35rem 0.7rem; + background: none; + border: none; + color: inherit; + cursor: pointer; +} + +.tl-tab-shape { + font-family: var(--mono); + font-size: 0.68rem; + padding: 0.05rem 0.3rem; + border: 1px solid var(--border-2); + border-radius: 3px; + color: var(--text-2); +} + +.tl-tab.is-current .tl-tab-shape { + border-color: var(--accent); + color: var(--accent-hi); +} + +.tl-tab-delete { + font: inherit; + font-size: 0.72rem; + padding: 0.2rem 0.5rem 0.2rem 0.2rem; + background: none; + border: none; + color: var(--text-3); + cursor: pointer; +} + +.tl-tab-delete:hover { + color: var(--danger); +} + +.tl-export-all { + margin: 0 0 0.3rem auto; + font-size: 0.8rem; + padding: 0.3rem 0.75rem; + white-space: nowrap; +} + +.tl-notice { + display: flex; + align-items: flex-start; + gap: 0.5rem; + padding: 0.4rem 1.1rem; + border-top: 1px solid var(--border); + font-size: 0.8rem; + color: var(--text-2); +} + +.tl-notice.is-warn { + color: var(--warn); +} + +.tl-notice-lines { + flex: 1; + min-width: 0; + overflow-wrap: anywhere; +} + .doc-timeline { background: linear-gradient(180deg, #181b21, #141619); border-top: 1px solid var(--border-2); @@ -1363,6 +1470,24 @@ body, gap: 0.5rem; } +.ci-short { + display: flex; + flex-direction: column; + gap: 0.45rem; + padding: 0.6rem; + border: 1px solid var(--border-2); + border-radius: var(--r-sm); + background: var(--bg-2); +} + +.ci-short-name { + flex: 1; + min-width: 0; + padding: 0.3rem 0.5rem; + font-family: var(--mono); + font-size: 0.82rem; +} + .ci-remove { align-self: flex-start; font: inherit; From 41841ac1ed3bab38b91e0c0d7b1bfbb4fed79db7 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 19:36:51 -0800 Subject: [PATCH 32/35] docs(docs): document export --all and shorts in clip ui Also logs the clip inspector getting no height on a short window as a backlog note; it predates the tabs. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +++ SKILL.md | 2 +- docs/backlog/ui-inspector-height.md | 5 +++++ llms.txt | 2 +- 4 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 docs/backlog/ui-inspector-height.md diff --git a/README.md b/README.md index 3b728d7..f09d0a5 100644 --- a/README.md +++ b/README.md @@ -176,10 +176,13 @@ clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off clip timeline --timeline best-1 export best-1.mp4 clip timeline list # main and every short +clip timeline export --all ./shorts # every short, as ./shorts/.mp4 ``` `short` cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone (where there's no clear pause it follows the transcript and warns), fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. +In `clip ui`, select words in a clip's transcript and press **Make short**: the short opens in a tab of its own above the timeline, next to main. **Export all shorts** renders every one. + --- ## Another example: a product demo from screen recordings diff --git a/SKILL.md b/SKILL.md index ef44f04..8eadb11 100644 --- a/SKILL.md +++ b/SKILL.md @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone, so it never keeps the next sentence's first syllable, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence (widen it to the sentence's edge unless the cut is meant) and an edge with no clear pause in the audio (music under the speech, or no break), which follows the transcript instead: look at that edge or listen to it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone, so it never keeps the next sentence's first syllable, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence (widen it to the sentence's edge unless the cut is meant) and an edge with no clear pause in the audio (music under the speech, or no break), which follows the transcript instead: look at that edge or listen to it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4` (`timeline export --all ` renders every short as `/.mp4`). `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/docs/backlog/ui-inspector-height.md b/docs/backlog/ui-inspector-height.md new file mode 100644 index 0000000..4efeff5 --- /dev/null +++ b/docs/backlog/ui-inspector-height.md @@ -0,0 +1,5 @@ +# Clip inspector has no room on a short window + +In `clip ui`, the viewer takes up to 44vh plus its bar, and the clip inspector (`.stage-detail`) gets whatever is left of `.main`. At a 1000 px tall window, `.main` is 468 px and the viewer fills all of it, so the inspector is 0 px tall: a selected clip's transcript, framing and Make short can't be reached. It works at 1500 px. Seen on 2026-11-15 while checking the shorts tabs; the layout was the same before them, and the tabs bar takes another ~30 px once a short exists. + +Fix: give the inspector a minimum height (or let the viewer shrink first), and check at 900 px tall. diff --git a/llms.txt b/llms.txt index fcc3577..382efa5 100644 --- a/llms.txt +++ b/llms.txt @@ -21,7 +21,7 @@ - **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time); on a vertical canvas they stay clear of the Shorts, Reels and TikTok overlays, and their width follows the font size - **Reframe:** `clip timeline reframe 9:16` turns a landscape cut vertical (also 1:1, 4:5); `--x` picks the part that shows, `clip timeline framing ` sets one clip's zoom and position - **Loudness:** `clip timeline loudness -14` normalizes the exported mp4 to -14 LUFS (or `-16` for podcasts) in two passes -- **Shorts from a long recording:** `clip timeline short best-1 ` (MCP: the `make_short` verb) turns a range of transcript words into a 9:16 short on a timeline of its own, captioned and at -14 LUFS, keeping the cuts made on the main timeline; `--timeline ` points any command at a short +- **Shorts from a long recording:** `clip timeline short best-1 ` (MCP: the `make_short` verb) turns a range of transcript words into a 9:16 short on a timeline of its own, captioned and at -14 LUFS, keeping the cuts made on the main timeline; `--timeline ` points any command at a short, and `clip timeline export --all ` renders every short. In `clip ui`, Make short works from a transcript selection and each short gets a tab - **Cut check:** `clip timeline check --sheet cuts.jpg` flags cuts that may clip a word and writes a contact sheet - **Export to pro editors:** `clip timeline export --format fcpxml` (Final Cut Pro, DaVinci Resolve) or `--format xmeml` (Premiere Pro; captions as .srt) - Every timeline edit is one undoable step (`clip timeline undo` / `redo`), from the CLI, the MCP server or the browser UI From 4bb87a692e38923ed7804033b4c096a0636ab0b7 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 19:40:47 -0800 Subject: [PATCH 33/35] fix(ui): re-check each short's media in export-all, and tie missing to its timeline The export-all route now resolves each short's files through confinedMediaMap, so a registered file swapped for a link out of the workspace is refused instead of rendered. useTimeline remembers which timeline answered 404, so switching to another short right after a fallback doesn't bounce back to main. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/confine-render-media.md | 9 ++++++ src/timeline/run-plan.ts | 12 +++++--- src/ui/server.ts | 9 ++++-- src/ui/web/src/hooks/useTimeline.ts | 7 +++-- tests/ui-timelines.test.ts | 42 ++++++++++++++++++++++++++-- 5 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 docs/backlog/confine-render-media.md diff --git a/docs/backlog/confine-render-media.md b/docs/backlog/confine-render-media.md new file mode 100644 index 0000000..b5ab9c8 --- /dev/null +++ b/docs/backlog/confine-render-media.md @@ -0,0 +1,9 @@ +# Re-check media against the workspace on every render for the UI and MCP + +`POST /api/timelines/export` re-checks each short's files with `confinedMediaMap`, so a registered file swapped for a symlink out of the workspace is refused. Three older render paths still take the registry as ingested (`buildMediaMap`): + +- `POST /api/timeline/export` (mp4) in `src/ui/server.ts`; +- `GET /api/timeline/frame` in `src/ui/server.ts`; +- the mp4 path of MCP `timeline_export` in `src/mcp/server.ts`. + +The transcript and project-file paths already confine. Pass `confinedMediaMap(comp)` on these three, and test each with a symlinked file, as `tests/ui-timelines.test.ts` does for export-all. Found by the Codex review of the UI shorts PR (2026-11-15). diff --git a/src/timeline/run-plan.ts b/src/timeline/run-plan.ts index 10f5028..e89b3af 100644 --- a/src/timeline/run-plan.ts +++ b/src/timeline/run-plan.ts @@ -8,9 +8,11 @@ import { } from '../ffmpeg/args/loudnorm.js'; import { filterGraphFileOption, requireDrawtext, runFfmpeg } from '../ffmpeg/run.js'; import { getWorkspace } from '../workspace.js'; -import { compileTimeline, type FfmpegPlan } from './compile.js'; +import { compileTimeline, type FfmpegPlan, type MediaInfo } from './compile.js'; +import type { Composition } from './composition.js'; import { readComposition } from './document-store.js'; import { buildMediaMap } from './media-registry.js'; +import type { MediaId } from './schema.js'; export interface LoudnessResult { targetLufs: number; @@ -168,14 +170,16 @@ export type TimelineExport = /** * Render each timeline to `/.mp4`, one at a time. A timeline that - * can't render is reported with its error, and the rest still render. + * can't render is reported with its error, and the rest still render. `media` + * resolves each document's files (default: the registry as ingested); callers + * acting for the UI pass one that re-checks them against the workspace. */ export async function exportTimelines( timelines: readonly string[], dir: string, + media: (comp: Composition) => Promise> = () => buildMediaMap(), ): Promise { await mkdir(dir, { recursive: true }); - const media = await buildMediaMap(); // Intermediates are named per run, so a concurrent export or frame preview // in the same workspace can't overwrite them. const run = randomBytes(4).toString('hex'); @@ -184,7 +188,7 @@ export async function exportTimelines( try { const comp = await readComposition(timeline); const plan = compileTimeline(comp, { - media, + media: await media(comp), dir: getWorkspace(), output: join(dir, `${timeline}.mp4`), tag: `${run}-${timeline}`, diff --git a/src/ui/server.ts b/src/ui/server.ts index 1408c62..319f47c 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -62,7 +62,12 @@ import { resolveInWorkspace, WorkspaceBoundaryError, } from '../workspace.js'; -import { clipConfinedMedia, confinedMediaRefs, makeVerbContext } from './timeline-tools.js'; +import { + clipConfinedMedia, + confinedMediaMap, + confinedMediaRefs, + makeVerbContext, +} from './timeline-tools.js'; /** Errors about the clip or its setup, not the server: silence removal or * transcription can't work on this clip, whisper.cpp isn't set up, a reframe @@ -543,7 +548,7 @@ export async function startUiServer(options: UiServerOptions = {}): Promise(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); - const [missing, setMissing] = useState(false); + // The timeline whose fetch last answered 404, so another one never inherits it. + const [missingTimeline, setMissingTimeline] = useState(null); // A response for a timeline switched away from is dropped, not shown. const wanted = useRef(timeline); wanted.current = timeline; @@ -44,7 +45,7 @@ export function useTimeline( try { const res = await fetch(withTimeline('/api/timeline', timeline)); if (wanted.current !== timeline) return; - setMissing(res.status === 404); + setMissingTimeline(res.status === 404 ? timeline : null); if (!res.ok) throw new Error(`HTTP ${res.status}`); const json = (await res.json()) as { document: Composition }; if (wanted.current !== timeline) return; @@ -65,5 +66,5 @@ export function useTimeline( }, [intervalMs, load]); const composition = loaded?.timeline === timeline ? loaded.doc : EMPTY; - return { composition, loading, error, missing, refresh: load }; + return { composition, loading, error, missing: missingTimeline === timeline, refresh: load }; } diff --git a/tests/ui-timelines.test.ts b/tests/ui-timelines.test.ts index e3be981..be3c613 100644 --- a/tests/ui-timelines.test.ts +++ b/tests/ui-timelines.test.ts @@ -1,7 +1,8 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { appendOp } from '../src/session/store.js'; import { emptyComposition } from '../src/timeline/composition.js'; import { listTimelines, @@ -9,7 +10,8 @@ import { readComposition, resetComposition, } from '../src/timeline/document-store.js'; -import { colorClip, videoTrack } from '../src/timeline/ops.js'; +import { colorClip, mediaClip, videoTrack } from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; import { startUiServer, type UiServer } from '../src/ui/server.js'; // The `clip ui` routes for named timelines: every /api/timeline route takes @@ -174,4 +176,40 @@ describe('/api/timelines', () => { const session = await call('GET', '/api/session'); expect(session.json.entries).toEqual([]); }); + + it('refuses a short whose media was swapped for a link out of the workspace', async () => { + const outside = await mkdtemp(join(tmpdir(), 'mmc-ui-outside-')); + try { + await writeFile(join(outside, 'private.mp4'), 'not for the export'); + const linked = join(workspace, 'talk.mp4'); + await symlink(join(outside, 'private.mp4'), linked); + await appendOp({ + tool: 'ingest', + args: { path: linked }, + result: { mediaId: 'm_0123456789ab', ref: { path: linked, audio: null, video: null } }, + }); + await mutateComposition( + [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ + id: 'c1', + mediaId: 'm_0123456789ab' as MediaId, + sourceOutSec: 2, + startSec: 0, + }), + }, + ], + { timeline: 'best-1', create: true }, + ); + const reply = await call('POST', '/api/timelines/export', {}); + expect(reply.json.exported).toEqual([ + { timeline: 'best-1', error: expect.stringContaining('outside the workspace') }, + ]); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); }); From 78aa9b2df19e075e0c09e48c7b6edc9dff6d1a66 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 20:31:35 -0800 Subject: [PATCH 34/35] docs(docs): update the test count and the Roadmap for 0.5.2 Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++--- llms.txt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f09d0a5..92f15a0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The open-source AI video editor for Claude Code and any MCP client: remove silen npm weekly downloads License: MIT Node 24+ - Tests: 859 passing + Tests: 965 passing

@@ -352,7 +352,7 @@ No — nowhere. The CLI, the browser UI, the Claude Code skill, and the MCP serv ### Capabilities & limits **Is it production-ready?** -This release is feature-complete for local editing — a dark timeline editor (view, edit, preview, export), silence removal, transcript editing, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, export to Final Cut Pro, DaVinci Resolve and Premiere Pro, 19 single-file tools, an MCP server, and 859 tests passing. The API surface is still pre-1.0 (tool schemas may change in minor ways before 1.0). Use it for real work; pin a version in CI. +This release is feature-complete for local editing — a dark timeline editor (view, edit, preview, export), silence removal, transcript editing, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, export to Final Cut Pro, DaVinci Resolve and Premiere Pro, 19 single-file tools, an MCP server, and 965 tests passing. The API surface is still pre-1.0 (tool schemas may change in minor ways before 1.0). Use it for real work; pin a version in CI. **What's the maximum video size or duration?** No hardcoded limit. FFmpeg streams the file rather than loading it into memory, and the browser UI streams uploads to disk — multi-GB recordings work fine. This release isn't tuned for projects with hundreds of clips or runtime above 30 minutes; for those, drive the CLI directly. @@ -454,7 +454,7 @@ auto-editor is a mature open-source command-line tool that cuts silence or motio ## Roadmap -- **Now (v0.5.0)** — silence removal, local transcription, edit by transcript, retake removal, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, click-free cuts, export to Final Cut Pro / DaVinci Resolve / Premiere Pro, on top of the dark timeline editor (view / edit / preview / export), 19 tools, MCP server, and snapshot/undo +- **Now (v0.5.2)** — vertical shorts from a long recording, silence removal, local transcription, edit by transcript, retake removal, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, click-free cuts, export to Final Cut Pro / DaVinci Resolve / Premiere Pro, on top of the dark timeline editor (view / edit / preview / export), 19 tools, MCP server, and snapshot/undo - **Next** — model picker, SSE for live session updates, AI Elements / shadcn migration - **1.0** — frozen tool schemas, MCP transport, published Anthropic skill, docs site - **Beyond** — desktop app (Electron), cloud rendering via [MakeMyClip.com](https://makemyclip.com) diff --git a/llms.txt b/llms.txt index 382efa5..21f2713 100644 --- a/llms.txt +++ b/llms.txt @@ -9,7 +9,7 @@ - **Backend:** FFmpeg (downloaded by `ffmpeg-static` at npm install; the Claude Desktop extension uses an installed FFmpeg); optional user-installed whisper.cpp for transcription - **Surfaces:** Claude Code skill · `clip` CLI · browser UI at `127.0.0.1:5573` · MCP server (`clip mcp`) - **Agent access:** Claude Code skill + MCP server (`clip mcp`); no API key — the editor embeds no language model, it runs on whichever Claude you already use -- **Tests:** 859 passing +- **Tests:** 965 passing - **License:** MIT (code); FFmpeg runs as a subprocess under its own license (GPLv3 builds from `ffmpeg-static`, except a nonfree Apple Silicon build; the `.mcpb` ships no FFmpeg) ## What v0.5.0 does From 72d3c31602f269cdc79bcfb87736e27c4eaf7de1 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 20:31:35 -0800 Subject: [PATCH 35/35] ci(ci): release the shorts work as 0.5.2, a patch The maintainer chose a patch for this release. The rule is removed again after 0.5.2 ships. Co-Authored-By: Claude Opus 5.5 --- .releaserc.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.releaserc.json b/.releaserc.json index 553148a..778a55b 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -7,7 +7,8 @@ "preset": "angular", "releaseRules": [ { "type": "chore", "scope": "deps", "release": "patch" }, - { "type": "chore", "scope": "security", "release": "patch" } + { "type": "chore", "scope": "security", "release": "patch" }, + { "type": "feat", "release": "patch" } ] } ],