From 18c8cf09c73c7e323aa0fb5b05dd5f1d832f476c Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:41:29 -0800 Subject: [PATCH 01/14] perf(timeline): render a media frame from a short window of its source The frame preview encoded the clip's whole segment to grab one frame, which took minutes on a long recording. A media clip's frame is now one step: seek the source to the frame and run the segment's own video chain on timestamps offset to the frame's time, so fades, captions and speed read the same time they do in the segment. Text and color cards keep the two-step path. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/frame-preview-short-window.md | 3 - src/timeline/compile.ts | 151 ++++++++++++++------- tests/timeline-introspect.test.ts | 89 +++++++++--- 3 files changed, 173 insertions(+), 70 deletions(-) delete mode 100644 docs/backlog/frame-preview-short-window.md diff --git a/docs/backlog/frame-preview-short-window.md b/docs/backlog/frame-preview-short-window.md deleted file mode 100644 index 177b62f..0000000 --- a/docs/backlog/frame-preview-short-window.md +++ /dev/null @@ -1,3 +0,0 @@ -# Frame preview encodes a short window - -`buildFrameAtPlan` (`src/timeline/compile.ts`) encodes the whole clip's segment to grab one frame, so every framing change in the browser UI re-encodes the full clip before the viewer updates. On a long uncut recording that takes seconds. Encode only a short window around the requested time (`-ss` before the input, `-t` of a few frames) and adjust the frame offset, keeping the same filter chain so the preview still matches export. diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 45dc785..dcf1bb3 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -399,6 +399,59 @@ function segPath(dir: string, index: number, clipId: string, tag?: string): stri return resolve(dir, `${SEG_PREFIX}-${tagInfix(tag)}${index}-${clipId}.mp4`); } +/** + * A clip's segment filters after its input: the video chain (framing onto the + * canvas, a text clip's own text, effects, then the text over it) and the audio + * effects. Shared by export and the frame preview, so a preview can't drift + * from what export draws. + */ +function clipChains( + comp: Composition, + clip: Clip, + index: number, + ctx: CompileContext, + overlays: TextClip[], + fadeSuppress: FadeSuppression = { in: false, out: false }, +): { chain: string; audio: string[]; textFiles: StepSideFile[] } { + const outDur = outputDuration(clip); + const { width, height, fps, background } = comp; + const { video: fx, audio } = effectFilters(clip, outDur, fadeSuppress); + const textFiles: StepSideFile[] = []; + + if (clip.kind === 'text') { + const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); + textFiles.push({ path: textfile, content: clip.text }); + fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); + } + + // Overlay times are timeline seconds; the segment runs in output time, which + // a speed effect scales. + const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; + const end = clipEndSec(clip); + for (const overlay of overlays) { + const from = Math.max(overlay.startSec, clip.startSec); + const to = Math.min(clipEndSec(overlay), end); + if (to - from <= ABUT_EPSILON) continue; + const textfile = resolve( + ctx.dir, + `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, + ); + textFiles.push({ path: textfile, content: overlay.text }); + const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; + fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); + } + + // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish + // for generated sources that are already canvas-sized, but keeps the chain uniform.) + const geometry = [ + ...framingFilters(clipFraming(comp, clip), width, height, background), + 'setsar=1', + `fps=${fps}`, + 'format=yuv420p', + ]; + return { chain: [...geometry, ...fx].join(','), audio, textFiles }; +} + /** * Normalize one clip to a self-contained segment: canvas-sized video at the * composition fps, effects baked in, and a guaranteed stereo AAC audio track @@ -419,8 +472,14 @@ function buildSegmentStep( const outDur = outputDuration(clip); const { width, height, fps, background } = comp; const out = segPath(ctx.dir, index, clip.id, ctx.tag); - const { video: fx, audio: afx } = effectFilters(clip, outDur, fadeSuppress); - const textFiles: StepSideFile[] = []; + const { chain: videoChain, audio: afx, textFiles } = clipChains( + comp, + clip, + index, + ctx, + overlays, + fadeSuppress, + ); const inputArgs: string[] = []; let videoSource: string; // filtergraph input label feeding the video chain, e.g. [0:v] @@ -475,38 +534,8 @@ function buildSegmentStep( ); videoSource = '[0:v]'; audioSource = '1:a'; - const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); - textFiles.push({ path: textfile, content: clip.text }); - fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); - } - - // Overlay times are timeline seconds; the segment runs in output time, which - // a speed effect scales. - const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; - const end = clipEndSec(clip); - for (const overlay of overlays) { - const from = Math.max(overlay.startSec, clip.startSec); - const to = Math.min(clipEndSec(overlay), end); - if (to - from <= ABUT_EPSILON) continue; - const textfile = resolve( - ctx.dir, - `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, - ); - textFiles.push({ path: textfile, content: overlay.text }); - const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; - fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); } - // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish - // for generated sources that are already canvas-sized, but keeps the chain uniform.) - const geometry = [ - ...framingFilters(clipFraming(comp, clip), width, height, background), - 'setsar=1', - `fps=${fps}`, - 'format=yuv420p', - ]; - - const videoChain = [...geometry, ...fx].join(','); // Always normalize audio to a uniform stereo/48k layout BEFORE any effects, so // the fold's concat/acrossfade can't silently downmix when a source clip is // mono (and so every segment exposes a real [a] output). aformat is a harmless @@ -692,18 +721,18 @@ function assertAbutting(clips: Clip[]): void { /** * Render ONE frame at timeline time `atSec` — the agent's "eyes" on the doc. - * Goes through the REAL segment path (`buildSegmentStep`) so the preview can't - * diverge from what export would produce, then extracts a still from that - * normalized segment. Two steps: encode the active clip's segment, then grab the - * frame at the mapped offset. Runs the SAME abutting check as export, so it fails - * (rather than guessing a clip) on an unexportable doc; throws `CompileError` past - * the end, in a gap, or on an overlap, and inherits the v1 single-video-track / - * no-compositing guards. + * Uses the segment's own video chain (`clipChains`) so the preview can't diverge + * from what export would produce. A media clip is one step: seek the source to + * the frame and run the chain on a window starting there, with the timestamps + * offset so fades and text windows read the time they would in the segment, so + * the cost doesn't grow with the clip's length. A text or color card (generated, + * and short) is two: encode its segment, then grab the frame from it. Runs the + * SAME abutting check as export, so it fails (rather than guessing a clip) on an + * unexportable doc; throws `CompileError` past the end, in a gap, or on an + * overlap, and inherits the v1 single-video-track / no-compositing guards. * - * Known v1 limits (flag, don't fix here): a frame inside a transition window - * shows the underlying clip, not the xfade blend; `-ss` is keyframe-accurate - * (off by up to a GOP) — fine for a thumbnail; and a long clip re-encodes a whole - * segment to grab one frame. + * Known v1 limit (flag, don't fix here): a frame inside a transition window + * shows the underlying clip, not the xfade blend. */ export function buildFrameAtPlan( comp: Composition, @@ -728,9 +757,6 @@ export function buildFrameAtPlan( } assertCompilable(clip); - // Encode the clip's segment exactly as export would, then extract the frame. - const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); - // Map doc-local time to the post-speed SEGMENT timebase. The ratio is 1 unless // the clip carries a speed effect, which shortens the segment relative to the // document extent — so a frame still lands on the source content the document @@ -741,9 +767,40 @@ export function buildFrameAtPlan( const segTime = clipDur > 0 ? (localDoc * outDur) / clipDur : 0; const lastFrame = Math.max(0, outDur - 1 / comp.fps); const clamped = Math.max(0, Math.min(segTime, lastFrame)); + const label = `frame:${clip.id}@${atSec}`; + if (clip.kind === 'media') { + const info = ctx.media.get(clip.mediaId); + if (!info) { + throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + // Seconds into the clip's source. The chain's speed `setpts` divides the + // offset back into segment time, which is what its fades and text read. + const local = Math.round(clamped * speedFactor(clip.effects) * 1000) / 1000; + const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); + const frameStep: FfmpegStep = { + label, + args: [ + '-y', + '-ss', + seconds(clip.sourceInSec + local), + '-i', + info.path, + '-filter_complex', + `[0:v]setpts=PTS-STARTPTS+${local}/TB,${chain}[v]`, + ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], + ctx.output, + ], + output: ctx.output, + textFiles, + }; + return { steps: [frameStep], output: ctx.output, durationSec: 0 }; + } + + // Encode the card's segment exactly as export would, then extract the frame. + const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { - label: `frame:${clip.id}@${atSec}`, + label, args: buildPreviewFrameArgs({ input: segStep.output, output: ctx.output, atSec: clamped }), output: ctx.output, textFiles: [], diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index e514515..c89df21 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { buildFrameAtPlan, CompileError, type MediaInfo } from '../src/timeline/compile.js'; +import { + buildFrameAtPlan, + CompileError, + compileTimeline, + type MediaInfo, +} from '../src/timeline/compile.js'; import { clipsAtTime, emptyComposition } from '../src/timeline/composition.js'; import { applyOps, mediaClip, textClip, videoTrack } from '../src/timeline/ops.js'; import type { MediaId } from '../src/timeline/schema.js'; @@ -65,18 +70,48 @@ describe('buildFrameAtPlan', () => { ]); } - it('encodes the clip segment then extracts the frame at the doc-local offset', () => { - const plan = buildFrameAtPlan(oneClip(), CTX('/ws/frame.jpg'), 3); - expect(plan.steps).toHaveLength(2); - const [seg, frame] = plan.steps; - expect(seg?.label).toBe('segment:c1'); - expect(frame?.label).toContain('frame:c1'); - expect(frame?.output).toBe('/ws/frame.jpg'); - // -ss -i … - expect(frame?.args.slice(0, 2)).toEqual(['-y', '-ss']); - expect(Number(frame?.args[2])).toBeCloseTo(3); - expect(frame?.args[4]).toBe(seg?.output); // reads the encoded segment - expect(frame?.args.at(-1)).toBe('/ws/frame.jpg'); + /** The `-filter_complex` graph of a step. */ + const graphOf = (args: string[] | undefined): string => + args?.[(args?.indexOf('-filter_complex') ?? -2) + 1] ?? ''; + + it('renders a media frame in one step from the source, through the segment chain', () => { + const doc = applyOps(oneClip(), [ + { op: 'addEffect', clipId: 'c1', effect: { type: 'fadeIn', durationSec: 1 } }, + ]); + const plan = buildFrameAtPlan(doc, CTX('/ws/frame.jpg'), 3); + expect(plan.steps).toHaveLength(1); + const [frame] = plan.steps; + expect(frame?.label).toBe('frame:c1@3'); + expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '3', '-i', '/in.mp4']); + expect(frame?.args.slice(-8)).toEqual([ + '-map', + '[v]', + '-an', + '-frames:v', + '1', + '-q:v', + '2', + '/ws/frame.jpg', + ]); + // The same video chain export runs, on timestamps offset to the frame's time. + const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); + const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); + expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS-STARTPTS+3/TB,${segmentChain}[v]`); + }); + + it('seeks into the source from the clip in-point and stops at its last frame', () => { + const doc = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId: M, sourceInSec: 100, sourceOutSec: 108, startSec: 0 }), + }, + ]); + const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; + // 30 fps: the last frame starts at 8 - 1/30 s. + expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.967']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.967\/TB,/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -90,9 +125,17 @@ describe('buildFrameAtPlan', () => { expect(graph).toContain("crop=w='min(iw,1080)':h='min(ih,1920)':x='(iw-ow)*0'"); }); - it('a tagged context gives the segment a collision-free path; untagged stays stable', () => { + it('a tagged context gives a card segment a collision-free path; untagged stays stable', () => { + const card = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: textClip({ id: 'c1', text: 'hi', durationSec: 8, startSec: 0 }), + }, + ]); const segOut = (tag?: string) => - buildFrameAtPlan(oneClip(), { ...CTX('/ws/frame.jpg'), tag }, 3).steps[0]?.output; + buildFrameAtPlan(card, { ...CTX('/ws/frame.jpg'), tag }, 3).steps[0]?.output; // Untagged keeps the deterministic name export also uses. expect(segOut()).toBe('/ws/tl-seg-0-c1.mp4'); // A tag is mixed in, so a preview can't overwrite/unlink an export's segment… @@ -106,12 +149,13 @@ describe('buildFrameAtPlan', () => { { op: 'addClip', trackId: 'v0', - clip: mediaClip({ id: 'c2', mediaId: M, sourceOutSec: 4, startSec: 8 }), + clip: mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1, sourceOutSec: 5, startSec: 8 }), }, ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) - expect(plan.steps[0]?.label).toBe('segment:c2'); - expect(Number(plan.steps[1]?.args[2])).toBeCloseTo(2); + expect(plan.steps[0]?.label).toBe('frame:c2@10'); + expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '3']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+2\/TB,/); }); it('maps doc-local time to the post-speed segment timebase for a speed clip', () => { @@ -130,8 +174,13 @@ describe('buildFrameAtPlan', () => { }), }, ]); - const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4); - expect(Number(plan.steps[1]?.args[2])).toBeCloseTo(2); + // The frame shows source second 4; the chain's setpts=PTS/2 turns the +4 s + // offset into segment time 2, which the fades and text windows read. + const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; + expect(frame?.args.slice(1, 3)).toEqual(['-ss', '4']); + const graph = graphOf(frame?.args); + expect(graph).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+4\/TB,/); + expect(graph).toContain('setpts=PTS/2'); }); it('works on a text clip (no media required)', () => { From 28529ff1acd40f11af4a40632cc0123586f924c4 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:44:21 -0800 Subject: [PATCH 02/14] fix(timeline): keep a sped-up clip on the canvas frame grid After setpts, a sped-up clip's frames reached the encoder off the canvas frame grid, and the encoder's frame dropping repeated some frames and shifted others by up to two frames. An fps filter right after the speed setpts picks the nearest frame for each slot, so export is steady and the frame preview shows the frame export plays. Adds the CI-gated check: the window frame matches the frame of the fully encoded clip at PSNR >= 40 dB mid-clip, inside a fade and at 2x, and a frame 9 minutes into a 10-minute clip renders in under a second. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 21 +-- .../integration/timeline-frame-window.test.ts | 161 ++++++++++++++++++ 2 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 tests/integration/timeline-frame-window.test.ts diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index dcf1bb3..d3c4d1b 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -322,6 +322,7 @@ interface FadeSuppression { function effectFilters( clip: Clip, outDur: number, + fps: number, suppress: FadeSuppression = { in: false, out: false }, ): FilterChains { const video: string[] = []; @@ -343,7 +344,10 @@ function effectFilters( if (clip.kind === 'media') { const f = speedFactor(clip.effects); if (f !== 1) { - video.push(`setpts=PTS/${f}`); + // Back to the canvas rate right away: retimed frames otherwise reach the + // encoder off its frame grid, and which ones it drops or repeats varies, + // so a frame preview couldn't show the frame export plays. + video.push(`setpts=PTS/${f}`, `fps=${fps}`); audio.push(atempoChain(f)); } } @@ -415,7 +419,7 @@ function clipChains( ): { chain: string; audio: string[]; textFiles: StepSideFile[] } { const outDur = outputDuration(clip); const { width, height, fps, background } = comp; - const { video: fx, audio } = effectFilters(clip, outDur, fadeSuppress); + const { video: fx, audio } = effectFilters(clip, outDur, fps, fadeSuppress); const textFiles: StepSideFile[] = []; if (clip.kind === 'text') { @@ -472,14 +476,11 @@ function buildSegmentStep( const outDur = outputDuration(clip); const { width, height, fps, background } = comp; const out = segPath(ctx.dir, index, clip.id, ctx.tag); - const { chain: videoChain, audio: afx, textFiles } = clipChains( - comp, - clip, - index, - ctx, - overlays, - fadeSuppress, - ); + const { + chain: videoChain, + audio: afx, + textFiles, + } = clipChains(comp, clip, index, ctx, overlays, fadeSuppress); const inputArgs: string[] = []; let videoSource: string; // filtergraph input label feeding the video chain, e.g. [0:v] diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts new file mode 100644 index 0000000..7c54764 --- /dev/null +++ b/tests/integration/timeline-frame-window.test.ts @@ -0,0 +1,161 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { buildPreviewFrameArgs } from '../../src/ffmpeg/args/preview.js'; +import { listFilters, runFfmpeg } from '../../src/ffmpeg/run.js'; +import { buildFrameAtPlan, compileTimeline, type MediaInfo } from '../../src/timeline/compile.js'; +import { + type Composition, + emptyComposition, + type MediaClip, +} from '../../src/timeline/composition.js'; +import { applyOps, mediaClip, textClip, videoTrack } from '../../src/timeline/ops.js'; +import { runPlan } from '../../src/timeline/run-plan.js'; +import type { MediaId } from '../../src/timeline/schema.js'; + +// The frame preview runs the segment's video chain on a window of the source +// starting at the frame. This checks it against the frame cut out of the fully +// encoded clip, which is what export plays, and that it doesn't grow with the +// clip's length. +const IS_CI = process.env.CI === 'true'; +const HAS_DRAWTEXT = IS_CI && (await listFilters()).has('drawtext'); +const M = 'm_aaaaaaaaaaaa' as MediaId; + +let workspace: string; +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-frame-window-')); +}); +afterEach(async () => { + await rm(workspace, { recursive: true, force: true }); +}); + +/** 720p30 test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +async function testPattern(path: string, seconds: number): Promise { + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=30:duration=${seconds}`], + ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', path], + ]); +} + +/** A vertical timeline holding one clip of the source, framed off-center. */ +function vertical(clip: MediaClip, withText = false): Composition { + return applyOps(emptyComposition({ width: 1080, height: 1920, fit: 'cover' }), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { op: 'addClip', trackId: 'v0', clip: { ...clip, transform: { ...clip.transform, x: 0.3 } } }, + ...(withText + ? [ + { op: 'addTrack' as const, track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip' as const, + trackId: 'captions', + clip: textClip({ + id: 't1', + text: 'Caption over the picture', + startSec: 2, + durationSec: 2, + style: { fontSize: 54, background: 'black@0.6', anchor: 'bottom-center' }, + }), + }, + ] + : []), + ]); +} + +/** PSNR between two images, dB. */ +async function psnr(a: string, b: string): Promise { + const { stderr } = await runFfmpeg([ + ...['-hide_banner', '-i', a, '-i', b, '-lavfi', 'psnr', '-f', 'null', '-'], + ]); + const match = /average:([\d.]+|inf)/.exec(stderr); + if (!match?.[1]) throw new Error(`No PSNR in:\n${stderr}`); + return match[1] === 'inf' ? Number.POSITIVE_INFINITY : Number(match[1]); +} + +describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg', () => { + it('matches the frame of the fully encoded clip: mid-clip, in a fade, at 2x', async () => { + const source = join(workspace, 'pattern.mp4'); + await testPattern(source, 20); + const media = new Map([[M, { path: source, hasAudio: true }]]); + const cases: { name: string; doc: Composition; atSec: number; exportSec: number }[] = [ + { + name: 'mid-clip', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 15, startSec: 0 }), + HAS_DRAWTEXT, + ), + atSec: 3, + exportSec: 3, + }, + { + name: 'inside a fade', + doc: vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 5, + sourceOutSec: 15, + startSec: 0, + effects: [{ type: 'fadeIn', durationSec: 2 }], + }), + ), + atSec: 1, + exportSec: 1, + }, + { + // 8 s of source at 2x plays in 4 s: timeline 4 s is export 2 s. + name: 'at 2x speed', + doc: vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 4, + sourceOutSec: 12, + startSec: 0, + effects: [ + { type: 'speed', factor: 2 }, + { type: 'fadeIn', durationSec: 3 }, + ], + }), + ), + atSec: 4, + exportSec: 2, + }, + ]; + + for (const { name, doc, atSec, exportSec } of cases) { + const tag = name.replace(/\W+/g, '-'); + const exported = join(workspace, `${tag}.mp4`); + await runPlan(compileTimeline(doc, { media, dir: workspace, output: exported })); + const full = join(workspace, `${tag}-full.jpg`); + await runFfmpeg(buildPreviewFrameArgs({ input: exported, output: full, atSec: exportSec })); + + const windowFrame = join(workspace, `${tag}-window.jpg`); + const plan = buildFrameAtPlan(doc, { media, dir: workspace, output: windowFrame }, atSec); + expect(plan.steps, name).toHaveLength(1); + await runPlan(plan); + + expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); + } + }, 180_000); + + it('renders a frame near the end of a 10-minute clip in under a second', async () => { + const piece = join(workspace, 'piece.mp4'); + const source = join(workspace, 'ten-minutes.mp4'); + await testPattern(piece, 30); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-stream_loop', '19', '-i', piece, '-c', 'copy', source], + ]); + const media = new Map([[M, { path: source, hasAudio: true }]]); + const doc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 600, startSec: 0 })); + const output = join(workspace, 'late.jpg'); + + const started = performance.now(); + await runPlan(buildFrameAtPlan(doc, { media, dir: workspace, output }, 540)); + expect(performance.now() - started).toBeLessThan(1000); + }, 120_000); +}); From 6488d7ad67635427ed5a152d65753684c7446b56 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:50:51 -0800 Subject: [PATCH 03/14] feat(timeline): outline a long recording as sentences timelineOutline lists the timeline's speech as numbered sentences, across silence-cut sub-clips, one 40 KB page at a time with nextFromSec. MCP timeline_transcript returns it when no clipId is given (view, fromSec and toSec added), and `clip timeline transcribe --sentences [--from --to]` prints one line per sentence. make_short now warns when a range starts or ends mid-sentence, and timeline_edit passes the warnings on. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- SKILL.md | 4 +- llms.txt | 2 +- src/cli-timeline.ts | 46 +++++-- src/mcp/server.ts | 50 +++++-- src/timeline/captions.ts | 8 +- src/timeline/document-store.ts | 12 +- src/timeline/shorts.ts | 38 +++++- src/timeline/transcript.ts | 167 ++++++++++++++++++++++- src/timeline/verbs.ts | 6 +- tests/mcp-server.test.ts | 61 ++++++++- tests/timeline-outline.test.ts | 240 +++++++++++++++++++++++++++++++++ tests/timeline-shorts.test.ts | 41 +++++- 13 files changed, 640 insertions(+), 38 deletions(-) create mode 100644 tests/timeline-outline.test.ts diff --git a/README.md b/README.md index 0dd305d..ff87911 100644 --- a/README.md +++ b/README.md @@ -161,8 +161,9 @@ The same steps from the CLI: ```bash clip timeline new clip timeline add-media podcast.mp4 # prints the clip id, e.g. clip_ec2f047d -clip timeline transcribe clip_ec2f047d # each word with its index and time +clip timeline transcribe clip_ec2f047d # transcribe it (once per file) clip timeline silence clip_ec2f047d # cut the pauses on main +clip timeline transcribe --sentences | head -80 # one line per sentence: time, word indices, text clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 as a 9:16 short clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off clip timeline --timeline best-1 export best-1.mp4 diff --git a/SKILL.md b/SKILL.md index a11612f..7553f49 100644 --- a/SKILL.md +++ b/SKILL.md @@ -323,7 +323,7 @@ To cut the pauses out of a talking clip already on the timeline, use `timeline s After cutting, run `timeline check` before exporting: it lists every hard cut whose audio is loud right at the join (usually a clipped word or a click) and writes a contact sheet (`--sheet out.jpg`) with the outgoing clip's last frame and the incoming clip's first for each cut. Open the sheet to see the jumps. A flagged cut is a reason to trim or move that boundary, not a verdict. Export fades the audio over 8 ms on each side of every hard cut, so a cut through steady sound doesn't click; the check still flags cuts in sound, since those can clip a word. -To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. +To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. For a long recording, read `timeline transcribe --sentences` instead: one line per sentence of the whole timeline (`12:34 #1400–1413 clip_ab12 text`, the time, the word indices, the clip it starts in), and `--from S --to S` for a stretch; pipe it through `head` to read a page at a time. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. To cut by words, use the `index` values from `timeline transcribe`: `timeline cut-words ` removes those words (inclusive) and moves later clips on the track up. Add `--transcript ` from the same `timeline transcribe` output so the cut is refused, not misplaced, if the clip was transcribed again since. The transcript also lists `retakes`, phrases said twice in a row; `timeline remove-retakes ` cuts the earlier take of each (`--dry-run` lists them first). Each is one edit that `timeline undo` reverses. A cut splits the clip, so transcribe the resulting clips again before the next cut, and run `timeline check` afterwards. @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence; widen it to the sentence's edge unless the cut is meant. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/llms.txt b/llms.txt index aa8b461..fd5e3a9 100644 --- a/llms.txt +++ b/llms.txt @@ -15,7 +15,7 @@ ## What v0.5.0 does - **Remove silence:** `clip timeline silence ` cuts the pauses out of a recording (`--sensitivity`, `--dry-run`) -- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine +- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine; `clip timeline transcribe --sentences` lists a long recording as numbered sentences - **Edit by transcript:** `clip timeline cut-words ` cuts a range of words - **Remove repeated takes:** `clip timeline remove-retakes ` cuts the earlier take of each phrase said twice in a row - **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index ceafd6e..e9f1b91 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -62,6 +62,7 @@ import { findRetakes, planWordCut, storedTranscriptForClip, + timelineOutline, transcribeClip, } from './timeline/transcript.js'; import { DEFAULT_VERB_TRACK as DEFAULT_TRACK, ensureTrack, trackEnd } from './timeline/verbs.js'; @@ -84,7 +85,7 @@ Every command works on the main timeline unless --timeline names another clip timeline remove clip timeline silence [--sensitivity 0..1] [--dry-run] clip timeline check [--sheet ] - clip timeline transcribe [--model ] [--language ] + clip timeline transcribe [] [--sentences] [--from S] [--to S] [--model ] [--language ] clip timeline cut-words [--transcript ] clip timeline remove-retakes [--dry-run] [--transcript ] clip timeline captions [] [--words N] [--track T] [--font N --color C --anchor A --background C|none] [--dry-run] @@ -109,7 +110,7 @@ interface Parsed { } /** Flags that take no value, so they don't swallow the next argument. */ -const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace']); +const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace', 'sentences']); function parseArgs(args: string[]): Parsed { const positional: string[] = []; @@ -171,6 +172,15 @@ function timelineId(value: string): string { return parsed.data; } +/** Timeline time as m:ss, or h:mm:ss from an hour. */ +function clock(sec: number): string { + const whole = Math.floor(sec); + const s = String(whole % 60).padStart(2, '0'); + const m = Math.floor(whole / 60) % 60; + const h = Math.floor(whole / 3600); + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${s}` : `${m}:${s}`; +} + function wordIndex(value: string | undefined): number { const n = Number(value); if (value === undefined || value.trim() === '' || !Number.isInteger(n) || n < 0) { @@ -610,18 +620,30 @@ export async function runTimeline(args: string[]): Promise { case 'transcribe': { const [clipId] = positional; - if (!clipId) { - throw new Error( - 'Usage: clip timeline transcribe [--model ] [--language ]', - ); - } const comp = await readComposition(timeline); - out( - await transcribeClip(comp, clipId, await buildMediaMap(), { - model: flags.model ? resolveInput(flags.model) : undefined, - language: flags.language || undefined, - }), + const opts = { + model: flags.model ? resolveInput(flags.model) : undefined, + language: flags.language || undefined, + }; + if (clipId && flags.sentences !== 'true') { + out(await transcribeClip(comp, clipId, await buildMediaMap(), opts)); + return; + } + // One line per sentence, cheap to read and to `head`: no page limit here. + const outline = await timelineOutline(comp, await buildMediaMap(), { + ...opts, + clipId, + fromSec: optionalNum('from', flags.from), + toSec: optionalNum('to', flags.to), + maxBytes: Number.POSITIVE_INFINITY, + }); + const lines = Object.entries(outline.transcriptIds).map( + ([mediaId, id]) => `transcript ${mediaId}: ${id}`, ); + for (const s of outline.sentences) { + lines.push(`${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId} ${s.text}`); + } + process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; } diff --git a/src/mcp/server.ts b/src/mcp/server.ts index db573d5..5d95c6a 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -32,7 +32,7 @@ import { reviewCuts } from '../timeline/review.js'; import { runPlan } from '../timeline/run-plan.js'; import { MAIN_TIMELINE, TimelineIdSchema } from '../timeline/schema.js'; import { ShortError } from '../timeline/shorts.js'; -import { transcribeClip } from '../timeline/transcript.js'; +import { timelineOutline, transcribeClip } from '../timeline/transcript.js'; import { EditableVerbSchema } from '../timeline/verbs.js'; import { doctor } from '../tools/doctor.js'; import { WhisperLanguageSchema } from '../transcribe/whisper.js'; @@ -218,15 +218,16 @@ export function buildMcpServer(): McpServer { title: 'Edit the timeline', description: 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). ' + - 'make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as "best-1" (never main). It plays the given word ranges (indices from timeline_transcript on the source timeline, default main) with a little padding, cut from the source\'s clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical), look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. ' + + "make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as \"best-1\" (never main). It plays the given word ranges (read the source timeline's sentences with timeline_transcript and no clipId; the source defaults to main) with a little padding, cut from the source's clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical); the result's warnings name a range that starts or ends mid-sentence. Look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. " + 'Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', inputSchema: { timeline: timelineInput, verbs: z.array(EditableVerbSchema).min(1) }, }, async ({ timeline = MAIN_TIMELINE, verbs }) => answeringTimelineErrors(async () => { - const { doc, ops } = await applyVerbs(verbs, makeVerbContext(), timeline); + const { doc, ops, warnings } = await applyVerbs(verbs, makeVerbContext(), timeline); return jsonResult({ applied: ops.length, + ...(warnings.length > 0 ? { warnings } : {}), document: await describeComposition(doc, timeline), }); }), @@ -235,21 +236,50 @@ export function buildMcpServer(): McpServer { server.registerTool( 'timeline_transcript', { - title: 'Transcribe a clip', + title: 'Transcribe the speech', description: - "Transcribe a media clip's speech with the local whisper.cpp and return its words with timeline times. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript. A word's `index` is stable for its source file.", + "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short (or cut_words). " + + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + + "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { timeline: timelineInput, - clipId: z.string().describe('Media clip to transcribe.'), + clipId: z + .string() + .optional() + .describe("Media clip to transcribe. Leave out for the whole timeline's sentences."), + view: z + .enum(['words', 'sentences']) + .optional() + .describe('words (default with a clipId) or sentences (default without one).'), + fromSec: z + .number() + .nonnegative() + .optional() + .describe("Sentences starting at or after this timeline time (a page's nextFromSec)."), + toSec: z + .number() + .positive() + .optional() + .describe('Sentences starting before this timeline time.'), language: WhisperLanguageSchema.optional(), }, }, - async ({ timeline = MAIN_TIMELINE, clipId, language }) => + async ({ timeline = MAIN_TIMELINE, clipId, view, fromSec, toSec, language }) => answeringTimelineErrors(async () => { const comp = await readComposition(timeline); - return jsonResult( - await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), - ); + const media = await confinedMediaMap(comp); + if ((view ?? (clipId ? 'words' : 'sentences')) === 'words') { + if (!clipId) { + return jsonResult({ + error: "The words view needs a clipId. Leave view out for the timeline's sentences.", + }); + } + return jsonResult(await transcribeClip(comp, clipId, media, { language })); + } + const outline = await timelineOutline(comp, media, { clipId, fromSec, toSec, language }); + // Compact, as the page size is counted: a long recording's outline is + // what has to fit in one result. + return { content: [{ type: 'text' as const, text: JSON.stringify(outline) }] }; }), ); diff --git a/src/timeline/captions.ts b/src/timeline/captions.ts index b16c45b..7c59c62 100644 --- a/src/timeline/captions.ts +++ b/src/timeline/captions.ts @@ -24,8 +24,8 @@ export const DEFAULT_CAPTION_WORDS = 7; /** Widest caption, in Latin characters (a CJK character counts as two). */ const MAX_CAPTION_WIDTH = 32; -/** A pause this long between words starts a new caption. */ -const PAUSE_SEC = 0.7; +/** A pause this long between words starts a new caption, or a new sentence. */ +export const PAUSE_SEC = 0.7; /** How long a caption stays up after its last word, unless the next one starts. */ const HOLD_SEC = 0.4; /** Shortest caption shown. */ @@ -270,7 +270,9 @@ export function joinWords(texts: string[]): string { return out; } -function endsSentence(text: string): boolean { +/** Whether a word ends a sentence: it ends in . ! ? … (or 。!?), maybe + * followed by a closing quote or bracket. */ +export function endsSentence(text: string): boolean { return /[.!?。!?…]["')\]」』]?$/u.test(text); } diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index d149aa2..2eb18b4 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -354,9 +354,14 @@ export async function applyVerbs( verbs: CompositionVerb[], ctx: VerbContext, timeline: string = MAIN_TIMELINE, -): Promise<{ doc: Composition; ops: CompositionOp[] }> { +): Promise<{ doc: Composition; ops: CompositionOp[]; warnings: string[] }> { const create = verbs[0]?.verb === 'make_short'; - const editCtx: VerbContext = { ...ctx, timeline }; + let warnings: string[] = []; + const editCtx: VerbContext = { + ...ctx, + timeline, + warn: (message) => warnings.push(message), + }; // The default append point (`trackEnd`) is baked from the snapshot we lower // against, so if another in-process writer commits between the read and the // apply, re-lower against the fresh doc rather than land overlapping clips. A @@ -366,6 +371,7 @@ export async function applyVerbs( // tail silently losing to a `CompositionConflictError`. for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { const current = await readForEdit(timeline, create); + warnings = []; const ops = await lowerVerbs(current, verbs, editCtx); try { const doc = await mutateComposition(ops, { @@ -373,7 +379,7 @@ export async function applyVerbs( timeline, create, }); - return { doc, ops }; + return { doc, ops, warnings }; } catch (err) { if (err instanceof CompositionConflictError && attempt < MAX_COMMIT_ATTEMPTS) continue; throw err; diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index febf983..e6d3c16 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -2,7 +2,9 @@ import { DEFAULT_CAPTION_TRACK, DEFAULT_CAPTION_WORDS, defaultCaptionStyle, + endsSentence, groupCaptions, + PAUSE_SEC, } from './captions.js'; import { speedFactor, videoLayers } from './compile.js'; import { @@ -281,7 +283,7 @@ export async function planShort( })), durationSec: roundMs(at), captions, - warnings: [], + warnings: windows.flatMap((window, i) => sentenceEdgeWarnings(window, i + 1)), }; } @@ -321,6 +323,40 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): }; } +/** + * Where a range starts or ends inside a sentence: right after a word that + * doesn't end one (with no real pause between), or right before the next word + * of the same sentence. Named by the words on either side; the short is still + * made, since a cut mid-sentence can be meant. + */ +function sentenceEdgeWarnings(window: WordWindow, rangeNumber: number): string[] { + const words = window.transcript.words; + const warnings: string[] = []; + const midSentence = ( + a: (typeof words)[number] | undefined, + b: (typeof words)[number] | undefined, + ) => + a !== undefined && + b !== undefined && + !endsSentence(a.text) && + b.startSec - a.endSec < PAUSE_SEC; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + if (midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber} starts mid-sentence, between "${before?.text}" and "${first?.text}" (word ${window.fromIndex}).`, + ); + } + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber} ends mid-sentence, between "${last?.text}" and "${after?.text}" (word ${window.toIndex}).`, + ); + } + return warnings; +} + /** * A window that picks up later in the same file than the one before it starts * where that one's padding ended, so two ranges of neighboring words play diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 5452d55..b2dba11 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -13,7 +13,8 @@ import { WhisperLanguageSchema, } from '../transcribe/whisper.js'; import { getWorkspace, resolveInWorkspace } from '../workspace.js'; -import { checkExportable, type MediaInfo, withoutTextOverlays } from './compile.js'; +import { endsSentence, joinWords, PAUSE_SEC } from './captions.js'; +import { checkExportable, type MediaInfo, videoLayers, withoutTextOverlays } from './compile.js'; import { type Composition, findClip, type MediaClip, makeClipId } from './composition.js'; import { applyOp, type CompositionOp, type SourceRange } from './ops.js'; import { type MediaId, MediaIdSchema } from './schema.js'; @@ -305,6 +306,170 @@ export async function transcribeClip( }; } +/** Longest sentence an outline line holds; whisper sometimes leaves out the + * punctuation, and a line has to end somewhere. */ +const MAX_SENTENCE_WORDS = 40; +/** Default size of one outline page, in bytes of compact JSON: about 10k + * tokens, which an MCP client takes in one result. */ +export const OUTLINE_PAGE_BYTES = 40_000; + +/** A word in timeline order, with the media clip it plays in. */ +export interface OutlineWord extends ClipWord { + clipId: string; +} + +/** One sentence of the timeline's speech. `fromIndex..toIndex` are word indices + * into its source's transcript, as `cut_words` and `make_short` take them, and + * `clipId` is the clip its first word plays in. */ +export interface OutlineSentence { + startSec: number; + endSec: number; + clipId: string; + fromIndex: number; + toIndex: number; + text: string; +} + +export interface TimelineOutline { + sentences: OutlineSentence[]; + /** The `transcriptId` of each source the sentences come from. */ + transcriptIds: Record; + /** Where the next page starts (pass it as `fromSec`); absent on the last page. */ + nextFromSec?: number; +} + +/** + * Group words (one source, timeline order) into sentences: a sentence ends at + * sentence-ending punctuation, before a pause longer than `PAUSE_SEC`, after + * `MAX_SENTENCE_WORDS` words, or where the timeline goes back in the source. + */ +export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { + const groups: OutlineWord[][] = []; + let current: OutlineWord[] = []; + for (const word of words) { + const prev = current[current.length - 1]; + if ( + prev && + (endsSentence(prev.text) || + word.startSec - prev.endSec > PAUSE_SEC || + word.index < prev.index || + current.length >= MAX_SENTENCE_WORDS) + ) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + return groups.flatMap((group) => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return []; + return [ + { + startSec: first.startSec, + endSec: last.endSec, + clipId: first.clipId, + fromIndex: first.index, + toIndex: last.index, + text: joinWords(group.map((w) => w.text)), + }, + ]; + }); +} + +export interface OutlineOptions extends TranscribeOptions { + /** Only this media clip's words. */ + clipId?: string; + /** Sentences starting at or after this timeline time. */ + fromSec?: number; + /** Sentences starting before this timeline time. */ + toSec?: number; + /** Page size in bytes of compact JSON (default `OUTLINE_PAGE_BYTES`). */ + maxBytes?: number; + /** How a source's transcript is had (default: `transcribeMedia`, cached). */ + transcribe?: (mediaId: MediaId, info: MediaInfo) => Promise; +} + +/** + * The timeline's speech as sentences, in timeline order: every media clip on + * the picture track, each source transcribed once (or read from its stored + * transcript). A sentence runs on across cuts between clips of the same source + * (a silence cut splits a clip mid-sentence), and a word a cut splits in two is + * listed once. A page holds at most `maxBytes` of compact JSON; `nextFromSec` + * says where the next one starts. + */ +export async function timelineOutline( + comp: Composition, + media: Map, + opts: OutlineOptions = {}, +): Promise { + const transcribe = opts.transcribe ?? ((mediaId, info) => transcribeMedia(mediaId, info, opts)); + const picture = videoLayers(comp).picture; + const clips = (picture?.clips ?? []) + .filter((c): c is MediaClip => c.kind === 'media') + .filter((c) => opts.clipId === undefined || c.id === opts.clipId) + .sort((a, b) => a.startSec - b.startSec); + if (opts.clipId !== undefined && clips.length === 0) { + // The clip isn't a media clip on the picture track: say why, as a lookup would. + locateMediaClip(comp, opts.clipId, media); + } + + const transcripts = new Map(); + const sentences: OutlineSentence[] = []; + let run: OutlineWord[] = []; + let runMedia: MediaId | undefined; + for (const clip of clips) { + let transcript = transcripts.get(clip.mediaId); + if (transcript === undefined) { + const info = media.get(clip.mediaId); + if (!info) { + throw new TranscriptError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + transcript = info.hasAudio ? await transcribe(clip.mediaId, info) : null; + transcripts.set(clip.mediaId, transcript); + } + if (!transcript) continue; + if (clip.mediaId !== runMedia) { + sentences.push(...outlineSentences(run)); + run = []; + runMedia = clip.mediaId; + } + for (const word of clipWords(transcript, clip)) { + // A word a cut runs through shows in the clips on both sides of it: one + // word, from its start in the first to its end in the last. + const last = run[run.length - 1]; + if (last?.index === word.index) run[run.length - 1] = { ...last, endSec: word.endSec }; + else run.push({ ...word, clipId: clip.id }); + } + } + sentences.push(...outlineSentences(run)); + + const transcriptIds: Record = {}; + for (const [mediaId, transcript] of transcripts) { + if (transcript) transcriptIds[mediaId] = transcriptIdOf(transcript); + } + const fromSec = opts.fromSec ?? 0; + const toSec = opts.toSec ?? Number.POSITIVE_INFINITY; + const inRange = sentences.filter((s) => s.startSec >= fromSec && s.startSec < toSec); + + const maxBytes = opts.maxBytes ?? OUTLINE_PAGE_BYTES; + // The page's own fields, with room for the largest nextFromSec. + let bytes = Buffer.byteLength( + JSON.stringify({ sentences: [], transcriptIds, nextFromSec: 99999.999 }), + ); + const page: OutlineSentence[] = []; + for (const sentence of inRange) { + const size = Buffer.byteLength(JSON.stringify(sentence)) + 1; + if (bytes + size > maxBytes && page.length > 0) { + return { sentences: page, transcriptIds, nextFromSec: sentence.startSec }; + } + page.push(sentence); + bytes += size; + } + return { sentences: page, transcriptIds }; +} + /** An inclusive range of transcript word indices. */ export interface WordRange { fromIndex: number; diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index f9167c0..4e8a308 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -305,7 +305,7 @@ const EDITABLE_VERB_SCHEMAS = [ .min(1) .max(20) .describe( - 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences; 15–60 s in all is typical.', + 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences from the timeline_transcript sentence outline; 15–60 s in all is typical. A range that starts or ends mid-sentence is made anyway, with a warning.', ), source: TimelineIdSchema.optional().describe( 'Timeline to cut from (default "main"). Its silence and retake cuts carry into the short.', @@ -403,6 +403,9 @@ export interface VerbContext { defaultTrack: string; /** The timeline being edited (default `main`); `applyVerbs` sets it. */ timeline?: string; + /** Hears what a verb did that the caller may not have meant (e.g. a short + * cut mid-sentence); `applyVerbs` collects them. */ + warn?: (message: string) => void; } const DEFAULT_TRACK = 'v0'; @@ -581,6 +584,7 @@ export async function lowerVerb( }, ctx.transcript, ); + for (const warning of plan.warnings) ctx.warn?.(warning); return plan.ops; } case 'transition': diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index 7fc7271..e9665fa 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -228,7 +228,7 @@ describe('MCP server', () => { describe('MCP shorts (named timelines)', () => { /** Main holds a 30 s registered recording with a stored transcript: nothing renders. */ - async function recordingOnMain(): Promise { + async function recordingOnMain(): Promise<{ mediaId: string }> { const source = join(workspace, 'talk.mp4'); await writeFile(source, 'video bytes'); const mediaId = makeMediaId(source); @@ -267,6 +267,7 @@ describe('MCP shorts (named timelines)', () => { clip: mediaClip({ id: 'c1', mediaId, sourceOutSec: 30, startSec: 0 }), }, ]); + return { mediaId }; } it('makes a short on its own timeline with timeline_edit, leaving main alone', async () => { @@ -289,6 +290,10 @@ describe('MCP shorts (named timelines)', () => { }, }), ); + expect(edit.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "word1" and "word2" (word 2).', + 'Range 1 ends mid-sentence, between "word6" and "word7" (word 6).', + ]); const doc = edit.document as Record; expect(doc.timeline).toBe('clip-1'); expect(doc.canvas).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); @@ -315,6 +320,58 @@ describe('MCP shorts (named timelines)', () => { } }); + it("outlines the timeline's sentences without a clipId, compactly", async () => { + const { mediaId } = await recordingOnMain(); + // A configured model the stored transcript matches, so nothing is transcribed. + const model = join(workspace, 'ggml-base.en.bin'); + await writeFile(model, 'model bytes'); + const m = await stat(model); + const transcriptFile = join(workspace, 'transcripts', `${mediaId}.json`); + const stored = JSON.parse(await readFile(transcriptFile, 'utf8')) as Transcript; + await writeFile( + transcriptFile, + JSON.stringify({ ...stored, modelKey: `${model}:${m.size}:${m.mtimeMs}` }), + ); + const env = { + model: process.env.MAKEMYCLIP_WHISPER_MODEL, + path: process.env.MAKEMYCLIP_WHISPER_PATH, + }; + process.env.MAKEMYCLIP_WHISPER_MODEL = model; + process.env.MAKEMYCLIP_WHISPER_PATH = join(workspace, 'no-whisper'); + const { client, close } = await connect(); + try { + const result = (await client.callTool({ name: 'timeline_transcript', arguments: {} })) as { + content: { type: string; text: string }[]; + }; + const text = result.content[0]?.text ?? ''; + expect(text).not.toContain('\n'); + const outline = JSON.parse(text) as { + sentences: { clipId: string; fromIndex: number; toIndex: number; text: string }[]; + transcriptIds: Record; + nextFromSec?: number; + }; + expect(outline.sentences).toEqual([ + expect.objectContaining({ clipId: 'c1', fromIndex: 0, toIndex: 19 }), + ]); + expect(Object.values(outline.transcriptIds)).toHaveLength(1); + expect(outline.nextFromSec).toBeUndefined(); + + const noClip = payload( + await client.callTool({ name: 'timeline_transcript', arguments: { view: 'words' } }), + ); + expect(noClip.error).toMatch(/needs a clipId/); + } finally { + await close(); + for (const [key, value] of [ + ['MAKEMYCLIP_WHISPER_MODEL', env.model], + ['MAKEMYCLIP_WHISPER_PATH', env.path], + ] as const) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } + }); + it('answers a missing timeline or a short on main with an error', async () => { await recordingOnMain(); const { client, close } = await connect(); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts new file mode 100644 index 0000000..81a99e9 --- /dev/null +++ b/tests/timeline-outline.test.ts @@ -0,0 +1,240 @@ +import { describe, expect, it } from 'vitest'; +import type { MediaInfo } from '../src/timeline/compile.js'; +import { type Composition, emptyComposition, type MediaClip } from '../src/timeline/composition.js'; +import { + applyOps, + type CompositionOp, + mediaClip, + textClip, + videoTrack, +} from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; +import { + clipWords, + type OutlineWord, + outlineSentences, + type Transcript, + TranscriptError, + timelineOutline, +} from '../src/timeline/transcript.js'; + +const M = 'm_aaaaaaaaaaaa' as MediaId; +const N = 'm_bbbbbbbbbbbb' as MediaId; +const O = 'm_cccccccccccc' as MediaId; +const MEDIA = new Map([ + [M, { path: '/talk.mp4', hasAudio: true }], + [N, { path: '/broll.mp4', hasAudio: false }], + [O, { path: '/guest.mp4', hasAudio: true }], +]); + +function transcriptOf(mediaId: MediaId, words: Transcript['words']): Transcript { + return { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-base.en.bin', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: 0, + sourceMtimeMs: 0, + words, + }; +} + +/** Abutting words, as whisper.cpp gives them: 0.4 s each from 0. */ +function spoken(texts: string[]): Transcript['words'] { + return texts.map((text, i) => ({ + text, + startSec: Math.round(i * 0.4 * 1000) / 1000, + endSec: Math.round((i + 1) * 0.4 * 1000) / 1000, + })); +} + +function timelineOf(clips: MediaClip[]): Composition { + return applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ...clips.map((clip): CompositionOp => ({ op: 'addClip', trackId: 'v0', clip })), + ]); +} + +const transcribeFrom = + (...transcripts: Transcript[]) => + async (mediaId: MediaId): Promise => { + const found = transcripts.find((t) => t.mediaId === mediaId); + if (!found) throw new TranscriptError(`no transcript for ${mediaId}`); + return found; + }; + +const ms = (sec: number): number => Math.round(sec * 1000) / 1000; +const word = (index: number, text: string, startSec: number, endSec: number): OutlineWord => ({ + index, + text, + startSec: ms(startSec), + endSec: ms(endSec), + clipId: 'c1', +}); + +describe('outlineSentences', () => { + it('ends a sentence at its punctuation', () => { + const words = ['Hello', 'there.', 'How', 'are', 'you?', 'Fine'].map((t, i) => + word(i, t, i * 0.4, (i + 1) * 0.4), + ); + expect(outlineSentences(words)).toEqual([ + { startSec: 0, endSec: 0.8, clipId: 'c1', fromIndex: 0, toIndex: 1, text: 'Hello there.' }, + { startSec: 0.8, endSec: 2, clipId: 'c1', fromIndex: 2, toIndex: 4, text: 'How are you?' }, + { startSec: 2, endSec: 2.4, clipId: 'c1', fromIndex: 5, toIndex: 5, text: 'Fine' }, + ]); + }); + + it('ends one at a long pause, after 40 words, and where the source goes back', () => { + const paused = [word(0, 'so', 0, 0.3), word(1, 'anyway', 1.2, 1.5)]; + expect(outlineSentences(paused).map((s) => s.text)).toEqual(['so', 'anyway']); + + const run = Array.from({ length: 45 }, (_, i) => word(i, 'and', i * 0.3, (i + 1) * 0.3)); + expect(outlineSentences(run).map((s) => [s.fromIndex, s.toIndex])).toEqual([ + [0, 39], + [40, 44], + ]); + + const reordered = [word(10, 'later', 0, 0.4), word(2, 'earlier', 0.4, 0.8)]; + expect(outlineSentences(reordered)).toHaveLength(2); + }); +}); + +describe('timelineOutline', () => { + const talk = transcriptOf( + M, + spoken(['First', 'point', 'here.', 'Second', 'point', 'runs', 'on', 'long.', 'Third', 'one.']), + ); + + it('keeps word indices across silence-cut sub-clips, listing a split word once', async () => { + // Cuts at 1.0–1.1 (through "here.", 0.8–1.2) and 2.7–3.0 (before "long.", 2.8–3.2). + const comp = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 1, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.1, sourceOutSec: 2.7, startSec: 1 }), + mediaClip({ id: 'c3', mediaId: M, sourceInSec: 3, sourceOutSec: 4, startSec: 2.6 }), + ]); + const outline = await timelineOutline(comp, MEDIA, { transcribe: transcribeFrom(talk) }); + expect(outline.sentences).toEqual([ + { + startSec: 0, + endSec: 1.1, + clipId: 'c1', + fromIndex: 0, + toIndex: 2, + text: 'First point here.', + }, + { + startSec: 1.1, + endSec: 2.8, + clipId: 'c2', + fromIndex: 3, + toIndex: 7, + text: 'Second point runs on long.', + }, + { startSec: 2.8, endSec: 3.6, clipId: 'c3', fromIndex: 8, toIndex: 9, text: 'Third one.' }, + ]); + // The same indices and times clipWords gives each clip. + const placed = comp.tracks[0]?.clips.flatMap((c) => + c.kind === 'media' ? clipWords(talk, c) : [], + ); + for (const s of outline.sentences) { + expect(placed?.find((w) => w.index === s.fromIndex)?.startSec).toBe(s.startSec); + expect(placed?.findLast((w) => w.index === s.toIndex)?.endSec).toBe(s.endSec); + } + expect(Object.keys(outline.transcriptIds)).toEqual([M]); + expect(outline.nextFromSec).toBeUndefined(); + }); + + it('reads each source once, skips one without sound, and breaks sentences between sources', async () => { + const guest = transcriptOf(O, spoken(['Hello', 'world'])); + const calls: MediaId[] = []; + const comp = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'o1', mediaId: O, sourceOutSec: 0.8, startSec: 0.8 }), + mediaClip({ id: 'b1', mediaId: N, sourceOutSec: 2, startSec: 1.6 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.2, sourceOutSec: 2, startSec: 3.6 }), + ]); + const outline = await timelineOutline(comp, MEDIA, { + transcribe: async (mediaId) => { + calls.push(mediaId); + return mediaId === O ? guest : talk; + }, + }); + expect(calls).toEqual([M, O]); + expect(outline.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex, s.text])).toEqual([ + ['c1', 0, 1, 'First point'], + ['o1', 0, 1, 'Hello world'], + ['c2', 3, 4, 'Second point'], + ]); + expect(Object.keys(outline.transcriptIds)).toEqual([M, O]); + }); + + it('pages by size and range, so pages put back together give every sentence once', async () => { + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 4, startSec: 0 })]); + const transcribe = transcribeFrom(talk); + const all = (await timelineOutline(comp, MEDIA, { transcribe })).sentences; + + const pages: string[] = []; + const seen = []; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec, maxBytes: 250 }); + pages.push(JSON.stringify(page)); + seen.push(...page.sentences); + fromSec = page.nextFromSec; + } + expect(pages.length).toBeGreaterThan(1); + for (const page of pages) expect(Buffer.byteLength(page)).toBeLessThanOrEqual(250); + expect(seen).toEqual(all); + + const middle = await timelineOutline(comp, MEDIA, { transcribe, fromSec: 1, toSec: 3 }); + expect(middle.sentences.map((s) => s.fromIndex)).toEqual([3]); + }); + + it('pages a 60-minute, 9,000-word recording at 40 KB', async () => { + const texts = Array.from({ length: 9000 }, (_, i) => + i % 14 === 13 ? `sentence${i}.` : `word${i}`, + ); + const hour = transcriptOf(M, spoken(texts)); + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 3600, startSec: 0 })]); + const transcribe = transcribeFrom(hour); + + let count = 0; + let pages = 0; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec }); + expect(Buffer.byteLength(JSON.stringify(page))).toBeLessThanOrEqual(40_000); + count += page.sentences.length; + pages++; + fromSec = page.nextFromSec; + } + expect(count).toBe(Math.ceil(9000 / 14)); + expect(pages).toBeGreaterThan(1); + }); + + it("outlines one clip, and says why a clip that isn't media can't be", async () => { + const comp = applyOps( + timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 1, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 1.2, sourceOutSec: 4, startSec: 1 }), + ]), + [ + { op: 'addTrack', track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip', + trackId: 'captions', + clip: textClip({ id: 't1', text: 'hi', startSec: 0, durationSec: 1 }), + }, + ], + ); + const transcribe = transcribeFrom(talk); + const one = await timelineOutline(comp, MEDIA, { transcribe, clipId: 'c1' }); + expect(one.sentences.map((s) => s.clipId)).toEqual(['c1']); + await expect(timelineOutline(comp, MEDIA, { transcribe, clipId: 't1' })).rejects.toThrow( + /text clip/, + ); + }); +}); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 0a110ae..b315fa8 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -366,7 +366,46 @@ describe('planShort: the edit', () => { }, ]); expect(short.captions).toBe(1); - expect(short.warnings).toEqual([]); + expect(short.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "w1" and "w2" (word 2).', + 'Range 1 ends mid-sentence, between "w4" and "w5" (word 4).', + ]); + }); + + it('warns only where a range cuts into a sentence', async () => { + // "One two. Three four five. Six" with a long pause before "Six". + const said = { + ...transcriptOf(M, [ + [1, 1.4], + [1.5, 1.9], + [2.2, 2.6], + [2.7, 3.1], + [3.2, 3.6], + [4.6, 5], + ]), + }; + said.words = said.words.map((w, i) => ({ + ...w, + text: ['One', 'two.', 'Three', 'four', 'five.', 'Six'][i] ?? w.text, + })); + const warnings = async (fromIndex: number, toIndex: number): Promise => + ( + await plan( + sourceOf([clipOf('c1', 0, 30, 0)]), + { ranges: [{ clipId: 'c1', fromIndex, toIndex }] }, + { transcripts: [said] }, + ) + ).warnings; + expect(await warnings(2, 4)).toEqual([]); // a whole sentence + expect(await warnings(0, 5)).toEqual([]); // the recording's edges + expect(await warnings(3, 4)).toEqual([ + 'Range 1 starts mid-sentence, between "Three" and "four" (word 3).', + ]); + expect(await warnings(2, 3)).toEqual([ + 'Range 1 ends mid-sentence, between "four" and "five." (word 3).', + ]); + // "five." ends the sentence; the 1 s pause before "Six" ends one too. + expect(await warnings(5, 5)).toEqual([]); }); it('captions only the asked-for words, not the word the lead reaches into', async () => { From c7866bfb7c175fcea0e9d20d47457ddb1aba4be0 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:59:44 -0800 Subject: [PATCH 04/14] fix(timeline): seek frames down to the microsecond, keep replayed words The window frame rounded its seek to the millisecond, which could land past a clip's last frame (no image) or its out-point; it now rounds down at microsecond precision. The outline merged a word repeated by consecutive clips as if a cut split it; it merges only when the next clip carries on later in the source, and a word played again starts a new sentence. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 10 +++++-- src/timeline/transcript.ts | 15 ++++++++--- .../integration/timeline-frame-window.test.ts | 9 ++++++- tests/timeline-introspect.test.ts | 5 ++-- tests/timeline-outline.test.ts | 27 +++++++++++++++++++ 5 files changed, 58 insertions(+), 8 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index d3c4d1b..8a732c3 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,10 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +function floorUs(sec: number): number { + return Math.floor(sec * 1e6 + 1e-6) / 1e6; +} + /** The font file for a text clip, with a missing font reported against the clip. * Text over the picture always gets a file: see `ffmpeg/font.ts`. */ function fontFor(ctx: CompileContext, clip: TextClip, always: boolean): string | undefined { @@ -777,14 +781,16 @@ export function buildFrameAtPlan( } // Seconds into the clip's source. The chain's speed `setpts` divides the // offset back into segment time, which is what its fades and text read. - const local = Math.round(clamped * speedFactor(clip.effects) * 1000) / 1000; + // Rounded down, to the microsecond: rounding up could seek past the clip's + // last frame (no image) or past its out-point. + const local = floorUs(clamped * speedFactor(clip.effects)); const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, args: [ '-y', '-ss', - seconds(clip.sourceInSec + local), + String(floorUs(clip.sourceInSec + local)), '-i', info.path, '-filter_complex', diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index b2dba11..f9c20bd 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -352,7 +352,7 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { prev && (endsSentence(prev.text) || word.startSec - prev.endSec > PAUSE_SEC || - word.index < prev.index || + word.index <= prev.index || current.length >= MAX_SENTENCE_WORDS) ) { groups.push(current); @@ -419,6 +419,7 @@ export async function timelineOutline( const sentences: OutlineSentence[] = []; let run: OutlineWord[] = []; let runMedia: MediaId | undefined; + let prevClip: MediaClip | undefined; for (const clip of clips) { let transcript = transcripts.get(clip.mediaId); if (transcript === undefined) { @@ -435,13 +436,21 @@ export async function timelineOutline( run = []; runMedia = clip.mediaId; } + // A clip that carries on later in the source than the one before it (a cut + // between them), rather than playing part of it again. + const continues = + prevClip?.mediaId === clip.mediaId && clip.sourceInSec >= prevClip.sourceOutSec - 1e-6; for (const word of clipWords(transcript, clip)) { // A word a cut runs through shows in the clips on both sides of it: one // word, from its start in the first to its end in the last. const last = run[run.length - 1]; - if (last?.index === word.index) run[run.length - 1] = { ...last, endSec: word.endSec }; - else run.push({ ...word, clipId: clip.id }); + if (continues && last?.index === word.index) { + run[run.length - 1] = { ...last, endSec: word.endSec }; + } else { + run.push({ ...word, clipId: clip.id }); + } } + prevClip = clip; } sentences.push(...outlineSentences(run)); diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 7c54764..e7d139a 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -140,6 +140,13 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); } + + // The very end of a clip that ends with its source: the seek must not pass + // the last frame, or there is no image. + const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); + const last = join(workspace, 'last.jpg'); + await runPlan(buildFrameAtPlan(whole, { media, dir: workspace, output: last }, 19.999)); + expect((await stat(last)).size).toBeGreaterThan(0); }, 180_000); it('renders a frame near the end of a 10-minute clip in under a second', async () => { diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index c89df21..6d6a872 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -110,8 +110,9 @@ describe('buildFrameAtPlan', () => { ]); const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.967']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.967\/TB,/); + // Rounded down, never past that frame or the clip's out-point. + expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.966666\/TB,/); }); it('frames the picture the way export does (fit, zoom, position)', () => { diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 81a99e9..9337355 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -147,6 +147,33 @@ describe('timelineOutline', () => { expect(outline.nextFromSec).toBeUndefined(); }); + it('keeps a word the timeline plays twice as two, where a clip replays the source', async () => { + const hello = transcriptOf(M, spoken(['Hello.', 'and', 'so'])); + const transcribe = transcribeFrom(hello); + const twice = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 0.4, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceOutSec: 0.4, startSec: 0.4 }), + ]); + expect( + (await timelineOutline(twice, MEDIA, { transcribe })).sentences.map((s) => [ + s.clipId, + s.startSec, + s.endSec, + ]), + ).toEqual([ + ['c1', 0, 0.4], + ['c2', 0.4, 0.8], + ]); + + const andAgain = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0.4, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 0.4, sourceOutSec: 1.2, startSec: 0.4 }), + ]); + expect( + (await timelineOutline(andAgain, MEDIA, { transcribe })).sentences.map((s) => s.text), + ).toEqual(['and', 'and so']); + }); + it('reads each source once, skips one without sound, and breaks sentences between sources', async () => { const guest = transcriptOf(O, spoken(['Hello', 'world'])); const calls: MediaId[] = []; From 390ad0cca19079d72471e09d8941fcce6c4df094 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:08:53 -0800 Subject: [PATCH 05/14] fix(timeline): read a frame preview from a second before the frame A source slower than the canvas shows a frame export repeats into later slots, and seeking straight to the frame skipped it (at a clip's end, no image at all). The window now reads from a second earlier and trims to the first frame at the frame's segment time, timestamped from the seek point. An outline of an explicit clip off the picture track lists its sentences, and the MCP outline no longer suggests cut_words for sentences that can span clips. Co-Authored-By: Claude Opus 5.5 --- src/mcp/server.ts | 2 +- src/timeline/compile.ts | 21 ++++++++++----- src/timeline/transcript.ts | 15 +++++------ .../integration/timeline-frame-window.test.ts | 27 ++++++++++++++++--- tests/timeline-introspect.test.ts | 27 ++++++++++++------- tests/timeline-outline.test.ts | 11 ++++++++ 6 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 5d95c6a..bd4f7f0 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -238,7 +238,7 @@ export function buildMcpServer(): McpServer { { title: 'Transcribe the speech', description: - "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short (or cut_words). " + + "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short. " + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 8a732c3..04e9233 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,9 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +/** How much of the source a frame preview reads before the frame. */ +const FRAME_PREROLL_SEC = 1; + function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -779,22 +782,28 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Seconds into the clip's source. The chain's speed `setpts` divides the - // offset back into segment time, which is what its fades and text read. - // Rounded down, to the microsecond: rounding up could seek past the clip's - // last frame (no image) or past its out-point. + // Where the window starts, in seconds into the clip's source: a little + // before the frame, so a source frame that export repeats into it (a source + // slower than the canvas) is still read. An input seek starts timestamps at + // the seek point, so adding `from` gives source time even when the first + // frame read comes after it. The chain's speed `setpts` divides + // the offset back into segment time, which its fades and text read, and + // `trim` then takes the first frame at the frame's segment time, as reading + // it out of the encoded segment would. Rounded down, to the microsecond, so + // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); + const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, args: [ '-y', '-ss', - String(floorUs(clip.sourceInSec + local)), + String(floorUs(clip.sourceInSec + from)), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS-STARTPTS+${local}/TB,${chain}[v]`, + `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${floorUs(clamped)}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index f9c20bd..3e22404 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -405,15 +405,12 @@ export async function timelineOutline( opts: OutlineOptions = {}, ): Promise { const transcribe = opts.transcribe ?? ((mediaId, info) => transcribeMedia(mediaId, info, opts)); - const picture = videoLayers(comp).picture; - const clips = (picture?.clips ?? []) - .filter((c): c is MediaClip => c.kind === 'media') - .filter((c) => opts.clipId === undefined || c.id === opts.clipId) - .sort((a, b) => a.startSec - b.startSec); - if (opts.clipId !== undefined && clips.length === 0) { - // The clip isn't a media clip on the picture track: say why, as a lookup would. - locateMediaClip(comp, opts.clipId, media); - } + const clips = + opts.clipId !== undefined + ? [locateMediaClip(comp, opts.clipId, media).clip] + : (videoLayers(comp).picture?.clips ?? []) + .filter((c): c is MediaClip => c.kind === 'media') + .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); const sentences: OutlineSentence[] = []; diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index e7d139a..2131c02 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -30,11 +30,11 @@ afterEach(async () => { await rm(workspace, { recursive: true, force: true }); }); -/** 720p30 test pattern (its counter and gradient move every frame) with a 2 s GOP. */ -async function testPattern(path: string, seconds: number): Promise { +/** 720p test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +async function testPattern(path: string, seconds: number, rate = 30): Promise { await runFfmpeg([ ...['-hide_banner', '-loglevel', 'error'], - ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=30:duration=${seconds}`], + ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=${rate}:duration=${seconds}`], ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], ...['-c:a', 'aac', '-shortest', path], @@ -141,6 +141,27 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), name).toBeGreaterThanOrEqual(40); } + // A 24 fps source on the 30 fps canvas: export repeats some source frames, + // and the preview shows the one export plays, up to the clip's very end. + const slow = join(workspace, 'slow.mp4'); + await testPattern(slow, 4, 24); + const slowMedia = new Map([[M, { path: slow, hasAudio: true }]]); + const slowDoc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 4, startSec: 0 })); + const slowExport = join(workspace, 'slow-export.mp4'); + await runPlan( + compileTimeline(slowDoc, { media: slowMedia, dir: workspace, output: slowExport }), + ); + for (const atSec of [1.5, 3.999]) { + const full = join(workspace, `slow-${atSec}-full.jpg`); + const exportSec = Math.min(atSec, 4 - 1 / 30); + await runFfmpeg(buildPreviewFrameArgs({ input: slowExport, output: full, atSec: exportSec })); + const windowFrame = join(workspace, `slow-${atSec}-window.jpg`); + await runPlan( + buildFrameAtPlan(slowDoc, { media: slowMedia, dir: workspace, output: windowFrame }, atSec), + ); + expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); + } + // The very end of a clip that ends with its source: the seek must not pass // the last frame, or there is no image. const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 6d6a872..a0272cc 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,7 +82,8 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '3', '-i', '/in.mp4']); + // Read from 1 s before the frame, then take the first frame at 3 s. + expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '2', '-i', '/in.mp4']); expect(frame?.args.slice(-8)).toEqual([ '-map', '[v]', @@ -96,7 +97,9 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS-STARTPTS+3/TB,${segmentChain}[v]`); + expect(graphOf(frame?.args)).toBe( + `[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`, + ); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { @@ -111,8 +114,9 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '107.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+7\.966666\/TB,/); + expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); + expect(graphOf(near?.args)).toMatch(/,trim=start=7\.966666\[v\]$/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -155,8 +159,9 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '3']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+2\/TB,/); + expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '2']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+1\/TB,/); + expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); it('maps doc-local time to the post-speed segment timebase for a speed clip', () => { @@ -175,13 +180,15 @@ describe('buildFrameAtPlan', () => { }), }, ]); - // The frame shows source second 4; the chain's setpts=PTS/2 turns the +4 s - // offset into segment time 2, which the fades and text windows read. + // The frame shows source second 4. Reading starts a second earlier, at 3; + // the chain's setpts=PTS/2 turns source time into segment time, which the + // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(1, 3)).toEqual(['-ss', '4']); + expect(frame?.args.slice(1, 3)).toEqual(['-ss', '3']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS-STARTPTS\+4\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+3\/TB,/); expect(graph).toContain('setpts=PTS/2'); + expect(graph).toMatch(/,trim=start=2\[v\]$/); }); it('works on a text clip (no media required)', () => { diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 9337355..ed104fb 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -260,6 +260,17 @@ describe('timelineOutline', () => { const transcribe = transcribeFrom(talk); const one = await timelineOutline(comp, MEDIA, { transcribe, clipId: 'c1' }); expect(one.sentences.map((s) => s.clipId)).toEqual(['c1']); + // A clip off the picture track (here a second media track) is still outlined. + const twoTracks = applyOps(comp, [ + { op: 'addTrack', track: videoTrack({ id: 'v1' }) }, + { + op: 'addClip', + trackId: 'v1', + clip: mediaClip({ id: 'x1', mediaId: M, sourceInSec: 1.2, sourceOutSec: 2, startSec: 0 }), + }, + ]); + const off = await timelineOutline(twoTracks, MEDIA, { transcribe, clipId: 'x1' }); + expect(off.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex])).toEqual([['x1', 3, 4]]); await expect(timelineOutline(comp, MEDIA, { transcribe, clipId: 't1' })).rejects.toThrow( /text clip/, ); From d00188e5b5eaf26ed61591cee91c5d6fd2e939de Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:14:38 -0800 Subject: [PATCH 06/14] style(timeline): format the frame-plan test Co-Authored-By: Claude Opus 5.5 --- tests/timeline-introspect.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index a0272cc..b28a605 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -97,9 +97,7 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`, - ); + expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { From 8d889cc8f0ba56f49dc86073a55c7b7606400dbc Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:26:38 -0800 Subject: [PATCH 07/14] fix(timeline): take the canvas frame at or after an off-grid preview time trim rounds an off-grid start to the nearest frame, so a preview between two frames could show the earlier one where export shows the later. The boundary is now the next canvas frame. The CI-gated test's pattern turns its hue 24 degrees a frame, so a preview off by one frame fails it (21 dB with the old boundary), and it adds a case between two frames. Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 5 ++++- .../integration/timeline-frame-window.test.ts | 18 ++++++++++++++++-- tests/timeline-introspect.test.ts | 8 +++++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 04e9233..f316251 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -793,6 +793,9 @@ export function buildFrameAtPlan( // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); + // The first canvas frame at or after the frame's time: `trim` rounds an + // off-grid start to the nearest frame, which could be the one before. + const firstFrame = Math.ceil(clamped * comp.fps - 1e-6) / comp.fps; const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, @@ -803,7 +806,7 @@ export function buildFrameAtPlan( '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${floorUs(clamped)}[v]`, + `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 2131c02..e69bcd3 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -30,11 +30,17 @@ afterEach(async () => { await rm(workspace, { recursive: true, force: true }); }); -/** 720p test pattern (its counter and gradient move every frame) with a 2 s GOP. */ +/** 720p test pattern with a 2 s GOP. Its hue turns 24 degrees a frame, so + * every frame differs all over and a frame off by one fails the PSNR check. */ async function testPattern(path: string, seconds: number, rate = 30): Promise { await runFfmpeg([ ...['-hide_banner', '-loglevel', 'error'], - ...['-f', 'lavfi', '-i', `testsrc=size=1280x720:rate=${rate}:duration=${seconds}`], + ...[ + '-f', + 'lavfi', + '-i', + `testsrc=size=1280x720:rate=${rate}:duration=${seconds},hue=h=t*${24 * rate}`, + ], ...['-f', 'lavfi', '-i', `sine=frequency=220:sample_rate=48000:duration=${seconds}`], ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], ...['-c:a', 'aac', '-shortest', path], @@ -90,6 +96,14 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' atSec: 3, exportSec: 3, }, + { + name: 'between two frames', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 15, startSec: 0 }), + ), + atSec: 3.01, + exportSec: 3.01, + }, { name: 'inside a fade', doc: vertical( diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index b28a605..e1642a4 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -114,7 +114,7 @@ describe('buildFrameAtPlan', () => { // Rounded down, never past that frame or the clip's out-point. expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); - expect(graphOf(near?.args)).toMatch(/,trim=start=7\.966666\[v\]$/); + expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); it('frames the picture the way export does (fit, zoom, position)', () => { @@ -147,6 +147,12 @@ describe('buildFrameAtPlan', () => { expect(segOut('frame-aaa')).not.toBe(segOut('frame-bbb')); }); + it('takes the first frame at or after a time between frames, as export would', () => { + // 30 fps: 3.01 s falls between the frames at 3.0 and 3.0333. + const graph = graphOf(buildFrameAtPlan(oneClip(), CTX('/ws/f.jpg'), 3.01).steps[0]?.args); + expect(graph).toMatch(/,trim=start=3\.0333333\d*\[v\]$/); + }); + it('selects the second clip and offsets into it for a time in the second clip', () => { const doc = applyOps(oneClip(), [ { From 59c957a4250ac433409bfa5085f2fa9a35668125 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:29:52 -0800 Subject: [PATCH 08/14] fix(timeline): keep a preview within the frames a segment encodes Encoding a clip to its length keeps round(length x fps) frames, so on a clip whose length is off the frame grid the next-frame boundary could pick a frame export leaves out. The boundary now stops at the segment's last frame, with a unit test and a CI-gated case (21 dB without it). Co-Authored-By: Claude Opus 5.5 --- src/timeline/compile.ts | 9 ++++++--- tests/integration/timeline-frame-window.test.ts | 9 +++++++++ tests/timeline-introspect.test.ts | 14 ++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index f316251..ac0ca4a 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -793,9 +793,12 @@ export function buildFrameAtPlan( // the seek never passes the clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); - // The first canvas frame at or after the frame's time: `trim` rounds an - // off-grid start to the nearest frame, which could be the one before. - const firstFrame = Math.ceil(clamped * comp.fps - 1e-6) / comp.fps; + // The first canvas frame at or after the frame's time (`trim` rounds an + // off-grid start to the nearest frame, which could be the one before), but + // no later than the segment's last frame: encoding to the clip's length + // keeps round(length x fps) frames. + const lastIndex = Math.max(0, Math.round(outDur * comp.fps) - 1); + const firstFrame = Math.min(Math.ceil(clamped * comp.fps - 1e-6), lastIndex) / comp.fps; const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { label, diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index e69bcd3..162fd0b 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -104,6 +104,15 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' atSec: 3.01, exportSec: 3.01, }, + { + // 1.01 s encodes 30 frames at 30 fps; the last one is at 29/30 s. + name: 'the last frame of a clip off the frame grid', + doc: vertical( + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 5, sourceOutSec: 6.01, startSec: 0 }), + ), + atSec: 1.009, + exportSec: 29 / 30, + }, { name: 'inside a fade', doc: vertical( diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index e1642a4..1476d44 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -153,6 +153,20 @@ describe('buildFrameAtPlan', () => { expect(graph).toMatch(/,trim=start=3\.0333333\d*\[v\]$/); }); + it("stops at the segment's last frame on a clip whose length is off the frame grid", () => { + // 1.01 s at 30 fps encodes 30 frames, the last at 29/30 s. + const short = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 1.01, startSec: 0 }), + }, + ]); + const graph = graphOf(buildFrameAtPlan(short, CTX('/ws/f.jpg'), 1.009).steps[0]?.args); + expect(graph).toMatch(/,trim=start=0\.9666666\d*\[v\]$/); + }); + it('selects the second clip and offsets into it for a time in the second clip', () => { const doc = applyOps(oneClip(), [ { From 34b7f6704ac55c8a0b862fe87872aefcd8f81431 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:40:51 -0800 Subject: [PATCH 09/14] fix(timeline): keep held frames in previews, tie-safe outline pages A frame preview now keeps every frame from the keyframe before its seek (-noaccurate_seek) instead of a 1 s preroll, so a sparse source, such as a screen recording holding a still, previews the frame export holds (19.5 dB before, now matching). Frames before the clip's in-point are dropped as export's seek drops them. The outline names each sentence's source when the timeline has more than one, so its transcriptId can be found, and a page never splits sentences that start at the same time, so paging always moves on. Co-Authored-By: Claude Opus 5.5 --- src/mcp/server.ts | 2 +- src/timeline/compile.ts | 28 ++++++++--------- src/timeline/transcript.ts | 29 ++++++++++++++---- .../integration/timeline-frame-window.test.ts | 18 +++++++++++ tests/timeline-introspect.test.ts | 29 +++++++++++------- tests/timeline-outline.test.ts | 30 +++++++++++++++++++ 6 files changed, 105 insertions(+), 31 deletions(-) diff --git a/src/mcp/server.ts b/src/mcp/server.ts index bd4f7f0..8ddc8b6 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -240,7 +240,7 @@ export function buildMcpServer(): McpServer { description: "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short. " + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + - "transcriptIds (by source) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", + "transcriptIds (by source; with more than one source each sentence names its mediaId) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { timeline: timelineInput, clipId: z diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index ac0ca4a..2657a68 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,9 +614,6 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } -/** How much of the source a frame preview reads before the frame. */ -const FRAME_PREROLL_SEC = 1; - function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -782,17 +779,17 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Where the window starts, in seconds into the clip's source: a little - // before the frame, so a source frame that export repeats into it (a source - // slower than the canvas) is still read. An input seek starts timestamps at - // the seek point, so adding `from` gives source time even when the first - // frame read comes after it. The chain's speed `setpts` divides - // the offset back into segment time, which its fades and text read, and - // `trim` then takes the first frame at the frame's segment time, as reading - // it out of the encoded segment would. Rounded down, to the microsecond, so - // the seek never passes the clip's last frame or its out-point. + // Seconds into the clip's source. The seek keeps every frame from the + // keyframe before it (`-noaccurate_seek`), so a frame that export repeats + // into this time (a source slower than the canvas, or a screen recording + // holding a still frame) is still read; its timestamps count from the seek + // point, so adding `local` gives source time from the clip's in-point. + // Frames before the in-point are dropped, as export's seek drops them. The + // chain's speed `setpts` turns source time into segment time, which its + // fades and text read, and `trim` takes the frame at the frame's segment + // time. Rounded down, to the microsecond, so the seek never passes the + // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); - const from = floorUs(Math.max(0, local - FRAME_PREROLL_SEC)); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length @@ -804,12 +801,13 @@ export function buildFrameAtPlan( label, args: [ '-y', + '-noaccurate_seek', '-ss', - String(floorUs(clip.sourceInSec + from)), + String(floorUs(clip.sourceInSec + local)), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${from}/TB,${chain},trim=start=${firstFrame}[v]`, + `[0:v]setpts=PTS+${local}/TB,trim=start=0,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 3e22404..686e83d 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -325,6 +325,9 @@ export interface OutlineSentence { startSec: number; endSec: number; clipId: string; + /** The sentence's source, named only when the timeline has more than one, + * so its `transcriptIds` entry can be found. */ + mediaId?: MediaId; fromIndex: number; toIndex: number; text: string; @@ -413,7 +416,7 @@ export async function timelineOutline( .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); - const sentences: OutlineSentence[] = []; + const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; let run: OutlineWord[] = []; let runMedia: MediaId | undefined; let prevClip: MediaClip | undefined; @@ -429,7 +432,7 @@ export async function timelineOutline( } if (!transcript) continue; if (clip.mediaId !== runMedia) { - sentences.push(...outlineSentences(run)); + if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); run = []; runMedia = clip.mediaId; } @@ -449,12 +452,16 @@ export async function timelineOutline( } prevClip = clip; } - sentences.push(...outlineSentences(run)); + if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); const transcriptIds: Record = {}; for (const [mediaId, transcript] of transcripts) { if (transcript) transcriptIds[mediaId] = transcriptIdOf(transcript); } + const named = Object.keys(transcriptIds).length > 1; + const sentences = bySource.flatMap(({ mediaId, sentences }) => + named ? sentences.map((s) => ({ ...s, mediaId })) : sentences, + ); const fromSec = opts.fromSec ?? 0; const toSec = opts.toSec ?? Number.POSITIVE_INFINITY; const inRange = sentences.filter((s) => s.startSec >= fromSec && s.startSec < toSec); @@ -465,10 +472,22 @@ export async function timelineOutline( JSON.stringify({ sentences: [], transcriptIds, nextFromSec: 99999.999 }), ); const page: OutlineSentence[] = []; - for (const sentence of inRange) { + for (const [i, sentence] of inRange.entries()) { const size = Buffer.byteLength(JSON.stringify(sentence)) + 1; if (bytes + size > maxBytes && page.length > 0) { - return { sentences: page, transcriptIds, nextFromSec: sentence.startSec }; + // The next page starts at a time, so sentences that start together stay + // on one page: split before them, or, when they fill this page from its + // first line, keep all of them here. + const tied = sentence.startSec; + const kept = page.filter((s) => s.startSec !== tied); + if (kept.length > 0) return { sentences: kept, transcriptIds, nextFromSec: tied }; + const rest = inRange.slice(i).findIndex((s) => s.startSec !== tied); + if (rest === -1) return { sentences: inRange, transcriptIds }; + return { + sentences: inRange.slice(0, i + rest), + transcriptIds, + nextFromSec: inRange[i + rest]?.startSec, + }; } page.push(sentence); bytes += size; diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 162fd0b..57d4a08 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -185,6 +185,24 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); } + // A sparse source (a frame every 5 s, like a screen recording holding a + // still): at 18 s export still shows the frame from 15 s. + const sparse = join(workspace, 'sparse.mp4'); + await testPattern(sparse, 25, 0.2); + const sparseMedia = new Map([[M, { path: sparse, hasAudio: true }]]); + const sparseDoc = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 25, startSec: 0 })); + const sparseExport = join(workspace, 'sparse-export.mp4'); + await runPlan( + compileTimeline(sparseDoc, { media: sparseMedia, dir: workspace, output: sparseExport }), + ); + const sparseFull = join(workspace, 'sparse-full.jpg'); + await runFfmpeg(buildPreviewFrameArgs({ input: sparseExport, output: sparseFull, atSec: 18 })); + const sparseWindow = join(workspace, 'sparse-window.jpg'); + await runPlan( + buildFrameAtPlan(sparseDoc, { media: sparseMedia, dir: workspace, output: sparseWindow }, 18), + ); + expect(await psnr(sparseFull, sparseWindow), 'sparse source').toBeGreaterThanOrEqual(40); + // The very end of a clip that ends with its source: the seek must not pass // the last frame, or there is no image. const whole = vertical(mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 20, startSec: 0 })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 1476d44..9d08532 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,8 +82,15 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - // Read from 1 s before the frame, then take the first frame at 3 s. - expect(frame?.args.slice(0, 5)).toEqual(['-y', '-ss', '2', '-i', '/in.mp4']); + // Seek to 3 s, keeping the frames from the keyframe before it. + expect(frame?.args.slice(0, 6)).toEqual([ + '-y', + '-noaccurate_seek', + '-ss', + '3', + '-i', + '/in.mp4', + ]); expect(frame?.args.slice(-8)).toEqual([ '-map', '[v]', @@ -97,7 +104,9 @@ describe('buildFrameAtPlan', () => { // The same video chain export runs, on timestamps offset to the frame's time. const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); - expect(graphOf(frame?.args)).toBe(`[0:v]setpts=PTS+2/TB,${segmentChain},trim=start=3[v]`); + expect(graphOf(frame?.args)).toBe( + `[0:v]setpts=PTS+3/TB,trim=start=0,${segmentChain},trim=start=3[v]`, + ); }); it('seeks into the source from the clip in-point and stops at its last frame', () => { @@ -112,8 +121,8 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(1, 3)).toEqual(['-ss', '106.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+6\.966666\/TB,/); + expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); + expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+7\.966666\/TB,trim=start=0,/); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -177,8 +186,8 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(1, 3)).toEqual(['-ss', '2']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+1\/TB,/); + expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); + expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+2\/TB,/); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -198,13 +207,13 @@ describe('buildFrameAtPlan', () => { }), }, ]); - // The frame shows source second 4. Reading starts a second earlier, at 3; + // The frame shows source second 4; // the chain's setpts=PTS/2 turns source time into segment time, which the // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(1, 3)).toEqual(['-ss', '3']); + expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+3\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+4\/TB,/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index ed104fb..34c1fd3 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -144,6 +144,7 @@ describe('timelineOutline', () => { expect(placed?.findLast((w) => w.index === s.toIndex)?.endSec).toBe(s.endSec); } expect(Object.keys(outline.transcriptIds)).toEqual([M]); + expect(outline.sentences.some((s) => 'mediaId' in s)).toBe(false); expect(outline.nextFromSec).toBeUndefined(); }); @@ -196,6 +197,8 @@ describe('timelineOutline', () => { ['c2', 3, 4, 'Second point'], ]); expect(Object.keys(outline.transcriptIds)).toEqual([M, O]); + // Two sources: each sentence names its own, to find its transcriptId. + expect(outline.sentences.map((s) => s.mediaId)).toEqual([M, O, M]); }); it('pages by size and range, so pages put back together give every sentence once', async () => { @@ -220,6 +223,33 @@ describe('timelineOutline', () => { expect(middle.sentences.map((s) => s.fromIndex)).toEqual([3]); }); + it('keeps sentences that start together on one page, so paging always moves on', async () => { + // Four one-word sentences at the same instant (zero-length words), then two more. + const tied = transcriptOf(M, [ + { text: 'Lead.', startSec: 0, endSec: 0.4 }, + ...['A.', 'B.', 'C.', 'D.'].map((text) => ({ text, startSec: 1, endSec: 1 })), + { text: 'After.', startSec: 1.2, endSec: 1.6 }, + { text: 'End.', startSec: 1.6, endSec: 2 }, + ]); + const comp = timelineOf([mediaClip({ id: 'c1', mediaId: M, sourceOutSec: 2, startSec: 0 })]); + const transcribe = transcribeFrom(tied); + const all = (await timelineOutline(comp, MEDIA, { transcribe })).sentences; + for (const maxBytes of [300, 450]) { + const seen = []; + const starts: number[] = []; + let fromSec: number | undefined = 0; + while (fromSec !== undefined) { + starts.push(fromSec); + const page = await timelineOutline(comp, MEDIA, { transcribe, fromSec, maxBytes }); + seen.push(...page.sentences); + fromSec = page.nextFromSec; + expect(starts.length).toBeLessThan(10); + } + expect(seen, `maxBytes ${maxBytes}`).toEqual(all); + expect(new Set(starts).size).toBe(starts.length); + } + }); + it('pages a 60-minute, 9,000-word recording at 40 KB', async () => { const texts = Array.from({ length: 9000 }, (_, i) => i % 14 === 13 ? `sentence${i}.` : `word${i}`, From 9b8029d2887c12a6e5ae61a3871ebe7e153dedff Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:48:46 -0800 Subject: [PATCH 10/14] fix(timeline): match export's frame timing tick for tick; outline ranges A frame preview restored its timestamps with a fractional offset while the seek rounds to whole stream ticks, so a 24 fps frame on a 30 fps canvas could land one canvas frame early (21 dB at 1.391 s). It now undoes the seek in whole ticks, leaving the timestamps export's own seek gives. An outline sentence ends where the timeline skips ahead over words it plays elsewhere (clips moved out of source order), so its index range never takes in words it doesn't show. The CLI sentence lines name the source when the timeline has more than one. Co-Authored-By: Claude Opus 5.5 --- src/cli-timeline.ts | 6 ++- src/timeline/compile.ts | 8 +++- src/timeline/transcript.ts | 45 +++++++++++++++---- .../integration/timeline-frame-window.test.ts | 4 +- tests/timeline-introspect.test.ts | 12 +++-- tests/timeline-outline.test.ts | 16 +++++++ 6 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e9f1b91..46130f8 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -641,7 +641,11 @@ export async function runTimeline(args: string[]): Promise { ([mediaId, id]) => `transcript ${mediaId}: ${id}`, ); for (const s of outline.sentences) { - lines.push(`${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId} ${s.text}`); + // With more than one source, its media id, to match a transcript line. + const source = s.mediaId ? ` ${s.mediaId}` : ''; + lines.push( + `${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId}${source} ${s.text}`, + ); } process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 2657a68..05cc08e 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -790,6 +790,7 @@ export function buildFrameAtPlan( // time. Rounded down, to the microsecond, so the seek never passes the // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); + const seek = floorUs(clip.sourceInSec + local); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length @@ -803,11 +804,14 @@ export function buildFrameAtPlan( '-y', '-noaccurate_seek', '-ss', - String(floorUs(clip.sourceInSec + local)), + String(seek), '-i', info.path, '-filter_complex', - `[0:v]setpts=PTS+${local}/TB,trim=start=0,${chain},trim=start=${firstFrame}[v]`, + // The seek shifted timestamps by (in-point + local) in whole stream + // ticks; shifting back by the same rounding leaves them as export's + // seek to the in-point does, tick for tick. + `[0:v]setpts=PTS+round(${seek}/TB)-round(${floorUs(clip.sourceInSec)}/TB),trim=start=0,${chain},trim=start=${firstFrame}[v]`, ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], ctx.output, ], diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 686e83d..cd2d742 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -344,9 +344,14 @@ export interface TimelineOutline { /** * Group words (one source, timeline order) into sentences: a sentence ends at * sentence-ending punctuation, before a pause longer than `PAUSE_SEC`, after - * `MAX_SENTENCE_WORDS` words, or where the timeline goes back in the source. + * `MAX_SENTENCE_WORDS` words, where the timeline goes back in the source, or + * where it skips ahead over words `played` (by index) somewhere else on it, so + * a sentence's index range never takes in words it doesn't show. */ -export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { +export function outlineSentences( + words: OutlineWord[], + played: ReadonlySet = new Set(), +): OutlineSentence[] { const groups: OutlineWord[][] = []; let current: OutlineWord[] = []; for (const word of words) { @@ -356,6 +361,7 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { (endsSentence(prev.text) || word.startSec - prev.endSec > PAUSE_SEC || word.index <= prev.index || + skipsPlayed(prev.index, word.index, played) || current.length >= MAX_SENTENCE_WORDS) ) { groups.push(current); @@ -381,6 +387,11 @@ export function outlineSentences(words: OutlineWord[]): OutlineSentence[] { }); } +function skipsPlayed(from: number, to: number, played: ReadonlySet): boolean { + for (let i = from + 1; i < to; i++) if (played.has(i)) return true; + return false; +} + export interface OutlineOptions extends TranscribeOptions { /** Only this media clip's words. */ clipId?: string; @@ -416,10 +427,9 @@ export async function timelineOutline( .sort((a, b) => a.startSec - b.startSec); const transcripts = new Map(); - const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; - let run: OutlineWord[] = []; - let runMedia: MediaId | undefined; - let prevClip: MediaClip | undefined; + const placed: { clip: MediaClip; words: ClipWord[] }[] = []; + /** Word indices each source plays anywhere in the outlined clips. */ + const played = new Map>(); for (const clip of clips) { let transcript = transcripts.get(clip.mediaId); if (transcript === undefined) { @@ -431,8 +441,25 @@ export async function timelineOutline( transcripts.set(clip.mediaId, transcript); } if (!transcript) continue; + const words = clipWords(transcript, clip); + placed.push({ clip, words }); + const indices = played.get(clip.mediaId) ?? new Set(); + for (const w of words) indices.add(w.index); + played.set(clip.mediaId, indices); + } + + const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; + let run: OutlineWord[] = []; + let runMedia: MediaId | undefined; + let prevClip: MediaClip | undefined; + const flush = (): void => { + if (runMedia) { + bySource.push({ mediaId: runMedia, sentences: outlineSentences(run, played.get(runMedia)) }); + } + }; + for (const { clip, words } of placed) { if (clip.mediaId !== runMedia) { - if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); + flush(); run = []; runMedia = clip.mediaId; } @@ -440,7 +467,7 @@ export async function timelineOutline( // between them), rather than playing part of it again. const continues = prevClip?.mediaId === clip.mediaId && clip.sourceInSec >= prevClip.sourceOutSec - 1e-6; - for (const word of clipWords(transcript, clip)) { + for (const word of words) { // A word a cut runs through shows in the clips on both sides of it: one // word, from its start in the first to its end in the last. const last = run[run.length - 1]; @@ -452,7 +479,7 @@ export async function timelineOutline( } prevClip = clip; } - if (runMedia) bySource.push({ mediaId: runMedia, sentences: outlineSentences(run) }); + flush(); const transcriptIds: Record = {}; for (const [mediaId, transcript] of transcripts) { diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index 57d4a08..ec4f692 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -174,7 +174,9 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' await runPlan( compileTimeline(slowDoc, { media: slowMedia, dir: workspace, output: slowExport }), ); - for (const atSec of [1.5, 3.999]) { + // 1.391 s: the seek's whole-tick rounding decides which side of a frame + // boundary the source frame at 1.4167 s lands on. + for (const atSec of [1.5, 1.391, 3.999]) { const full = join(workspace, `slow-${atSec}-full.jpg`); const exportSec = Math.min(atSec, 4 - 1 / 30); await runFfmpeg(buildPreviewFrameArgs({ input: slowExport, output: full, atSec: exportSec })); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index 9d08532..ba8c8fb 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -105,7 +105,7 @@ describe('buildFrameAtPlan', () => { const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+3/TB,trim=start=0,${segmentChain},trim=start=3[v]`, + `[0:v]setpts=PTS+round(3/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, ); }); @@ -122,7 +122,9 @@ describe('buildFrameAtPlan', () => { // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); - expect(graphOf(near?.args)).toMatch(/^\[0:v\]setpts=PTS\+7\.966666\/TB,trim=start=0,/); + expect(graphOf(near?.args)).toMatch( + /^\[0:v\]setpts=PTS\+round\(107\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, + ); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -187,7 +189,9 @@ describe('buildFrameAtPlan', () => { const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); - expect(graphOf(plan.steps[0]?.args)).toMatch(/^\[0:v\]setpts=PTS\+2\/TB,/); + expect(graphOf(plan.steps[0]?.args)).toMatch( + /^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(1\/TB\),/, + ); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -213,7 +217,7 @@ describe('buildFrameAtPlan', () => { const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+4\/TB,/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(4\/TB\)-round\(0\/TB\),/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); diff --git a/tests/timeline-outline.test.ts b/tests/timeline-outline.test.ts index 34c1fd3..dabf8d7 100644 --- a/tests/timeline-outline.test.ts +++ b/tests/timeline-outline.test.ts @@ -148,6 +148,22 @@ describe('timelineOutline', () => { expect(outline.nextFromSec).toBeUndefined(); }); + it('ends a sentence where the timeline skips words it plays elsewhere', async () => { + const run = transcriptOf(M, spoken('a b c d e f g h i j k l'.split(' '))); + // Words 0–1, then 10–11, then 2–9: clips moved out of source order. + const moved = timelineOf([ + mediaClip({ id: 'c1', mediaId: M, sourceInSec: 0, sourceOutSec: 0.8, startSec: 0 }), + mediaClip({ id: 'c2', mediaId: M, sourceInSec: 4, sourceOutSec: 4.8, startSec: 0.8 }), + mediaClip({ id: 'c3', mediaId: M, sourceInSec: 0.8, sourceOutSec: 4, startSec: 1.6 }), + ]); + const outline = await timelineOutline(moved, MEDIA, { transcribe: transcribeFrom(run) }); + expect(outline.sentences.map((s) => [s.clipId, s.fromIndex, s.toIndex])).toEqual([ + ['c1', 0, 1], + ['c2', 10, 11], + ['c3', 2, 9], + ]); + }); + it('keeps a word the timeline plays twice as two, where a clip replays the source', async () => { const hello = transcriptOf(M, spoken(['Hello.', 'and', 'so'])); const transcribe = transcribeFrom(hello); From 19e6220ee6a1e0e041e4a1c8c3646ca12fd0f6af Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 16:55:45 -0800 Subject: [PATCH 11/14] fix(timeline): preroll slow-motion previews; end the sentence listing on a closed pipe A slow-motion frame can need the source frame before the keyframe its own time falls just after; the preview now seeks a second early as well as keeping the frames from the keyframe before (21 dB before at 0.25x). `clip timeline transcribe --sentences | head` ended with an unhandled EPIPE once head closed the pipe on a long listing; a closed pipe now just ends it. Co-Authored-By: Claude Opus 5.5 --- src/cli-timeline.ts | 4 +++ src/timeline/compile.ts | 15 +++++---- .../integration/timeline-frame-window.test.ts | 31 +++++++++++++++++++ tests/timeline-introspect.test.ts | 18 +++++------ 4 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index 46130f8..8c3731d 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -647,6 +647,10 @@ export async function runTimeline(args: string[]): Promise { `${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId}${source} ${s.text}`, ); } + // `| head` closes the pipe early: that ends the listing, not with an error. + process.stdout.on('error', (err: NodeJS.ErrnoException) => { + if (err.code !== 'EPIPE') throw err; + }); process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; } diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 05cc08e..7f21fe1 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -614,6 +614,9 @@ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +/** How much of the source a frame preview reads before the frame, at least. */ +const FRAME_PREROLL_SEC = 1; + function floorUs(sec: number): number { return Math.floor(sec * 1e6 + 1e-6) / 1e6; } @@ -779,18 +782,18 @@ export function buildFrameAtPlan( if (!info) { throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); } - // Seconds into the clip's source. The seek keeps every frame from the - // keyframe before it (`-noaccurate_seek`), so a frame that export repeats - // into this time (a source slower than the canvas, or a screen recording - // holding a still frame) is still read; its timestamps count from the seek - // point, so adding `local` gives source time from the clip's in-point. + // Seconds into the clip's source. The seek lands a second before that and + // keeps every frame from the keyframe before it (`-noaccurate_seek`), so a + // frame that export repeats into this time (a source slower than the + // canvas, slow motion, or a screen recording holding a still frame) is + // still read; undoing the seek gives source time from the clip's in-point. // Frames before the in-point are dropped, as export's seek drops them. The // chain's speed `setpts` turns source time into segment time, which its // fades and text read, and `trim` takes the frame at the frame's segment // time. Rounded down, to the microsecond, so the seek never passes the // clip's last frame or its out-point. const local = floorUs(clamped * speedFactor(clip.effects)); - const seek = floorUs(clip.sourceInSec + local); + const seek = floorUs(clip.sourceInSec + Math.max(0, local - FRAME_PREROLL_SEC)); // The first canvas frame at or after the frame's time (`trim` rounds an // off-grid start to the nearest frame, which could be the one before), but // no later than the segment's last frame: encoding to the clip's length diff --git a/tests/integration/timeline-frame-window.test.ts b/tests/integration/timeline-frame-window.test.ts index ec4f692..258cc21 100644 --- a/tests/integration/timeline-frame-window.test.ts +++ b/tests/integration/timeline-frame-window.test.ts @@ -187,6 +187,37 @@ describe.skipIf(!IS_CI)('frame preview from a short window, against real FFmpeg' expect(await psnr(full, windowFrame), `24 fps at ${atSec}`).toBeGreaterThanOrEqual(40); } + // Slow motion: at 0.25x a canvas frame can need the source frame before the + // keyframe the frame's own time falls just after (2.5 s here, a 60-frame GOP). + const slowMotion = vertical( + mediaClip({ + id: 'c1', + mediaId: M, + sourceInSec: 0.01, + sourceOutSec: 4, + startSec: 0, + effects: [{ type: 'speed', factor: 0.25 }], + }), + ); + const slowMotionExport = join(workspace, 'slow-motion.mp4'); + await runPlan( + compileTimeline(slowMotion, { media: slowMedia, dir: workspace, output: slowMotionExport }), + ); + const slowMotionFull = join(workspace, 'slow-motion-full.jpg'); + // Timeline 2.491 s of a 0.25x clip plays at 9.964 s of its segment. + await runFfmpeg( + buildPreviewFrameArgs({ input: slowMotionExport, output: slowMotionFull, atSec: 9.964 }), + ); + const slowMotionWindow = join(workspace, 'slow-motion-window.jpg'); + await runPlan( + buildFrameAtPlan( + slowMotion, + { media: slowMedia, dir: workspace, output: slowMotionWindow }, + 2.491, + ), + ); + expect(await psnr(slowMotionFull, slowMotionWindow), 'slow motion').toBeGreaterThanOrEqual(40); + // A sparse source (a frame every 5 s, like a screen recording holding a // still): at 18 s export still shows the frame from 15 s. const sparse = join(workspace, 'sparse.mp4'); diff --git a/tests/timeline-introspect.test.ts b/tests/timeline-introspect.test.ts index ba8c8fb..5049355 100644 --- a/tests/timeline-introspect.test.ts +++ b/tests/timeline-introspect.test.ts @@ -82,12 +82,12 @@ describe('buildFrameAtPlan', () => { expect(plan.steps).toHaveLength(1); const [frame] = plan.steps; expect(frame?.label).toBe('frame:c1@3'); - // Seek to 3 s, keeping the frames from the keyframe before it. + // Seek a second before 3 s, keeping the frames from the keyframe before that. expect(frame?.args.slice(0, 6)).toEqual([ '-y', '-noaccurate_seek', '-ss', - '3', + '2', '-i', '/in.mp4', ]); @@ -105,7 +105,7 @@ describe('buildFrameAtPlan', () => { const exported = graphOf(compileTimeline(doc, CTX('/ws/out.mp4')).steps[0]?.args); const segmentChain = exported.slice('[0:v]'.length, exported.indexOf('[v]')); expect(graphOf(frame?.args)).toBe( - `[0:v]setpts=PTS+round(3/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, + `[0:v]setpts=PTS+round(2/TB)-round(0/TB),trim=start=0,${segmentChain},trim=start=3[v]`, ); }); @@ -121,9 +121,9 @@ describe('buildFrameAtPlan', () => { const near = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 7.999).steps[0]; // 30 fps: the last frame starts at 8 - 1/30 s. // Rounded down, never past that frame or the clip's out-point. - expect(near?.args.slice(2, 4)).toEqual(['-ss', '107.966666']); + expect(near?.args.slice(2, 4)).toEqual(['-ss', '106.966666']); expect(graphOf(near?.args)).toMatch( - /^\[0:v\]setpts=PTS\+round\(107\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, + /^\[0:v\]setpts=PTS\+round\(106\.966666\/TB\)-round\(100\/TB\),trim=start=0,/, ); expect(graphOf(near?.args)).toMatch(/,trim=start=7\.96666666\d*\[v\]$/); }); @@ -188,9 +188,9 @@ describe('buildFrameAtPlan', () => { ]); const plan = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 10); // 2s into c2 ([8,12)) expect(plan.steps[0]?.label).toBe('frame:c2@10'); - expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '3']); + expect(plan.steps[0]?.args.slice(2, 4)).toEqual(['-ss', '2']); expect(graphOf(plan.steps[0]?.args)).toMatch( - /^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(1\/TB\),/, + /^\[0:v\]setpts=PTS\+round\(2\/TB\)-round\(1\/TB\),/, ); expect(graphOf(plan.steps[0]?.args)).toMatch(/,trim=start=2\[v\]$/); }); @@ -215,9 +215,9 @@ describe('buildFrameAtPlan', () => { // the chain's setpts=PTS/2 turns source time into segment time, which the // fades and text windows read, and trim takes the frame at segment time 2. const frame = buildFrameAtPlan(doc, CTX('/ws/f.jpg'), 4).steps[0]; - expect(frame?.args.slice(2, 4)).toEqual(['-ss', '4']); + expect(frame?.args.slice(2, 4)).toEqual(['-ss', '3']); const graph = graphOf(frame?.args); - expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(4\/TB\)-round\(0\/TB\),/); + expect(graph).toMatch(/^\[0:v\]setpts=PTS\+round\(3\/TB\)-round\(0\/TB\),/); expect(graph).toContain('setpts=PTS/2'); expect(graph).toMatch(/,trim=start=2\[v\]$/); }); From 7665a7f34dc0290230d8ebdd6f417eda856c498f Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 17:52:08 -0800 Subject: [PATCH 12/14] feat(timeline): cut a short's edges in the pause the audio shows whisper.cpp gives each pause to the word before it and runs late at sentence ends, so a short cut by the transcript kept up to 0.35 s of the next sentence. make_short now measures the source's levels around each edge (one short FFmpeg read, 10 ms RMS frames), finds the pause the transcript boundary points at by a threshold set from those levels, and cuts 0.1 s into it. An edge with no clear pause keeps the transcript rule and warns; a word whisper places past the cut keeps its caption. On the 30-sentence say recording every sentence-short is clean: no edge clipped, none keeping a neighbor's speech (before: 18 ends kept the next sentence, 3 starts were clipped). On jfk.wav every cut lands in a pause. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- SKILL.md | 2 +- docs/backlog/cut-words-pause-edges.md | 3 + docs/backlog/short-edges-snap-to-pause.md | 3 - src/cli-timeline.ts | 11 +- src/ffmpeg/args/speech-levels.ts | 42 +++++++ src/index.ts | 13 ++ src/timeline/pauses.ts | 140 +++++++++++++++++++++ src/timeline/shorts.ts | 147 ++++++++++++++++++---- src/timeline/verbs.ts | 7 +- src/ui/timeline-tools.ts | 7 ++ tests/integration/timeline-pauses.test.ts | 57 +++++++++ tests/mcp-server.test.ts | 2 + tests/timeline-captions.test.ts | 1 + tests/timeline-pauses.test.ts | 137 ++++++++++++++++++++ tests/timeline-shorts.test.ts | 128 +++++++++++++++++-- tests/timeline-verbs.test.ts | 2 + 17 files changed, 664 insertions(+), 40 deletions(-) create mode 100644 docs/backlog/cut-words-pause-edges.md delete mode 100644 docs/backlog/short-edges-snap-to-pause.md create mode 100644 src/ffmpeg/args/speech-levels.ts create mode 100644 src/timeline/pauses.ts create mode 100644 tests/integration/timeline-pauses.test.ts create mode 100644 tests/timeline-pauses.test.ts diff --git a/README.md b/README.md index ff87911..0910a66 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ clip timeline --timeline best-1 export best-1.mp4 clip timeline list # main and every short ``` -`short` starts 0.25 s before the first word and ends with the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. +`short` cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone (where there's no clear pause it follows the transcript and warns), fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. --- diff --git a/SKILL.md b/SKILL.md index 7553f49..3e8389e 100644 --- a/SKILL.md +++ b/SKILL.md @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence; widen it to the sentence's edge unless the cut is meant. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone, so it never keeps the next sentence's first syllable, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence (widen it to the sentence's edge unless the cut is meant) and an edge with no clear pause in the audio (music under the speech, or no break), which follows the transcript instead: look at that edge or listen to it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/docs/backlog/cut-words-pause-edges.md b/docs/backlog/cut-words-pause-edges.md new file mode 100644 index 0000000..5c88d73 --- /dev/null +++ b/docs/backlog/cut-words-pause-edges.md @@ -0,0 +1,3 @@ +# Cut words at the pause + +`cut_words` and `remove_retakes` cut from the first word's start to the last word's end as whisper.cpp gives them (`planWordCut` in `src/timeline/transcript.ts`). whisper gives each pause to the word before it and runs late at sentence edges. On a 30-sentence test recording, cutting a sentence would also remove up to 0.35 s of the next sentence's first syllable, at 18 of 29 boundaries. Place those cut edges in the pause the audio shows, as `make_short` does (`src/timeline/pauses.ts`), through `VerbContext.speechLevels`. diff --git a/docs/backlog/short-edges-snap-to-pause.md b/docs/backlog/short-edges-snap-to-pause.md deleted file mode 100644 index 3bcb401..0000000 --- a/docs/backlog/short-edges-snap-to-pause.md +++ /dev/null @@ -1,3 +0,0 @@ -# Snap short edges to the pause - -whisper.cpp (`-ml 1 -sow`) gives each pause to the word before it, so at a sentence edge its words run late. On a 30-sentence test recording (macOS `say`, model base.en), a short ending on a sentence's last word ran past the pause into the next sentence in 18 of 29 cases, by up to 0.35 s. Even with the 0.25 s lead, 3 of 29 starts lost up to 0.1 s of the first word. No fixed padding fixes both. Move each edge of a range to the quietest point of the audio near it: run a silencedetect pass over a second or so around the edge and keep the edge inside the pause between the range and its neighbors. `-dtw base.en` didn't change the per-word offsets in a quick check. diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index 8773c7d..72633ac 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -45,6 +45,7 @@ import { } from './timeline/interchange.js'; import { buildMediaMap, buildMediaRefs } from './timeline/media-registry.js'; import { colorClip, mediaClip, textClip } from './timeline/ops.js'; +import { measureSpeechLevels } from './timeline/pauses.js'; import { AspectSchema, type FramingFit, @@ -572,7 +573,15 @@ export async function runTimeline(args: string[]): Promise { flags.loudness === undefined ? undefined : loudnessValue(flags.loudness, usage), replace: flags.replace === 'true', }, - (comp, clipId, transcriptId) => storedTranscriptForClip(comp, clipId, media, transcriptId), + { + transcript: (comp, clipId, transcriptId) => + storedTranscriptForClip(comp, clipId, media, transcriptId), + levels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + const info = clip?.kind === 'media' ? media.get(clip.mediaId) : undefined; + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, + }, ); const report = { timeline: target, diff --git a/src/ffmpeg/args/speech-levels.ts b/src/ffmpeg/args/speech-levels.ts new file mode 100644 index 0000000..6db6fa7 --- /dev/null +++ b/src/ffmpeg/args/speech-levels.ts @@ -0,0 +1,42 @@ +export interface SpeechLevelsArgs { + input: string; + /** Where to start measuring, in source seconds. */ + fromSec: number; + durationSec: number; +} + +/** Samples per measured frame at 16 kHz: 10 ms. */ +export const LEVEL_FRAME_SAMPLES = 160; +export const LEVEL_SAMPLE_RATE = 16000; + +/** + * The RMS level of every 10 ms of a source's first audio stream over a window, + * printed to stderr as `pts_time:…` / `lavfi.astats.Overall.RMS_level=…` lines. + * Mono at 16 kHz is plenty to tell speech from a pause, and the window is a few + * seconds, so it runs in well under a tenth of a second. + */ +export function buildSpeechLevelsArgs({ input, fromSec, durationSec }: SpeechLevelsArgs): string[] { + return [ + '-hide_banner', + '-nostats', + '-ss', + String(fromSec), + '-t', + String(durationSec), + '-i', + input, + '-map', + '0:a:0', + '-af', + [ + `aresample=${LEVEL_SAMPLE_RATE}`, + 'aformat=channel_layouts=mono', + `asetnsamples=n=${LEVEL_FRAME_SAMPLES}:p=0`, + 'astats=metadata=1:reset=1', + 'ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + ].join(','), + '-f', + 'null', + '-', + ]; +} diff --git a/src/index.ts b/src/index.ts index bea7419..7206d6b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -199,6 +199,17 @@ export { textClip, videoTrack, } from './timeline/ops.js'; +export { + endInPause, + findPauses, + measureSpeechLevels, + PAUSE_KEEP_SEC, + type Pause, + parseSpeechLevels, + pauseAt, + type SpeechLevels, + startInPause, +} from './timeline/pauses.js'; export { type Aspect, AspectSchema, @@ -251,7 +262,9 @@ export { type ShortPlan, type ShortRange, type ShortRequest, + type ShortSources, type ShortTranscriptLookup, + type SpeechLevelsLookup, } from './timeline/shorts.js'; export { analyzeSilence, diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts new file mode 100644 index 0000000..0cfa147 --- /dev/null +++ b/src/timeline/pauses.ts @@ -0,0 +1,140 @@ +import { + buildSpeechLevelsArgs, + LEVEL_FRAME_SAMPLES, + LEVEL_SAMPLE_RATE, +} from '../ffmpeg/args/speech-levels.js'; +import { runFfmpeg } from '../ffmpeg/run.js'; + +/** + * Pauses in speech, found in the audio itself. whisper.cpp's word edges run + * late at sentence ends: it gives each pause to the word before it, and the + * next word can start a third of a second into its own speech. A cut placed by + * the transcript alone then keeps the start of the next sentence. Measured + * levels show where the voice actually stops, so an edge is moved into the + * pause the transcript points at. + */ + +/** RMS level of each 10 ms of a source's audio, dBFS, from `fromSec` on. */ +export interface SpeechLevels { + fromSec: number; + frameSec: number; + db: number[]; +} + +/** A stretch of a source quiet enough to be a pause, in source seconds. */ +export interface Pause { + startSec: number; + endSec: number; +} + +/** Room tone kept between a cut and the speech beside it. */ +export const PAUSE_KEEP_SEC = 0.1; +/** How far before the transcript's boundary the pause may lie: whisper runs + * up to about 0.75 s late there. */ +const SEARCH_BEFORE_SEC = 0.9; +/** …and after it, for a boundary whisper put early. */ +const SEARCH_AFTER_SEC = 0.05; +/** Measured around a boundary, for a noise floor and speech level of its own. */ +const MEASURE_BEFORE_SEC = 1.5; +const MEASURE_AFTER_SEC = 1; +/** Quiet means this far over the window's floor (its 5th percentile)… */ +const ABOVE_FLOOR_DB = 10; +/** …and this far under its speech (its 95th percentile). */ +const BELOW_SPEECH_DB = 20; +/** The shortest quiet stretch counted as a pause. */ +const MIN_PAUSE_SEC = 0.08; +/** Sound shorter than this inside a pause (a click, a lip noise) doesn't end it. */ +const MAX_BLIP_SEC = 0.04; +/** Two pauses at least this long in reach make the boundary ambiguous. */ +const COMPETING_PAUSE_SEC = 0.12; +const SILENT_DB = -120; + +/** Read the per-frame levels `buildSpeechLevelsArgs` prints. */ +export function parseSpeechLevels(stderr: string, fromSec: number): SpeechLevels { + const db: number[] = []; + for (const match of stderr.matchAll(/lavfi\.astats\.Overall\.RMS_level=(\S+)/g)) { + const value = Number(match[1]); + db.push(Number.isFinite(value) ? Math.max(value, SILENT_DB) : SILENT_DB); + } + return { fromSec, frameSec: LEVEL_FRAME_SAMPLES / LEVEL_SAMPLE_RATE, db }; +} + +/** Measure a source's levels around a boundary (runs FFmpeg). */ +export async function measureSpeechLevels( + path: string, + boundarySec: number, +): Promise { + const fromSec = Math.max(0, boundarySec - MEASURE_BEFORE_SEC); + const durationSec = boundarySec + MEASURE_AFTER_SEC - fromSec; + const { stderr } = await runFfmpeg(buildSpeechLevelsArgs({ input: path, fromSec, durationSec })); + return parseSpeechLevels(stderr, fromSec); +} + +/** Every pause in the levels, by a threshold set from the levels themselves. */ +export function findPauses(levels: SpeechLevels): Pause[] { + const sorted = [...levels.db].sort((a, b) => a - b); + const at = (q: number): number => sorted[Math.floor(q * (sorted.length - 1))] ?? SILENT_DB; + const threshold = Math.min(at(0.05) + ABOVE_FLOOR_DB, at(0.95) - BELOW_SPEECH_DB); + const quiet = levels.db.map((level) => level <= threshold); + const maxBlip = Math.round(MAX_BLIP_SEC / levels.frameSec); + // A short burst of sound between two quiet stretches is part of the pause. + for (let i = 0; i < quiet.length; ) { + if (quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && !quiet[j]) j++; + if (i > 0 && j < quiet.length && j - i <= maxBlip) quiet.fill(true, i, j); + i = j; + } + const pauses: Pause[] = []; + for (let i = 0; i < quiet.length; ) { + if (!quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && quiet[j]) j++; + if ((j - i) * levels.frameSec >= MIN_PAUSE_SEC - 1e-9) { + pauses.push({ + startSec: levels.fromSec + i * levels.frameSec, + endSec: levels.fromSec + j * levels.frameSec, + }); + } + i = j; + } + return pauses; +} + +/** + * The pause at a transcript boundary: the longest pause reaching into + * [boundary − 0.9 s, boundary + 0.05 s], but not before the start of the word + * the boundary ends. `none` when there is none (speech runs on, or music under + * it); `ambiguous` when two pauses of 0.12 s or more are in reach, since the + * transcript can't say which one it means. + */ +export function pauseAt( + levels: SpeechLevels, + boundarySec: number, + wordBeforeStartSec: number, +): Pause | 'none' | 'ambiguous' { + const from = Math.max(boundarySec - SEARCH_BEFORE_SEC, wordBeforeStartSec - SEARCH_AFTER_SEC); + const to = boundarySec + SEARCH_AFTER_SEC; + const inReach = findPauses(levels).filter((p) => p.endSec > from && p.startSec < to); + if (inReach.length === 0) return 'none'; + if (inReach.filter((p) => p.endSec - p.startSec >= COMPETING_PAUSE_SEC - 1e-9).length > 1) { + return 'ambiguous'; + } + return inReach.reduce((a, b) => (b.endSec - b.startSec > a.endSec - a.startSec ? b : a)); +} + +/** Where a short ends in a pause: a little room tone after the voice stops. */ +export function endInPause(pause: Pause): number { + return pause.startSec + Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} + +/** Where a short starts in a pause: a little room tone before the voice. */ +export function startInPause(pause: Pause): number { + return pause.endSec - Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index e6d3c16..1e5b216 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -16,6 +16,7 @@ import { makeClipId, } from './composition.js'; import { applyOps, type CompositionOp, mediaClip, textClip, videoTrack } from './ops.js'; +import { endInPause, pauseAt, type SpeechLevels, startInPause } from './pauses.js'; import { type Aspect, canvasFor, planReframe } from './reframe.js'; import { MAIN_TIMELINE, type MediaId } from './schema.js'; import { clipWords, type Transcript } from './transcript.js'; @@ -40,6 +41,8 @@ export const SHORT_LEAD_SEC = 0.25; export const DEFAULT_SHORT_LUFS = -14; const SHORT_TRACK = 'v0'; +/** A range whose snapped edges leave less than this keeps the transcript's. */ +const MIN_SNAPPED_SEC = 0.2; /** A piece shorter than this is dropped: too short to hear, just a flash. */ const MIN_PIECE_SEC = 0.05; const SAME_SEC = 1e-6; @@ -101,6 +104,23 @@ export type ShortTranscriptLookup = ( transcriptId?: string, ) => Promise<{ clip: MediaClip; transcript: Transcript }>; +/** The audio levels of a media clip's source around a time, in source seconds + * (runs FFmpeg); null when the source has no sound. */ +export type SpeechLevelsLookup = ( + comp: Composition, + clipId: string, + aroundSec: number, +) => Promise; + +/** What `planShort` reads beyond the documents. */ +export interface ShortSources { + transcript: ShortTranscriptLookup; + /** Moves each edge into the pause the audio shows there. Without it, edges + * follow the transcript. */ + levels?: SpeechLevelsLookup; + mintId?: () => string; +} + /** Refuse a short on `main` or on the timeline it is cut from. */ export function checkShortTimelines(target: string, source: string = MAIN_TIMELINE): void { if (target === MAIN_TIMELINE) { @@ -136,8 +156,9 @@ interface Piece extends ShortPiece { speed: number; sourceClipId: string; transcript: Transcript; - /** The word ranges the piece plays, which are what it captions. */ - asked: { fromIndex: number; toIndex: number }[]; + /** The word ranges the piece plays, which are what it captions, with the + * source window each was cut to. */ + asked: { fromIndex: number; toIndex: number; start: number; end: number }[]; } /** @@ -150,9 +171,9 @@ export async function planShort( target: Composition, source: Composition, request: ShortRequest, - lookup: ShortTranscriptLookup, - mintId: () => string = makeClipId, + sources: ShortSources, ): Promise { + const mintId = sources.mintId ?? makeClipId; if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { throw new ShortError( @@ -167,9 +188,16 @@ export async function planShort( } const windows: WordWindow[] = []; - for (const range of request.ranges) { - const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); - windows.push(wordWindow(transcript, range, clip)); + const warnings: string[] = []; + for (const [i, range] of request.ranges.entries()) { + const { clip, transcript } = await sources.transcript(source, range.clipId, range.transcriptId); + const window = wordWindow(transcript, range, clip); + warnings.push(...sentenceEdgeWarnings(window, i + 1)); + windows.push( + sources.levels + ? await snapToPauses(window, source, clip.id, sources.levels, i + 1, warnings) + : window, + ); } const pieces: Piece[] = []; windows.forEach((window, i) => { @@ -246,9 +274,19 @@ export async function planShort( // isn't part of the short even when another range asks for it. const style = defaultCaptionStyle(applyOps(target, ops)); const cues = placed.flatMap(({ piece, clip }) => { - const words = piece.transcript.words.filter((_, i) => - piece.asked.some((r) => i >= r.fromIndex && i <= r.toIndex), - ); + // A word whisper places past the cut (it runs late at a sentence end) is + // pulled inside it, so its caption isn't lost. + const words = piece.transcript.words.flatMap((w, i) => { + const range = piece.asked.find((r) => i >= r.fromIndex && i <= r.toIndex); + if (!range) return []; + if (w.startSec >= range.end) { + return [{ ...w, startSec: Math.max(range.start, range.end - 0.25), endSec: range.end }]; + } + if (w.endSec <= range.start) { + return [{ ...w, startSec: range.start, endSec: Math.min(range.end, range.start + 0.25) }]; + } + return [w]; + }); return groupCaptions( clipWords({ ...piece.transcript, words }, clip), clipEndSec(clip), @@ -283,7 +321,7 @@ export async function planShort( })), durationSec: roundMs(at), captions, - warnings: windows.flatMap((window, i) => sentenceEdgeWarnings(window, i + 1)), + warnings, }; } @@ -323,6 +361,68 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): }; } +/** + * Move a range's edges into the pauses its source's audio shows at its + * transcript boundaries: a little room tone before the first word and after + * the last, never the next sentence's first syllable. An edge with no clear + * pause (speech runs on, music under it, or two pauses the transcript can't + * tell apart) keeps the transcript rule, with a warning unless the range was + * cut mid-sentence there anyway. + */ +async function snapToPauses( + window: WordWindow, + source: Composition, + clipId: string, + levelsFor: SpeechLevelsLookup, + rangeNumber: number, + warnings: string[], +): Promise { + const words = window.transcript.words; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (!first || !last) return window; + + let { start, end } = window; + let startLevels: SpeechLevels | null; + let endLevels: SpeechLevels | null; + try { + startLevels = await levelsFor(source, clipId, first.startSec); + endLevels = startLevels && (await levelsFor(source, clipId, last.endSec)); + } catch { + // FFmpeg couldn't read the audio; the short is still made, by the transcript. + warnings.push( + `Range ${rangeNumber}: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.`, + ); + return window; + } + if (!startLevels) return window; // no sound to measure + const startPause = pauseAt(startLevels, first.startSec, before?.startSec ?? 0); + if (typeof startPause === 'object') start = startInPause(startPause); + else if (before && !midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber}: no clear pause before "${first.text}", so its start follows the transcript and may clip that word or keep the end of "${before.text}". Look at it or listen.`, + ); + } + const endPause = endLevels ? pauseAt(endLevels, last.endSec, last.startSec) : 'none'; + if (typeof endPause === 'object') end = endInPause(endPause); + else if (after && !midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber}: no clear pause after "${last.text}", so its end follows the transcript and may keep the start of "${after.text}". Look at it or listen.`, + ); + } + // A pause found on the wrong side of a short range would leave nothing. + if (end - start < MIN_SNAPPED_SEC) return window; + return { ...window, start: roundMs(start), end: roundMs(end) }; +} + +/** Whether two neighboring words belong to one sentence: no sentence end and + * no real pause between them. */ +function midSentence(a: { text: string; endSec: number }, b: { startSec: number }): boolean { + return !endsSentence(a.text) && b.startSec - a.endSec < PAUSE_SEC; +} + /** * Where a range starts or ends inside a sentence: right after a word that * doesn't end one (with no real pause between), or right before the next word @@ -332,26 +432,18 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): function sentenceEdgeWarnings(window: WordWindow, rangeNumber: number): string[] { const words = window.transcript.words; const warnings: string[] = []; - const midSentence = ( - a: (typeof words)[number] | undefined, - b: (typeof words)[number] | undefined, - ) => - a !== undefined && - b !== undefined && - !endsSentence(a.text) && - b.startSec - a.endSec < PAUSE_SEC; const before = words[window.fromIndex - 1]; const first = words[window.fromIndex]; - if (midSentence(before, first)) { + if (before && first && midSentence(before, first)) { warnings.push( - `Range ${rangeNumber} starts mid-sentence, between "${before?.text}" and "${first?.text}" (word ${window.fromIndex}).`, + `Range ${rangeNumber} starts mid-sentence, between "${before.text}" and "${first.text}" (word ${window.fromIndex}).`, ); } const last = words[window.toIndex]; const after = words[window.toIndex + 1]; - if (midSentence(last, after)) { + if (last && after && midSentence(last, after)) { warnings.push( - `Range ${rangeNumber} ends mid-sentence, between "${last?.text}" and "${after?.text}" (word ${window.toIndex}).`, + `Range ${rangeNumber} ends mid-sentence, between "${last.text}" and "${after.text}" (word ${window.toIndex}).`, ); } return warnings; @@ -400,7 +492,14 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { speed: speedFactor(clip.effects), sourceClipId: clip.id, transcript: window.transcript, - asked: [{ fromIndex: window.fromIndex, toIndex: window.toIndex }], + asked: [ + { + fromIndex: window.fromIndex, + toIndex: window.toIndex, + start: window.start, + end: window.end, + }, + ], }); taken.push([a, b]); } diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index 91e81b5..6e96d54 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -23,7 +23,7 @@ import { planReframe, } from './reframe.js'; import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from './schema.js'; -import { checkShortTimelines, planShort } from './shorts.js'; +import { checkShortTimelines, planShort, type SpeechLevelsLookup } from './shorts.js'; import { SensitivitySchema, type SilenceCut } from './silence.js'; import { clipWords, @@ -402,6 +402,9 @@ export interface VerbContext { ) => Promise<{ clip: MediaClip; transcript: Transcript; media: Map }>; /** Read another timeline of the workspace (make_short's source). */ readTimeline: (timeline: string) => Promise; + /** A media clip's source audio levels around a time (runs FFmpeg), for + * placing a short's edges in pauses; null when the source has no sound. */ + speechLevels: SpeechLevelsLookup; /** Track a clip is placed on when the verb doesn't say. */ defaultTrack: string; /** The timeline being edited (default `main`); `applyVerbs` sets it. */ @@ -585,7 +588,7 @@ export async function lowerVerb( targetLufs: verb.targetLufs, replace: verb.replace, }, - ctx.transcript, + { transcript: ctx.transcript, levels: ctx.speechLevels }, ); for (const warning of plan.warnings) ctx.warn?.(warning); return plan.ops; diff --git a/src/ui/timeline-tools.ts b/src/ui/timeline-tools.ts index f1ac55d..881628b 100644 --- a/src/ui/timeline-tools.ts +++ b/src/ui/timeline-tools.ts @@ -9,6 +9,7 @@ import { } from '../timeline/composition.js'; import { readComposition } from '../timeline/document-store.js'; import { buildMediaMap, buildMediaRefs } from '../timeline/media-registry.js'; +import { measureSpeechLevels } from '../timeline/pauses.js'; import { ownFraming } from '../timeline/reframe.js'; import type { MediaId, MediaRef } from '../timeline/schema.js'; import { analyzeSilence } from '../timeline/silence.js'; @@ -35,6 +36,12 @@ export function makeVerbContext(): VerbContext { return { mediaId: result.mediaId, durationSec: result.ref.durationSec }; }, readTimeline: (timeline) => readComposition(timeline), + speechLevels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + if (clip?.kind !== 'media') return null; + const info = (await clipConfinedMedia(comp, clipId)).get(clip.mediaId); + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, analyzeSilence: async (comp, clipId, sensitivity) => analyzeSilence(comp, clipId, { media: await clipConfinedMedia(comp, clipId), sensitivity }), transcript: async (comp, clipId, transcriptId) => { diff --git a/tests/integration/timeline-pauses.test.ts b/tests/integration/timeline-pauses.test.ts new file mode 100644 index 0000000..2488b31 --- /dev/null +++ b/tests/integration/timeline-pauses.test.ts @@ -0,0 +1,57 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { runFfmpeg } from '../../src/ffmpeg/run.js'; +import { measureSpeechLevels, pauseAt } from '../../src/timeline/pauses.js'; + +// The levels a short's edges are placed by, measured by real FFmpeg on an +// encoded file: a tone sounding 0.1–0.7 s of every second, over quiet noise. +const IS_CI = process.env.CI === 'true'; + +let workspace: string; +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-pauses-')); +}); +afterEach(async () => { + await rm(workspace, { recursive: true, force: true }); +}); + +describe.skipIf(!IS_CI)('pauses against real FFmpeg', () => { + it('finds the pause a late transcript boundary points at, inside the real one', async () => { + const source = join(workspace, 'talk.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'color=c=gray:s=320x240:r=30:d=8'], + ...[ + '-f', + 'lavfi', + '-i', + "aevalsrc='0.3*sin(2*PI*220*t)*between(mod(t,1),0.1,0.7)+0.001*(random(0)-0.5)':s=48000:d=8", + ], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', source], + ]); + + // "Speech" stops at 3.7 and starts again at 4.1; a transcript running + // 0.25 s late puts the boundary at 4.35, its last word from 3.35. + const levels = await measureSpeechLevels(source, 4.35); + const pause = pauseAt(levels, 4.35, 3.35); + if (typeof pause !== 'object') throw new Error(`no pause found: ${pause}`); + // Inside the real pause, by at most 40 ms (AAC smears a sharp ending by + // about 30 ms): a cut placed from it never moves toward the speech. + expect(pause.startSec).toBeGreaterThanOrEqual(3.7 - 1e-6); + expect(pause.startSec).toBeLessThanOrEqual(3.74); + expect(pause.endSec).toBeLessThanOrEqual(4.1 + 1e-6); + expect(pause.endSec).toBeGreaterThanOrEqual(4.06); + + // Where the tone never stops there is no pause to find. + const steady = join(workspace, 'steady.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'sine=frequency=220:sample_rate=48000:duration=4'], + ...['-c:a', 'aac', steady], + ]); + expect(pauseAt(await measureSpeechLevels(steady, 2), 2, 1.5)).toBe('none'); + }, 60_000); +}); diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index e9665fa..62a31d8 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -293,6 +293,8 @@ describe('MCP shorts (named timelines)', () => { expect(edit.warnings).toEqual([ 'Range 1 starts mid-sentence, between "word1" and "word2" (word 2).', 'Range 1 ends mid-sentence, between "word6" and "word7" (word 6).', + // The fixture's "video" is a few bytes, which FFmpeg can't measure. + "Range 1: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.", ]); const doc = edit.document as Record; expect(doc.timeline).toBe('clip-1'); diff --git a/tests/timeline-captions.test.ts b/tests/timeline-captions.test.ts index 4e47a19..79dfd99 100644 --- a/tests/timeline-captions.test.ts +++ b/tests/timeline-captions.test.ts @@ -322,6 +322,7 @@ describe('caption_speech verb', () => { const { clip, transcript } = await lookup(new Map([[M, RECORDING]]))(clipId); return { clip, transcript, media: new Map() }; }, + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, diff --git a/tests/timeline-pauses.test.ts b/tests/timeline-pauses.test.ts new file mode 100644 index 0000000..fbf6359 --- /dev/null +++ b/tests/timeline-pauses.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from 'vitest'; +import { buildSpeechLevelsArgs } from '../src/ffmpeg/args/speech-levels.js'; +import { + endInPause, + findPauses, + parseSpeechLevels, + pauseAt, + type SpeechLevels, + startInPause, +} from '../src/timeline/pauses.js'; + +/** Levels built from [seconds, dB] stretches, 10 ms a frame, from `fromSec`. */ +function levelsOf(stretches: [number, number][], fromSec = 10): SpeechLevels { + const db = stretches.flatMap(([sec, level]) => + Array.from({ length: Math.round(sec / 0.01) }, () => level), + ); + return { fromSec, frameSec: 0.01, db }; +} + +const near = (value: number): number => Math.round(value * 1000) / 1000; + +describe('buildSpeechLevelsArgs', () => { + it('measures 10 ms of mono 16 kHz audio a frame over the window', () => { + expect(buildSpeechLevelsArgs({ input: '/in.mp4', fromSec: 8.5, durationSec: 2.5 })).toEqual([ + '-hide_banner', + '-nostats', + '-ss', + '8.5', + '-t', + '2.5', + '-i', + '/in.mp4', + '-map', + '0:a:0', + '-af', + 'aresample=16000,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + '-f', + 'null', + '-', + ]); + }); +}); + +describe('parseSpeechLevels', () => { + it('reads one level a frame, digital silence as -120 dB', () => { + const stderr = [ + '[Parsed_ametadata_4 @ 0x1] frame:0 pts:0 pts_time:0', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-12.703566', + '[Parsed_ametadata_4 @ 0x1] frame:1 pts:160 pts_time:0.01', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-inf', + '[Parsed_ametadata_4 @ 0x1] frame:2 pts:320 pts_time:0.02', + '[Parsed_ametadata_4 @ 0x1] lavfi.astats.Overall.RMS_level=-48.5', + ].join('\n'); + expect(parseSpeechLevels(stderr, 3)).toEqual({ + fromSec: 3, + frameSec: 0.01, + db: [-12.703566, -120, -48.5], + }); + }); +}); + +describe('findPauses', () => { + it('finds pauses by the window’s own floor and speech level', () => { + // A noisy room: speech at -18 dB, room tone at -45 dB. + const levels = levelsOf([ + [1, -18], + [0.3, -45], + [0.8, -18], + [0.05, -45], // too short to be a pause + [0.5, -18], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([[11, 11.3]]); + }); + + it('keeps a pause whole across a click shorter than 40 ms', () => { + const levels = levelsOf([ + [1, -20], + [0.15, -60], + [0.03, -25], + [0.15, -60], + [1, -20], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([ + [11, 11.33], + ]); + }); + + it('finds none in speech that never stops, or in a steady level', () => { + expect(findPauses(levelsOf([[3, -20]]))).toEqual([]); + const stepped = levelsOf([ + [1, -26], + [1, -20], + [1, -26], + ]); + expect(findPauses(stepped)).toEqual([]); + }); +}); + +describe('pauseAt', () => { + // Speech, a 0.4 s pause from 11.2, speech again: whisper puts the boundary at + // 11.75, inside the next sentence, and the last word started at 10.9. + const levels = levelsOf([ + [1.2, -20], + [0.4, -60], + [1, -20], + ]); + + it('finds the pause a late transcript boundary points at', () => { + const pause = pauseAt(levels, 11.75, 10.9); + expect(pause).toEqual({ startSec: expect.closeTo(11.2, 6), endSec: expect.closeTo(11.6, 6) }); + if (typeof pause !== 'object') throw new Error('expected a pause'); + expect(near(endInPause(pause))).toBe(11.3); + expect(near(startInPause(pause))).toBe(11.5); + }); + + it('keeps half a short pause on each side', () => { + expect(near(endInPause({ startSec: 5, endSec: 5.1 }))).toBe(5.05); + expect(near(startInPause({ startSec: 5, endSec: 5.1 }))).toBe(5.05); + }); + + it('looks no earlier than the start of the word the boundary ends', () => { + // The last word started at 11.65, after the pause: there's none for it. + expect(pauseAt(levels, 11.75, 11.65)).toBe('none'); + }); + + it('says none when speech runs on, and ambiguous when two pauses compete', () => { + expect(pauseAt(levelsOf([[3, -20]]), 11.5, 11)).toBe('none'); + const two = levelsOf([ + [1, -20], + [0.2, -60], + [0.3, -20], + [0.2, -60], + [1, -20], + ]); + expect(pauseAt(two, 11.7, 10.5)).toBe('ambiguous'); + }); +}); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index b315fa8..c2d3410 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -99,8 +99,7 @@ async function plan( opts.target ?? emptyComposition(), source, { captions: false, ...request }, - lookupFor(opts.transcripts ?? [SPACED]), - counter(), + { transcript: lookupFor(opts.transcripts ?? [SPACED]), mintId: counter() }, ); } @@ -333,8 +332,7 @@ describe('planShort: the edit', () => { emptyComposition(), whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], x: 0.3 }, - lookupFor([SPACED]), - counter(), + { transcript: lookupFor([SPACED]), mintId: counter() }, ); expect(short.ops).toEqual([ { op: 'setCanvas', width: 1080, height: 1920, fps: 30, background: 'black', fit: 'cover' }, @@ -420,8 +418,7 @@ describe('planShort: the edit', () => { emptyComposition(), whole, { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, - lookupFor([abutting]), - counter(), + { transcript: lookupFor([abutting]), mintId: counter() }, ); expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }]); const captions = addedClips(short.ops).filter((c) => c.kind === 'text'); @@ -443,8 +440,7 @@ describe('planShort: the edit', () => { { clipId: 'c1', fromIndex: 0, toIndex: 0 }, ], }, - lookupFor([abutting]), - counter(), + { transcript: lookupFor([abutting]), mintId: counter() }, ); expect(short.pieces).toEqual([ { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }, @@ -507,6 +503,120 @@ describe('planShort: the edit', () => { }); }); +/** Speech at [start, end] spans at -20 dB over -60 dB room tone, measured the way + * the levels capability measures it: 1.5 s before a time to 1 s after. */ +function levelsFrom(spans: [number, number][]): SpeechLevelsLookup { + return async (_comp, _clipId, aroundSec) => { + const fromSec = Math.max(0, aroundSec - 1.5); + const frames = Math.round((aroundSec + 1 - fromSec) / 0.01); + const db = Array.from({ length: frames }, (_, i) => { + const t = fromSec + (i + 0.5) * 0.01; + return spans.some(([a, b]) => t >= a && t < b) ? -20 : -60; + }); + return { fromSec, frameSec: 0.01, db }; + }; +} + +describe('planShort: edges in the pauses', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + // Word i is spoken from i+0.1 to i+0.9 s; whisper, 0.3 s late as at sentence + // ends, puts it at [i+0.3, i+1.3), each word running into the next. + const late = (texts: string[]): Transcript => { + const t = transcriptOf( + M, + texts.map((_, i): [number, number] => [i + 0.3, i + 1.3]), + ); + return { ...t, words: t.words.map((w, i) => ({ ...w, text: `${w.text}${texts[i]}` })) }; + }; + const spoken = Array.from({ length: 10 }, (_, i): [number, number] => [i + 0.1, i + 0.9]); + const sentences = (t: Transcript): Transcript => ({ + ...t, + words: t.words.map((w, i) => ({ ...w, text: `w${i}.` })), + }); + + async function snap( + transcript: Transcript, + levels: SpeechLevelsLookup | undefined, + range = { clipId: 'c1', fromIndex: 3, toIndex: 5 }, + captions = false, + ) { + return planShort( + emptyComposition(), + whole, + { ranges: [range], captions }, + { transcript: lookupFor([transcript]), levels, mintId: counter() }, + ); + } + + it('moves both edges into the pauses, off the next sentence', async () => { + const short = await snap(late(Array(10).fill('')), levelsFrom(spoken)); + // The pause before word 3 is 2.9–3.1 and the one after word 5 is 5.9–6.1: + // 0.1 s of room tone inside each, where the transcript alone gives 3.05–6.3. + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3, sourceOutSec: 6 }]); + expect(short.warnings.some((w) => w.includes('pause'))).toBe(false); + }); + + it('keeps the transcript edges with a warning where the audio shows no pause', async () => { + const talking = levelsFrom([[0, 30]]); + const short = await snap(sentences(late(Array(10).fill(''))), talking); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([ + 'Range 1: no clear pause before "w3.", so its start follows the transcript and may clip that word or keep the end of "w2.". Look at it or listen.', + 'Range 1: no clear pause after "w5.", so its end follows the transcript and may keep the start of "w6.". Look at it or listen.', + ]); + }); + + it('warns once where a range was cut mid-sentence anyway', async () => { + const short = await snap(late(Array(10).fill('')), levelsFrom([[0, 30]])); + expect(short.warnings).toEqual([ + 'Range 1 starts mid-sentence, between "w2" and "w3" (word 3).', + 'Range 1 ends mid-sentence, between "w5" and "w6" (word 5).', + ]); + }); + + it('follows the transcript, quietly, for a source without sound', async () => { + const short = await snap(late(Array(10).fill('.')), async () => null); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([]); + }); + + it('still makes the short when the audio can’t be measured', async () => { + const short = await snap(late(Array(10).fill('.')), async () => { + throw new Error('FFmpeg failed'); + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.05, sourceOutSec: 6.3 }]); + expect(short.warnings).toEqual([ + "Range 1: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.", + ]); + }); + + it('keeps the caption of a last word whisper places past the cut', async () => { + // Speech of word 5 ends at 5.6; whisper starts it at 5.8. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 4.3], + [4.3, 5.8], + [5.8, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 5.1], + [5.2, 5.6], + [6.2, 7], + ]; + const short = await snap(transcript, levelsFrom(speech), undefined, true); + expect(short.pieces.at(-1)?.sourceOutSec).toBe(5.7); + const texts = addedClips(short.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts.join(' ')).toContain('w5'); + }); +}); + describe('checkShortTimelines', () => { it('refuses main as the short, and the short as its own source', () => { expect(() => checkShortTimelines('main')).toThrow(/not main/); @@ -529,6 +639,7 @@ describe('make_short verb', () => { ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), media: new Map(), }), + speechLevels: async () => null, readTimeline: async (id) => { const doc = sources[id]; if (!doc) throw new NoTimelineError(id); @@ -606,6 +717,7 @@ describe('make_short on the store', () => { media: new Map(), }), readTimeline: (id) => readComposition(id), + speechLevels: async () => null, defaultTrack: 'v0', }; const makeShort = { diff --git a/tests/timeline-verbs.test.ts b/tests/timeline-verbs.test.ts index afd7f96..f2be506 100644 --- a/tests/timeline-verbs.test.ts +++ b/tests/timeline-verbs.test.ts @@ -30,6 +30,7 @@ const ctx = (durationSec = 10): VerbContext => ({ transcript: async () => { throw new Error('not used'); }, + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, @@ -416,6 +417,7 @@ describe('applyVerbs (op-aware + undoable)', () => { openGate = resolve; }); const barrierCtx: VerbContext = { + speechLevels: async () => null, readTimeline: async () => { throw new Error('not used'); }, From dabc615f3c6f3ec55e782fd2c222ee2e506d9d09 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 17:59:36 -0800 Subject: [PATCH 13/14] fix(timeline): measure pauses on padded audio; keep pulled words in order Levels are measured with the transcription's aresample=async=1:first_pts=0, so a late audio start or a timestamp gap can't shift a pause; digital silence counts as quiet but is left out of the floor estimate, or room tone between sentences would read as sound. A word pulled inside a cut keeps its place after the word before it, and the speech check uses the pulled words, so a one-word range whisper places past its cut is kept. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/retake-test-model.md | 3 ++ src/ffmpeg/args/speech-levels.ts | 4 +- src/timeline/pauses.ts | 9 +++- src/timeline/shorts.ts | 53 ++++++++++++++------ tests/integration/timeline-pauses.test.ts | 23 +++++++++ tests/timeline-pauses.test.ts | 15 +++++- tests/timeline-shorts.test.ts | 60 +++++++++++++++++++++++ 7 files changed, 149 insertions(+), 18 deletions(-) create mode 100644 docs/backlog/retake-test-model.md diff --git a/docs/backlog/retake-test-model.md b/docs/backlog/retake-test-model.md new file mode 100644 index 0000000..16fe1aa --- /dev/null +++ b/docs/backlog/retake-test-model.md @@ -0,0 +1,3 @@ +# Retake integration test depends on the model + +`tests/integration/timeline-transcript.test.ts` ("removes a repeated take as one undoable edit") runs whenever whisper.cpp and a model are set up. With `ggml-base.en.bin` it fails on `feat/shorts` too: the model's transcript of the test's spoken phrase has no immediate repeat, so `retakes` is empty. Make the test independent of the model (assert on a stored transcript with a known repeat, as the unit tests do) or pin the model it was written for. diff --git a/src/ffmpeg/args/speech-levels.ts b/src/ffmpeg/args/speech-levels.ts index 6db6fa7..cafb438 100644 --- a/src/ffmpeg/args/speech-levels.ts +++ b/src/ffmpeg/args/speech-levels.ts @@ -29,7 +29,9 @@ export function buildSpeechLevelsArgs({ input, fromSec, durationSec }: SpeechLev '0:a:0', '-af', [ - `aresample=${LEVEL_SAMPLE_RATE}`, + // A late start or a timestamp gap becomes silence, as for transcription, + // so frame n is always n x 10 ms after the seek point. + `aresample=${LEVEL_SAMPLE_RATE}:async=1:first_pts=0`, 'aformat=channel_layouts=mono', `asetnsamples=n=${LEVEL_FRAME_SAMPLES}:p=0`, 'astats=metadata=1:reset=1', diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts index 0cfa147..efc7170 100644 --- a/src/timeline/pauses.ts +++ b/src/timeline/pauses.ts @@ -48,6 +48,8 @@ const MAX_BLIP_SEC = 0.04; /** Two pauses at least this long in reach make the boundary ambiguous. */ const COMPETING_PAUSE_SEC = 0.12; const SILENT_DB = -120; +/** At or below this, a frame is digital silence, not a room. */ +const DIGITAL_SILENCE_DB = -100; /** Read the per-frame levels `buildSpeechLevelsArgs` prints. */ export function parseSpeechLevels(stderr: string, fromSec: number): SpeechLevels { @@ -72,10 +74,13 @@ export async function measureSpeechLevels( /** Every pause in the levels, by a threshold set from the levels themselves. */ export function findPauses(levels: SpeechLevels): Pause[] { - const sorted = [...levels.db].sort((a, b) => a - b); + // Digital silence (padding where the audio starts late, or a muted stretch) + // is quiet, but it isn't the room's floor: leave it out of the estimate, or + // the room tone between sentences would count as sound. + const sorted = levels.db.filter((level) => level > DIGITAL_SILENCE_DB).sort((a, b) => a - b); const at = (q: number): number => sorted[Math.floor(q * (sorted.length - 1))] ?? SILENT_DB; const threshold = Math.min(at(0.05) + ABOVE_FLOOR_DB, at(0.95) - BELOW_SPEECH_DB); - const quiet = levels.db.map((level) => level <= threshold); + const quiet = levels.db.map((level) => level <= DIGITAL_SILENCE_DB || level <= threshold); const maxBlip = Math.round(MAX_BLIP_SEC / levels.frameSec); // A short burst of sound between two quiet stretches is part of the pause. for (let i = 0; i < quiet.length; ) { diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 1e5b216..4671873 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -43,6 +43,10 @@ export const DEFAULT_SHORT_LUFS = -14; const SHORT_TRACK = 'v0'; /** A range whose snapped edges leave less than this keeps the transcript's. */ const MIN_SNAPPED_SEC = 0.2; +/** How long a word pulled inside a cut lasts. */ +const PULLED_WORD_SEC = 0.25; +/** The least a word starts after the one before it. */ +const WORD_ORDER_SEC = 0.01; /** A piece shorter than this is dropped: too short to hear, just a flash. */ const MIN_PIECE_SEC = 0.05; const SAME_SEC = 1e-6; @@ -274,19 +278,9 @@ export async function planShort( // isn't part of the short even when another range asks for it. const style = defaultCaptionStyle(applyOps(target, ops)); const cues = placed.flatMap(({ piece, clip }) => { - // A word whisper places past the cut (it runs late at a sentence end) is - // pulled inside it, so its caption isn't lost. - const words = piece.transcript.words.flatMap((w, i) => { - const range = piece.asked.find((r) => i >= r.fromIndex && i <= r.toIndex); - if (!range) return []; - if (w.startSec >= range.end) { - return [{ ...w, startSec: Math.max(range.start, range.end - 0.25), endSec: range.end }]; - } - if (w.endSec <= range.start) { - return [{ ...w, startSec: range.start, endSec: Math.min(range.end, range.start + 0.25) }]; - } - return [w]; - }); + const words = piece.asked.flatMap((r) => + wordsInside(piece.transcript.words.slice(r.fromIndex, r.toIndex + 1), r.start, r.end), + ); return groupCaptions( clipWords({ ...piece.transcript, words }, clip), clipEndSec(clip), @@ -414,7 +408,38 @@ async function snapToPauses( } // A pause found on the wrong side of a short range would leave nothing. if (end - start < MIN_SNAPPED_SEC) return window; - return { ...window, start: roundMs(start), end: roundMs(end) }; + start = roundMs(start); + end = roundMs(end); + return { ...window, start, end, words: wordsInside(window.words, start, end) }; +} + +/** + * A range's words kept inside the window it was cut to, in order. A word + * whisper places past an edge (it runs late at a sentence end) is pulled just + * inside, so the range still holds its speech and the word keeps its caption; + * no word starts before the one ahead of it, or a caption shown a word at a + * time would lose that one. + */ +function wordsInside( + words: W[], + start: number, + end: number, +): W[] { + let prevStart = Number.NEGATIVE_INFINITY; + return words.map((w) => { + let startSec = w.startSec; + let endSec = w.endSec; + if (startSec >= end) { + startSec = Math.max(start, end - PULLED_WORD_SEC); + endSec = end; + } else if (endSec <= start) { + startSec = start; + endSec = Math.min(end, start + PULLED_WORD_SEC); + } + startSec = Math.min(Math.max(startSec, prevStart + WORD_ORDER_SEC), endSec - WORD_ORDER_SEC); + prevStart = startSec; + return startSec === w.startSec && endSec === w.endSec ? w : { ...w, startSec, endSec }; + }); } /** Whether two neighboring words belong to one sentence: no sentence end and diff --git a/tests/integration/timeline-pauses.test.ts b/tests/integration/timeline-pauses.test.ts index 2488b31..e8641d3 100644 --- a/tests/integration/timeline-pauses.test.ts +++ b/tests/integration/timeline-pauses.test.ts @@ -45,6 +45,29 @@ describe.skipIf(!IS_CI)('pauses against real FFmpeg', () => { expect(pause.endSec).toBeLessThanOrEqual(4.1 + 1e-6); expect(pause.endSec).toBeGreaterThanOrEqual(4.06); + // Audio that starts a second into the file: the levels still count from + // the seek point, so the pause lands where it is (1.7–2.1), not 0.15 s early. + const late = join(workspace, 'late.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'color=c=gray:s=320x240:r=30:d=8'], + ...['-itsoffset', '1'], + ...[ + '-f', + 'lavfi', + '-i', + "aevalsrc='0.3*sin(2*PI*220*t)*between(mod(t,1),0.1,0.7)+0.001*(random(0)-0.5)':s=48000:d=7", + ], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', late], + ]); + const shifted = pauseAt(await measureSpeechLevels(late, 2.35), 2.35, 1.35); + if (typeof shifted !== 'object') throw new Error(`no pause found: ${shifted}`); + expect(shifted.startSec).toBeGreaterThanOrEqual(1.7 - 1e-6); + expect(shifted.startSec).toBeLessThanOrEqual(1.74); + expect(shifted.endSec).toBeLessThanOrEqual(2.1 + 1e-6); + expect(shifted.endSec).toBeGreaterThanOrEqual(2.06); + // Where the tone never stops there is no pause to find. const steady = join(workspace, 'steady.mp4'); await runFfmpeg([ diff --git a/tests/timeline-pauses.test.ts b/tests/timeline-pauses.test.ts index fbf6359..7819066 100644 --- a/tests/timeline-pauses.test.ts +++ b/tests/timeline-pauses.test.ts @@ -33,7 +33,7 @@ describe('buildSpeechLevelsArgs', () => { '-map', '0:a:0', '-af', - 'aresample=16000,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + 'aresample=16000:async=1:first_pts=0,aformat=channel_layouts=mono,asetnsamples=n=160:p=0,astats=metadata=1:reset=1,ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', '-f', 'null', '-', @@ -85,6 +85,19 @@ describe('findPauses', () => { ]); }); + it('sets the floor by the room, not by digital silence before the audio starts', () => { + const levels = levelsOf([ + [0.5, -120], // padding: the audio starts late + [1, -20], + [0.3, -62], // room tone between sentences + [1, -20], + ]); + expect(findPauses(levels).map((p) => [near(p.startSec), near(p.endSec)])).toEqual([ + [10, 10.5], + [11.5, 11.8], + ]); + }); + it('finds none in speech that never stops, or in a steady level', () => { expect(findPauses(levelsOf([[3, -20]]))).toEqual([]); const stepped = levelsOf([ diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index c2d3410..cea617b 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -590,6 +590,66 @@ describe('planShort: edges in the pauses', () => { ]); }); + it('keeps every word’s caption, in order, one word at a time', async () => { + // Cut at 5.7: the word before the last starts at 5.6, the last at 5.8. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 5.6], + [5.6, 5.8], + [5.8, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 4.8], + [4.9, 5.6], + [6.2, 7], + ]; + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 5 }], maxWords: 1 }, + { transcript: lookupFor([transcript]), levels: levelsFrom(speech), mintId: counter() }, + ); + expect(short.pieces.at(-1)?.sourceOutSec).toBe(5.7); + const texts = addedClips(short.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts).toEqual(['w3', 'w4', 'w5']); + }); + + it('keeps a one-word range whose word whisper places past its cut', async () => { + // The word is spoken 5.2–5.6; whisper says 5.72–6.4. Its start finds two + // pauses (ambiguous, so the transcript's), its end the pause at 5.6. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 4.3], + [4.3, 5.72], + [5.72, 6.4], + [6.4, 7.4], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 4.9], + [5.2, 5.6], + [6.2, 7], + ]; + const short = await snap(transcript, levelsFrom(speech), { + clipId: 'c1', + fromIndex: 5, + toIndex: 5, + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 5.47, sourceOutSec: 5.7 }]); + }); + it('keeps the caption of a last word whisper places past the cut', async () => { // Speech of word 5 ends at 5.6; whisper starts it at 5.8. const transcript = transcriptOf(M, [ From c3a0a2d8d07977ec33019116391eae32d63bf4b0 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 18:10:20 -0800 Subject: [PATCH 14/14] fix(timeline): pause after the last word, not before; captions once, in order A pause that ends by the time the last word starts is the one before it, so an accurately timed last word with a long pause ahead of it is no longer cut off. A range across a source split captions its words once, and words crowded against a cut move earlier to make room instead of sharing a start, so no caption shown a word at a time is lost. Co-Authored-By: Claude Opus 5.5 --- src/timeline/pauses.ts | 7 +++--- src/timeline/shorts.ts | 43 +++++++++++++++++++++++------------ tests/timeline-pauses.test.ts | 13 +++++++++++ tests/timeline-shorts.test.ts | 38 +++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 17 deletions(-) diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts index efc7170..ebda2a4 100644 --- a/src/timeline/pauses.ts +++ b/src/timeline/pauses.ts @@ -114,8 +114,8 @@ export function findPauses(levels: SpeechLevels): Pause[] { /** * The pause at a transcript boundary: the longest pause reaching into - * [boundary − 0.9 s, boundary + 0.05 s], but not before the start of the word - * the boundary ends. `none` when there is none (speech runs on, or music under + * [boundary − 0.9 s, boundary + 0.05 s] that is still going after the word + * the boundary ends has started (not the pause before that word). `none` when there is none (speech runs on, or music under * it); `ambiguous` when two pauses of 0.12 s or more are in reach, since the * transcript can't say which one it means. */ @@ -124,7 +124,8 @@ export function pauseAt( boundarySec: number, wordBeforeStartSec: number, ): Pause | 'none' | 'ambiguous' { - const from = Math.max(boundarySec - SEARCH_BEFORE_SEC, wordBeforeStartSec - SEARCH_AFTER_SEC); + // A pause that ends by the time that word starts is the one before it. + const from = Math.max(boundarySec - SEARCH_BEFORE_SEC, wordBeforeStartSec + SEARCH_AFTER_SEC); const to = boundarySec + SEARCH_AFTER_SEC; const inReach = findPauses(levels).filter((p) => p.endSec > from && p.startSec < to); if (inReach.length === 0) return 'none'; diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 4671873..608f8dc 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -425,19 +425,28 @@ function wordsInside( start: number, end: number, ): W[] { - let prevStart = Number.NEGATIVE_INFINITY; - return words.map((w) => { - let startSec = w.startSec; - let endSec = w.endSec; - if (startSec >= end) { - startSec = Math.max(start, end - PULLED_WORD_SEC); - endSec = end; - } else if (endSec <= start) { - startSec = start; - endSec = Math.min(end, start + PULLED_WORD_SEC); - } - startSec = Math.min(Math.max(startSec, prevStart + WORD_ORDER_SEC), endSec - WORD_ORDER_SEC); - prevStart = startSec; + const starts = words.map((w) => { + if (w.startSec >= end) return Math.max(start, end - PULLED_WORD_SEC); + if (w.endSec <= start) return start; + return w.startSec; + }); + // In order going forward, then back from the cut: a word pushed up against + // the end makes room by moving the ones before it earlier. + for (let i = 1; i < starts.length; i++) { + starts[i] = Math.max(starts[i] ?? 0, (starts[i - 1] ?? 0) + WORD_ORDER_SEC); + } + for (let i = starts.length - 1; i >= 0; i--) { + const limit = i === starts.length - 1 ? end : (starts[i + 1] ?? end); + starts[i] = Math.max(start, Math.min(starts[i] ?? 0, limit - WORD_ORDER_SEC)); + } + return words.map((w, i) => { + const startSec = starts[i] ?? w.startSec; + const endSec = + w.startSec >= end + ? end + : w.endSec <= start + ? Math.min(end, start + PULLED_WORD_SEC) + : Math.max(w.endSec, startSec + WORD_ORDER_SEC); return startSec === w.startSec && endSec === w.endSec ? w : { ...w, startSec, endSec }; }); } @@ -575,7 +584,13 @@ function joinContinuous(pieces: Piece[]): Piece[] { joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec, - asked: [...prev.asked, ...piece.asked], + // One range across a split comes back once, or its captions would repeat. + asked: [ + ...prev.asked, + ...piece.asked.filter( + (r) => !prev.asked.some((q) => q.fromIndex === r.fromIndex && q.toIndex === r.toIndex), + ), + ], }; } else { joined.push(piece); diff --git a/tests/timeline-pauses.test.ts b/tests/timeline-pauses.test.ts index 7819066..df9952f 100644 --- a/tests/timeline-pauses.test.ts +++ b/tests/timeline-pauses.test.ts @@ -136,6 +136,19 @@ describe('pauseAt', () => { expect(pauseAt(levels, 11.75, 11.65)).toBe('none'); }); + it('never takes the pause before the last word for the one after it', () => { + // An accurate last word, 15–15.5, a long pause before it and a short one after. + const around = levelsOf([ + [4.7, -20], + [0.3, -60], + [0.5, -20], + [0.1, -60], + [1, -20], + ]); + const pause = pauseAt(around, 15.5, 15); + expect(pause).toEqual({ startSec: expect.closeTo(15.5, 6), endSec: expect.closeTo(15.6, 6) }); + }); + it('says none when speech runs on, and ambiguous when two pauses compete', () => { expect(pauseAt(levelsOf([[3, -20]]), 11.5, 11)).toBe('none'); const two = levelsOf([ diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index cea617b..b8a2656 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -294,6 +294,13 @@ describe('planShort: cut from the source timeline', () => { const split = sourceOf([clipOf('c1', 0, 4, 0), clipOf('c2', 4, 10, 4)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); + // …and captions the range once. + const captioned = await plan(split, { + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + captions: true, + }); + const texts = addedClips(captioned.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts).toEqual(['w2 w3 w4']); }); it('keeps color and volume, and drops the fades of the source clip', async () => { @@ -650,6 +657,37 @@ describe('planShort: edges in the pauses', () => { expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 5.47, sourceOutSec: 5.7 }]); }); + it('keeps both captions when the last two words crowd the cut', async () => { + // Cut at 6.0; whisper puts the last two words at 5.99 and 6.1. + const transcript = transcriptOf(M, [ + [0.3, 1.3], + [1.3, 2.3], + [2.3, 3.3], + [3.3, 5.99], + [5.99, 6.1], + [6.1, 6.5], + [6.5, 7.5], + ]); + const speech: [number, number][] = [ + [0.1, 0.9], + [1.1, 1.9], + [2.1, 2.9], + [3.1, 3.9], + [4.1, 5], + [5.1, 5.9], + [6.3, 7], + ]; + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 5 }], maxWords: 1 }, + { transcript: lookupFor([transcript]), levels: levelsFrom(speech), mintId: counter() }, + ); + expect(short.pieces.at(-1)?.sourceOutSec).toBe(6); + const texts = addedClips(short.ops).flatMap((c) => (c.kind === 'text' ? [c.text] : [])); + expect(texts).toEqual(['w3', 'w4', 'w5']); + }); + it('keeps the caption of a last word whisper places past the cut', async () => { // Speech of word 5 ends at 5.6; whisper starts it at 5.8. const transcript = transcriptOf(M, [