From df882f2b2667992be50a6de720a4b57769201134 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 14:58:41 -0800 Subject: [PATCH 1/6] feat(timeline): add named timelines and make_short A workspace now holds several timelines: main stays at composition.json, others live in timelines// with their own rev, lock and undo history. The make_short verb builds a vertical short on its own timeline from transcript word ranges of a source timeline: padded edges, cut from the source's clips so its silence and retake cuts carry in, a 9:16 cover canvas, -14 LUFS and captions, as one undoable edit. The CLI gains --timeline, `short`, `list` and `delete`; every MCP timeline tool takes an optional timeline id and timeline_show lists them. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/segment-duration-rounding.md | 3 + src/cli-timeline.ts | 294 ++++++++++--- src/index.ts | 20 + src/mcp/server.ts | 189 +++++---- src/timeline/document-store.ts | 321 ++++++++++---- src/timeline/schema.ts | 14 + src/timeline/shorts.ts | 341 +++++++++++++++ src/timeline/verbs.ts | 92 +++- src/ui/server.ts | 6 +- src/ui/timeline-tools.ts | 2 + tests/document-store.test.ts | 134 +++++- tests/integration/timeline-shorts.test.ts | 159 +++++++ tests/mcp-server.test.ts | 133 +++++- tests/timeline-captions.test.ts | 3 + tests/timeline-shorts.test.ts | 492 ++++++++++++++++++++++ tests/timeline-verbs.test.ts | 6 + 16 files changed, 2000 insertions(+), 209 deletions(-) create mode 100644 docs/backlog/segment-duration-rounding.md create mode 100644 src/timeline/shorts.ts create mode 100644 tests/integration/timeline-shorts.test.ts create mode 100644 tests/timeline-shorts.test.ts diff --git a/docs/backlog/segment-duration-rounding.md b/docs/backlog/segment-duration-rounding.md new file mode 100644 index 0000000..46e8c92 --- /dev/null +++ b/docs/backlog/segment-duration-rounding.md @@ -0,0 +1,3 @@ +# Segment duration rounding + +Export renders each clip as its own segment and joins them, and each segment comes out a little longer than its clip: the video rounds up to a whole frame and the AAC audio to a whole 1024-sample frame. A three-piece short planned at 16.200 s exported as 488 video frames (16.267 s at 30 fps) and 16.25 s of audio. Check whether the audio-video offset accumulates over a timeline with hundreds of cuts (a silence-cut hour), and if it does, trim each segment to a frame-aligned length or carry the remainder into the next segment. diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e623c1e..ceafd6e 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -9,6 +9,7 @@ import { import { AnchorSchema, type Clip, + type Composition, clipDuration, clipEndSec, clipsAtTime, @@ -24,11 +25,15 @@ import { type TextStyle, } from './timeline/composition.js'; import { + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, undoLastDocOp, } from './timeline/document-store.js'; import { @@ -49,6 +54,8 @@ import { } from './timeline/reframe.js'; import { reviewCuts } from './timeline/review.js'; import { runPlan } from './timeline/run-plan.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './timeline/schema.js'; +import { checkShortTimelines, planShort, type ShortRange } from './timeline/shorts.js'; import { analyzeSilence, DEFAULT_SENSITIVITY, SensitivitySchema } from './timeline/silence.js'; import { clipWords, @@ -63,6 +70,9 @@ import { getWorkspace, newOutputPath, resolveInput } from './workspace.js'; const TIMELINE_HELP = `clip timeline — build and export a non-destructive composition +Every command works on the main timeline unless --timeline names another +(a short made with \`short\` is a timeline of its own). + clip timeline new [--width W --height H --fps F --background C --fit contain|cover] clip timeline add-media [ ] [--start S] [--track T] clip timeline add-text [--start S] [--track T] [--font N --color C --anchor A --background C] @@ -81,6 +91,9 @@ const TIMELINE_HELP = `clip timeline — build and export a non-destructive comp clip timeline reframe <9:16|1:1|4:5|16:9|WxH> [--fit cover|contain] [--x 0..1] [--y 0..1] [--dry-run] clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset] clip timeline loudness <-14|-16|LUFS|off> + clip timeline short [ …] [--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] [--loudness LUFS|off] [--replace] [--dry-run] + clip timeline list + clip timeline delete clip timeline show clip timeline at clip timeline frame [] @@ -96,7 +109,7 @@ interface Parsed { } /** Flags that take no value, so they don't swallow the next argument. */ -const BOOLEAN_FLAGS = new Set(['dry-run', 'reset']); +const BOOLEAN_FLAGS = new Set(['dry-run', 'reset', 'no-captions', 'replace']); function parseArgs(args: string[]): Parsed { const positional: string[] = []; @@ -148,6 +161,36 @@ function framingFitFlag(value: string | undefined): FramingFit | undefined { return parsed.data; } +function timelineId(value: string): string { + const parsed = TimelineIdSchema.safeParse(value); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${value}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; +} + +function wordIndex(value: string | undefined): number { + const n = Number(value); + if (value === undefined || value.trim() === '' || !Number.isInteger(n) || n < 0) { + throw new Error(`A word index is a whole number from 0, got "${value}".`); + } + return n; +} + +/** `off`, or a loudness target within the export's range. */ +function loudnessValue(value: string, usage: string): number | null { + if (value === 'off') return null; + const parsed = TargetLufsSchema.safeParse(Number(value)); + if (!parsed.success) { + throw new Error( + `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, + ); + } + return parsed.data; +} + /** A short, human-readable label for a clip (for `show` / `at`). */ function clipLabel(clip: Clip): string { switch (clip.kind) { @@ -161,8 +204,10 @@ function clipLabel(clip: Clip): string { } export async function runTimeline(args: string[]): Promise { - const [sub, ...rest] = args; - const { positional, flags } = parseArgs(rest); + const parsed = parseArgs(args); + const [sub, ...positional] = parsed.positional; + const { flags } = parsed; + const timeline = flags.timeline !== undefined ? timelineId(flags.timeline) : MAIN_TIMELINE; switch (sub) { case undefined: @@ -180,8 +225,16 @@ export async function runTimeline(args: string[]): Promise { background: flags.background, fit: fitFlag(flags.fit), }), + timeline, ); - out({ created: true, width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }); + out({ + created: true, + timeline, + width: comp.width, + height: comp.height, + fps: comp.fps, + fit: comp.fit, + }); return; } @@ -197,7 +250,7 @@ export async function runTimeline(args: string[]): Promise { }); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = mediaClip({ id: makeClipId(), @@ -206,7 +259,9 @@ export async function runTimeline(args: string[]): Promise { sourceOutSec: num(outSec, result.ref.durationSec), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'media', clipId: clip.id, mediaId: result.mediaId, trackId, startSec: start }); return; } @@ -217,7 +272,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error('Usage: clip timeline add-text '); } const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const style: Record = {}; if (flags.font) style.fontSize = num(flags.font, 48); @@ -231,7 +286,9 @@ export async function runTimeline(args: string[]): Promise { startSec: start, style, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'text', clipId: clip.id, trackId, startSec: start }); return; } @@ -240,7 +297,7 @@ export async function runTimeline(args: string[]): Promise { const [duration] = positional; if (duration === undefined) throw new Error('Usage: clip timeline add-color '); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = colorClip({ id: makeClipId(), @@ -248,7 +305,9 @@ export async function runTimeline(args: string[]): Promise { durationSec: num(duration, 2), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'color', clipId: clip.id, trackId, startSec: start }); return; } @@ -258,17 +317,20 @@ export async function runTimeline(args: string[]): Promise { if (!afterClipId) throw new Error('Usage: clip timeline transition [] []'); const trackId = flags.track || DEFAULT_TRACK; - await mutateComposition([ - { - op: 'addTransition', - trackId, - transition: { - afterClipId, - kind: (kind as never) ?? 'fade', - durationSec: num(duration, 1), + await mutateComposition( + [ + { + op: 'addTransition', + trackId, + transition: { + afterClipId, + kind: (kind as never) ?? 'fade', + durationSec: num(duration, 1), + }, }, - }, - ]); + ], + { timeline }, + ); out({ added: 'transition', afterClipId, kind: kind ?? 'fade', trackId }); return; } @@ -278,9 +340,10 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || inSec === undefined || outSec === undefined) { throw new Error('Usage: clip timeline trim '); } - await mutateComposition([ - { op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }, - ]); + await mutateComposition( + [{ op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }], + { timeline }, + ); out({ trimmed: clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }); return; } @@ -289,9 +352,17 @@ export async function runTimeline(args: string[]): Promise { const [clipId, startSec] = positional; if (!clipId || startSec === undefined) throw new Error('Usage: clip timeline move '); - await mutateComposition([ - { op: 'moveClip', clipId, startSec: num(startSec, 0), toTrackId: flags.track || undefined }, - ]); + await mutateComposition( + [ + { + op: 'moveClip', + clipId, + startSec: num(startSec, 0), + toTrackId: flags.track || undefined, + }, + ], + { timeline }, + ); out({ moved: clipId, startSec: num(startSec, 0), trackId: flags.track || undefined }); return; } @@ -301,7 +372,9 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || atSec === undefined) throw new Error('Usage: clip timeline split '); const newClipId = makeClipId(); - await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }]); + await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }], { + timeline, + }); out({ split: clipId, into: [clipId, newClipId], atSec: num(atSec, 0) }); return; } @@ -309,7 +382,7 @@ export async function runTimeline(args: string[]): Promise { case 'remove': { const [clipId] = positional; if (!clipId) throw new Error('Usage: clip timeline remove '); - await mutateComposition([{ op: 'removeClip', clipId }]); + await mutateComposition([{ op: 'removeClip', clipId }], { timeline }); out({ removed: clipId }); return; } @@ -324,7 +397,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error(`--sensitivity must be between 0 and 1, got "${flags.sensitivity}".`); } const sensitivity = parsed.data; - const comp = await readComposition(); + const comp = await readComposition(timeline); const cut = await analyzeSilence(comp, clipId, { media: await buildMediaMap(), sensitivity }); const report = { clipId, @@ -342,7 +415,7 @@ export async function runTimeline(args: string[]): Promise { return; } // Analysis read `comp` outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, clipIds: [clipId, ...cut.op.newClipIds], rev: doc.rev }); return; } @@ -359,7 +432,7 @@ export async function runTimeline(args: string[]): Promise { if (!aspect.success && !size) { throw new Error(`Unknown shape "${shape}": use 9:16, 1:1, 4:5, 16:9 or WIDTHxHEIGHT.`); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const plan = planReframe(comp, { ...(aspect.success ? { aspect: aspect.data } @@ -373,7 +446,7 @@ export async function runTimeline(args: string[]): Promise { out({ ...report, applied: false, ...(flags['dry-run'] ? { dryRun: true } : {}) }); return; } - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } @@ -385,7 +458,7 @@ export async function runTimeline(args: string[]): Promise { 'Usage: clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const ops = planFraming(comp, { clipId, fit: framingFitFlag(flags.fit), @@ -398,7 +471,7 @@ export async function runTimeline(args: string[]): Promise { out({ clipId, applied: false, message: 'Nothing to change.' }); return; } - const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev, timeline }); const framed = findClip(doc, clipId); out({ clipId, @@ -413,30 +486,122 @@ export async function runTimeline(args: string[]): Promise { const [value] = positional; const usage = 'Usage: clip timeline loudness '; if (!value) throw new Error(usage); - let targetLufs: number | null = null; - if (value !== 'off') { - const parsed = TargetLufsSchema.safeParse(Number(value)); - if (!parsed.success) { - throw new Error( - `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, - ); - } - targetLufs = parsed.data; - } - const comp = await readComposition(); + const targetLufs = loudnessValue(value, usage); + const comp = await readComposition(timeline); if ((comp.loudness?.targetLufs ?? null) === targetLufs) { out({ loudness: comp.loudness, applied: false }); return; } const doc = await mutateComposition([{ op: 'setLoudness', targetLufs }], { expectedBaseRev: comp.rev, + timeline, }); out({ loudness: doc.loudness, applied: true, rev: doc.rev }); return; } + case 'short': { + const [id, ...rangeArgs] = positional; + const usage = + 'Usage: clip timeline short [ …] ' + + '[--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] ' + + '[--loudness LUFS|off] [--replace] [--dry-run]'; + if (!id || rangeArgs.length === 0 || rangeArgs.length % 3 !== 0) throw new Error(usage); + const target = timelineId(id); + if (flags.timeline !== undefined && timeline !== target) { + throw new Error( + `\`short\` builds the timeline named by its first argument ("${target}"), not --timeline. Use --source for the timeline to cut from.`, + ); + } + const sourceId = flags.source !== undefined ? timelineId(flags.source) : MAIN_TIMELINE; + checkShortTimelines(target, sourceId); + const ranges: ShortRange[] = []; + for (let i = 0; i < rangeArgs.length; i += 3) { + ranges.push({ + clipId: rangeArgs[i] ?? '', + fromIndex: wordIndex(rangeArgs[i + 1]), + toIndex: wordIndex(rangeArgs[i + 2]), + }); + } + const aspect = flags.aspect === undefined ? undefined : AspectSchema.safeParse(flags.aspect); + if (aspect && !aspect.success) { + throw new Error(`--aspect must be 9:16, 1:1, 4:5 or 16:9, got "${flags.aspect}".`); + } + const maxWords = num(flags.words, DEFAULT_CAPTION_WORDS); + if (!Number.isInteger(maxWords) || maxWords < 1 || maxWords > 12) { + throw new CaptionError( + `--words must be a whole number from 1 to 12, got "${flags.words}".`, + ); + } + + let current: Composition; + try { + current = await readComposition(target); + } catch (err) { + if (!(err instanceof NoTimelineError)) throw err; + current = emptyComposition(); + } + const media = await buildMediaMap(); + const plan = await planShort( + current, + await readComposition(sourceId), + { + ranges, + aspect: aspect?.data, + x: optionalNum('x', flags.x), + y: optionalNum('y', flags.y), + captions: flags['no-captions'] !== 'true', + maxWords, + targetLufs: + flags.loudness === undefined ? undefined : loudnessValue(flags.loudness, usage), + replace: flags.replace === 'true', + }, + (comp, clipId, transcriptId) => storedTranscriptForClip(comp, clipId, media, transcriptId), + ); + const report = { + timeline: target, + source: sourceId, + durationSec: plan.durationSec, + pieces: plan.pieces, + captions: plan.captions, + warnings: plan.warnings, + }; + if (flags['dry-run']) { + out({ ...report, applied: false, dryRun: true }); + return; + } + // The source and transcripts were read outside the write lock; reject if + // the short's timeline moved since. + const doc = await mutateComposition(plan.ops, { + expectedBaseRev: current.rev, + timeline: target, + create: true, + }); + out({ + ...report, + applied: true, + canvas: { width: doc.width, height: doc.height, fit: doc.fit }, + loudness: doc.loudness, + rev: doc.rev, + }); + return; + } + + case 'list': + out({ timelines: await listTimelines() }); + return; + + case 'delete': { + const [id] = positional; + if (!id) throw new Error('Usage: clip timeline delete '); + const target = timelineId(id); + await deleteTimeline(target); + out({ deleted: target }); + return; + } + case 'check': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const sheetPath = flags.sheet ? resolveInput(flags.sheet) : newOutputPath('cut-sheet', 'jpg'); out(await reviewCuts(comp, media, sheetPath, getWorkspace())); @@ -450,7 +615,7 @@ export async function runTimeline(args: string[]): Promise { 'Usage: clip timeline transcribe [--model ] [--language ]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); out( await transcribeClip(comp, clipId, await buildMediaMap(), { model: flags.model ? resolveInput(flags.model) : undefined, @@ -471,7 +636,7 @@ export async function runTimeline(args: string[]): Promise { : 'Usage: clip timeline remove-retakes [--dry-run]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const { clip, transcript } = await storedTranscriptForClip( comp, @@ -492,7 +657,7 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcript was read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([op], { expectedBaseRev: comp.rev, timeline }); out({ clipId, applied: true, ranges, rev: doc.rev }); return; } @@ -511,7 +676,7 @@ export async function runTimeline(args: string[]): Promise { if (flags.background) style.background = flags.background === 'none' ? null : flags.background; if (flags.anchor) style.anchor = AnchorSchema.parse(flags.anchor); - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const plan = await planCaptions( comp, @@ -530,15 +695,16 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcripts were read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } case 'show': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const { exportable, blockers } = checkExportable(comp, await buildMediaMap(), getWorkspace()); out({ + timeline, rev: comp.rev, durationSec: compositionDuration(comp), canvas: { width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }, @@ -575,7 +741,7 @@ export async function runTimeline(args: string[]): Promise { const [at] = positional; if (!at) throw new Error('Usage: clip timeline at '); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); out({ atSec, clips: clipsAtTime(comp, atSec).map((h) => ({ @@ -593,7 +759,7 @@ export async function runTimeline(args: string[]): Promise { const [at, output] = positional; if (!at) throw new Error('Usage: clip timeline frame []'); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-frame', 'jpg'); const plan = buildFrameAtPlan( @@ -607,7 +773,7 @@ export async function runTimeline(args: string[]): Promise { } case 'undo': { - const { undone, doc, label } = await undoLastDocOp(); + const { undone, doc, label } = await undoLastDocOp(timeline); out( undone ? { undone: true, label, rev: doc.rev } @@ -617,7 +783,7 @@ export async function runTimeline(args: string[]): Promise { } case 'redo': { - const { redone, doc, label } = await redoDocOp(); + const { redone, doc, label } = await redoDocOp(timeline); out( redone ? { redone: true, label, rev: doc.rev } @@ -627,7 +793,7 @@ export async function runTimeline(args: string[]): Promise { } case 'log': { - const log = await readDocOpLog(); + const log = await readDocOpLog(timeline); out({ rev: log.rev, cursor: log.cursor, @@ -653,10 +819,14 @@ export async function runTimeline(args: string[]): Promise { const target = output ? resolveInput(output) : newOutputPath('timeline-export', interchangeExtension(parsed.data)); - const result = await exportInterchange(await readComposition(), await buildMediaRefs(), { - format: parsed.data, - output: target, - }); + const result = await exportInterchange( + await readComposition(timeline), + await buildMediaRefs(), + { + format: parsed.data, + output: target, + }, + ); out({ exported: target, format: result.format, @@ -666,7 +836,7 @@ export async function runTimeline(args: string[]): Promise { }); return; } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-export', 'mp4'); const plan = compileTimeline(comp, { media, dir: getWorkspace(), output: finalOutput }); diff --git a/src/index.ts b/src/index.ts index 76eb806..30de026 100644 --- a/src/index.ts +++ b/src/index.ts @@ -165,12 +165,17 @@ export { CompositionCorruptError, compositionOpsPath, compositionPath, + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, overwriteComposition, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, + type TimelineListing, undoLastDocOp, writeComposition, writeCompositionIfUnchanged, @@ -225,6 +230,7 @@ export { export { type LoudnessResult, type RunPlanResult, runPlan } from './timeline/run-plan.js'; export { AudioStreamSchema, + MAIN_TIMELINE, type MediaId, MediaIdSchema, type MediaRef, @@ -232,8 +238,22 @@ export { makeMediaId, type Timecode, TimecodeSchema, + TimelineIdSchema, VideoStreamSchema, } from './timeline/schema.js'; +export { + checkShortTimelines, + DEFAULT_SHORT_LUFS, + planShort, + SHORT_LEAD_SEC, + SHORT_TAIL_SEC, + ShortError, + type ShortPiece, + type ShortPlan, + type ShortRange, + type ShortRequest, + type ShortTranscriptLookup, +} from './timeline/shorts.js'; export { analyzeSilence, DEFAULT_SENSITIVITY, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 7416f5d..db573d5 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -15,6 +15,8 @@ import { import type { Composition } from '../timeline/composition.js'; import { applyVerbs, + listTimelines, + NoTimelineError, readComposition, redoDocOp, undoLastDocOp, @@ -28,6 +30,8 @@ import { import { buildMediaMap } from '../timeline/media-registry.js'; import { reviewCuts } from '../timeline/review.js'; import { runPlan } from '../timeline/run-plan.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; import { transcribeClip } from '../timeline/transcript.js'; import { EditableVerbSchema } from '../timeline/verbs.js'; import { doctor } from '../tools/doctor.js'; @@ -98,12 +102,30 @@ async function jpegContent( */ async function describeComposition( comp: Composition, -): Promise { + timeline: string, +): Promise { const media = await buildMediaMap(); const { exportable, blockers } = checkExportable(comp, media, getWorkspace()); - return { ...summarizeComposition(comp), exportable, blockers }; + return { timeline, ...summarizeComposition(comp), exportable, blockers }; } +/** Answer a missing timeline or a short that can't be made with `{ error }`: + * both are the agent's to fix, like an export it can't render. */ +async function answeringTimelineErrors( + run: () => Promise, +): Promise> { + try { + return await run(); + } catch (err) { + if (err instanceof NoTimelineError || err instanceof ShortError) { + return jsonResult({ error: err.message }); + } + throw err; + } +} + +const timelineInput = TimelineIdSchema.optional(); + /** * Build the MakeMyClip Editor MCP server — a second front door onto the SAME * op-aware, undoable CompositionDoc the CLI, the `clip ui`, and the Claude Code @@ -119,8 +141,9 @@ export function buildMcpServer(): McpServer { { title: 'Show the timeline', description: - 'Read the current timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', + 'Read a timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers) — plus `timelines`, the ids of every timeline in the workspace (main and each short). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', inputSchema: { + timeline: timelineInput, atSec: z .number() .nonnegative() @@ -130,35 +153,37 @@ export function buildMcpServer(): McpServer { ), }, }, - async ({ atSec }) => { - const comp = await readComposition(); - const summary = jsonResult(await describeComposition(comp)); - if (atSec === undefined) return summary; - const output = newOutputPath('mcp-frame', 'jpg'); - try { - const plan = buildFrameAtPlan( - comp, - { - media: await confinedMediaMap(comp), - dir: getWorkspace(), - output, - tag: `mcp${randomBytes(4).toString('hex')}`, - }, - atSec, - ); - await runPlan(plan); - return { content: [...summary.content, await jpegContent(output)] }; - } catch (err) { - if (!(err instanceof CompileError)) throw err; - const frameError = { - type: 'text' as const, - text: JSON.stringify({ frameError: err.message }), - }; - return { content: [...summary.content, frameError] }; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE, atSec }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const timelines = (await listTimelines()).map((t) => t.id); + const summary = jsonResult({ ...(await describeComposition(comp, timeline)), timelines }); + if (atSec === undefined) return summary; + const output = newOutputPath('mcp-frame', 'jpg'); + try { + const plan = buildFrameAtPlan( + comp, + { + media: await confinedMediaMap(comp), + dir: getWorkspace(), + output, + tag: `mcp${randomBytes(4).toString('hex')}`, + }, + atSec, + ); + await runPlan(plan); + return { content: [...summary.content, await jpegContent(output)] }; + } catch (err) { + if (!(err instanceof CompileError)) throw err; + const frameError = { + type: 'text' as const, + text: JSON.stringify({ frameError: err.message }), + }; + return { content: [...summary.content, frameError] }; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -167,22 +192,24 @@ export function buildMcpServer(): McpServer { title: 'Check the cuts', description: "Review every hard cut (abutting clips with no transition). Reports cuts whose audio is loud right at the edge, which usually means a clipped word or a click, and returns a contact sheet image: one row per cut, the outgoing clip's last frame then the incoming clip's first. Run it after cutting (e.g. remove_silence) and before exporting.", + inputSchema: { timeline: timelineInput }, }, - async () => { - const comp = await readComposition(); - const media = await confinedMediaMap(comp); - const output = newOutputPath('cut-sheet', 'jpg'); - try { - const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); - const summary = jsonResult({ - ...review, - sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, - }); - return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const media = await confinedMediaMap(comp); + const output = newOutputPath('cut-sheet', 'jpg'); + try { + const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); + const summary = jsonResult({ + ...review, + sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, + }); + return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -190,13 +217,19 @@ export function buildMcpServer(): McpServer { { title: 'Edit the timeline', description: - 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', - inputSchema: { verbs: z.array(EditableVerbSchema).min(1) }, - }, - async ({ verbs }) => { - const { doc, ops } = await applyVerbs(verbs, makeVerbContext()); - return jsonResult({ applied: ops.length, document: await describeComposition(doc) }); + 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). ' + + 'make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as "best-1" (never main). It plays the given word ranges (indices from timeline_transcript on the source timeline, default main) with a little padding, cut from the source\'s clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical), look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. ' + + 'Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', + inputSchema: { timeline: timelineInput, verbs: z.array(EditableVerbSchema).min(1) }, }, + async ({ timeline = MAIN_TIMELINE, verbs }) => + answeringTimelineErrors(async () => { + const { doc, ops } = await applyVerbs(verbs, makeVerbContext(), timeline); + return jsonResult({ + applied: ops.length, + document: await describeComposition(doc, timeline), + }); + }), ); server.registerTool( @@ -206,34 +239,46 @@ export function buildMcpServer(): McpServer { description: "Transcribe a media clip's speech with the local whisper.cpp and return its words with timeline times. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript. A word's `index` is stable for its source file.", inputSchema: { + timeline: timelineInput, clipId: z.string().describe('Media clip to transcribe.'), language: WhisperLanguageSchema.optional(), }, }, - async ({ clipId, language }) => { - const comp = await readComposition(); - return jsonResult( - await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), - ); - }, + async ({ timeline = MAIN_TIMELINE, clipId, language }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + return jsonResult( + await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), + ); + }), ); server.registerTool( 'timeline_undo', - { title: 'Undo', description: 'Undo the most recent timeline edit.' }, - async () => { - const { undone, label } = await undoLastDocOp(); - return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + { + title: 'Undo', + description: 'Undo the most recent edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { undone, label } = await undoLastDocOp(timeline); + return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + }), ); server.registerTool( 'timeline_redo', - { title: 'Redo', description: 'Redo the most recently undone timeline edit.' }, - async () => { - const { redone, label } = await redoDocOp(); - return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + { + title: 'Redo', + description: 'Redo the most recently undone edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { redone, label } = await redoDocOp(timeline); + return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + }), ); server.registerTool( @@ -241,10 +286,11 @@ export function buildMcpServer(): McpServer { { title: 'Export the timeline', description: - 'Export the current timeline document to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + + 'Export a timeline (default main; pass `timeline` for a short) to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + 'fcpxml writes an FCPXML project for Final Cut Pro and DaVinci Resolve; xmeml writes Final Cut Pro 7 XML for Premiere Pro (Resolve reads it too), with text as .srt caption files beside it. ' + 'Both reference the source media by absolute path, refuse what the editor file cannot hold with the fix to make, and return notes on what is carried only in part.', inputSchema: { + timeline: timelineInput, format: z .enum(['mp4', 'fcpxml', 'xmeml']) .optional() @@ -255,9 +301,9 @@ export function buildMcpServer(): McpServer { .describe('Output file inside the workspace (relative paths resolve against it).'), }, }, - async ({ format = 'mp4', output }) => { - const comp = await readComposition(); + async ({ timeline = MAIN_TIMELINE, format = 'mp4', output }) => { try { + const comp = await readComposition(timeline); if (format !== 'mp4') { const target = output ? resolveInWorkspace(output) @@ -290,7 +336,8 @@ export function buildMcpServer(): McpServer { if ( err instanceof CompileError || err instanceof InterchangeError || - err instanceof WorkspaceBoundaryError + err instanceof WorkspaceBoundaryError || + err instanceof NoTimelineError ) { return jsonResult({ error: err.message }); } diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index d3f71c1..2a16dc1 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -1,10 +1,20 @@ import { randomBytes } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { readdir, readFile, rm } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; import { atomicWriteFile } from '../session/store.js'; -import { createRevisionedStore } from '../storage/revisioned-store.js'; -import { ensureWorkspace, getWorkspace } from '../workspace.js'; -import { type Composition, CompositionSchema, emptyComposition } from './composition.js'; +import { createRevisionedStore, type RevisionedStore } from '../storage/revisioned-store.js'; +import { + ensureWorkspace, + getWorkspace, + resolveInWorkspace, + WorkspaceBoundaryError, +} from '../workspace.js'; +import { + type Composition, + CompositionSchema, + compositionDuration, + emptyComposition, +} from './composition.js'; import { type DocOpLog, type DocOpLogEntry, @@ -17,17 +27,67 @@ import { undo as undoLog, } from './doc-op-log.js'; import { applyOps, applyOpsTracked, type CompositionOp } from './ops.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './schema.js'; import { type CompositionVerb, lowerVerbs, type VerbContext } from './verbs.js'; +/* + * A workspace holds several named timelines. `main` stays at the workspace root + * (`composition.json` + `composition-ops.json`), so existing workspaces and older + * `clip` versions keep working; every other timeline keeps the same two files in + * `timelines//`. Each timeline has its own rev, write lock and undo history. + */ const COMPOSITION_FILE = 'composition.json'; const COMPOSITION_OPS_FILE = 'composition-ops.json'; +const TIMELINES_DIR = 'timelines'; + +/** Thrown for a malformed timeline id, or for an operation `main` can't take. */ +export class TimelineIdError extends Error { + constructor(message: string) { + super(message); + this.name = 'TimelineIdError'; + } +} + +/** Thrown when a timeline other than `main` doesn't exist. Only `make_short` and + * `clip timeline new --timeline` create one, so a typo can't edit an empty document. */ +export class NoTimelineError extends Error { + readonly timeline: string; + constructor(timeline: string) { + super( + `No timeline "${timeline}" in the workspace. List them with \`clip timeline list\` ` + + `(timeline_show reports them too). make_short creates a short's timeline.`, + ); + this.name = 'NoTimelineError'; + this.timeline = timeline; + } +} -export function compositionPath(): string { - return resolve(getWorkspace(), COMPOSITION_FILE); +function checkedTimelineId(timeline: string): string { + const parsed = TimelineIdSchema.safeParse(timeline); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${timeline}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; } -export function compositionOpsPath(): string { - return resolve(getWorkspace(), COMPOSITION_OPS_FILE); +function timelineFile(timeline: string, file: string): string { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) return resolve(getWorkspace(), file); + // Checked against the workspace: `timelines/` could be a symlink out of it. + return resolveInWorkspace(join(TIMELINES_DIR, timeline, file)); +} + +export function compositionPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_FILE); +} + +export function compositionOpsPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_OPS_FILE); +} + +function isMissingFile(err: unknown): boolean { + return err instanceof Error && 'code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'); } /** Thrown when composition.json exists but cannot be parsed/validated — surfaced, @@ -65,31 +125,52 @@ export class CompositionConflictError extends Error { } } -export async function readComposition(): Promise { +/** The stored document, or null when the timeline has none yet. */ +async function readStoredComposition(timeline: string): Promise { await ensureWorkspace(); + const path = compositionPath(timeline); let raw: string; try { - raw = await readFile(compositionPath(), 'utf-8'); + raw = await readFile(path, 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyComposition(); + if (isMissingFile(err)) return null; throw err; } try { return CompositionSchema.parse(JSON.parse(raw)); } catch (err) { - throw new CompositionCorruptError(compositionPath(), { cause: err }); + throw new CompositionCorruptError(path, { cause: err }); } } -export async function writeComposition(comp: Composition): Promise { - await atomicWriteFile(compositionPath(), `${JSON.stringify(comp, null, 2)}\n`); +/** The document an edit starts from. A missing `main` is an empty document (a + * fresh workspace); any other missing timeline is an error unless the edit + * `create`s it. */ +async function readForEdit(timeline: string, create: boolean): Promise { + const comp = await readStoredComposition(timeline); + if (comp) return comp; + if (create || timeline === MAIN_TIMELINE) return emptyComposition(); + throw new NoTimelineError(timeline); +} + +/** Read a timeline's document (default `main`). Throws `NoTimelineError` for a + * timeline other than `main` that doesn't exist. */ +export async function readComposition(timeline: string = MAIN_TIMELINE): Promise { + return readForEdit(timeline, false); } -/** On-disk rev, treating a corrupt file as rev 0 — a deliberate overwrite is - * about to discard it anyway. Non-corruption read errors still propagate. */ -async function readCompositionRevTolerant(): Promise { +export async function writeComposition( + comp: Composition, + timeline: string = MAIN_TIMELINE, +): Promise { + await atomicWriteFile(compositionPath(timeline), `${JSON.stringify(comp, null, 2)}\n`); +} + +/** On-disk rev, treating a corrupt or missing file as rev 0 — a deliberate + * overwrite is about to replace it anyway. Other read errors still propagate. */ +async function readCompositionRevTolerant(timeline: string): Promise { try { - return (await readComposition()).rev; + return (await readStoredComposition(timeline))?.rev ?? 0; } catch (err) { if (err instanceof CompositionCorruptError) return 0; throw err; @@ -97,16 +178,26 @@ async function readCompositionRevTolerant(): Promise { } // Serialization + compare-and-swap, shared with the session store (see -// `storage/revisioned-store.ts`) so co-editing the doc (the agent + `clip ui`) -// can't silently lose updates. -const docStore = createRevisionedStore({ - read: readComposition, - write: writeComposition, - getRev: (comp) => comp.rev, - withRev: (comp, rev) => ({ ...comp, rev }), - readRevTolerant: readCompositionRevTolerant, - onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), -}); +// `storage/revisioned-store.ts`) so co-editing a timeline (the agent + `clip ui`) +// can't silently lose updates. One store per timeline, so each has its own lock. +const stores = new Map>(); + +function storeFor(timeline: string): RevisionedStore { + const id = checkedTimelineId(timeline); + let store = stores.get(id); + if (!store) { + store = createRevisionedStore({ + read: () => readComposition(id), + write: (comp) => writeComposition(comp, id), + getRev: (comp) => comp.rev, + withRev: (comp, rev) => ({ ...comp, rev }), + readRevTolerant: () => readCompositionRevTolerant(id), + onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), + }); + stores.set(id, store); + } + return store; +} // ─── op-log (undo/redo) ────────────────────────────────────────────────────── // @@ -130,13 +221,13 @@ function labelForOps(ops: CompositionOp[]): string { /** Read the raw op-log file. Missing OR corrupt → an empty log: undo history is a * convenience, never worth failing a read for. The caller reconciles it against * the authoritative document rev. */ -async function readDocOpLogFile(): Promise { +async function readDocOpLogFile(timeline: string): Promise { await ensureWorkspace(); let raw: string; try { - raw = await readFile(compositionOpsPath(), 'utf-8'); + raw = await readFile(compositionOpsPath(timeline), 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyDocOpLog(0); + if (isMissingFile(err)) return emptyDocOpLog(0); throw err; } try { @@ -146,8 +237,8 @@ async function readDocOpLogFile(): Promise { } } -async function writeDocOpLog(log: DocOpLog): Promise { - await atomicWriteFile(compositionOpsPath(), serializeDocOpLog(log)); +async function writeDocOpLog(log: DocOpLog, timeline: string): Promise { + await atomicWriteFile(compositionOpsPath(timeline), serializeDocOpLog(log)); } /** The op-log reconciled against the current document rev — the trustworthy view @@ -155,10 +246,10 @@ async function writeDocOpLog(log: DocOpLog): Promise { * inside the store's exclusive section so an interleaved IN-PROCESS commit can't * make a consistent log momentarily read as empty (cross-process reads stay * best-effort, as everywhere in this store). */ -export async function readDocOpLog(): Promise { - return docStore.runExclusive(async () => { - const docRev = (await readComposition()).rev; - return reconcile(await readDocOpLogFile(), docRev); +export async function readDocOpLog(timeline: string = MAIN_TIMELINE): Promise { + return storeFor(timeline).runExclusive(async () => { + const docRev = (await readComposition(timeline)).rev; + return reconcile(await readDocOpLogFile(timeline), docRev); }); } @@ -176,19 +267,21 @@ export async function readDocOpLog(): Promise { * shipping surface). */ async function commitDocAndLog( + timeline: string, + create: boolean, derive: ( doc: Composition, log: DocOpLog, ) => { doc: Composition; log: DocOpLog; result: T } | null, ): Promise<{ doc: Composition; result: T } | null> { - return docStore.runExclusive(async () => { + return storeFor(timeline).runExclusive(async () => { for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const log = reconcile(await readDocOpLogFile(), current.rev); + const current = await readForEdit(timeline, create); + const log = reconcile(await readDocOpLogFile(timeline), current.rev); const derived = derive(current, log); if (derived === null) return null; - const observed = (await readComposition()).rev; + const observed = (await readForEdit(timeline, create)).rev; if (observed !== current.rev) { if (attempt === MAX_COMMIT_ATTEMPTS) { throw new CompositionConflictError(current.rev, observed); @@ -199,8 +292,8 @@ async function commitDocAndLog( const nextRev = current.rev + 1; const committedDoc = { ...derived.doc, rev: nextRev }; const committedLog = { ...derived.log, rev: nextRev }; - await writeDocOpLog(committedLog); - await writeComposition(committedDoc); + await writeDocOpLog(committedLog, timeline); + await writeComposition(committedDoc, timeline); return { doc: committedDoc, result: derived.result }; } throw new CompositionConflictError(-1, -1); @@ -211,14 +304,18 @@ async function commitDocAndLog( * Read → apply ops → write, returning the new composition, and record the batch * (with its computed inverse) to the op-log so the edit is undoable. Serialized + * rev compare-and-swap via the shared revisioned store, so the agent and the UI - * can co-edit without lost updates. An empty batch is a true no-op. + * can co-edit without lost updates. An empty batch is a true no-op. `timeline` + * defaults to `main`; `create` lets the edit start a timeline that doesn't exist + * yet (from an empty document) instead of throwing `NoTimelineError`. */ export async function mutateComposition( ops: CompositionOp[], - opts?: { expectedBaseRev?: number }, + opts?: { expectedBaseRev?: number; timeline?: string; create?: boolean }, ): Promise { - if (ops.length === 0) return readComposition(); - const committed = await commitDocAndLog((current, log) => { + const timeline = opts?.timeline ?? MAIN_TIMELINE; + const create = opts?.create ?? false; + if (ops.length === 0) return readForEdit(timeline, create); + const committed = await commitDocAndLog(timeline, create, (current, log) => { // Optimistic-concurrency guard for callers whose ops were lowered against a // specific revision (e.g. `applyVerbs`, where a default append point is baked // from a snapshot read): if the doc moved under us, reject so the caller can @@ -235,7 +332,7 @@ export async function mutateComposition( }; return { doc, log: recordOps(log, entry), result: null }; }); - return committed ? committed.doc : readComposition(); + return committed ? committed.doc : readForEdit(timeline, create); } /** @@ -243,12 +340,16 @@ export async function mutateComposition( * op-aware mutation path the agent and `clip ui` share. Lowering (which ingests * files and mints ids) runs outside the write lock; `mutateComposition` then * validates, applies, and records the ops (so the edit lands in the undo stack). - * Returns the new document and the ops that were applied. + * Returns the new document and the ops that were applied. Edits `timeline` + * (default `main`), which must exist unless the batch starts with `make_short`. */ export async function applyVerbs( verbs: CompositionVerb[], ctx: VerbContext, + timeline: string = MAIN_TIMELINE, ): Promise<{ doc: Composition; ops: CompositionOp[] }> { + const create = verbs[0]?.verb === 'make_short'; + const editCtx: VerbContext = { ...ctx, timeline }; // The default append point (`trackEnd`) is baked from the snapshot we lower // against, so if another in-process writer commits between the read and the // apply, re-lower against the fresh doc rather than land overlapping clips. A @@ -257,10 +358,14 @@ export async function applyVerbs( // a burst of N concurrent edits each re-lower and land instead of the (N-cap) // tail silently losing to a `CompositionConflictError`. for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const ops = await lowerVerbs(current, verbs, ctx); + const current = await readForEdit(timeline, create); + const ops = await lowerVerbs(current, verbs, editCtx); try { - const doc = await mutateComposition(ops, { expectedBaseRev: current.rev }); + const doc = await mutateComposition(ops, { + expectedBaseRev: current.rev, + timeline, + create, + }); return { doc, ops }; } catch (err) { if (err instanceof CompositionConflictError && attempt < MAX_COMMIT_ATTEMPTS) continue; @@ -273,33 +378,33 @@ export async function applyVerbs( /** Undo the most recent recorded edit (apply its inverse), moving the op-log * cursor back. `undone: false` when there is nothing to undo. */ -export async function undoLastDocOp(): Promise<{ +export async function undoLastDocOp(timeline: string = MAIN_TIMELINE): Promise<{ undone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = undoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.inverse), log: step.log, result: step.entry.label }; }); - if (!committed) return { undone: false, doc: await readComposition(), label: null }; + if (!committed) return { undone: false, doc: await readComposition(timeline), label: null }; return { undone: true, doc: committed.doc, label: committed.result }; } /** Redo the most recently undone edit (re-apply its forward ops), moving the * cursor forward. `redone: false` when there is nothing to redo. */ -export async function redoDocOp(): Promise<{ +export async function redoDocOp(timeline: string = MAIN_TIMELINE): Promise<{ redone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = redoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.forward), log: step.log, result: step.entry.label }; }); - if (!committed) return { redone: false, doc: await readComposition(), label: null }; + if (!committed) return { redone: false, doc: await readComposition(timeline), label: null }; return { redone: true, doc: committed.doc, label: committed.result }; } @@ -318,9 +423,10 @@ export async function redoDocOp(): Promise<{ export async function writeCompositionIfUnchanged( comp: Composition, expectedRev: number, + timeline: string = MAIN_TIMELINE, ): Promise { - const committed = await docStore.writeIfUnchanged(comp, expectedRev); - await writeDocOpLog(emptyDocOpLog(committed.rev)); + const committed = await storeFor(timeline).writeIfUnchanged(comp, expectedRev); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); return committed; } @@ -332,12 +438,12 @@ export async function writeCompositionIfUnchanged( * replay an inverse against the wrong document. Log-first ordering (the seeded * log is empty anyway, so a crash resolves to the same empty history). */ -async function overwriteDocAndResetLog(comp: Composition): Promise { - return docStore.runExclusive(async () => { - const baseRev = await readCompositionRevTolerant(); +async function overwriteDocAndResetLog(comp: Composition, timeline: string): Promise { + return storeFor(timeline).runExclusive(async () => { + const baseRev = await readCompositionRevTolerant(timeline); const committed: Composition = { ...comp, rev: baseRev + 1 }; - await writeDocOpLog(emptyDocOpLog(committed.rev)); - await writeComposition(committed); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); + await writeComposition(committed, timeline); return committed; }); } @@ -348,12 +454,85 @@ async function overwriteDocAndResetLog(comp: Composition): Promise * recovered). Resets undo history — a recovered document has none — and advances * `rev` so a stale log can never collide with the new revision. */ -export function overwriteComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); +export function overwriteComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); } /** Start a fresh timeline (`clip timeline new`): replace the document, clear undo - * history, and advance `rev` past the previous one (never reuse a rev). */ -export function resetComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); + * history, and advance `rev` past the previous one (never reuse a rev). Creates + * `timeline` when it doesn't exist yet. */ +export function resetComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); +} + +// ─── the workspace's timelines ─────────────────────────────────────────────── + +/** A timeline in the workspace, or one whose document can't be read. */ +export type TimelineListing = + | { id: string; width: number; height: number; durationSec: number; rev: number } + | { id: string; error: string }; + +/** `main` first, then every `timelines//` holding a document, by id. A + * document that can't be read is listed with its error instead of failing the + * list; a `timelines/` folder that leads out of the workspace is refused. */ +export async function listTimelines(): Promise { + await ensureWorkspace(); + const listed: TimelineListing[] = []; + for (const id of [MAIN_TIMELINE, ...(await timelineFolders())]) { + try { + const comp = await readStoredComposition(id); + if (!comp && id !== MAIN_TIMELINE) continue; + const doc = comp ?? emptyComposition(); + listed.push({ + id, + width: doc.width, + height: doc.height, + durationSec: compositionDuration(doc), + rev: doc.rev, + }); + } catch (err) { + if (!(err instanceof CompositionCorruptError || err instanceof WorkspaceBoundaryError)) { + throw err; + } + listed.push({ id, error: err.message }); + } + } + return listed; +} + +async function timelineFolders(): Promise { + let names: string[]; + try { + names = await readdir(resolveInWorkspace(TIMELINES_DIR)); + } catch (err) { + if (isMissingFile(err)) return []; + throw err; + } + return names + .filter((name) => name !== MAIN_TIMELINE && TimelineIdSchema.safeParse(name).success) + .sort(); +} + +/** Remove a timeline and its undo history. `main` can't be deleted. */ +export async function deleteTimeline(timeline: string): Promise { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) { + throw new TimelineIdError( + "The main timeline can't be deleted. Start it over with `clip timeline new`.", + ); + } + await storeFor(timeline).runExclusive(async () => { + try { + if (!(await readStoredComposition(timeline))) throw new NoTimelineError(timeline); + } catch (err) { + // A corrupt document is deleted like any other. + if (!(err instanceof CompositionCorruptError)) throw err; + } + await rm(dirname(compositionPath(timeline)), { recursive: true, force: true }); + }); } diff --git a/src/timeline/schema.ts b/src/timeline/schema.ts index 88b07c5..76881ae 100644 --- a/src/timeline/schema.ts +++ b/src/timeline/schema.ts @@ -9,6 +9,20 @@ export const MediaIdSchema = z.string().regex(/^m_[a-f0-9]{12}$/, { message: 'MediaId must be of the form m_<12 hex chars>', }); +/** The timeline every workspace starts with, kept at `composition.json`. */ +export const MAIN_TIMELINE = 'main'; + +/** A named timeline in the workspace: `main`, or a short such as `best-1`. */ +export const TimelineIdSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9-]{0,47}$/, { + message: + 'A timeline id is 1–48 lowercase letters, digits or hyphens, starting with a letter or digit (e.g. "best-1").', + }) + .describe( + 'Timeline to work on (default "main"). A short made with make_short is a timeline of its own.', + ); + export const VideoStreamSchema = z.object({ codec: z.string(), width: z.number().int().positive(), diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts new file mode 100644 index 0000000..5fb7637 --- /dev/null +++ b/src/timeline/shorts.ts @@ -0,0 +1,341 @@ +import { planCaptions } from './captions.js'; +import { speedFactor, videoLayers } from './compile.js'; +import { + type Clip, + type Composition, + type Effect, + type MediaClip, + makeClipId, +} from './composition.js'; +import { applyOps, type CompositionOp, mediaClip, videoTrack } from './ops.js'; +import { type Aspect, canvasFor, planReframe } from './reframe.js'; +import { MAIN_TIMELINE, type MediaId } from './schema.js'; +import type { Transcript } from './transcript.js'; + +/** + * Shorts: a vertical clip cut out of a longer timeline by transcript word ranges, + * built on a timeline of its own. It is cut from the source timeline's clips, not + * from the raw file, so silence and retake cuts already made there carry in. It + * is a snapshot: later edits to the source don't flow into it. The agent or the + * person picks the words; nothing here scores highlights. + */ + +/** Kept before a range's first word, so its attack isn't clipped. */ +export const SHORT_LEAD_SEC = 0.1; +/** Kept after a range's last word, so its release and the breath after it aren't clipped. */ +export const SHORT_TAIL_SEC = 0.25; +/** Loudness a short is normalized to unless asked otherwise: the vertical platforms' -14 LUFS. */ +export const DEFAULT_SHORT_LUFS = -14; + +const SHORT_TRACK = 'v0'; +/** A piece shorter than this is dropped: too short to hear, just a flash. */ +const MIN_PIECE_SEC = 0.05; +const SAME_SEC = 1e-6; + +/** Words `fromIndex..toIndex` (inclusive) of the transcript behind a media clip + * on the source timeline. */ +export interface ShortRange { + clipId: string; + fromIndex: number; + toIndex: number; + /** The `transcriptId` the indices were read from; refused if it changed. */ + transcriptId?: string; +} + +export interface ShortRequest { + /** Played in this order, so a later moment can open the short as its hook. */ + ranges: ShortRange[]; + aspect?: Aspect; + /** Horizontal position of the picture in the frame: 0 left, 0.5 center, 1 right. */ + x?: number; + y?: number; + /** Burn in captions (default true). */ + captions?: boolean; + maxWords?: number; + /** Default -14; null leaves loudness off. */ + targetLufs?: number | null; + /** Rebuild a timeline that already holds a short. */ + replace?: boolean; +} + +/** A window of a source file the short plays, in source seconds. */ +export interface ShortPiece { + mediaId: MediaId; + sourceInSec: number; + sourceOutSec: number; +} + +export interface ShortPlan { + ops: CompositionOp[]; + pieces: ShortPiece[]; + durationSec: number; + /** Captions written (0 with captions off). */ + captions: number; + warnings: string[]; +} + +/** Thrown when a short can't be made from the request. */ +export class ShortError extends Error { + constructor(message: string) { + super(message); + this.name = 'ShortError'; + } +} + +/** A media clip of `comp` and its source's stored transcript (never transcribed here). */ +export type ShortTranscriptLookup = ( + comp: Composition, + clipId: string, + transcriptId?: string, +) => Promise<{ clip: MediaClip; transcript: Transcript }>; + +/** Refuse a short on `main` or on the timeline it is cut from. */ +export function checkShortTimelines(target: string, source: string = MAIN_TIMELINE): void { + if (target === MAIN_TIMELINE) { + throw new ShortError( + 'A short is built on a timeline of its own: pick a new timeline id such as "best-1", not main.', + ); + } + if (source === target) { + throw new ShortError( + `A short can't be cut from its own timeline "${target}". Cut it from main or another timeline.`, + ); + } +} + +interface WordWindow { + /** Padded edges, source seconds. */ + start: number; + end: number; + /** First word's start and last word's end. */ + speechStart: number; + speechEnd: number; +} + +interface Piece extends ShortPiece { + effects: Effect[]; +} + +/** + * Plan the edit that builds a short on `target` from `source`: for each range, + * the source timeline's clips under its words (padded), laid end to end on a + * fresh picture track, on a 9:16 canvas filled edge to edge, normalized to + * -14 LUFS and captioned. One batch, so one undo takes the whole short away. + */ +export async function planShort( + target: Composition, + source: Composition, + request: ShortRequest, + lookup: ShortTranscriptLookup, + mintId: () => string = makeClipId, +): Promise { + if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); + if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { + throw new ShortError( + 'That timeline already holds a short. Pass replace to rebuild it, or pick a new timeline id.', + ); + } + const picture = videoLayers(source).picture; + if (!picture) { + throw new ShortError( + 'The source timeline has no picture track to cut from: it is empty or has more than one.', + ); + } + + const pieces: Piece[] = []; + for (const range of request.ranges) { + const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); + const window = wordWindow(transcript, range, clip.id); + const found = piecesUnder(picture.clips, clip.mediaId, window); + if (found.length === 0) { + throw new ShortError( + `Words ${range.fromIndex}..${range.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, + ); + } + pieces.push(...found); + } + const joined = joinContinuous(pieces); + + const ops: CompositionOp[] = target.tracks.map( + (t): CompositionOp => ({ op: 'removeTrack', trackId: t.id }), + ); + const canvas = canvasFor(source, request.aspect ?? '9:16'); + ops.push( + { + op: 'setCanvas', + width: canvas.width, + height: canvas.height, + fps: source.fps, + background: source.background, + fit: 'cover', + }, + { op: 'addTrack', track: videoTrack({ id: SHORT_TRACK }) }, + ); + let at = 0; + for (const piece of joined) { + ops.push({ + op: 'addClip', + trackId: SHORT_TRACK, + clip: mediaClip({ + id: mintId(), + mediaId: piece.mediaId, + sourceInSec: piece.sourceInSec, + sourceOutSec: piece.sourceOutSec, + // Rounded to the microsecond: clean numbers, far inside the export's + // abutting tolerance. + startSec: Math.round(at * 1e6) / 1e6, + effects: piece.effects, + }), + }); + at += piece.sourceOutSec - piece.sourceInSec; + } + + if (request.x !== undefined || request.y !== undefined) { + const framed = planReframe(applyOps(target, ops), { + width: canvas.width, + height: canvas.height, + fit: 'cover', + x: request.x, + y: request.y, + }); + ops.push(...framed.ops); + } + + const targetLufs = request.targetLufs === undefined ? DEFAULT_SHORT_LUFS : request.targetLufs; + if ((target.loudness?.targetLufs ?? null) !== targetLufs) { + ops.push({ op: 'setLoudness', targetLufs }); + } + + let captions = 0; + if (request.captions !== false) { + const built = applyOps(target, ops); + const plan = await planCaptions( + built, + { maxWords: request.maxWords }, + (clipId, transcriptId) => lookup(built, clipId, transcriptId), + mintId, + ); + ops.push(...plan.ops); + captions = plan.captions; + } + + return { + ops, + pieces: joined.map(({ mediaId, sourceInSec, sourceOutSec }) => ({ + mediaId, + sourceInSec, + sourceOutSec, + })), + durationSec: roundMs(at), + captions, + warnings: [], + }; +} + +/** + * The source window a range plays: its first word's start minus the lead to its + * last word's end plus the tail. Each side stops at the neighboring word's edge, + * so the padding never takes in part of a word outside the range, and at the + * start of the file. + */ +function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): WordWindow { + const first = transcript.words[range.fromIndex]; + const last = transcript.words[range.toIndex]; + if (!first || !last || range.toIndex < range.fromIndex) { + throw new ShortError( + `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clipId}" ` + + `(it has ${transcript.words.length} words, numbered from 0).`, + ); + } + const before = transcript.words[range.fromIndex - 1]; + const after = transcript.words[range.toIndex + 1]; + const start = Math.min( + first.startSec, + Math.max(first.startSec - SHORT_LEAD_SEC, before?.endSec ?? 0, 0), + ); + const end = Math.max( + last.endSec, + Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), + ); + return { + start: roundMs(start), + end: roundMs(end), + speechStart: first.startSec, + speechEnd: last.endSec, + }; +} + +/** + * The parts of a window that the source timeline's picture clips of one file + * play, in timeline order. A moment the timeline plays twice is taken once, and + * a part holding only padding (no word of the range) is left out, since it would + * only add a jump cut. + */ +function piecesUnder(clips: readonly Clip[], mediaId: MediaId, window: WordWindow): Piece[] { + const taken: [number, number][] = []; + const pieces: Piece[] = []; + for (const clip of [...clips].sort((a, b) => a.startSec - b.startSec)) { + if (clip.kind !== 'media' || clip.mediaId !== mediaId) continue; + const lo = Math.max(window.start, clip.sourceInSec); + const hi = Math.min(window.end, clip.sourceOutSec); + if (hi <= lo) continue; + for (const [a, b] of subtract([lo, hi], taken)) { + if (b - a < MIN_PIECE_SEC || b < window.speechStart || a > window.speechEnd) continue; + const speed = speedFactor(clip.effects); + if (speed !== 1) { + throw new ShortError( + `Clip "${clip.id}" on the source timeline plays at ${speed}x, and a short can't include a speed change yet. ` + + 'Remove its speed effect or pick words outside it.', + ); + } + pieces.push({ + mediaId, + sourceInSec: a, + sourceOutSec: b, + // Fades belonged to the source clip's edges, not the short's. + effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), + }); + taken.push([a, b]); + } + } + return pieces; +} + +/** `[lo, hi]` minus the `taken` intervals, in order. */ +function subtract([lo, hi]: [number, number], taken: [number, number][]): [number, number][] { + let parts: [number, number][] = [[lo, hi]]; + for (const [a, b] of taken) { + parts = parts.flatMap(([p, q]): [number, number][] => { + if (b <= p || a >= q) return [[p, q]]; + const rest: [number, number][] = []; + if (a > p) rest.push([p, a]); + if (b < q) rest.push([b, q]); + return rest; + }); + } + return parts; +} + +/** Merge pieces that continue one another in the same file with the same + * effects (a split, not a cut, on the source), so the short has no cut there. */ +function joinContinuous(pieces: Piece[]): Piece[] { + const joined: Piece[] = []; + for (const piece of pieces) { + const prev = joined[joined.length - 1]; + if ( + prev && + prev.mediaId === piece.mediaId && + Math.abs(prev.sourceOutSec - piece.sourceInSec) < SAME_SEC && + JSON.stringify(prev.effects) === JSON.stringify(piece.effects) + ) { + joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; + } else { + joined.push(piece); + } + } + return joined; +} + +function roundMs(sec: number): number { + return Math.round(sec * 1000) / 1000; +} diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index b72e9ec..f9167c0 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -21,7 +21,8 @@ import { planFraming, planReframe, } from './reframe.js'; -import type { MediaId } from './schema.js'; +import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from './schema.js'; +import { checkShortTimelines, planShort } from './shorts.js'; import { SensitivitySchema, type SilenceCut } from './silence.js'; import { clipWords, @@ -277,6 +278,71 @@ const EDITABLE_VERB_SCHEMAS = [ 'Normalize the exported mp4 to this integrated loudness in LUFS: -14 for YouTube, Shorts, Reels and TikTok, -16 for podcasts. null turns it off. Applies to mp4 export only.', ), }), + z.object({ + verb: z.literal('make_short'), + ranges: z + .array( + z.object({ + clipId: z + .string() + .describe( + 'A media clip on the source timeline whose transcript the indices come from.', + ), + fromIndex: z + .number() + .int() + .nonnegative() + .describe('First word of the range: its `index` from timeline_transcript.'), + toIndex: z.number().int().nonnegative().describe('Last word of the range (inclusive).'), + transcriptId: z + .string() + .optional() + .describe( + 'The `transcriptId` the indices were read from; refused if the clip was transcribed again since.', + ), + }), + ) + .min(1) + .max(20) + .describe( + 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences; 15–60 s in all is typical.', + ), + source: TimelineIdSchema.optional().describe( + 'Timeline to cut from (default "main"). Its silence and retake cuts carry into the short.', + ), + aspect: AspectSchema.optional().describe('Output shape (default 9:16).'), + x: z + .number() + .min(0) + .max(1) + .optional() + .describe( + 'Horizontal position of the picture: 0 shows the left edge of the source, 0.5 the center, 1 the right edge. Put the speaker in frame.', + ), + y: z.number().min(0).max(1).optional().describe('Vertical position: 0 top, 1 bottom.'), + captions: z + .boolean() + .optional() + .describe('Burn in captions from the transcript (default true).'), + maxWords: z + .number() + .int() + .min(1) + .max(12) + .optional() + .describe( + `Most words per caption (default ${DEFAULT_CAPTION_WORDS}); 1 shows one word at a time.`, + ), + targetLufs: TargetLufsSchema.nullable() + .optional() + .describe( + 'Export loudness in LUFS (default -14, for Shorts, Reels and TikTok); null for off.', + ), + replace: z + .boolean() + .optional() + .describe('Rebuild this timeline when it already holds a short (default: refuse).'), + }), z.object({ verb: z.literal('transition'), afterClipId: z @@ -331,8 +397,12 @@ export interface VerbContext { clipId: string, transcriptId?: string, ) => Promise<{ clip: MediaClip; transcript: Transcript; media: Map }>; + /** Read another timeline of the workspace (make_short's source). */ + readTimeline: (timeline: string) => Promise; /** Track a clip is placed on when the verb doesn't say. */ defaultTrack: string; + /** The timeline being edited (default `main`); `applyVerbs` sets it. */ + timeline?: string; } const DEFAULT_TRACK = 'v0'; @@ -493,6 +563,26 @@ export async function lowerVerb( return (comp.loudness?.targetLufs ?? null) === verb.targetLufs ? [] : [{ op: 'setLoudness', targetLufs: verb.targetLufs }]; + case 'make_short': { + const sourceId = verb.source ?? MAIN_TIMELINE; + checkShortTimelines(ctx.timeline ?? MAIN_TIMELINE, sourceId); + const plan = await planShort( + comp, + await ctx.readTimeline(sourceId), + { + ranges: verb.ranges, + aspect: verb.aspect, + x: verb.x, + y: verb.y, + captions: verb.captions, + maxWords: verb.maxWords, + targetLufs: verb.targetLufs, + replace: verb.replace, + }, + ctx.transcript, + ); + return plan.ops; + } case 'transition': return [ { diff --git a/src/ui/server.ts b/src/ui/server.ts index 735a0d4..26b2859 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -42,6 +42,7 @@ import { CompositionOpError } from '../timeline/ops.js'; import { ReframeError } from '../timeline/reframe.js'; import { runPlan } from '../timeline/run-plan.js'; import type { MediaId } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; import { SilenceAnalysisError } from '../timeline/silence.js'; import { TranscriptError, transcribeClip } from '../timeline/transcript.js'; import { CompositionVerbSchema } from '../timeline/verbs.js'; @@ -60,11 +61,12 @@ import { import { clipConfinedMedia, confinedMediaRefs, makeVerbContext } from './timeline-tools.js'; /** Errors about the clip or its setup, not the server: silence removal or - * transcription can't work on this clip, whisper.cpp isn't set up, or a - * reframe asks for a size or position that can't be. */ + * transcription can't work on this clip, whisper.cpp isn't set up, a reframe + * asks for a size or position that can't be, or a short can't be made. */ function isClipConditionError(err: unknown): boolean { return ( err instanceof ReframeError || + err instanceof ShortError || err instanceof SilenceAnalysisError || err instanceof TranscriptError || err instanceof TranscriptionUnavailableError diff --git a/src/ui/timeline-tools.ts b/src/ui/timeline-tools.ts index 15235b6..f1ac55d 100644 --- a/src/ui/timeline-tools.ts +++ b/src/ui/timeline-tools.ts @@ -7,6 +7,7 @@ import { type Fit, findClip, } from '../timeline/composition.js'; +import { readComposition } from '../timeline/document-store.js'; import { buildMediaMap, buildMediaRefs } from '../timeline/media-registry.js'; import { ownFraming } from '../timeline/reframe.js'; import type { MediaId, MediaRef } from '../timeline/schema.js'; @@ -33,6 +34,7 @@ export function makeVerbContext(): VerbContext { }); return { mediaId: result.mediaId, durationSec: result.ref.durationSec }; }, + readTimeline: (timeline) => readComposition(timeline), analyzeSilence: async (comp, clipId, sensitivity) => analyzeSilence(comp, clipId, { media: await clipConfinedMedia(comp, clipId), sensitivity }), transcript: async (comp, clipId, transcriptId) => { diff --git a/tests/document-store.test.ts b/tests/document-store.test.ts index 1a6c647..f8430bc 100644 --- a/tests/document-store.test.ts +++ b/tests/document-store.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -6,16 +6,25 @@ import { emptyComposition } from '../src/timeline/composition.js'; import { CompositionConflictError, CompositionCorruptError, + compositionOpsPath, compositionPath, + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, overwriteComposition, readComposition, + readDocOpLog, + redoDocOp, resetComposition, + TimelineIdError, + undoLastDocOp, writeComposition, writeCompositionIfUnchanged, } from '../src/timeline/document-store.js'; import { mediaClip, videoTrack } from '../src/timeline/ops.js'; import type { MediaId } from '../src/timeline/schema.js'; +import { WorkspaceBoundaryError } from '../src/workspace.js'; const M1 = 'm_aaaaaaaaaaaa' as MediaId; @@ -141,3 +150,126 @@ describe('overwriteComposition (recovery primitive)', () => { expect(reread.rev).toBe(1); }); }); + +describe('named timelines', () => { + it('keeps main at the workspace root and every other timeline in timelines//', () => { + expect(compositionPath()).toBe(join(workspace, 'composition.json')); + expect(compositionPath('main')).toBe(join(workspace, 'composition.json')); + expect(compositionOpsPath('main')).toBe(join(workspace, 'composition-ops.json')); + expect(compositionPath('best-1')).toBe( + join(workspace, 'timelines', 'best-1', 'composition.json'), + ); + expect(compositionOpsPath('best-1')).toBe( + join(workspace, 'timelines', 'best-1', 'composition-ops.json'), + ); + }); + + it('refuses a malformed timeline id before touching a path', () => { + for (const bad of ['../main', 'Best', '', '-x', 'a/b', 'x'.repeat(49)]) { + expect(() => compositionPath(bad)).toThrow(TimelineIdError); + } + }); + + it('throws NoTimelineError for a missing timeline other than main', async () => { + await expect(readComposition('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 1280 }], { timeline: 'nope' }), + ).rejects.toBeInstanceOf(NoTimelineError); + await expect(undoLastDocOp('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(redoDocOp('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(readDocOpLog('nope')).rejects.toBeInstanceOf(NoTimelineError); + await expect(writeCompositionIfUnchanged(emptyComposition(), 0, 'nope')).rejects.toBeInstanceOf( + NoTimelineError, + ); + expect((await readComposition('main')).tracks).toEqual([]); + }); + + it('creates a timeline with create or reset, then keeps its own rev and undo history', async () => { + await mutateComposition([{ op: 'setCanvas', width: 1280 }]); // main rev 1 + await mutateComposition([{ op: 'setCanvas', height: 720 }]); // main rev 2 + + const short = await mutateComposition([{ op: 'setCanvas', width: 1080, height: 1920 }], { + timeline: 'best-1', + create: true, + }); + expect(short.rev).toBe(1); + await resetComposition(emptyComposition({ fps: 24 }), 'best-2'); + expect((await readComposition('best-2')).fps).toBe(24); + + await mutateComposition([{ op: 'setCanvas', fps: 60 }], { timeline: 'best-1' }); + expect((await readComposition('best-1')).rev).toBe(2); + expect((await readComposition()).rev).toBe(2); + expect((await readDocOpLog('best-1')).entries).toHaveLength(2); + + await undoLastDocOp('best-1'); + expect((await readComposition('best-1')).fps).toBe(30); + expect((await readComposition()).height).toBe(720); + expect((await readDocOpLog()).cursor).toBe(2); + }); + + it('serializes writes per timeline, so concurrent edits of two timelines both land', async () => { + await resetComposition(emptyComposition(), 'best-1'); + await Promise.all([ + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'a' }) }], { + timeline: 'best-1', + }), + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'b' }) }], { + timeline: 'best-1', + }), + mutateComposition([{ op: 'addTrack', track: videoTrack({ id: 'm' }) }]), + ]); + expect((await readComposition('best-1')).tracks.map((t) => t.id).sort()).toEqual(['a', 'b']); + expect((await readComposition()).tracks.map((t) => t.id)).toEqual(['m']); + }); + + it('lists main first, then each timeline by id, and a corrupt one with its error', async () => { + await resetComposition(emptyComposition({ width: 1080, height: 1920 }), 'zeta'); + await resetComposition(emptyComposition(), 'alpha'); + await resetComposition(emptyComposition(), 'broken'); + await writeFile(compositionPath('broken'), 'not json'); + await mkdir(join(workspace, 'timelines', 'empty-folder'), { recursive: true }); + await writeFile(join(workspace, 'timelines', 'stray.txt'), 'x'); + + const listed = await listTimelines(); + expect(listed.map((t) => t.id)).toEqual(['main', 'alpha', 'broken', 'zeta']); + expect(listed[0]).toEqual({ id: 'main', width: 1920, height: 1080, durationSec: 0, rev: 0 }); + expect(listed[3]).toMatchObject({ id: 'zeta', width: 1080, height: 1920, rev: 1 }); + expect(listed[2]).toMatchObject({ id: 'broken', error: expect.stringMatching(/corrupt/) }); + }); + + it('deletes a timeline, refuses main, and reports a missing one', async () => { + await resetComposition(emptyComposition(), 'best-1'); + await deleteTimeline('best-1'); + await expect(readComposition('best-1')).rejects.toBeInstanceOf(NoTimelineError); + expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); + + await expect(deleteTimeline('best-1')).rejects.toBeInstanceOf(NoTimelineError); + await expect(deleteTimeline('main')).rejects.toBeInstanceOf(TimelineIdError); + }); + + it('deletes a corrupt timeline', async () => { + await resetComposition(emptyComposition(), 'broken'); + await writeFile(compositionPath('broken'), 'not json'); + await deleteTimeline('broken'); + expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); + }); + + it('refuses a timelines/ folder that is a symlink out of the workspace', async () => { + const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); + try { + await mkdir(join(outside, 'best-1')); + await writeFile( + join(outside, 'best-1', 'composition.json'), + JSON.stringify(emptyComposition()), + ); + await symlink(outside, join(workspace, 'timelines')); + await expect(readComposition('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 2 }], { timeline: 'best-2', create: true }), + ).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect(listTimelines()).rejects.toBeInstanceOf(WorkspaceBoundaryError); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration/timeline-shorts.test.ts b/tests/integration/timeline-shorts.test.ts new file mode 100644 index 0000000..a5476af --- /dev/null +++ b/tests/integration/timeline-shorts.test.ts @@ -0,0 +1,159 @@ +import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { runTimeline } from '../../src/cli-timeline.js'; +import { probe } from '../../src/ffmpeg/probe.js'; +import { listFilters, runFfmpeg } from '../../src/ffmpeg/run.js'; +import { readComposition } from '../../src/timeline/document-store.js'; +import type { Transcript } from '../../src/timeline/transcript.js'; + +// A short cut from a 10-minute recording, end to end through the CLI: the +// source's cuts carry in, the export is 9:16 at -14 LUFS, main is untouched and +// one undo takes the short away. The transcript is a fixture, so no whisper. +const IS_CI = process.env.CI === 'true'; +// CI's FFmpeg build has no drawtext, so captions are burned in only where it does. +const HAS_DRAWTEXT = IS_CI && (await listFilters()).has('drawtext'); + +let workspace: string; +let saved: string | undefined; + +beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-shorts-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); +}); + +/** Run a `clip timeline` subcommand and parse the JSON it prints. */ +async function cli>(args: string[]): Promise { + let printed = ''; + const spy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { + printed += String(chunk); + return true; + }); + try { + await runTimeline(args); + } finally { + spy.mockRestore(); + } + return JSON.parse(printed) as T; +} + +/** Integrated loudness of a file, LUFS, measured by ebur128. */ +async function integratedLufs(path: string): Promise { + const { stderr } = await runFfmpeg([ + ...['-hide_banner', '-nostats', '-i', path, '-map', '0:a:0'], + ...['-af', 'ebur128', '-f', 'null', '-'], + ]); + const summary = stderr.slice(stderr.lastIndexOf('Summary:')); + const match = /I:\s+(-?[\d.]+) LUFS/.exec(summary); + if (!match?.[1]) throw new Error(`No integrated loudness in:\n${summary}`); + return Number(match[1]); +} + +/** 10 minutes of 720p30 test pattern over a quiet tone stepping between two + * levels (about -26 LUFS), with a 2 s GOP: a 30 s piece looped by stream copy, + * which takes seconds instead of minutes to make. */ +async function tenMinuteRecording(path: string): Promise { + const piece = join(workspace, 'piece.mp4'); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-f', 'lavfi', '-i', 'testsrc=size=1280x720:rate=30:duration=30'], + ...['-f', 'lavfi', '-i', "aevalsrc='(0.05+0.05*lt(mod(t,4),2))*sin(2*PI*220*t)':s=48000:d=30"], + ...['-c:v', 'libx264', '-preset', 'ultrafast', '-g', '60', '-pix_fmt', 'yuv420p'], + ...['-c:a', 'aac', '-shortest', piece], + ]); + await runFfmpeg([ + ...['-hide_banner', '-loglevel', 'error'], + ...['-stream_loop', '19', '-i', piece, '-c', 'copy', path], + ]); +} + +/** Word i spoken from i+0.2 s to i+0.7 s, as whisper would store it. */ +async function storeTranscript(mediaId: string, source: string): Promise { + const file = await stat(source); + const transcript: Transcript = { + version: 1, + mediaId: mediaId as Transcript['mediaId'], + language: 'en', + source: 'whisper-cpp', + model: 'fixture', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: file.size, + sourceMtimeMs: file.mtimeMs, + words: Array.from({ length: 599 }, (_, i) => ({ + text: i % 8 === 7 ? `word${i}.` : `word${i}`, + startSec: i + 0.2, + endSec: i + 0.7, + })), + }; + await mkdir(join(workspace, 'transcripts'), { recursive: true }); + await writeFile(join(workspace, 'transcripts', `${mediaId}.json`), JSON.stringify(transcript)); +} + +describe.skipIf(!IS_CI)('shorts against real FFmpeg', () => { + it('cuts a 9:16 short at -14 LUFS from a 10-minute recording, leaving main alone', async () => { + const source = join(workspace, 'podcast.mp4'); + await tenMinuteRecording(source); + await cli(['new', '--width', '1280', '--height', '720']); + const added = await cli<{ clipId: string; mediaId: string }>(['add-media', source]); + await storeTranscript(added.mediaId, source); + // A retake cut on main (words 545..547) carries into the short. + await cli(['cut-words', added.clipId, '545', '547']); + const main = await readComposition(); + + const short = await cli<{ + durationSec: number; + pieces: { sourceInSec: number; sourceOutSec: number }[]; + canvas: { width: number; height: number; fit: string }; + captions: number; + rev: number; + }>([ + 'short', + 'best-1', + ...[added.clipId, '540', '552'], + ...[added.clipId, '100', '105'], + ...['--x', '0.3'], + ...(HAS_DRAWTEXT ? [] : ['--no-captions']), + ]); + expect(short.pieces).toEqual([ + expect.objectContaining({ sourceInSec: 540.1, sourceOutSec: 545.2 }), + expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.95 }), + expect.objectContaining({ sourceInSec: 100.1, sourceOutSec: 105.95 }), + ]); + expect(short.durationSec).toBe(16.2); + expect(short.canvas).toEqual({ width: 720, height: 1280, fit: 'cover' }); + if (HAS_DRAWTEXT) expect(short.captions).toBeGreaterThan(0); + + const output = join(workspace, 'best-1.mp4'); + const exported = await cli<{ loudness: { mode: string } }>([ + ...['--timeline', 'best-1', 'export', output], + ]); + expect(exported.loudness.mode).not.toBe('skipped'); + const probed = await probe(output); + expect(probed.video).toMatchObject({ width: 720, height: 1280 }); + // Each piece is its own segment, which export rounds up to a whole video + // frame (and AAC frame): up to one frame per piece, plus one AAC frame. + const slack = short.pieces.length / 30 + 1024 / 48000; + expect(Math.abs(probed.durationSec - short.durationSec)).toBeLessThanOrEqual(slack); + expect(Math.abs((await integratedLufs(output)) + 14)).toBeLessThan(1); + + expect(await readComposition()).toEqual(main); + const listed = await cli<{ timelines: { id: string }[] }>(['list']); + expect(listed.timelines.map((t) => t.id)).toEqual(['main', 'best-1']); + + const undo = await cli<{ undone: boolean }>(['--timeline', 'best-1', 'undo']); + expect(undo.undone).toBe(true); + const emptied = await cli<{ tracks: unknown[] }>(['--timeline', 'best-1', 'show']); + expect(emptied.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(main.rev); + }, 240_000); +}); diff --git a/tests/mcp-server.test.ts b/tests/mcp-server.test.ts index 3b69f1c..7fc7271 100644 --- a/tests/mcp-server.test.ts +++ b/tests/mcp-server.test.ts @@ -1,4 +1,4 @@ -import { mkdtemp, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; import { createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -6,6 +6,11 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js'; import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildMcpServer } from '../src/mcp/server.js'; +import { appendOp } from '../src/session/store.js'; +import { mutateComposition, readComposition } from '../src/timeline/document-store.js'; +import { mediaClip, videoTrack } from '../src/timeline/ops.js'; +import { makeMediaId } from '../src/timeline/schema.js'; +import type { Transcript } from '../src/timeline/transcript.js'; const pkg = createRequire(import.meta.url)('../package.json') as { version: string }; @@ -220,3 +225,129 @@ describe('MCP server', () => { } }); }); + +describe('MCP shorts (named timelines)', () => { + /** Main holds a 30 s registered recording with a stored transcript: nothing renders. */ + async function recordingOnMain(): Promise { + const source = join(workspace, 'talk.mp4'); + await writeFile(source, 'video bytes'); + const mediaId = makeMediaId(source); + await appendOp({ + tool: 'ingest', + args: { path: source }, + result: { + mediaId, + ref: { path: source, durationSec: 30, audio: { codec: 'aac' }, video: { codec: 'h264' } }, + }, + }); + const file = await stat(source); + const transcript: Transcript = { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-tiny.en.bin', + modelKey: 'fixture', + requestedLanguage: 'auto', + sourceSize: file.size, + sourceMtimeMs: file.mtimeMs, + words: Array.from({ length: 20 }, (_, i) => ({ + text: `word${i}`, + startSec: i + 1, + endSec: i + 1.5, + })), + }; + await mkdir(join(workspace, 'transcripts'), { recursive: true }); + await writeFile(join(workspace, 'transcripts', `${mediaId}.json`), JSON.stringify(transcript)); + await mutateComposition([ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ id: 'c1', mediaId, sourceOutSec: 30, startSec: 0 }), + }, + ]); + } + + it('makes a short on its own timeline with timeline_edit, leaving main alone', async () => { + await recordingOnMain(); + const mainBefore = await readComposition(); + const { client, close } = await connect(); + try { + const edit = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { + timeline: 'clip-1', + verbs: [ + { + verb: 'make_short', + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 6 }], + x: 0.4, + }, + ], + }, + }), + ); + const doc = edit.document as Record; + expect(doc.timeline).toBe('clip-1'); + expect(doc.canvas).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); + expect(doc.loudness).toEqual({ targetLufs: -14 }); + expect(doc.exportable).toBe(true); + expect((doc.tracks as { id: string }[]).map((t) => t.id)).toEqual(['v0', 'captions']); + expect(await readComposition()).toEqual(mainBefore); + + const show = payload(await client.callTool({ name: 'timeline_show', arguments: {} })); + expect(show.timeline).toBe('main'); + expect(show.timelines).toEqual(['main', 'clip-1']); + + const undo = payload( + await client.callTool({ name: 'timeline_undo', arguments: { timeline: 'clip-1' } }), + ); + expect(undo.undone).toBe(true); + const emptied = payload( + await client.callTool({ name: 'timeline_show', arguments: { timeline: 'clip-1' } }), + ); + expect(emptied.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(mainBefore.rev); + } finally { + await close(); + } + }); + + it('answers a missing timeline or a short on main with an error', async () => { + await recordingOnMain(); + const { client, close } = await connect(); + try { + for (const name of ['timeline_show', 'timeline_check', 'timeline_undo', 'timeline_export']) { + const result = payload(await client.callTool({ name, arguments: { timeline: 'nope' } })); + expect(result.error).toMatch(/No timeline "nope"/); + } + const typo = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { timeline: 'nope', verbs: [{ verb: 'remove', clipId: 'c1' }] }, + }), + ); + expect(typo.error).toMatch(/No timeline "nope"/); + + const onMain = payload( + await client.callTool({ + name: 'timeline_edit', + arguments: { + verbs: [{ verb: 'make_short', ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 6 }] }], + }, + }), + ); + expect(onMain.error).toMatch(/not main/); + + const badId = await client.callTool({ + name: 'timeline_show', + arguments: { timeline: '../main' }, + }); + expect(badId.isError).toBe(true); + } finally { + await close(); + } + }); +}); diff --git a/tests/timeline-captions.test.ts b/tests/timeline-captions.test.ts index 1b944b3..4e47a19 100644 --- a/tests/timeline-captions.test.ts +++ b/tests/timeline-captions.test.ts @@ -322,6 +322,9 @@ describe('caption_speech verb', () => { const { clip, transcript } = await lookup(new Map([[M, RECORDING]]))(clipId); return { clip, transcript, media: new Map() }; }, + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', }); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts new file mode 100644 index 0000000..c2d1267 --- /dev/null +++ b/tests/timeline-shorts.test.ts @@ -0,0 +1,492 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + type Composition, + emptyComposition, + findClip, + type MediaClip, +} from '../src/timeline/composition.js'; +import { + applyVerbs, + NoTimelineError, + readComposition, + readDocOpLog, + resetComposition, + undoLastDocOp, +} from '../src/timeline/document-store.js'; +import { + applyOps, + type CompositionOp, + mediaClip, + textClip, + videoTrack, +} from '../src/timeline/ops.js'; +import type { MediaId } from '../src/timeline/schema.js'; +import { + checkShortTimelines, + planShort, + ShortError, + type ShortRequest, + type ShortTranscriptLookup, +} from '../src/timeline/shorts.js'; +import { NoTranscriptError, type Transcript, TranscriptError } from '../src/timeline/transcript.js'; +import { EditableVerbSchema, lowerVerb, type VerbContext } from '../src/timeline/verbs.js'; + +const M = 'm_aaaaaaaaaaaa' as MediaId; +const N = 'm_bbbbbbbbbbbb' as MediaId; + +function transcriptOf(mediaId: MediaId, spans: [number, number][]): Transcript { + return { + version: 1, + mediaId, + language: 'en', + source: 'whisper-cpp', + model: 'ggml-tiny.en.bin', + modelKey: '', + requestedLanguage: 'auto', + sourceSize: 0, + sourceMtimeMs: 0, + words: spans.map(([startSec, endSec], i) => ({ text: `w${i}`, startSec, endSec })), + }; +} + +/** Word i spoken from i+1 s to i+1.5 s: half-second pauses between words. */ +const SPACED = transcriptOf( + M, + Array.from({ length: 20 }, (_, i): [number, number] => [i + 1, i + 1.5]), +); + +/** A recording on the source timeline's picture track. */ +function sourceOf(clips: MediaClip[], canvas: Partial = {}): Composition { + return applyOps(emptyComposition({ width: 1920, height: 1080, ...canvas }), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ...clips.map((clip): CompositionOp => ({ op: 'addClip', trackId: 'v0', clip })), + ]); +} + +const clipOf = ( + id: string, + sourceInSec: number, + sourceOutSec: number, + startSec: number, + extra: Partial = {}, +): MediaClip => + mediaClip({ id, mediaId: M, sourceInSec, sourceOutSec, startSec, effects: extra.effects }); + +function lookupFor(transcripts: Transcript[]): ShortTranscriptLookup { + return async (comp, clipId) => { + const found = findClip(comp, clipId); + if (found?.clip.kind !== 'media') throw new TranscriptError(`No clip "${clipId}"`); + const transcript = transcripts.find((t) => t.mediaId === found.clip.mediaId); + if (!transcript) throw new NoTranscriptError(`Clip "${clipId}" has no transcript yet.`); + return { clip: found.clip as MediaClip, transcript }; + }; +} + +function counter(prefix = 's'): () => string { + let n = 0; + return () => `${prefix}${++n}`; +} + +async function plan( + source: Composition, + request: ShortRequest, + opts: { target?: Composition; transcripts?: Transcript[] } = {}, +) { + return planShort( + opts.target ?? emptyComposition(), + source, + { captions: false, ...request }, + lookupFor(opts.transcripts ?? [SPACED]), + counter(), + ); +} + +type AddClipOp = Extract; +const addedClips = (ops: CompositionOp[]): AddClipOp['clip'][] => + ops.filter((o): o is AddClipOp => o.op === 'addClip').map((o) => o.clip); + +describe('planShort: edges', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + + it('pads a range 0.10 s before its first word and 0.25 s after its last', async () => { + const short = await plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.durationSec).toBe(2.85); + }); + + it('stops the padding at the neighboring words and at the start of the file', async () => { + const tight = transcriptOf(M, [ + [0.03, 0.3], + [0.32, 0.6], + [0.65, 0.9], + [0.95, 1.2], + ]); + const middle = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, + { transcripts: [tight] }, + ); + expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.3, sourceOutSec: 0.95 }]); + + const first = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 0 }] }, + { transcripts: [tight] }, + ); + expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.32 }]); + }); + + it('stops at the end of the source clip', async () => { + const short = await plan(sourceOf([clipOf('c1', 0, 5.6, 0)]), { + ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }], + }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + }); + + it('refuses word indices outside the transcript', async () => { + await expect( + plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 18, toIndex: 25 }] }), + ).rejects.toThrow(/aren't in the transcript.*20 words/); + await expect( + plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 4, toIndex: 2 }] }), + ).rejects.toBeInstanceOf(ShortError); + }); +}); + +describe('planShort: cut from the source timeline', () => { + it('keeps the silence cuts made on the source: three sub-clips give three pieces', async () => { + const cut = sourceOf([ + clipOf('c1', 0, 3.7, 0), + clipOf('c2', 3.9, 5.7, 3.7), + clipOf('c3', 5.9, 12, 5.5), + ]); + const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 6 }] }); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.7 }, + { mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.7 }, + { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.75 }, + ]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.8, 2.6]); + expect(short.durationSec).toBe(4.45); + }); + + it('refuses words that were cut out of the source timeline', async () => { + const retakeCut = sourceOf([clipOf('c1', 0, 3.7, 0), clipOf('c3', 5.9, 12, 3.7)]); + await expect( + plan(retakeCut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }), + ).rejects.toThrow(/cut out of the source timeline/); + }); + + it('leaves out a sliver that holds only padding', async () => { + // The tail padding of word 4 (ends 5.5) reaches 5.75, just into c2. + const cut = sourceOf([clipOf('c1', 0, 5.6, 0), clipOf('c2', 5.7, 12, 5.6)]); + const short = await plan(cut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + }); + + it('plays the ranges in the order given', async () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const short = await plan(whole, { + ranges: [ + { clipId: 'c1', fromIndex: 6, toIndex: 7 }, + { clipId: 'c1', fromIndex: 1, toIndex: 2 }, + ], + }); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 6.9, sourceOutSec: 8.75 }, + { mediaId: M, sourceInSec: 1.9, sourceOutSec: 3.75 }, + ]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.85]); + }); + + it('cuts ranges from two recordings', async () => { + const other = transcriptOf(N, [[0.5, 1]]); + const two = sourceOf([ + clipOf('c1', 0, 30, 0), + mediaClip({ id: 'd1', mediaId: N, sourceOutSec: 2, startSec: 30 }), + ]); + const short = await plan( + two, + { + ranges: [ + { clipId: 'd1', fromIndex: 0, toIndex: 0 }, + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + ], + }, + { transcripts: [SPACED, other] }, + ); + expect(short.pieces.map((p) => p.mediaId)).toEqual([N, M]); + }); + + it('takes a moment the source plays twice only once', async () => { + const repeated = sourceOf([clipOf('c1', 0, 10, 0), clipOf('c2', 2, 6, 10)]); + const short = await plan(repeated, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + }); + + it('joins pieces that a split (not a cut) left continuous', async () => { + const split = sourceOf([clipOf('c1', 0, 4, 0), clipOf('c2', 4, 10, 4)]); + const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + }); + + it('keeps color and volume, and drops the fades of the source clip', async () => { + const source = sourceOf([ + clipOf('c1', 0, 30, 0, { + effects: [ + { type: 'fadeIn', durationSec: 1 }, + { type: 'adjust', brightness: 0.1, contrast: 1, saturation: 1 }, + { type: 'volume', gain: 1.5 }, + ], + }), + ]); + const short = await plan(source, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); + expect(addedClips(short.ops)[0]?.effects.map((e) => e.type)).toEqual(['adjust', 'volume']); + }); + + it('refuses a range over a clip with a speed change', async () => { + const fast = sourceOf([clipOf('c1', 0, 30, 0, { effects: [{ type: 'speed', factor: 2 }] })]); + await expect( + plan(fast, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }), + ).rejects.toThrow(/2x.*speed/); + }); + + it('refuses a source timeline with no picture track', async () => { + await expect( + plan(emptyComposition(), { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }), + ).rejects.toThrow(/no picture track/); + }); +}); + +describe('planShort: the edit', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + + it('builds a 9:16 cover canvas, the pieces, the framing, -14 LUFS and captions', async () => { + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], x: 0.3 }, + lookupFor([SPACED]), + counter(), + ); + expect(short.ops).toEqual([ + { op: 'setCanvas', width: 1080, height: 1920, fps: 30, background: 'black', fit: 'cover' }, + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + { + op: 'addClip', + trackId: 'v0', + clip: mediaClip({ + id: 's1', + mediaId: M, + sourceInSec: 2.9, + sourceOutSec: 5.75, + startSec: 0, + }), + }, + { op: 'setTransform', clipId: 's1', transform: { x: 0.3 } }, + { op: 'setLoudness', targetLufs: -14 }, + { op: 'addTrack', track: videoTrack({ id: 'captions' }) }, + { + op: 'addClip', + trackId: 'captions', + clip: textClip({ + id: 's2', + text: 'w2 w3 w4', + startSec: 0.1, + durationSec: 2.75, + style: { fontSize: 54, color: 'white', background: 'black@0.6', anchor: 'bottom-center' }, + }), + }, + ]); + expect(short.captions).toBe(1); + expect(short.warnings).toEqual([]); + }); + + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { + const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); + const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; + const vertical = await plan(hd, { ranges: [range] }); + expect(vertical.ops[0]).toMatchObject({ width: 720, height: 1280, fps: 25 }); + const square = await plan(hd, { ranges: [range], aspect: '1:1' }); + expect(square.ops[0]).toMatchObject({ width: 720, height: 720 }); + }); + + it('leaves loudness off with null, and captions out when asked', async () => { + const short = await plan(whole, { + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + targetLufs: null, + }); + expect(short.ops.map((o) => o.op)).toEqual(['setCanvas', 'addTrack', 'addClip']); + expect(short.captions).toBe(0); + }); + + it('refuses a timeline that already holds a short, and rebuilds it with replace', async () => { + const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; + const first = await plan(whole, { ranges: [range] }); + const made = applyOps(emptyComposition(), first.ops); + await expect(plan(whole, { ranges: [range] }, { target: made })).rejects.toThrow(/replace/); + + const rebuilt = await plan(whole, { ranges: [range], replace: true }, { target: made }); + // Loudness is already -14, so only the tracks and clips are rebuilt. + expect(rebuilt.ops.map((o) => o.op)).toEqual([ + 'removeTrack', + 'setCanvas', + 'addTrack', + 'addClip', + ]); + }); + + it('reuses a timeline left empty by an undo without replace', async () => { + const emptied = applyOps(emptyComposition(), [ + { op: 'addTrack', track: videoTrack({ id: 'v0' }) }, + ]); + const short = await plan( + whole, + { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }, + { target: emptied }, + ); + expect(short.ops[0]).toEqual({ op: 'removeTrack', trackId: 'v0' }); + expect(() => applyOps(emptied, short.ops)).not.toThrow(); + }); +}); + +describe('checkShortTimelines', () => { + it('refuses main as the short, and the short as its own source', () => { + expect(() => checkShortTimelines('main')).toThrow(/not main/); + expect(() => checkShortTimelines('best-1', 'best-1')).toThrow(/its own timeline/); + expect(() => checkShortTimelines('best-1')).not.toThrow(); + expect(() => checkShortTimelines('best-2', 'best-1')).not.toThrow(); + }); +}); + +describe('make_short verb', () => { + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const ctxFor = (timeline?: string, sources: Record = {}): VerbContext => ({ + ingest: async () => { + throw new Error('not used'); + }, + analyzeSilence: async () => { + throw new Error('not used'); + }, + transcript: async (comp, clipId, transcriptId) => ({ + ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), + media: new Map(), + }), + readTimeline: async (id) => { + const doc = sources[id]; + if (!doc) throw new NoTimelineError(id); + return doc; + }, + defaultTrack: 'v0', + timeline, + }); + const verb = { + verb: 'make_short' as const, + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + }; + + it('lowers to the planned short, cut from main by default', async () => { + const ops = await lowerVerb(emptyComposition(), verb, ctxFor('best-1', { main: whole })); + const doc = applyOps(emptyComposition(), ops); + expect(doc).toMatchObject({ width: 1080, height: 1920, fit: 'cover' }); + expect(doc.loudness).toEqual({ targetLufs: -14 }); + expect(doc.tracks.map((t) => t.id)).toEqual(['v0', 'captions']); + }); + + it('cuts from another timeline with source', async () => { + const ops = await lowerVerb( + emptyComposition(), + { ...verb, source: 'long-cut' }, + ctxFor('best-1', { 'long-cut': whole }), + ); + expect(ops.length).toBeGreaterThan(0); + }); + + it('refuses main as its target', async () => { + await expect( + lowerVerb(emptyComposition(), verb, ctxFor(undefined, { main: whole })), + ).rejects.toBeInstanceOf(ShortError); + }); + + it('is an editable verb with bounded ranges and settings', () => { + const ok = (extra: Record): boolean => + EditableVerbSchema.safeParse({ ...verb, ...extra }).success; + expect(ok({})).toBe(true); + expect(ok({ ranges: [] })).toBe(false); + expect(ok({ ranges: Array.from({ length: 21 }, () => verb.ranges[0]) })).toBe(false); + expect(ok({ ranges: [{ clipId: 'c1', fromIndex: -1, toIndex: 2 }] })).toBe(false); + expect(ok({ x: 1.5 })).toBe(false); + expect(ok({ maxWords: 13 })).toBe(false); + expect(ok({ targetLufs: -40 })).toBe(false); + expect(ok({ targetLufs: null, captions: false, replace: true, aspect: '4:5' })).toBe(true); + expect(ok({ source: 'Main Cut' })).toBe(false); + }); +}); + +describe('make_short on the store', () => { + let workspace: string; + let saved: string | undefined; + beforeEach(async () => { + workspace = await mkdtemp(join(tmpdir(), 'mmc-shorts-test-')); + saved = process.env.MAKEMYCLIP_WORKSPACE; + process.env.MAKEMYCLIP_WORKSPACE = workspace; + }); + afterEach(async () => { + if (saved === undefined) delete process.env.MAKEMYCLIP_WORKSPACE; + else process.env.MAKEMYCLIP_WORKSPACE = saved; + await rm(workspace, { recursive: true, force: true }); + }); + + const ctx: VerbContext = { + ingest: async () => { + throw new Error('not used'); + }, + analyzeSilence: async () => { + throw new Error('not used'); + }, + transcript: async (comp, clipId, transcriptId) => ({ + ...(await lookupFor([SPACED])(comp, clipId, transcriptId)), + media: new Map(), + }), + readTimeline: (id) => readComposition(id), + defaultTrack: 'v0', + }; + const makeShort = { + verb: 'make_short' as const, + ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }], + }; + + it('creates the short as one undoable edit and leaves main alone', async () => { + const main = await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + + const { doc } = await applyVerbs([makeShort], ctx, 'best-1'); + expect(doc).toMatchObject({ width: 1080, height: 1920, rev: 1 }); + expect(await readComposition()).toEqual(main); + expect((await readDocOpLog('best-1')).entries).toHaveLength(1); + + const undo = await undoLastDocOp('best-1'); + expect(undo.undone).toBe(true); + expect(undo.doc.tracks).toEqual([]); + expect((await readComposition()).rev).toBe(main.rev); + }); + + it('creates a timeline only when the batch starts with make_short', async () => { + await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + const addText = { verb: 'add_text' as const, text: 'Hook', durationSec: 1 }; + await expect(applyVerbs([addText], ctx, 'best-1')).rejects.toBeInstanceOf(NoTimelineError); + await expect(applyVerbs([addText, makeShort], ctx, 'best-1')).rejects.toBeInstanceOf( + NoTimelineError, + ); + + const { doc } = await applyVerbs([makeShort, { ...addText, track: 'hook' }], ctx, 'best-1'); + expect(doc.tracks.map((t) => t.id)).toEqual(['v0', 'captions', 'hook']); + }); + + it('refuses to make a short on main', async () => { + await resetComposition(sourceOf([clipOf('c1', 0, 30, 0)])); + await expect(applyVerbs([makeShort], ctx)).rejects.toBeInstanceOf(ShortError); + }); +}); diff --git a/tests/timeline-verbs.test.ts b/tests/timeline-verbs.test.ts index 88a4a3c..afd7f96 100644 --- a/tests/timeline-verbs.test.ts +++ b/tests/timeline-verbs.test.ts @@ -30,6 +30,9 @@ const ctx = (durationSec = 10): VerbContext => ({ transcript: async () => { throw new Error('not used'); }, + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', }); @@ -413,6 +416,9 @@ describe('applyVerbs (op-aware + undoable)', () => { openGate = resolve; }); const barrierCtx: VerbContext = { + readTimeline: async () => { + throw new Error('not used'); + }, defaultTrack: 'v0', analyzeSilence: async () => { throw new Error('not used'); From 13adac9b9d0a3a18e73073872322e1ab5d3f3505 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:03:26 -0800 Subject: [PATCH 2/6] fix(timeline): keep padding-only and replayed padding out of shorts A piece that only touches a range's words (they were cut out exactly) is dropped, so make_short refuses instead of playing padding alone. A range that continues the one before it in the same file starts where that one's padding ended, so neighboring words play straight through. Co-Authored-By: Claude Opus 5.5 --- src/timeline/shorts.ts | 88 +++++++++++++++++++++++++---------- tests/timeline-shorts.test.ts | 49 +++++++++++++++++++ 2 files changed, 113 insertions(+), 24 deletions(-) diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 5fb7637..0bfef72 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -104,6 +104,7 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI } interface WordWindow { + mediaId: MediaId; /** Padded edges, source seconds. */ start: number; end: number; @@ -114,6 +115,9 @@ interface WordWindow { interface Piece extends ShortPiece { effects: Effect[]; + /** The source clip's speed, refused only if the piece is played. */ + speed: number; + sourceClipId: string; } /** @@ -142,17 +146,28 @@ export async function planShort( ); } - const pieces: Piece[] = []; + const windows: WordWindow[] = []; for (const range of request.ranges) { const { clip, transcript } = await lookup(source, range.clipId, range.transcriptId); - const window = wordWindow(transcript, range, clip.id); - const found = piecesUnder(picture.clips, clip.mediaId, window); + windows.push(wordWindow(transcript, range, clip)); + } + const pieces: Piece[] = []; + windows.forEach((window, i) => { + const range = request.ranges[i]; + const found = piecesUnder(picture.clips, withoutReplay(window, windows[i - 1])); if (found.length === 0) { throw new ShortError( - `Words ${range.fromIndex}..${range.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, + `Words ${range?.fromIndex}..${range?.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, ); } pieces.push(...found); + }); + const sped = pieces.find((p) => p.speed !== 1); + if (sped) { + throw new ShortError( + `Clip "${sped.sourceClipId}" on the source timeline plays at ${sped.speed}x, and a short can't include a speed change yet. ` + + 'Remove its speed effect or pick words outside it.', + ); } const joined = joinContinuous(pieces); @@ -238,12 +253,12 @@ export async function planShort( * so the padding never takes in part of a word outside the range, and at the * start of the file. */ -function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): WordWindow { +function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): WordWindow { const first = transcript.words[range.fromIndex]; const last = transcript.words[range.toIndex]; if (!first || !last || range.toIndex < range.fromIndex) { throw new ShortError( - `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clipId}" ` + + `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clip.id}" ` + `(it has ${transcript.words.length} words, numbered from 0).`, ); } @@ -258,6 +273,7 @@ function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), ); return { + mediaId: clip.mediaId, start: roundMs(start), end: roundMs(end), speechStart: first.startSec, @@ -266,39 +282,62 @@ function wordWindow(transcript: Transcript, range: ShortRange, clipId: string): } /** - * The parts of a window that the source timeline's picture clips of one file - * play, in timeline order. A moment the timeline plays twice is taken once, and - * a part holding only padding (no word of the range) is left out, since it would - * only add a jump cut. + * A window that picks up later in the same file than the one before it starts + * where that one's padding ended, so two ranges of neighboring words play + * straight through instead of replaying the padding they share. A range that + * repeats or goes back to earlier words is left as asked. + */ +function withoutReplay(window: WordWindow, before: WordWindow | undefined): WordWindow { + if ( + !before || + before.mediaId !== window.mediaId || + window.speechStart < before.speechEnd || + window.start >= before.end + ) { + return window; + } + return { ...window, start: Math.min(before.end, window.speechStart) }; +} + +/** + * The parts of a window that the source timeline's picture clips of its file + * play, in timeline order. A moment the timeline plays twice is taken once. + * Parts a split left continuous are joined first; then a part holding none of + * the range's words (only padding) is left out, since it would only add a jump + * cut, as is a sliver too short to see. */ -function piecesUnder(clips: readonly Clip[], mediaId: MediaId, window: WordWindow): Piece[] { +function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { const taken: [number, number][] = []; - const pieces: Piece[] = []; + const parts: Piece[] = []; for (const clip of [...clips].sort((a, b) => a.startSec - b.startSec)) { - if (clip.kind !== 'media' || clip.mediaId !== mediaId) continue; + if (clip.kind !== 'media' || clip.mediaId !== window.mediaId) continue; const lo = Math.max(window.start, clip.sourceInSec); const hi = Math.min(window.end, clip.sourceOutSec); if (hi <= lo) continue; for (const [a, b] of subtract([lo, hi], taken)) { - if (b - a < MIN_PIECE_SEC || b < window.speechStart || a > window.speechEnd) continue; - const speed = speedFactor(clip.effects); - if (speed !== 1) { - throw new ShortError( - `Clip "${clip.id}" on the source timeline plays at ${speed}x, and a short can't include a speed change yet. ` + - 'Remove its speed effect or pick words outside it.', - ); - } - pieces.push({ - mediaId, + parts.push({ + mediaId: window.mediaId, sourceInSec: a, sourceOutSec: b, // Fades belonged to the source clip's edges, not the short's. effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), + speed: speedFactor(clip.effects), + sourceClipId: clip.id, }); taken.push([a, b]); } } - return pieces; + return joinContinuous(parts).filter( + (p) => p.sourceOutSec - p.sourceInSec >= MIN_PIECE_SEC && holdsSpeech(p, window), + ); +} + +/** Whether a piece plays any of the range's words, not just touches their edge. */ +function holdsSpeech(piece: Piece, window: WordWindow): boolean { + if (window.speechEnd > window.speechStart) { + return piece.sourceOutSec > window.speechStart && piece.sourceInSec < window.speechEnd; + } + return piece.sourceInSec <= window.speechStart && piece.sourceOutSec >= window.speechEnd; } /** `[lo, hi]` minus the `taken` intervals, in order. */ @@ -326,6 +365,7 @@ function joinContinuous(pieces: Piece[]): Piece[] { prev && prev.mediaId === piece.mediaId && Math.abs(prev.sourceOutSec - piece.sourceInSec) < SAME_SEC && + prev.speed === piece.speed && JSON.stringify(prev.effects) === JSON.stringify(piece.effects) ) { joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index c2d1267..49006cf 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -187,6 +187,55 @@ describe('planShort: cut from the source timeline', () => { expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); }); + it('refuses words cut out exactly, where only padding touches them', async () => { + // Word 2 (3.0–3.5) was cut out exactly: c1 ends at its start, c2 starts at its end. + const exact = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3.5, 30, 3)]); + await expect( + plan(exact, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }), + ).rejects.toThrow(/cut out of the source timeline/); + }); + + it('keeps the lead padding across a split at the first word', async () => { + const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); + const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.75 }]); + }); + + it('plays two ranges of neighboring words straight through, without replaying padding', async () => { + const close = transcriptOf(M, [ + [1, 1.5], + [1.7, 2.2], + [2.9, 3.4], + ]); + const whole = sourceOf([clipOf('c1', 0, 30, 0)]); + const short = await plan( + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + ], + }, + { transcripts: [close] }, + ); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }]); + + const repeated = await plan( + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + { clipId: 'c1', fromIndex: 0, toIndex: 1 }, + ], + }, + { transcripts: [close] }, + ); + expect(repeated.pieces).toEqual([ + { mediaId: M, sourceInSec: 1.6, sourceOutSec: 2.45 }, + { mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }, + ]); + }); + it('plays the ranges in the order given', async () => { const whole = sourceOf([clipOf('c1', 0, 30, 0)]); const short = await plan(whole, { From b6688cba6914d83cd10e276d7b89372b00d616af Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:05:05 -0800 Subject: [PATCH 3/6] docs(docs): describe shorts in the README, SKILL.md and llms.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the long recording → shorts quick start, the `short` command, the make_short verb in the MCP lists and an FAQ entry, and polishes the missing-timeline and non-empty-short errors for CLI users. Co-Authored-By: Claude Opus 5.5 --- README.md | 32 ++++++++++++++++++++++++++++++-- SKILL.md | 5 ++++- llms.txt | 3 ++- mcpb/manifest.json | 2 +- src/timeline/document-store.ts | 4 ++-- src/timeline/shorts.ts | 2 +- 6 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 29bfeef..73d4bae 100644 --- a/README.md +++ b/README.md @@ -41,12 +41,13 @@ Version 0.5.0 turns a raw talking-head recording into a clean, captioned cut: ve | **Captions** | Burned-in captions timed to the words, as short phrases or one word at a time | `clip timeline captions` | | **Reframe** | Turns a landscape cut into 9:16, 1:1 or 4:5, filling the frame and choosing the part that shows; zoom and position per clip | `clip timeline reframe` | | **Loudness** | Evens out the exported mp4's volume to a target (-14 LUFS for YouTube, Shorts, Reels and TikTok; -16 for podcasts), measured in two passes | `clip timeline loudness` | +| **Shorts from a long recording** | Turns ranges of words from a long recording into a vertical short on a timeline of its own: captioned, at -14 LUFS, with the cuts already made on the recording; the long cut stays as it is | `clip timeline short` | | **Cut check** | Flags cuts where a word may be clipped, with a contact sheet of the frames on each side | `clip timeline check` | | **Export to Final Cut Pro and DaVinci Resolve** | FCPXML 1.10 project | `clip timeline export --format fcpxml` | | **Export to Premiere Pro** | Final Cut Pro 7 XML project, captions as `.srt` files | `clip timeline export --format xmeml` | | **Render** | mp4 of the finished cut | `clip timeline export` | -All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check isn't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. +All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check and shorts aren't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. --- @@ -149,6 +150,29 @@ A cut splits the clip into pieces: the first keeps its id, and `clip timeline sh --- +## Quick start: a long recording → vertical shorts + +Each short is a timeline of its own, cut from the main timeline's clips, so the pauses and retakes you cut there stay cut. In Claude Code or Claude Desktop, ask: + +> *Transcribe podcast.mp4 and cut the pauses. Then find the three strongest moments, 20 to 60 seconds each, and make each one a short.* + +The same steps from the CLI: + +```bash +clip timeline new +clip timeline add-media podcast.mp4 # prints the clip id, e.g. clip_ec2f047d +clip timeline transcribe clip_ec2f047d # each word with its index and time +clip timeline silence clip_ec2f047d # cut the pauses on main +clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 as a 9:16 short +clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off +clip timeline --timeline best-1 export best-1.mp4 +clip timeline list # main and every short +``` + +`short` keeps 0.10 s before the first word and 0.25 s after the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. + +--- + ## Another example: a product demo from screen recordings Five screen recordings → one polished product demo. Type this into Claude Code (or Claude Desktop, via the MCP server): @@ -374,7 +398,7 @@ MakeMyClip Editor is built for that job: an open-source video editor that Claude ### Is there an MCP server for video editing? -Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. +Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, make shorts, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. ### How do I remove silence from a video automatically? @@ -388,6 +412,10 @@ Yes. MakeMyClip Editor transcribes with whisper.cpp on your machine; install it Yes. After transcription, MakeMyClip Editor's `clip timeline captions` (or the UI's Captions button) puts captions timed to the words on a track above the video, and export burns them in. Captions come as short phrases or one word at a time, with your choice of size, color, position, and background. Non-Latin scripts use an installed font that has the glyphs. In the Premiere export, captions become `.srt` files. On Linux, the bundled FFmpeg can't draw text, so set `MAKEMYCLIP_FFMPEG_PATH` to an FFmpeg built with libfreetype and libharfbuzz. +### Can it cut Shorts or Reels out of a podcast? + +Yes. Transcribe the recording, then `clip timeline short best-1 ` (or the `make_short` verb from chat) turns that range of words into a vertical short on a timeline of its own: 9:16, filling the frame, captioned, normalized to -14 LUFS, with the silence and retake cuts already made on the recording. Make as many as you like; the long cut stays untouched, and each short exports on its own. You or the agent pick the moments: nothing scores highlights for you, and nothing is uploaded. + ### Can I finish an AI edit in DaVinci Resolve, Premiere Pro or Final Cut Pro? Yes. MakeMyClip Editor's `clip timeline export cut.fcpxml` writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve, and `clip timeline export cut.xml` writes Final Cut Pro 7 XML for Premiere Pro. Both keep the cuts, gaps, fades, volume, and cross dissolves, and point at your original files by path. Anything the format can't hold (speed, color adjustments, chroma key, a reframe's fill, zoom or position) is refused with the fix to make. diff --git a/SKILL.md b/SKILL.md index f28502d..e3bb63b 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: editor -description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. +description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; cut clips or shorts from a long video or podcast; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. license: MIT compatibility: Requires Node 24+. FFmpeg is auto-downloaded via ffmpeg-static on first use; override with MAKEMYCLIP_FFMPEG_PATH. Transcription (transcript cuts, retakes, captions) needs a user-installed whisper.cpp and a model file set in MAKEMYCLIP_WHISPER_MODEL. metadata: @@ -22,6 +22,7 @@ Trigger this skill when the user asks to: - Cut the silence, pauses, or dead air out of a recording ("tighten it up", "remove the gaps") - Remove repeated takes or bad takes, or cut a sentence or words by the transcript - Clean up a talking-head video, tutorial, lecture, or video podcast +- Cut clips or vertical shorts (Shorts, Reels, TikTok) out of a long video, webinar or podcast - Add captions or subtitles timed to the speech, or titles and lower-thirds - Zoom, pan, or apply Ken Burns to a region of a clip - Assemble multiple clips into one (concat, montage) @@ -332,6 +333,8 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it keeps 0.10 s before the first word and 0.25 s after the last, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. + To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. ### Open the local UI (implemented) diff --git a/llms.txt b/llms.txt index e245b0a..aa8b461 100644 --- a/llms.txt +++ b/llms.txt @@ -21,6 +21,7 @@ - **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time) - **Reframe:** `clip timeline reframe 9:16` turns a landscape cut vertical (also 1:1, 4:5); `--x` picks the part that shows, `clip timeline framing ` sets one clip's zoom and position - **Loudness:** `clip timeline loudness -14` normalizes the exported mp4 to -14 LUFS (or `-16` for podcasts) in two passes +- **Shorts from a long recording:** `clip timeline short best-1 ` (MCP: the `make_short` verb) turns a range of transcript words into a 9:16 short on a timeline of its own, captioned and at -14 LUFS, keeping the cuts made on the main timeline; `--timeline ` points any command at a short - **Cut check:** `clip timeline check --sheet cuts.jpg` flags cuts that may clip a word and writes a contact sheet - **Export to pro editors:** `clip timeline export --format fcpxml` (Final Cut Pro, DaVinci Resolve) or `--format xmeml` (Premiere Pro; captions as .srt) - Every timeline edit is one undoable step (`clip timeline undo` / `redo`), from the CLI, the MCP server or the browser UI @@ -52,7 +53,7 @@ Plus session meta-ops not in the tool registry: snapshot, undo, inspect, delete. - Different category from Runway/Pika (generative AI), Premiere/DaVinci (heavy NLEs) — but it exports its cut to Final Cut Pro, Resolve and Premiere for finishing - Overlaps with auto-editor (open-source CLI: silence cuts, transcript edits, NLE export); MakeMyClip Editor adds a shared timeline document that an agent and a person (browser UI) both edit, with undo per step - Transcription is local (whisper.cpp); nothing is uploaded -- Single workspace folder holds composition.json (the timeline), session.json (the op log), output files, snapshots/ — fully version-controllable +- Single workspace folder holds composition.json (the main timeline), timelines/ (shorts and other named timelines), session.json (the op log), output files, snapshots/ — fully version-controllable - Stream-copy for lossless trim/split/concat; re-encodes only when filter graphs require it ## Architecture diff --git a/mcpb/manifest.json b/mcpb/manifest.json index 2841441..dc6e9c5 100644 --- a/mcpb/manifest.json +++ b/mcpb/manifest.json @@ -40,7 +40,7 @@ }, { "name": "timeline_edit", - "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, transition." + "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, make a vertical short from transcript words, transition." }, { "name": "timeline_check", diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index 2a16dc1..b5ff3ed 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -55,7 +55,7 @@ export class NoTimelineError extends Error { constructor(timeline: string) { super( `No timeline "${timeline}" in the workspace. List them with \`clip timeline list\` ` + - `(timeline_show reports them too). make_short creates a short's timeline.`, + '(timeline_show reports them too). A short gets its timeline from make_short (`clip timeline short`).', ); this.name = 'NoTimelineError'; this.timeline = timeline; @@ -493,7 +493,7 @@ export async function listTimelines(): Promise { id, width: doc.width, height: doc.height, - durationSec: compositionDuration(doc), + durationSec: Math.round(compositionDuration(doc) * 1000) / 1000, rev: doc.rev, }); } catch (err) { diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 0bfef72..0fbb9ab 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -136,7 +136,7 @@ export async function planShort( if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { throw new ShortError( - 'That timeline already holds a short. Pass replace to rebuild it, or pick a new timeline id.', + 'That timeline already holds a short. Pass replace (`--replace`) to rebuild it, or pick a new timeline id.', ); } const picture = videoLayers(source).picture; From bc8e1d9fee05127dcf6e0b6d220f1c1d8e239c51 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:14:53 -0800 Subject: [PATCH 4/6] fix(timeline): tune short edges to whisper.cpp's word timing whisper.cpp words abut, each pause going to the word before it, so the neighbor-word clamp removed all padding. Measured on a 30-sentence say recording with base.en and on jfk.wav: a range now starts 0.25 s before its first word (stopping halfway into the word before) and ends with its last word, which already runs through the pause after it. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- SKILL.md | 2 +- docs/backlog/short-edges-snap-to-pause.md | 3 ++ src/index.ts | 1 - src/timeline/shorts.ts | 32 +++++++------ tests/integration/timeline-shorts.test.ts | 8 ++-- tests/timeline-shorts.test.ts | 58 +++++++++++------------ 7 files changed, 55 insertions(+), 51 deletions(-) create mode 100644 docs/backlog/short-edges-snap-to-pause.md diff --git a/README.md b/README.md index 73d4bae..0dd305d 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ clip timeline --timeline best-1 export best-1.mp4 clip timeline list # main and every short ``` -`short` keeps 0.10 s before the first word and 0.25 s after the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. +`short` starts 0.25 s before the first word and ends with the last, fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. --- diff --git a/SKILL.md b/SKILL.md index e3bb63b..a11612f 100644 --- a/SKILL.md +++ b/SKILL.md @@ -333,7 +333,7 @@ To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline ref To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. -To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it keeps 0.10 s before the first word and 0.25 s after the last, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then pick each moment by its words and run `timeline short `, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it starts 0.25 s before the first word and ends with the last (whisper's last word already covers the pause after it), fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4`. `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. diff --git a/docs/backlog/short-edges-snap-to-pause.md b/docs/backlog/short-edges-snap-to-pause.md new file mode 100644 index 0000000..3bcb401 --- /dev/null +++ b/docs/backlog/short-edges-snap-to-pause.md @@ -0,0 +1,3 @@ +# Snap short edges to the pause + +whisper.cpp (`-ml 1 -sow`) gives each pause to the word before it, so at a sentence edge its words run late. On a 30-sentence test recording (macOS `say`, model base.en), a short ending on a sentence's last word ran past the pause into the next sentence in 18 of 29 cases, by up to 0.35 s. Even with the 0.25 s lead, 3 of 29 starts lost up to 0.1 s of the first word. No fixed padding fixes both. Move each edge of a range to the quietest point of the audio near it: run a silencedetect pass over a second or so around the edge and keep the edge inside the pause between the range and its neighbors. `-dtw base.en` didn't change the per-word offsets in a quick check. diff --git a/src/index.ts b/src/index.ts index 30de026..bea7419 100644 --- a/src/index.ts +++ b/src/index.ts @@ -246,7 +246,6 @@ export { DEFAULT_SHORT_LUFS, planShort, SHORT_LEAD_SEC, - SHORT_TAIL_SEC, ShortError, type ShortPiece, type ShortPlan, diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 0fbb9ab..3cc598c 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -20,10 +20,14 @@ import type { Transcript } from './transcript.js'; * person picks the words; nothing here scores highlights. */ -/** Kept before a range's first word, so its attack isn't clipped. */ -export const SHORT_LEAD_SEC = 0.1; -/** Kept after a range's last word, so its release and the breath after it aren't clipped. */ -export const SHORT_TAIL_SEC = 0.25; +/** + * Kept before a range's first word. whisper.cpp hands each pause to the word + * before it, so the next word tends to start late: on test speech its first + * words started up to 0.35 s after the voice did. 0.25 s left 3 of 29 sentence + * starts short by at most 0.1 s. A range ends where its last word does, with no + * padding: that word already runs on through the pause after it. + */ +export const SHORT_LEAD_SEC = 0.25; /** Loudness a short is normalized to unless asked otherwise: the vertical platforms' -14 LUFS. */ export const DEFAULT_SHORT_LUFS = -14; @@ -248,10 +252,9 @@ export async function planShort( } /** - * The source window a range plays: its first word's start minus the lead to its - * last word's end plus the tail. Each side stops at the neighboring word's edge, - * so the padding never takes in part of a word outside the range, and at the - * start of the file. + * The source window a range plays: its first word's start minus the lead, to + * its last word's end. The lead stops halfway into the word before (whisper.cpp + * words abut, so that word also holds the pause) and at the start of the file. */ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): WordWindow { const first = transcript.words[range.fromIndex]; @@ -263,19 +266,18 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); } const before = transcript.words[range.fromIndex - 1]; - const after = transcript.words[range.toIndex + 1]; const start = Math.min( first.startSec, - Math.max(first.startSec - SHORT_LEAD_SEC, before?.endSec ?? 0, 0), - ); - const end = Math.max( - last.endSec, - Math.min(last.endSec + SHORT_TAIL_SEC, after?.startSec ?? Number.POSITIVE_INFINITY), + Math.max( + first.startSec - SHORT_LEAD_SEC, + before ? (before.startSec + before.endSec) / 2 : 0, + 0, + ), ); return { mediaId: clip.mediaId, start: roundMs(start), - end: roundMs(end), + end: last.endSec, speechStart: first.startSec, speechEnd: last.endSec, }; diff --git a/tests/integration/timeline-shorts.test.ts b/tests/integration/timeline-shorts.test.ts index a5476af..753c7d3 100644 --- a/tests/integration/timeline-shorts.test.ts +++ b/tests/integration/timeline-shorts.test.ts @@ -125,11 +125,11 @@ describe.skipIf(!IS_CI)('shorts against real FFmpeg', () => { ...(HAS_DRAWTEXT ? [] : ['--no-captions']), ]); expect(short.pieces).toEqual([ - expect.objectContaining({ sourceInSec: 540.1, sourceOutSec: 545.2 }), - expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.95 }), - expect.objectContaining({ sourceInSec: 100.1, sourceOutSec: 105.95 }), + expect.objectContaining({ sourceInSec: 539.95, sourceOutSec: 545.2 }), + expect.objectContaining({ sourceInSec: 547.7, sourceOutSec: 552.7 }), + expect.objectContaining({ sourceInSec: 99.95, sourceOutSec: 105.7 }), ]); - expect(short.durationSec).toBe(16.2); + expect(short.durationSec).toBe(16); expect(short.canvas).toEqual({ width: 720, height: 1280, fit: 'cover' }); if (HAS_DRAWTEXT) expect(short.captions).toBeGreaterThan(0); diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 49006cf..e67e3f5 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -111,13 +111,13 @@ const addedClips = (ops: CompositionOp[]): AddClipOp['clip'][] => describe('planShort: edges', () => { const whole = sourceOf([clipOf('c1', 0, 30, 0)]); - it('pads a range 0.10 s before its first word and 0.25 s after its last', async () => { + it('starts a range 0.25 s before its first word and ends it with its last', async () => { const short = await plan(whole, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); - expect(short.durationSec).toBe(2.85); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); + expect(short.durationSec).toBe(2.75); }); - it('stops the padding at the neighboring words and at the start of the file', async () => { + it('stops the lead halfway into the word before, and at the start of the file', async () => { const tight = transcriptOf(M, [ [0.03, 0.3], [0.32, 0.6], @@ -129,21 +129,21 @@ describe('planShort: edges', () => { { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, { transcripts: [tight] }, ); - expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.3, sourceOutSec: 0.95 }]); + expect(middle.pieces).toEqual([{ mediaId: M, sourceInSec: 0.165, sourceOutSec: 0.9 }]); const first = await plan( whole, { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 0 }] }, { transcripts: [tight] }, ); - expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.32 }]); + expect(first.pieces).toEqual([{ mediaId: M, sourceInSec: 0, sourceOutSec: 0.3 }]); }); it('stops at the end of the source clip', async () => { - const short = await plan(sourceOf([clipOf('c1', 0, 5.6, 0)]), { + const short = await plan(sourceOf([clipOf('c1', 0, 5.3, 0)]), { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }], }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.75, sourceOutSec: 5.3 }]); }); it('refuses word indices outside the transcript', async () => { @@ -165,12 +165,12 @@ describe('planShort: cut from the source timeline', () => { ]); const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 6 }] }); expect(short.pieces).toEqual([ - { mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.7 }, + { mediaId: M, sourceInSec: 2.75, sourceOutSec: 3.7 }, { mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.7 }, - { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.75 }, + { mediaId: M, sourceInSec: 5.9, sourceOutSec: 7.5 }, ]); - expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.8, 2.6]); - expect(short.durationSec).toBe(4.45); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 0.95, 2.75]); + expect(short.durationSec).toBe(4.35); }); it('refuses words that were cut out of the source timeline', async () => { @@ -181,10 +181,10 @@ describe('planShort: cut from the source timeline', () => { }); it('leaves out a sliver that holds only padding', async () => { - // The tail padding of word 4 (ends 5.5) reaches 5.75, just into c2. - const cut = sourceOf([clipOf('c1', 0, 5.6, 0), clipOf('c2', 5.7, 12, 5.6)]); - const short = await plan(cut, { ranges: [{ clipId: 'c1', fromIndex: 3, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.9, sourceOutSec: 5.6 }]); + // The lead of word 3 (starts 4.0) reaches back to 3.75, just into the end of c1. + const cut = sourceOf([clipOf('c1', 0, 3.8, 0), clipOf('c2', 3.85, 12, 3.8)]); + const short = await plan(cut, { ranges: [{ clipId: 'c2', fromIndex: 3, toIndex: 4 }] }); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 3.85, sourceOutSec: 5.5 }]); }); it('refuses words cut out exactly, where only padding touches them', async () => { @@ -198,7 +198,7 @@ describe('planShort: cut from the source timeline', () => { it('keeps the lead padding across a split at the first word', async () => { const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 3.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 3.5 }]); }); it('plays two ranges of neighboring words straight through, without replaying padding', async () => { @@ -218,7 +218,7 @@ describe('planShort: cut from the source timeline', () => { }, { transcripts: [close] }, ); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2.2 }]); const repeated = await plan( whole, @@ -231,8 +231,8 @@ describe('planShort: cut from the source timeline', () => { { transcripts: [close] }, ); expect(repeated.pieces).toEqual([ - { mediaId: M, sourceInSec: 1.6, sourceOutSec: 2.45 }, - { mediaId: M, sourceInSec: 0.9, sourceOutSec: 2.45 }, + { mediaId: M, sourceInSec: 1.45, sourceOutSec: 2.2 }, + { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2.2 }, ]); }); @@ -245,10 +245,10 @@ describe('planShort: cut from the source timeline', () => { ], }); expect(short.pieces).toEqual([ - { mediaId: M, sourceInSec: 6.9, sourceOutSec: 8.75 }, - { mediaId: M, sourceInSec: 1.9, sourceOutSec: 3.75 }, + { mediaId: M, sourceInSec: 6.75, sourceOutSec: 8.5 }, + { mediaId: M, sourceInSec: 1.75, sourceOutSec: 3.5 }, ]); - expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.85]); + expect(addedClips(short.ops).map((c) => c.startSec)).toEqual([0, 1.75]); }); it('cuts ranges from two recordings', async () => { @@ -273,13 +273,13 @@ describe('planShort: cut from the source timeline', () => { it('takes a moment the source plays twice only once', async () => { const repeated = sourceOf([clipOf('c1', 0, 10, 0), clipOf('c2', 2, 6, 10)]); const short = await plan(repeated, { ranges: [{ clipId: 'c2', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); }); it('joins pieces that a split (not a cut) left continuous', async () => { const split = sourceOf([clipOf('c1', 0, 4, 0), clipOf('c2', 4, 10, 4)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 4 }] }); - expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.9, sourceOutSec: 5.75 }]); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 2.75, sourceOutSec: 5.5 }]); }); it('keeps color and volume, and drops the fades of the source clip', async () => { @@ -330,8 +330,8 @@ describe('planShort: the edit', () => { clip: mediaClip({ id: 's1', mediaId: M, - sourceInSec: 2.9, - sourceOutSec: 5.75, + sourceInSec: 2.75, + sourceOutSec: 5.5, startSec: 0, }), }, @@ -344,8 +344,8 @@ describe('planShort: the edit', () => { clip: textClip({ id: 's2', text: 'w2 w3 w4', - startSec: 0.1, - durationSec: 2.75, + startSec: 0.25, + durationSec: 2.5, style: { fontSize: 54, color: 'white', background: 'black@0.6', anchor: 'bottom-center' }, }), }, From 7dec1b1bcfdeb17b632b590280612805037276bc Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:23:42 -0800 Subject: [PATCH 5/6] fix(timeline): confine timeline folders and caption only the chosen words A timeline's folder is now checked against the workspace too, not only its files, so a timelines/ symlink out of the workspace can't receive temp files or a recursive delete through files that link back in. Captions on a short come only from the chosen words, not the word the lead padding reaches into, and a range whose words were all cut is refused even when a clip still covers the gap between them. Co-Authored-By: Claude Opus 5.5 --- docs/backlog/review-test-timeout.md | 3 +++ src/timeline/document-store.ts | 13 ++++++++--- src/timeline/shorts.ts | 30 ++++++++++++++++++++----- tests/document-store.test.ts | 29 ++++++++++++++++++++++++ tests/timeline-shorts.test.ts | 35 +++++++++++++++++++++++++++++ 5 files changed, 101 insertions(+), 9 deletions(-) create mode 100644 docs/backlog/review-test-timeout.md diff --git a/docs/backlog/review-test-timeout.md b/docs/backlog/review-test-timeout.md new file mode 100644 index 0000000..955a01f --- /dev/null +++ b/docs/backlog/review-test-timeout.md @@ -0,0 +1,3 @@ +# Review integration test timeout + +`tests/integration/timeline-review.test.ts` ("flags a cut planted mid-tone…") runs under vitest's default 5 s timeout but renders several FFmpeg steps, so it sometimes times out on a busy machine (seen once in two runs on `feat/shorts` at 18511f1). Give it an explicit timeout like the other integration tests (`120_000`). diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index b5ff3ed..d149aa2 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -1,6 +1,6 @@ import { randomBytes } from 'node:crypto'; import { readdir, readFile, rm } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { join, resolve } from 'node:path'; import { atomicWriteFile } from '../session/store.js'; import { createRevisionedStore, type RevisionedStore } from '../storage/revisioned-store.js'; import { @@ -72,9 +72,16 @@ function checkedTimelineId(timeline: string): string { return parsed.data; } +/** A non-main timeline's folder, checked against the workspace: `timelines/` + * or the folder could be a symlink out of it, and writes (temp files) and + * deletes act on the folder itself, not only on the files in it. */ +function timelineDir(timeline: string): string { + return resolveInWorkspace(join(TIMELINES_DIR, checkedTimelineId(timeline))); +} + function timelineFile(timeline: string, file: string): string { if (checkedTimelineId(timeline) === MAIN_TIMELINE) return resolve(getWorkspace(), file); - // Checked against the workspace: `timelines/` could be a symlink out of it. + timelineDir(timeline); return resolveInWorkspace(join(TIMELINES_DIR, timeline, file)); } @@ -533,6 +540,6 @@ export async function deleteTimeline(timeline: string): Promise { // A corrupt document is deleted like any other. if (!(err instanceof CompositionCorruptError)) throw err; } - await rm(dirname(compositionPath(timeline)), { recursive: true, force: true }); + await rm(timelineDir(timeline), { recursive: true, force: true }); }); } diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index 3cc598c..bd5a43f 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -109,6 +109,10 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI interface WordWindow { mediaId: MediaId; + /** The range's words, by index into its transcript. */ + fromIndex: number; + toIndex: number; + words: { startSec: number; endSec: number }[]; /** Padded edges, source seconds. */ start: number; end: number; @@ -228,10 +232,19 @@ export async function planShort( let captions = 0; if (request.captions !== false) { const built = applyOps(target, ops); + // Only the asked-for words: the lead reaches into the word before, which + // whisper.cpp stretches over the pause, and it isn't part of the short. const plan = await planCaptions( built, { maxWords: request.maxWords }, - (clipId, transcriptId) => lookup(built, clipId, transcriptId), + async (clipId, transcriptId) => { + const found = await lookup(built, clipId, transcriptId); + const asked = windows.filter((w) => w.mediaId === found.transcript.mediaId); + const words = found.transcript.words.filter((_, i) => + asked.some((w) => i >= w.fromIndex && i <= w.toIndex), + ); + return { ...found, transcript: { ...found.transcript, words } }; + }, mintId, ); ops.push(...plan.ops); @@ -276,6 +289,9 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); return { mediaId: clip.mediaId, + fromIndex: range.fromIndex, + toIndex: range.toIndex, + words: transcript.words.slice(range.fromIndex, range.toIndex + 1), start: roundMs(start), end: last.endSec, speechStart: first.startSec, @@ -334,12 +350,14 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { ); } -/** Whether a piece plays any of the range's words, not just touches their edge. */ +/** Whether a piece plays any of the range's words, not just touches one's edge + * or the gap between two that were cut. */ function holdsSpeech(piece: Piece, window: WordWindow): boolean { - if (window.speechEnd > window.speechStart) { - return piece.sourceOutSec > window.speechStart && piece.sourceInSec < window.speechEnd; - } - return piece.sourceInSec <= window.speechStart && piece.sourceOutSec >= window.speechEnd; + return window.words.some((w) => + w.endSec > w.startSec + ? piece.sourceOutSec > w.startSec && piece.sourceInSec < w.endSec + : piece.sourceInSec <= w.startSec && piece.sourceOutSec >= w.endSec, + ); } /** `[lo, hi]` minus the `taken` intervals, in order. */ diff --git a/tests/document-store.test.ts b/tests/document-store.test.ts index f8430bc..d3226f8 100644 --- a/tests/document-store.test.ts +++ b/tests/document-store.test.ts @@ -254,6 +254,35 @@ describe('named timelines', () => { expect((await listTimelines()).map((t) => t.id)).toEqual(['main']); }); + it('refuses a timeline folder out of the workspace even when its files link back in', async () => { + const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); + try { + // timelines/ leads out; the files in it link back to documents inside. + const kept = join(workspace, 'kept'); + await mkdir(kept); + await writeFile(join(kept, 'composition.json'), JSON.stringify(emptyComposition())); + await writeFile(join(kept, 'composition-ops.json'), '{}'); + await mkdir(join(outside, 'best-1')); + for (const file of ['composition.json', 'composition-ops.json']) { + await symlink(join(kept, file), join(outside, 'best-1', file)); + } + await symlink(outside, join(workspace, 'timelines')); + + await expect(readComposition('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect( + mutateComposition([{ op: 'setCanvas', width: 2 }], { timeline: 'best-1' }), + ).rejects.toBeInstanceOf(WorkspaceBoundaryError); + await expect(deleteTimeline('best-1')).rejects.toBeInstanceOf(WorkspaceBoundaryError); + const { readdir } = await import('node:fs/promises'); + expect((await readdir(join(outside, 'best-1'))).sort()).toEqual([ + 'composition-ops.json', + 'composition.json', + ]); + } finally { + await rm(outside, { recursive: true, force: true }); + } + }); + it('refuses a timelines/ folder that is a symlink out of the workspace', async () => { const outside = await mkdtemp(join(tmpdir(), 'mmc-outside-')); try { diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index e67e3f5..8ac8798 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -195,6 +195,21 @@ describe('planShort: cut from the source timeline', () => { ).rejects.toThrow(/cut out of the source timeline/); }); + it('refuses a range whose words were all cut, leaving only the gap between them', async () => { + const apart = transcriptOf(M, [ + [1, 1.5], + [4, 4.5], + ]); + const gapOnly = sourceOf([clipOf('c1', 1.5, 4, 0)]); + await expect( + plan( + gapOnly, + { ranges: [{ clipId: 'c1', fromIndex: 0, toIndex: 1 }] }, + { transcripts: [apart] }, + ), + ).rejects.toThrow(/cut out of the source timeline/); + }); + it('keeps the lead padding across a split at the first word', async () => { const split = sourceOf([clipOf('c1', 0, 3, 0), clipOf('c2', 3, 30, 3)]); const short = await plan(split, { ranges: [{ clipId: 'c1', fromIndex: 2, toIndex: 2 }] }); @@ -354,6 +369,26 @@ describe('planShort: the edit', () => { expect(short.warnings).toEqual([]); }); + it('captions only the asked-for words, not the word the lead reaches into', async () => { + // whisper.cpp words abut: w0 runs through the pause before w1. + const abutting = transcriptOf(M, [ + [0, 1], + [1, 1.5], + [1.5, 2], + [2, 3], + ]); + const short = await planShort( + emptyComposition(), + whole, + { ranges: [{ clipId: 'c1', fromIndex: 1, toIndex: 2 }] }, + lookupFor([abutting]), + counter(), + ); + expect(short.pieces).toEqual([{ mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }]); + const captions = addedClips(short.ops).filter((c) => c.kind === 'text'); + expect(captions.map((c) => (c.kind === 'text' ? c.text : ''))).toEqual(['w1 w2']); + }); + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 }; From 638e0ff2488bf9050a136ae3c0753524177f4b59 Mon Sep 17 00:00:00 2001 From: Slava Yultyyev Date: Sun, 15 Nov 2026 15:30:54 -0800 Subject: [PATCH 6/6] fix(timeline): caption each piece of a short from its own range Captions came from the union of every range's words, so with reordered neighboring ranges one range's word could show in another's lead. Each piece now captions only the ranges it plays, from the transcripts the ranges were already read with. Co-Authored-By: Claude Opus 5.5 --- src/timeline/shorts.ts | 102 ++++++++++++++++++++++------------ tests/timeline-shorts.test.ts | 31 +++++++++++ 2 files changed, 96 insertions(+), 37 deletions(-) diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts index bd5a43f..febf983 100644 --- a/src/timeline/shorts.ts +++ b/src/timeline/shorts.ts @@ -1,16 +1,22 @@ -import { planCaptions } from './captions.js'; +import { + DEFAULT_CAPTION_TRACK, + DEFAULT_CAPTION_WORDS, + defaultCaptionStyle, + groupCaptions, +} from './captions.js'; import { speedFactor, videoLayers } from './compile.js'; import { type Clip, type Composition, + clipEndSec, type Effect, type MediaClip, makeClipId, } from './composition.js'; -import { applyOps, type CompositionOp, mediaClip, videoTrack } from './ops.js'; +import { applyOps, type CompositionOp, mediaClip, textClip, videoTrack } from './ops.js'; import { type Aspect, canvasFor, planReframe } from './reframe.js'; import { MAIN_TIMELINE, type MediaId } from './schema.js'; -import type { Transcript } from './transcript.js'; +import { clipWords, type Transcript } from './transcript.js'; /** * Shorts: a vertical clip cut out of a longer timeline by transcript word ranges, @@ -109,6 +115,7 @@ export function checkShortTimelines(target: string, source: string = MAIN_TIMELI interface WordWindow { mediaId: MediaId; + transcript: Transcript; /** The range's words, by index into its transcript. */ fromIndex: number; toIndex: number; @@ -126,6 +133,9 @@ interface Piece extends ShortPiece { /** The source clip's speed, refused only if the piece is played. */ speed: number; sourceClipId: string; + transcript: Transcript; + /** The word ranges the piece plays, which are what it captions. */ + asked: { fromIndex: number; toIndex: number }[]; } /** @@ -195,23 +205,21 @@ export async function planShort( { op: 'addTrack', track: videoTrack({ id: SHORT_TRACK }) }, ); let at = 0; - for (const piece of joined) { - ops.push({ - op: 'addClip', - trackId: SHORT_TRACK, - clip: mediaClip({ - id: mintId(), - mediaId: piece.mediaId, - sourceInSec: piece.sourceInSec, - sourceOutSec: piece.sourceOutSec, - // Rounded to the microsecond: clean numbers, far inside the export's - // abutting tolerance. - startSec: Math.round(at * 1e6) / 1e6, - effects: piece.effects, - }), + const placed = joined.map((piece) => { + const clip = mediaClip({ + id: mintId(), + mediaId: piece.mediaId, + sourceInSec: piece.sourceInSec, + sourceOutSec: piece.sourceOutSec, + // Rounded to the microsecond: clean numbers, far inside the export's + // abutting tolerance. + startSec: Math.round(at * 1e6) / 1e6, + effects: piece.effects, }); + ops.push({ op: 'addClip', trackId: SHORT_TRACK, clip }); at += piece.sourceOutSec - piece.sourceInSec; - } + return { piece, clip }; + }); if (request.x !== undefined || request.y !== undefined) { const framed = planReframe(applyOps(target, ops), { @@ -231,24 +239,37 @@ export async function planShort( let captions = 0; if (request.captions !== false) { - const built = applyOps(target, ops); - // Only the asked-for words: the lead reaches into the word before, which - // whisper.cpp stretches over the pause, and it isn't part of the short. - const plan = await planCaptions( - built, - { maxWords: request.maxWords }, - async (clipId, transcriptId) => { - const found = await lookup(built, clipId, transcriptId); - const asked = windows.filter((w) => w.mediaId === found.transcript.mediaId); - const words = found.transcript.words.filter((_, i) => - asked.some((w) => i >= w.fromIndex && i <= w.toIndex), - ); - return { ...found, transcript: { ...found.transcript, words } }; - }, - mintId, - ); - ops.push(...plan.ops); - captions = plan.captions; + // Each piece captions only its own range's words: the lead reaches into the + // word before, which whisper.cpp stretches over the pause, and that word + // isn't part of the short even when another range asks for it. + const style = defaultCaptionStyle(applyOps(target, ops)); + const cues = placed.flatMap(({ piece, clip }) => { + const words = piece.transcript.words.filter((_, i) => + piece.asked.some((r) => i >= r.fromIndex && i <= r.toIndex), + ); + return groupCaptions( + clipWords({ ...piece.transcript, words }, clip), + clipEndSec(clip), + request.maxWords ?? DEFAULT_CAPTION_WORDS, + ); + }); + if (cues.length > 0) { + ops.push({ op: 'addTrack', track: videoTrack({ id: DEFAULT_CAPTION_TRACK }) }); + for (const cue of cues) { + ops.push({ + op: 'addClip', + trackId: DEFAULT_CAPTION_TRACK, + clip: textClip({ + id: mintId(), + text: cue.text, + startSec: cue.startSec, + durationSec: roundMs(cue.endSec - cue.startSec), + style, + }), + }); + } + } + captions = cues.length; } return { @@ -289,6 +310,7 @@ function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): ); return { mediaId: clip.mediaId, + transcript, fromIndex: range.fromIndex, toIndex: range.toIndex, words: transcript.words.slice(range.fromIndex, range.toIndex + 1), @@ -341,6 +363,8 @@ function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), speed: speedFactor(clip.effects), sourceClipId: clip.id, + transcript: window.transcript, + asked: [{ fromIndex: window.fromIndex, toIndex: window.toIndex }], }); taken.push([a, b]); } @@ -388,7 +412,11 @@ function joinContinuous(pieces: Piece[]): Piece[] { prev.speed === piece.speed && JSON.stringify(prev.effects) === JSON.stringify(piece.effects) ) { - joined[joined.length - 1] = { ...prev, sourceOutSec: piece.sourceOutSec }; + joined[joined.length - 1] = { + ...prev, + sourceOutSec: piece.sourceOutSec, + asked: [...prev.asked, ...piece.asked], + }; } else { joined.push(piece); } diff --git a/tests/timeline-shorts.test.ts b/tests/timeline-shorts.test.ts index 8ac8798..0a110ae 100644 --- a/tests/timeline-shorts.test.ts +++ b/tests/timeline-shorts.test.ts @@ -389,6 +389,37 @@ describe('planShort: the edit', () => { expect(captions.map((c) => (c.kind === 'text' ? c.text : ''))).toEqual(['w1 w2']); }); + it("captions each piece from its own range, even when another range asks for the lead's word", async () => { + const abutting = transcriptOf(M, [ + [0, 1], + [1, 2], + [2, 3], + ]); + const short = await planShort( + emptyComposition(), + whole, + { + ranges: [ + { clipId: 'c1', fromIndex: 1, toIndex: 1 }, + { clipId: 'c1', fromIndex: 0, toIndex: 0 }, + ], + }, + lookupFor([abutting]), + counter(), + ); + expect(short.pieces).toEqual([ + { mediaId: M, sourceInSec: 0.75, sourceOutSec: 2 }, + { mediaId: M, sourceInSec: 0, sourceOutSec: 1 }, + ]); + const captions = addedClips(short.ops).flatMap((c) => + c.kind === 'text' ? [{ text: c.text, startSec: c.startSec }] : [], + ); + expect(captions).toEqual([ + { text: 'w1', startSec: 0.25 }, + { text: 'w0', startSec: 1.25 }, + ]); + }); + it('keeps the source short side: a 720p source makes 720x1280, 1:1 makes a square', async () => { const hd = sourceOf([clipOf('c1', 0, 30, 0)], { width: 1280, height: 720, fps: 25 }); const range = { clipId: 'c1', fromIndex: 2, toIndex: 4 };