diff --git a/.releaserc.json b/.releaserc.json index 553148a..778a55b 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -7,7 +7,8 @@ "preset": "angular", "releaseRules": [ { "type": "chore", "scope": "deps", "release": "patch" }, - { "type": "chore", "scope": "security", "release": "patch" } + { "type": "chore", "scope": "security", "release": "patch" }, + { "type": "feat", "release": "patch" } ] } ], diff --git a/README.md b/README.md index 05f0790..92f15a0 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ The open-source AI video editor for Claude Code and any MCP client: remove silen npm weekly downloads License: MIT Node 24+ - Tests: 859 passing + Tests: 965 passing

@@ -40,12 +40,13 @@ Version 0.5.0 turns a raw talking-head recording into a clean, captioned cut: ve | **Captions** | Burned-in captions timed to the words, as short phrases or one word at a time | `clip timeline captions` | | **Reframe** | Turns a landscape cut into 9:16, 1:1 or 4:5, filling the frame and choosing the part that shows; zoom and position per clip | `clip timeline reframe` | | **Loudness** | Evens out the exported mp4's volume to a target (-14 LUFS for YouTube, Shorts, Reels and TikTok; -16 for podcasts), measured in two passes | `clip timeline loudness` | +| **Shorts from a long recording** | Turns ranges of words from a long recording into a vertical short on a timeline of its own: captioned, at -14 LUFS, with the cuts already made on the recording; the long cut stays as it is | `clip timeline short` | | **Cut check** | Flags cuts where a word may be clipped, with a contact sheet of the frames on each side | `clip timeline check` | | **Export to Final Cut Pro and DaVinci Resolve** | FCPXML 1.10 project | `clip timeline export --format fcpxml` | | **Export to Premiere Pro** | Final Cut Pro 7 XML project, captions as `.srt` files | `clip timeline export --format xmeml` | | **Render** | mp4 of the finished cut | `clip timeline export` | -All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check isn't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. +All of it works from chat (the Claude Code skill or the MCP server), the CLI, and the browser UI; the cut check and shorts aren't in the UI yet. Every edit is one step on a non-destructive timeline that `undo` and `redo` walk through. Underneath sit [19 single-file FFmpeg tools](#all-19-tools) for everything else: trim, split, concat, transitions, text and title cards, zoom/pan, overlay, speed, color, chroma key, stabilize, and render. --- @@ -157,6 +158,33 @@ A cut splits the clip into pieces: the first keeps its id, and `clip timeline sh --- +## Quick start: a long recording → vertical shorts + +Each short is a timeline of its own, cut from the main timeline's clips, so the pauses and retakes you cut there stay cut. In Claude Code or Claude Desktop, ask: + +> *Transcribe podcast.mp4 and cut the pauses. Then find the three strongest moments, 20 to 60 seconds each, and make each one a short.* + +The same steps from the CLI: + +```bash +clip timeline new +clip timeline add-media podcast.mp4 # prints the clip id, e.g. clip_ec2f047d +clip timeline transcribe clip_ec2f047d # transcribe it (once per file) +clip timeline silence clip_ec2f047d # cut the pauses on main +clip timeline transcribe --sentences | head -80 # one line per sentence: time, word indices, text +clip timeline short best-1 clip_ec2f047d 1400 1478 --x 0.4 # words 1400–1478 as a 9:16 short +clip timeline --timeline best-1 frame 3 check.jpg # look at it; move --x if the speaker is off +clip timeline --timeline best-1 export best-1.mp4 +clip timeline list # main and every short +clip timeline export --all ./shorts # every short, as ./shorts/.mp4 +``` + +`short` cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone (where there's no clear pause it follows the transcript and warns), fills the 9:16 frame (`--x` picks the part that shows), sets -14 LUFS and burns in captions. Give several ` ` ranges to join moments, in the order you give them. Any command takes `--timeline ` to work on a short, `clip timeline --timeline best-1 undo` takes the short apart again, and `short best-1 … --replace` rebuilds it. + +In `clip ui`, select words in a clip's transcript and press **Make short**: the short opens in a tab of its own above the timeline, next to main. **Export all shorts** renders every one. + +--- + ## Another example: a product demo from screen recordings Five screen recordings → one polished product demo. Type this into Claude Code (or Claude Desktop, via the MCP server): @@ -324,7 +352,7 @@ No — nowhere. The CLI, the browser UI, the Claude Code skill, and the MCP serv ### Capabilities & limits **Is it production-ready?** -This release is feature-complete for local editing — a dark timeline editor (view, edit, preview, export), silence removal, transcript editing, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, export to Final Cut Pro, DaVinci Resolve and Premiere Pro, 19 single-file tools, an MCP server, and 859 tests passing. The API surface is still pre-1.0 (tool schemas may change in minor ways before 1.0). Use it for real work; pin a version in CI. +This release is feature-complete for local editing — a dark timeline editor (view, edit, preview, export), silence removal, transcript editing, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, export to Final Cut Pro, DaVinci Resolve and Premiere Pro, 19 single-file tools, an MCP server, and 965 tests passing. The API surface is still pre-1.0 (tool schemas may change in minor ways before 1.0). Use it for real work; pin a version in CI. **What's the maximum video size or duration?** No hardcoded limit. FFmpeg streams the file rather than loading it into memory, and the browser UI streams uploads to disk — multi-GB recordings work fine. This release isn't tuned for projects with hundreds of clips or runtime above 30 minutes; for those, drive the CLI directly. @@ -386,7 +414,7 @@ MakeMyClip Editor is built for that job: an open-source video editor that Claude ### Is there an MCP server for video editing? -Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. +Yes. `npx -y @makemyclip/editor mcp` starts MakeMyClip Editor's local MCP server over stdio for Claude Desktop, Cursor, or any MCP client. Its tools show the timeline, edit it (add media, trim, split, remove silence, cut words, remove retakes, caption speech, reframe to 9:16, set loudness, make shorts, transitions), check the cuts, transcribe, undo, redo, and export to mp4, FCPXML or Premiere XML. Media stays in one workspace folder, and no API key is needed. ### How do I remove silence from a video automatically? @@ -398,7 +426,11 @@ Yes. MakeMyClip Editor transcribes with whisper.cpp on your machine; install it ### Can Claude add captions or subtitles to a video? -Yes. After transcription, MakeMyClip Editor's `clip timeline captions` (or the UI's Captions button) puts captions timed to the words on a track above the video, and export burns them in. Captions come as short phrases or one word at a time, with your choice of size, color, position, and background. Non-Latin scripts use an installed font that has the glyphs. In the Premiere export, captions become `.srt` files. On Linux, the bundled FFmpeg can't draw text, so set `MAKEMYCLIP_FFMPEG_PATH` to an FFmpeg built with libfreetype and libharfbuzz. +Yes. After transcription, MakeMyClip Editor's `clip timeline captions` (or the UI's Captions button) puts captions timed to the words on a track above the video, and export burns them in. Captions come as short phrases or one word at a time, with your choice of size, color, position, and background. On a vertical canvas they sit above the bottom quarter, clear of the Shorts, Reels and TikTok buttons, and a line is never wider than fits at its font size. Non-Latin scripts use an installed font that has the glyphs. In the Premiere export, captions become `.srt` files. On Linux, the bundled FFmpeg can't draw text, so set `MAKEMYCLIP_FFMPEG_PATH` to an FFmpeg built with libfreetype and libharfbuzz. + +### Can it cut Shorts or Reels out of a podcast? + +Yes. Transcribe the recording, then `clip timeline short best-1 ` (or the `make_short` verb from chat) turns that range of words into a vertical short on a timeline of its own: 9:16, filling the frame, captioned, normalized to -14 LUFS, with the silence and retake cuts already made on the recording. Make as many as you like; the long cut stays untouched, and each short exports on its own. You or the agent pick the moments: nothing scores highlights for you, and nothing is uploaded. ### Can I finish an AI edit in DaVinci Resolve, Premiere Pro or Final Cut Pro? @@ -422,7 +454,7 @@ auto-editor is a mature open-source command-line tool that cuts silence or motio ## Roadmap -- **Now (v0.5.0)** — silence removal, local transcription, edit by transcript, retake removal, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, click-free cuts, export to Final Cut Pro / DaVinci Resolve / Premiere Pro, on top of the dark timeline editor (view / edit / preview / export), 19 tools, MCP server, and snapshot/undo +- **Now (v0.5.2)** — vertical shorts from a long recording, silence removal, local transcription, edit by transcript, retake removal, captions, 9:16 / 1:1 / 4:5 reframing, loudness normalization, click-free cuts, export to Final Cut Pro / DaVinci Resolve / Premiere Pro, on top of the dark timeline editor (view / edit / preview / export), 19 tools, MCP server, and snapshot/undo - **Next** — model picker, SSE for live session updates, AI Elements / shadcn migration - **1.0** — frozen tool schemas, MCP transport, published Anthropic skill, docs site - **Beyond** — desktop app (Electron), cloud rendering via [MakeMyClip.com](https://makemyclip.com) diff --git a/SKILL.md b/SKILL.md index f28502d..8eadb11 100644 --- a/SKILL.md +++ b/SKILL.md @@ -1,6 +1,6 @@ --- name: editor -description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. +description: Local, open-source video editor that agents drive with FFmpeg and whisper.cpp. Use when the user wants to edit a video file - cut out silence, pauses or dead air; remove repeated takes or cut words using the transcript; add captions or subtitles; make a vertical 9:16 (Shorts, Reels, TikTok), square or 4:5 version; cut clips or shorts from a long video or podcast; even out the volume (loudness normalization); trim, split, join or crossfade clips; add text, title cards or zoom; clean up a talking-head video, screen recording, tutorial or podcast; render an mp4; or export the cut to Final Cut Pro, DaVinci Resolve or Premiere Pro (FCPXML / XML). Runs offline with no API key; every timeline edit can be undone; no generative AI. license: MIT compatibility: Requires Node 24+. FFmpeg is auto-downloaded via ffmpeg-static on first use; override with MAKEMYCLIP_FFMPEG_PATH. Transcription (transcript cuts, retakes, captions) needs a user-installed whisper.cpp and a model file set in MAKEMYCLIP_WHISPER_MODEL. metadata: @@ -22,6 +22,7 @@ Trigger this skill when the user asks to: - Cut the silence, pauses, or dead air out of a recording ("tighten it up", "remove the gaps") - Remove repeated takes or bad takes, or cut a sentence or words by the transcript - Clean up a talking-head video, tutorial, lecture, or video podcast +- Cut clips or vertical shorts (Shorts, Reels, TikTok) out of a long video, webinar or podcast - Add captions or subtitles timed to the speech, or titles and lower-thirds - Zoom, pan, or apply Ken Burns to a region of a clip - Assemble multiple clips into one (concat, montage) @@ -322,16 +323,18 @@ To cut the pauses out of a talking clip already on the timeline, use `timeline s After cutting, run `timeline check` before exporting: it lists every hard cut whose audio is loud right at the join (usually a clipped word or a click) and writes a contact sheet (`--sheet out.jpg`) with the outgoing clip's last frame and the incoming clip's first for each cut. Open the sheet to see the jumps. A flagged cut is a reason to trim or move that boundary, not a verdict. Export fades the audio over 8 ms on each side of every hard cut, so a cut through steady sound doesn't click; the check still flags cuts in sound, since those can clip a word. -To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. +To read what's said in a clip, run `timeline transcribe `: it returns the clip's words with their timeline times, so you can find a sentence before cutting around it. For a long recording, read `timeline transcribe --sentences` instead: one line per sentence of the whole timeline (`12:34 #1400–1413 clip_ab12 text`, the time, the word indices, the clip it starts in), and `--from S --to S` for a stretch; pipe it through `head` to read a page at a time. Transcription is optional and runs a local whisper.cpp; if it isn't set up, the command says how (install whisper.cpp, download a model, set `MAKEMYCLIP_WHISPER_MODEL`). The first run per source file takes a while; later runs reuse the stored transcript. Whisper tends to leave out filler words like "um", so don't rely on the transcript to find them. To cut by words, use the `index` values from `timeline transcribe`: `timeline cut-words ` removes those words (inclusive) and moves later clips on the track up. Add `--transcript ` from the same `timeline transcribe` output so the cut is refused, not misplaced, if the clip was transcribed again since. The transcript also lists `retakes`, phrases said twice in a row; `timeline remove-retakes ` cuts the earlier take of each (`--dry-run` lists them first). Each is one edit that `timeline undo` reverses. A cut splits the clip, so transcribe the resulting clips again before the next cut, and run `timeline check` afterwards. -To caption speech, run `timeline captions` once the cuts are done: it turns the stored transcripts into text clips on a `captions` track above the video, timed to the words, and export burns them in. With no clip id it captions every transcribed clip and replaces the whole caption track; `timeline captions ` replaces only the captions over that clip. Captions are short phrases (up to 7 words, broken at pauses and sentence ends); `--words 1` shows one word at a time. Style with `--font --color --anchor --background ` (default white on a dark box, bottom center). Captions don't move with later cuts, so run it again after one. It's one edit that `timeline undo` reverses, and `timeline frame ` shows a captioned frame. Text outside Latin (Cyrillic, Chinese, Japanese, Korean) is drawn with an installed font that has its glyphs; if none does, export says which characters are missing — install a font (for CJK, Noto Sans CJK) or set `MAKEMYCLIP_FALLBACK_FONT` to a font file. Nothing is downloaded. +To caption speech, run `timeline captions` once the cuts are done: it turns the stored transcripts into text clips on a `captions` track above the video, timed to the words, and export burns them in. With no clip id it captions every transcribed clip and replaces the whole caption track; `timeline captions ` replaces only the captions over that clip. Captions are short phrases (up to 7 words, broken at pauses and sentence ends); `--words 1` shows one word at a time. Style with `--font --color --anchor --background ` (default white on a dark box, bottom center). On a vertical (portrait) canvas captions sit above the bottom quarter and keep 14% in from each side, clear of the Shorts, Reels and TikTok overlays; `--margin ` sets the distance from the bottom (or top) edge. A line never gets wider than fits at its font size (22 characters at the default size on 1080x1920), so a bigger `--font` makes shorter lines. Captions don't move with later cuts, so run it again after one. It's one edit that `timeline undo` reverses, and `timeline frame ` shows a captioned frame. Text outside Latin (Cyrillic, Chinese, Japanese, Korean) is drawn with an installed font that has its glyphs; if none does, export says which characters are missing — install a font (for CJK, Noto Sans CJK) or set `MAKEMYCLIP_FALLBACK_FONT` to a font file. Nothing is downloaded. To make a vertical (9:16), square (1:1) or feed (4:5) version, run `timeline reframe 9:16`. It changes the canvas shape, keeping its short side (1920x1080 becomes 1080x1920), and fills it: `--fit cover` (the default) crops what overflows, `--fit contain` shows the whole picture with bars. Look at `timeline frame ` first to see where the speaker is, then pass `--x` (0 shows the left edge of the source, 0.5 the center, 1 the right edge; `--y` likewise, top to bottom) to keep them in frame. `--x` sets every media clip on the timeline now; clips added later start centered. To frame one clip differently, run `timeline framing --x 0.3` (also `--scale` 1–4 to punch in, `--fit cover|contain|inherit`, `--reset`). Check with `timeline frame ` after; the `timeline check` contact sheet shows the source frames, not the crop. Captions keep their size, since the short side doesn't change. Each is one edit that `timeline undo` reverses. A reframed cut exports as mp4; the editor-project export refuses it. To even out the volume of the exported mp4, run `timeline loudness -14` (YouTube, Shorts, Reels, TikTok) or `-16` (podcasts); `off` turns it off. Export then measures the finished mp4 and normalizes it to that integrated loudness in a second pass (EBU R128, true peak at most -1.5 dBTP), and reports `loudness` with the measured before and after values. `mode: dynamic` means loudnorm couldn't apply one gain (steady or very short audio, a wide loudness range, or peaks); `skipped` means the audio was silent or too short to measure. It's one setting on the timeline that `timeline undo` reverses, it applies to the mp4 only, and it doesn't even out clips recorded at different levels. +To cut vertical shorts out of a long recording, transcribe it and make the cuts you want everywhere (pauses, retakes) on the main timeline first, then read `timeline transcribe --sentences`, pick each moment as whole sentences and run `timeline short ` with their word indices, where `` is a new name such as `best-1`. Pick whole sentences, 15–60 s in all; you choose the moments, the editor doesn't score them. The short is a timeline of its own, cut from main's clips (main's cuts carry in, main itself doesn't change): it cuts each edge inside the pause the audio shows there, keeping 0.1 s of room tone, so it never keeps the next sentence's first syllable, fills a 9:16 frame, sets -14 LUFS and burns in captions. More ` ` triples join several moments in the order given, so a later line can open the short as its hook. Options: `--x 0..1` to keep the speaker in frame, `--aspect 1:1|4:5`, `--words N` per caption, `--no-captions`, `--loudness `, `--source `, `--dry-run` (prints the pieces), `--replace` to rebuild it. Its `warnings` name a range that starts or ends mid-sentence (widen it to the sentence's edge unless the cut is meant) and an edge with no clear pause in the audio (music under the speech, or no break), which follows the transcript instead: look at that edge or listen to it. Every other command works on a short with `--timeline `: look at it with `timeline --timeline best-1 frame 3 check.jpg`, adjust with `timeline --timeline best-1 reframe 9:16 --x 0.3`, and render it with `timeline --timeline best-1 export best-1.mp4` (`timeline export --all

` renders every short as `/.mp4`). `timeline list` shows every timeline, `timeline --timeline best-1 undo` takes the short apart, and `timeline delete best-1` removes it. Over MCP, it's the `make_short` verb in `timeline_edit` with `timeline` set to the new id; every timeline tool takes `timeline`, and `timeline_show` lists them all. + To hand the cut to an editor app, export the timeline as an editor project instead of a render. `timeline export cut.fcpxml` (or `--format fcpxml`) writes FCPXML 1.10 for Final Cut Pro and DaVinci Resolve; `timeline export cut.xml` (or `--format xmeml`) writes Final Cut Pro 7 XML for Premiere Pro, which Resolve reads too. Premiere doesn't import FCPXML, so use `xmeml` for it. Both reference the source files by absolute path (nothing is copied, so keep the sources where they are), place camera files by their start timecode, and keep the mp4's timing: clips, gaps, fades, volume, fade/dissolve transitions (as cross dissolves) and black or colored cards. Text over the picture becomes titles in FCPXML; in the Premiere XML it goes to `.srt` caption files next to the XML, since Premiere drops that format's titles. What the file can't hold — speed, color adjustments, chroma key, a reframe's fill, zoom or position (set the timeline to `--fit contain` and reset clip framing to carry it), rotation, wipe/slide/circle transitions, audio-only tracks, media on two tracks, a frame rate editors don't offer — is refused with the fix to make; what it holds only in part (text styling, caption boxes) is listed in the output's `notes`. ### Open the local UI (implemented) diff --git a/docs/backlog/caption-safe-zone.md b/docs/backlog/caption-safe-zone.md deleted file mode 100644 index 7a7d784..0000000 --- a/docs/backlog/caption-safe-zone.md +++ /dev/null @@ -1,3 +0,0 @@ -# Caption safe zone for vertical video - -Captions sit 40 px from the bottom edge (`EDGE` in `src/timeline/compile.ts`) whatever the canvas shape. On a 9:16 export, the Shorts, Reels and TikTok overlays (caption text, buttons) cover roughly the bottom fifth of the frame, so bottom-anchored captions land under them. Raise the default caption position on portrait canvases, or add a safe-zone offset to the caption style. diff --git a/docs/backlog/caption-width-follows-font-size.md b/docs/backlog/caption-width-follows-font-size.md deleted file mode 100644 index 6b0e133..0000000 --- a/docs/backlog/caption-width-follows-font-size.md +++ /dev/null @@ -1,3 +0,0 @@ -# Caption width follows the font size - -`groupCaptions` breaks captions at 32 Latin characters (`MAX_CAPTION_WIDTH` in `src/timeline/captions.ts`) whatever the font size, and drawtext doesn't wrap. With the default size (the canvas short side / 20) a caption fits, but a larger `--font` makes long captions run off both edges. Derive the limit from the effective font size and the canvas width. diff --git a/docs/backlog/caption-width-font-metrics.md b/docs/backlog/caption-width-font-metrics.md new file mode 100644 index 0000000..b34fc58 --- /dev/null +++ b/docs/backlog/caption-width-font-metrics.md @@ -0,0 +1,3 @@ +# Caption width from the font's own metrics + +Caption lines are fitted by `textWidthEm` in `src/timeline/captions.ts`, an estimate per character class that errs wide for Arial and DejaVu Sans (W and m at 1 em, other capitals 0.8, non-Latin letters 0.95–1.1). A font wider than those, or a glyph outside the classes, can still overflow the safe area. Read the advance widths from the font file the caption is drawn with (`fontFileFor` in `src/ffmpeg/font.ts` already parses its character map) and fit lines by them. diff --git a/docs/backlog/confine-render-media.md b/docs/backlog/confine-render-media.md new file mode 100644 index 0000000..b5ab9c8 --- /dev/null +++ b/docs/backlog/confine-render-media.md @@ -0,0 +1,9 @@ +# Re-check media against the workspace on every render for the UI and MCP + +`POST /api/timelines/export` re-checks each short's files with `confinedMediaMap`, so a registered file swapped for a symlink out of the workspace is refused. Three older render paths still take the registry as ingested (`buildMediaMap`): + +- `POST /api/timeline/export` (mp4) in `src/ui/server.ts`; +- `GET /api/timeline/frame` in `src/ui/server.ts`; +- the mp4 path of MCP `timeline_export` in `src/mcp/server.ts`. + +The transcript and project-file paths already confine. Pass `confinedMediaMap(comp)` on these three, and test each with a symlinked file, as `tests/ui-timelines.test.ts` does for export-all. Found by the Codex review of the UI shorts PR (2026-11-15). diff --git a/docs/backlog/cut-words-pause-edges.md b/docs/backlog/cut-words-pause-edges.md new file mode 100644 index 0000000..5c88d73 --- /dev/null +++ b/docs/backlog/cut-words-pause-edges.md @@ -0,0 +1,3 @@ +# Cut words at the pause + +`cut_words` and `remove_retakes` cut from the first word's start to the last word's end as whisper.cpp gives them (`planWordCut` in `src/timeline/transcript.ts`). whisper gives each pause to the word before it and runs late at sentence edges. On a 30-sentence test recording, cutting a sentence would also remove up to 0.35 s of the next sentence's first syllable, at 18 of 29 boundaries. Place those cut edges in the pause the audio shows, as `make_short` does (`src/timeline/pauses.ts`), through `VerbContext.speechLevels`. diff --git a/docs/backlog/frame-preview-short-window.md b/docs/backlog/frame-preview-short-window.md deleted file mode 100644 index 177b62f..0000000 --- a/docs/backlog/frame-preview-short-window.md +++ /dev/null @@ -1,3 +0,0 @@ -# Frame preview encodes a short window - -`buildFrameAtPlan` (`src/timeline/compile.ts`) encodes the whole clip's segment to grab one frame, so every framing change in the browser UI re-encodes the full clip before the viewer updates. On a long uncut recording that takes seconds. Encode only a short window around the requested time (`-ss` before the input, `-t` of a few frames) and adjust the frame offset, keeping the same filter chain so the preview still matches export. diff --git a/docs/backlog/retake-test-model.md b/docs/backlog/retake-test-model.md new file mode 100644 index 0000000..16fe1aa --- /dev/null +++ b/docs/backlog/retake-test-model.md @@ -0,0 +1,3 @@ +# Retake integration test depends on the model + +`tests/integration/timeline-transcript.test.ts` ("removes a repeated take as one undoable edit") runs whenever whisper.cpp and a model are set up. With `ggml-base.en.bin` it fails on `feat/shorts` too: the model's transcript of the test's spoken phrase has no immediate repeat, so `retakes` is empty. Make the test independent of the model (assert on a stored transcript with a known repeat, as the unit tests do) or pin the model it was written for. diff --git a/docs/backlog/review-test-timeout.md b/docs/backlog/review-test-timeout.md new file mode 100644 index 0000000..955a01f --- /dev/null +++ b/docs/backlog/review-test-timeout.md @@ -0,0 +1,3 @@ +# Review integration test timeout + +`tests/integration/timeline-review.test.ts` ("flags a cut planted mid-tone…") runs under vitest's default 5 s timeout but renders several FFmpeg steps, so it sometimes times out on a busy machine (seen once in two runs on `feat/shorts` at 18511f1). Give it an explicit timeout like the other integration tests (`120_000`). diff --git a/docs/backlog/segment-duration-rounding.md b/docs/backlog/segment-duration-rounding.md new file mode 100644 index 0000000..46e8c92 --- /dev/null +++ b/docs/backlog/segment-duration-rounding.md @@ -0,0 +1,3 @@ +# Segment duration rounding + +Export renders each clip as its own segment and joins them, and each segment comes out a little longer than its clip: the video rounds up to a whole frame and the AAC audio to a whole 1024-sample frame. A three-piece short planned at 16.200 s exported as 488 video frames (16.267 s at 30 fps) and 16.25 s of audio. Check whether the audio-video offset accumulates over a timeline with hundreds of cuts (a silence-cut hour), and if it does, trim each segment to a frame-aligned length or carry the remainder into the next segment. diff --git a/docs/backlog/timeline-color-validation.md b/docs/backlog/timeline-color-validation.md deleted file mode 100644 index 643dac7..0000000 --- a/docs/backlog/timeline-color-validation.md +++ /dev/null @@ -1,3 +0,0 @@ -# Timeline color validation - -Timeline `color` / `background` fields are bare `z.string()` (src/timeline/composition.ts, ops.ts) and `quoteColor` in src/timeline/compile.ts passes non-`#` values into the filtergraph unescaped, so a value like `red,drawtext=...` adds a filter; validate them as hex or a named-color allowlist in the schema. diff --git a/docs/backlog/ui-inspector-height.md b/docs/backlog/ui-inspector-height.md new file mode 100644 index 0000000..4efeff5 --- /dev/null +++ b/docs/backlog/ui-inspector-height.md @@ -0,0 +1,5 @@ +# Clip inspector has no room on a short window + +In `clip ui`, the viewer takes up to 44vh plus its bar, and the clip inspector (`.stage-detail`) gets whatever is left of `.main`. At a 1000 px tall window, `.main` is 468 px and the viewer fills all of it, so the inspector is 0 px tall: a selected clip's transcript, framing and Make short can't be reached. It works at 1500 px. Seen on 2026-11-15 while checking the shorts tabs; the layout was the same before them, and the tabs bar takes another ~30 px once a short exists. + +Fix: give the inspector a minimum height (or let the viewer shrink first), and check at 900 px tall. diff --git a/llms.txt b/llms.txt index e245b0a..21f2713 100644 --- a/llms.txt +++ b/llms.txt @@ -9,18 +9,19 @@ - **Backend:** FFmpeg (downloaded by `ffmpeg-static` at npm install; the Claude Desktop extension uses an installed FFmpeg); optional user-installed whisper.cpp for transcription - **Surfaces:** Claude Code skill · `clip` CLI · browser UI at `127.0.0.1:5573` · MCP server (`clip mcp`) - **Agent access:** Claude Code skill + MCP server (`clip mcp`); no API key — the editor embeds no language model, it runs on whichever Claude you already use -- **Tests:** 859 passing +- **Tests:** 965 passing - **License:** MIT (code); FFmpeg runs as a subprocess under its own license (GPLv3 builds from `ffmpeg-static`, except a nonfree Apple Silicon build; the `.mcpb` ships no FFmpeg) ## What v0.5.0 does - **Remove silence:** `clip timeline silence ` cuts the pauses out of a recording (`--sensitivity`, `--dry-run`) -- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine +- **Transcribe locally:** `clip timeline transcribe ` returns each word with its timeline time, via whisper.cpp on the user's machine; `clip timeline transcribe --sentences` lists a long recording as numbered sentences - **Edit by transcript:** `clip timeline cut-words ` cuts a range of words - **Remove repeated takes:** `clip timeline remove-retakes ` cuts the earlier take of each phrase said twice in a row -- **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time) +- **Captions:** `clip timeline captions` burns in captions timed to the words (short phrases or one word at a time); on a vertical canvas they stay clear of the Shorts, Reels and TikTok overlays, and their width follows the font size - **Reframe:** `clip timeline reframe 9:16` turns a landscape cut vertical (also 1:1, 4:5); `--x` picks the part that shows, `clip timeline framing ` sets one clip's zoom and position - **Loudness:** `clip timeline loudness -14` normalizes the exported mp4 to -14 LUFS (or `-16` for podcasts) in two passes +- **Shorts from a long recording:** `clip timeline short best-1 ` (MCP: the `make_short` verb) turns a range of transcript words into a 9:16 short on a timeline of its own, captioned and at -14 LUFS, keeping the cuts made on the main timeline; `--timeline ` points any command at a short, and `clip timeline export --all ` renders every short. In `clip ui`, Make short works from a transcript selection and each short gets a tab - **Cut check:** `clip timeline check --sheet cuts.jpg` flags cuts that may clip a word and writes a contact sheet - **Export to pro editors:** `clip timeline export --format fcpxml` (Final Cut Pro, DaVinci Resolve) or `--format xmeml` (Premiere Pro; captions as .srt) - Every timeline edit is one undoable step (`clip timeline undo` / `redo`), from the CLI, the MCP server or the browser UI @@ -52,7 +53,7 @@ Plus session meta-ops not in the tool registry: snapshot, undo, inspect, delete. - Different category from Runway/Pika (generative AI), Premiere/DaVinci (heavy NLEs) — but it exports its cut to Final Cut Pro, Resolve and Premiere for finishing - Overlaps with auto-editor (open-source CLI: silence cuts, transcript edits, NLE export); MakeMyClip Editor adds a shared timeline document that an agent and a person (browser UI) both edit, with undo per step - Transcription is local (whisper.cpp); nothing is uploaded -- Single workspace folder holds composition.json (the timeline), session.json (the op log), output files, snapshots/ — fully version-controllable +- Single workspace folder holds composition.json (the main timeline), timelines/ (shorts and other named timelines), session.json (the op log), output files, snapshots/ — fully version-controllable - Stream-copy for lossless trim/split/concat; re-encodes only when filter graphs require it ## Architecture diff --git a/mcpb/manifest.json b/mcpb/manifest.json index 2841441..dc6e9c5 100644 --- a/mcpb/manifest.json +++ b/mcpb/manifest.json @@ -40,7 +40,7 @@ }, { "name": "timeline_edit", - "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, transition." + "description": "Apply editing verbs as one undoable change: add media, text or color, trim, move, split, remove, remove silence, cut words or retakes, caption speech, reframe to 9:16 / 1:1 / 4:5, frame a clip, set loudness, make a vertical short from transcript words, transition." }, { "name": "timeline_check", diff --git a/src/cli-timeline.ts b/src/cli-timeline.ts index e623c1e..41485d2 100644 --- a/src/cli-timeline.ts +++ b/src/cli-timeline.ts @@ -1,5 +1,11 @@ import { appendOp } from './session/store.js'; -import { CaptionError, DEFAULT_CAPTION_WORDS, planCaptions } from './timeline/captions.js'; +import { + CaptionError, + captionSideMargin, + DEFAULT_CAPTION_WORDS, + planCaptions, +} from './timeline/captions.js'; +import { invalidColorMessage, isColor } from './timeline/color.js'; import { buildFrameAtPlan, checkExportable, @@ -9,6 +15,7 @@ import { import { AnchorSchema, type Clip, + type Composition, clipDuration, clipEndSec, clipsAtTime, @@ -24,11 +31,15 @@ import { type TextStyle, } from './timeline/composition.js'; import { + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, undoLastDocOp, } from './timeline/document-store.js'; import { @@ -39,6 +50,7 @@ import { } from './timeline/interchange.js'; import { buildMediaMap, buildMediaRefs } from './timeline/media-registry.js'; import { colorClip, mediaClip, textClip } from './timeline/ops.js'; +import { measureSpeechLevels } from './timeline/pauses.js'; import { AspectSchema, type FramingFit, @@ -48,13 +60,16 @@ import { planReframe, } from './timeline/reframe.js'; import { reviewCuts } from './timeline/review.js'; -import { runPlan } from './timeline/run-plan.js'; +import { exportTimelines, runPlan } from './timeline/run-plan.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './timeline/schema.js'; +import { checkShortTimelines, planShort, type ShortRange } from './timeline/shorts.js'; import { analyzeSilence, DEFAULT_SENSITIVITY, SensitivitySchema } from './timeline/silence.js'; import { clipWords, findRetakes, planWordCut, storedTranscriptForClip, + timelineOutline, transcribeClip, } from './timeline/transcript.js'; import { DEFAULT_VERB_TRACK as DEFAULT_TRACK, ensureTrack, trackEnd } from './timeline/verbs.js'; @@ -63,6 +78,9 @@ import { getWorkspace, newOutputPath, resolveInput } from './workspace.js'; const TIMELINE_HELP = `clip timeline — build and export a non-destructive composition +Every command works on the main timeline unless --timeline names another +(a short made with \`short\` is a timeline of its own). + clip timeline new [--width W --height H --fps F --background C --fit contain|cover] clip timeline add-media [ ] [--start S] [--track T] clip timeline add-text [--start S] [--track T] [--font N --color C --anchor A --background C] @@ -74,13 +92,16 @@ const TIMELINE_HELP = `clip timeline — build and export a non-destructive comp clip timeline remove clip timeline silence [--sensitivity 0..1] [--dry-run] clip timeline check [--sheet ] - clip timeline transcribe [--model ] [--language ] + clip timeline transcribe [] [--sentences] [--from S] [--to S] [--model ] [--language ] clip timeline cut-words [--transcript ] clip timeline remove-retakes [--dry-run] [--transcript ] - clip timeline captions [] [--words N] [--track T] [--font N --color C --anchor A --background C|none] [--dry-run] + clip timeline captions [] [--words N] [--track T] [--font N --color C --anchor A --background C|none --margin PX] [--dry-run] clip timeline reframe <9:16|1:1|4:5|16:9|WxH> [--fit cover|contain] [--x 0..1] [--y 0..1] [--dry-run] clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset] clip timeline loudness <-14|-16|LUFS|off> + clip timeline short [ …] [--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] [--loudness LUFS|off] [--replace] [--dry-run] + clip timeline list + clip timeline delete clip timeline show clip timeline at clip timeline frame [] @@ -88,6 +109,7 @@ const TIMELINE_HELP = `clip timeline — build and export a non-destructive comp clip timeline redo clip timeline log clip timeline export [] [--format mp4|fcpxml|xmeml] + clip timeline export --all (every short, as /.mp4) `; interface Parsed { @@ -96,7 +118,7 @@ interface Parsed { } /** Flags that take no value, so they don't swallow the next argument. */ -const BOOLEAN_FLAGS = new Set(['dry-run', 'reset']); +const BOOLEAN_FLAGS = new Set(['all', 'dry-run', 'reset', 'no-captions', 'replace', 'sentences']); function parseArgs(args: string[]): Parsed { const positional: string[] = []; @@ -134,6 +156,11 @@ function optionalNum(name: string, value: string | undefined): number | undefine return n; } +function colorFlag(name: string, value: string | undefined): string | undefined { + if (value === undefined || isColor(value)) return value; + throw new Error(`--${name} ${invalidColorMessage(value)}`); +} + function fitFlag(value: string | undefined): Fit | undefined { if (value === undefined) return undefined; const parsed = FitSchema.safeParse(value); @@ -148,6 +175,45 @@ function framingFitFlag(value: string | undefined): FramingFit | undefined { return parsed.data; } +function timelineId(value: string): string { + const parsed = TimelineIdSchema.safeParse(value); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${value}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; +} + +/** Timeline time as m:ss, or h:mm:ss from an hour. */ +function clock(sec: number): string { + const whole = Math.floor(sec); + const s = String(whole % 60).padStart(2, '0'); + const m = Math.floor(whole / 60) % 60; + const h = Math.floor(whole / 3600); + return h > 0 ? `${h}:${String(m).padStart(2, '0')}:${s}` : `${m}:${s}`; +} + +function wordIndex(value: string | undefined): number { + const n = Number(value); + if (value === undefined || value.trim() === '' || !Number.isInteger(n) || n < 0) { + throw new Error(`A word index is a whole number from 0, got "${value}".`); + } + return n; +} + +/** `off`, or a loudness target within the export's range. */ +function loudnessValue(value: string, usage: string): number | null { + if (value === 'off') return null; + const parsed = TargetLufsSchema.safeParse(Number(value)); + if (!parsed.success) { + throw new Error( + `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, + ); + } + return parsed.data; +} + /** A short, human-readable label for a clip (for `show` / `at`). */ function clipLabel(clip: Clip): string { switch (clip.kind) { @@ -161,8 +227,10 @@ function clipLabel(clip: Clip): string { } export async function runTimeline(args: string[]): Promise { - const [sub, ...rest] = args; - const { positional, flags } = parseArgs(rest); + const parsed = parseArgs(args); + const [sub, ...positional] = parsed.positional; + const { flags } = parsed; + const timeline = flags.timeline !== undefined ? timelineId(flags.timeline) : MAIN_TIMELINE; switch (sub) { case undefined: @@ -177,11 +245,19 @@ export async function runTimeline(args: string[]): Promise { width: flags.width ? num(flags.width, 1920) : undefined, height: flags.height ? num(flags.height, 1080) : undefined, fps: flags.fps ? num(flags.fps, 30) : undefined, - background: flags.background, + background: colorFlag('background', flags.background), fit: fitFlag(flags.fit), }), + timeline, ); - out({ created: true, width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }); + out({ + created: true, + timeline, + width: comp.width, + height: comp.height, + fps: comp.fps, + fit: comp.fit, + }); return; } @@ -197,7 +273,7 @@ export async function runTimeline(args: string[]): Promise { }); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = mediaClip({ id: makeClipId(), @@ -206,7 +282,9 @@ export async function runTimeline(args: string[]): Promise { sourceOutSec: num(outSec, result.ref.durationSec), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'media', clipId: clip.id, mediaId: result.mediaId, trackId, startSec: start }); return; } @@ -217,7 +295,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error('Usage: clip timeline add-text '); } const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const style: Record = {}; if (flags.font) style.fontSize = num(flags.font, 48); @@ -231,7 +309,9 @@ export async function runTimeline(args: string[]): Promise { startSec: start, style, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'text', clipId: clip.id, trackId, startSec: start }); return; } @@ -240,15 +320,17 @@ export async function runTimeline(args: string[]): Promise { const [duration] = positional; if (duration === undefined) throw new Error('Usage: clip timeline add-color '); const trackId = flags.track || DEFAULT_TRACK; - const comp = await readComposition(); + const comp = await readComposition(timeline); const start = flags.start !== undefined ? num(flags.start, 0) : trackEnd(comp, trackId); const clip = colorClip({ id: makeClipId(), - color: flags.color, + color: colorFlag('color', flags.color), durationSec: num(duration, 2), startSec: start, }); - await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }]); + await mutateComposition([...ensureTrack(comp, trackId), { op: 'addClip', trackId, clip }], { + timeline, + }); out({ added: 'color', clipId: clip.id, trackId, startSec: start }); return; } @@ -258,17 +340,20 @@ export async function runTimeline(args: string[]): Promise { if (!afterClipId) throw new Error('Usage: clip timeline transition [] []'); const trackId = flags.track || DEFAULT_TRACK; - await mutateComposition([ - { - op: 'addTransition', - trackId, - transition: { - afterClipId, - kind: (kind as never) ?? 'fade', - durationSec: num(duration, 1), + await mutateComposition( + [ + { + op: 'addTransition', + trackId, + transition: { + afterClipId, + kind: (kind as never) ?? 'fade', + durationSec: num(duration, 1), + }, }, - }, - ]); + ], + { timeline }, + ); out({ added: 'transition', afterClipId, kind: kind ?? 'fade', trackId }); return; } @@ -278,9 +363,10 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || inSec === undefined || outSec === undefined) { throw new Error('Usage: clip timeline trim '); } - await mutateComposition([ - { op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }, - ]); + await mutateComposition( + [{ op: 'setTrim', clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }], + { timeline }, + ); out({ trimmed: clipId, sourceInSec: num(inSec, 0), sourceOutSec: num(outSec, 0) }); return; } @@ -289,9 +375,17 @@ export async function runTimeline(args: string[]): Promise { const [clipId, startSec] = positional; if (!clipId || startSec === undefined) throw new Error('Usage: clip timeline move '); - await mutateComposition([ - { op: 'moveClip', clipId, startSec: num(startSec, 0), toTrackId: flags.track || undefined }, - ]); + await mutateComposition( + [ + { + op: 'moveClip', + clipId, + startSec: num(startSec, 0), + toTrackId: flags.track || undefined, + }, + ], + { timeline }, + ); out({ moved: clipId, startSec: num(startSec, 0), trackId: flags.track || undefined }); return; } @@ -301,7 +395,9 @@ export async function runTimeline(args: string[]): Promise { if (!clipId || atSec === undefined) throw new Error('Usage: clip timeline split '); const newClipId = makeClipId(); - await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }]); + await mutateComposition([{ op: 'splitClip', clipId, atSec: num(atSec, 0), newClipId }], { + timeline, + }); out({ split: clipId, into: [clipId, newClipId], atSec: num(atSec, 0) }); return; } @@ -309,7 +405,7 @@ export async function runTimeline(args: string[]): Promise { case 'remove': { const [clipId] = positional; if (!clipId) throw new Error('Usage: clip timeline remove '); - await mutateComposition([{ op: 'removeClip', clipId }]); + await mutateComposition([{ op: 'removeClip', clipId }], { timeline }); out({ removed: clipId }); return; } @@ -324,7 +420,7 @@ export async function runTimeline(args: string[]): Promise { throw new Error(`--sensitivity must be between 0 and 1, got "${flags.sensitivity}".`); } const sensitivity = parsed.data; - const comp = await readComposition(); + const comp = await readComposition(timeline); const cut = await analyzeSilence(comp, clipId, { media: await buildMediaMap(), sensitivity }); const report = { clipId, @@ -342,7 +438,7 @@ export async function runTimeline(args: string[]): Promise { return; } // Analysis read `comp` outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([cut.op], { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, clipIds: [clipId, ...cut.op.newClipIds], rev: doc.rev }); return; } @@ -359,7 +455,7 @@ export async function runTimeline(args: string[]): Promise { if (!aspect.success && !size) { throw new Error(`Unknown shape "${shape}": use 9:16, 1:1, 4:5, 16:9 or WIDTHxHEIGHT.`); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const plan = planReframe(comp, { ...(aspect.success ? { aspect: aspect.data } @@ -373,7 +469,7 @@ export async function runTimeline(args: string[]): Promise { out({ ...report, applied: false, ...(flags['dry-run'] ? { dryRun: true } : {}) }); return; } - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } @@ -385,7 +481,7 @@ export async function runTimeline(args: string[]): Promise { 'Usage: clip timeline framing [--fit cover|contain|inherit] [--scale 0.1..4] [--x 0..1] [--y 0..1] [--reset]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const ops = planFraming(comp, { clipId, fit: framingFitFlag(flags.fit), @@ -398,7 +494,7 @@ export async function runTimeline(args: string[]): Promise { out({ clipId, applied: false, message: 'Nothing to change.' }); return; } - const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(ops, { expectedBaseRev: comp.rev, timeline }); const framed = findClip(doc, clipId); out({ clipId, @@ -413,30 +509,130 @@ export async function runTimeline(args: string[]): Promise { const [value] = positional; const usage = 'Usage: clip timeline loudness '; if (!value) throw new Error(usage); - let targetLufs: number | null = null; - if (value !== 'off') { - const parsed = TargetLufsSchema.safeParse(Number(value)); - if (!parsed.success) { - throw new Error( - `Loudness must be off or a LUFS value from ${LOUDNESS_MIN_LUFS} to ${LOUDNESS_MAX_LUFS}, got "${value}". ${usage}`, - ); - } - targetLufs = parsed.data; - } - const comp = await readComposition(); + const targetLufs = loudnessValue(value, usage); + const comp = await readComposition(timeline); if ((comp.loudness?.targetLufs ?? null) === targetLufs) { out({ loudness: comp.loudness, applied: false }); return; } const doc = await mutateComposition([{ op: 'setLoudness', targetLufs }], { expectedBaseRev: comp.rev, + timeline, }); out({ loudness: doc.loudness, applied: true, rev: doc.rev }); return; } + case 'short': { + const [id, ...rangeArgs] = positional; + const usage = + 'Usage: clip timeline short [ …] ' + + '[--source T] [--aspect 9:16|1:1|4:5|16:9] [--x 0..1] [--y 0..1] [--words N] [--no-captions] ' + + '[--loudness LUFS|off] [--replace] [--dry-run]'; + if (!id || rangeArgs.length === 0 || rangeArgs.length % 3 !== 0) throw new Error(usage); + const target = timelineId(id); + if (flags.timeline !== undefined && timeline !== target) { + throw new Error( + `\`short\` builds the timeline named by its first argument ("${target}"), not --timeline. Use --source for the timeline to cut from.`, + ); + } + const sourceId = flags.source !== undefined ? timelineId(flags.source) : MAIN_TIMELINE; + checkShortTimelines(target, sourceId); + const ranges: ShortRange[] = []; + for (let i = 0; i < rangeArgs.length; i += 3) { + ranges.push({ + clipId: rangeArgs[i] ?? '', + fromIndex: wordIndex(rangeArgs[i + 1]), + toIndex: wordIndex(rangeArgs[i + 2]), + }); + } + const aspect = flags.aspect === undefined ? undefined : AspectSchema.safeParse(flags.aspect); + if (aspect && !aspect.success) { + throw new Error(`--aspect must be 9:16, 1:1, 4:5 or 16:9, got "${flags.aspect}".`); + } + const maxWords = num(flags.words, DEFAULT_CAPTION_WORDS); + if (!Number.isInteger(maxWords) || maxWords < 1 || maxWords > 12) { + throw new CaptionError( + `--words must be a whole number from 1 to 12, got "${flags.words}".`, + ); + } + + let current: Composition; + try { + current = await readComposition(target); + } catch (err) { + if (!(err instanceof NoTimelineError)) throw err; + current = emptyComposition(); + } + const media = await buildMediaMap(); + const plan = await planShort( + current, + await readComposition(sourceId), + { + ranges, + aspect: aspect?.data, + x: optionalNum('x', flags.x), + y: optionalNum('y', flags.y), + captions: flags['no-captions'] !== 'true', + maxWords, + targetLufs: + flags.loudness === undefined ? undefined : loudnessValue(flags.loudness, usage), + replace: flags.replace === 'true', + }, + { + transcript: (comp, clipId, transcriptId) => + storedTranscriptForClip(comp, clipId, media, transcriptId), + levels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + const info = clip?.kind === 'media' ? media.get(clip.mediaId) : undefined; + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, + }, + ); + const report = { + timeline: target, + source: sourceId, + durationSec: plan.durationSec, + pieces: plan.pieces, + captions: plan.captions, + warnings: plan.warnings, + }; + if (flags['dry-run']) { + out({ ...report, applied: false, dryRun: true }); + return; + } + // The source and transcripts were read outside the write lock; reject if + // the short's timeline moved since. + const doc = await mutateComposition(plan.ops, { + expectedBaseRev: current.rev, + timeline: target, + create: true, + }); + out({ + ...report, + applied: true, + canvas: { width: doc.width, height: doc.height, fit: doc.fit }, + loudness: doc.loudness, + rev: doc.rev, + }); + return; + } + + case 'list': + out({ timelines: await listTimelines() }); + return; + + case 'delete': { + const [id] = positional; + if (!id) throw new Error('Usage: clip timeline delete '); + const target = timelineId(id); + await deleteTimeline(target); + out({ deleted: target }); + return; + } + case 'check': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const sheetPath = flags.sheet ? resolveInput(flags.sheet) : newOutputPath('cut-sheet', 'jpg'); out(await reviewCuts(comp, media, sheetPath, getWorkspace())); @@ -445,18 +641,38 @@ export async function runTimeline(args: string[]): Promise { case 'transcribe': { const [clipId] = positional; - if (!clipId) { - throw new Error( - 'Usage: clip timeline transcribe [--model ] [--language ]', - ); + const comp = await readComposition(timeline); + const opts = { + model: flags.model ? resolveInput(flags.model) : undefined, + language: flags.language || undefined, + }; + if (clipId && flags.sentences !== 'true') { + out(await transcribeClip(comp, clipId, await buildMediaMap(), opts)); + return; } - const comp = await readComposition(); - out( - await transcribeClip(comp, clipId, await buildMediaMap(), { - model: flags.model ? resolveInput(flags.model) : undefined, - language: flags.language || undefined, - }), + // One line per sentence, cheap to read and to `head`: no page limit here. + const outline = await timelineOutline(comp, await buildMediaMap(), { + ...opts, + clipId, + fromSec: optionalNum('from', flags.from), + toSec: optionalNum('to', flags.to), + maxBytes: Number.POSITIVE_INFINITY, + }); + const lines = Object.entries(outline.transcriptIds).map( + ([mediaId, id]) => `transcript ${mediaId}: ${id}`, ); + for (const s of outline.sentences) { + // With more than one source, its media id, to match a transcript line. + const source = s.mediaId ? ` ${s.mediaId}` : ''; + lines.push( + `${clock(s.startSec)} #${s.fromIndex}–${s.toIndex} ${s.clipId}${source} ${s.text}`, + ); + } + // `| head` closes the pipe early: that ends the listing, not with an error. + process.stdout.on('error', (err: NodeJS.ErrnoException) => { + if (err.code !== 'EPIPE') throw err; + }); + process.stdout.write(lines.length > 0 ? `${lines.join('\n')}\n` : ''); return; } @@ -471,7 +687,7 @@ export async function runTimeline(args: string[]): Promise { : 'Usage: clip timeline remove-retakes [--dry-run]', ); } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const { clip, transcript } = await storedTranscriptForClip( comp, @@ -492,7 +708,7 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcript was read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition([op], { expectedBaseRev: comp.rev }); + const doc = await mutateComposition([op], { expectedBaseRev: comp.rev, timeline }); out({ clipId, applied: true, ranges, rev: doc.rev }); return; } @@ -511,7 +727,16 @@ export async function runTimeline(args: string[]): Promise { if (flags.background) style.background = flags.background === 'none' ? null : flags.background; if (flags.anchor) style.anchor = AnchorSchema.parse(flags.anchor); - const comp = await readComposition(); + const comp = await readComposition(timeline); + const margin = optionalNum('margin', flags.margin); + if (margin !== undefined) { + if (!Number.isInteger(margin) || margin < 0) { + throw new CaptionError( + `--margin must be a whole number of pixels, got "${flags.margin}".`, + ); + } + style.margin = { x: captionSideMargin(comp), y: margin }; + } const media = await buildMediaMap(); const plan = await planCaptions( comp, @@ -530,15 +755,16 @@ export async function runTimeline(args: string[]): Promise { return; } // The transcripts were read outside the write lock; reject if the doc moved since. - const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev }); + const doc = await mutateComposition(plan.ops, { expectedBaseRev: comp.rev, timeline }); out({ ...report, applied: true, rev: doc.rev }); return; } case 'show': { - const comp = await readComposition(); + const comp = await readComposition(timeline); const { exportable, blockers } = checkExportable(comp, await buildMediaMap(), getWorkspace()); out({ + timeline, rev: comp.rev, durationSec: compositionDuration(comp), canvas: { width: comp.width, height: comp.height, fps: comp.fps, fit: comp.fit }, @@ -575,7 +801,7 @@ export async function runTimeline(args: string[]): Promise { const [at] = positional; if (!at) throw new Error('Usage: clip timeline at '); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); out({ atSec, clips: clipsAtTime(comp, atSec).map((h) => ({ @@ -593,7 +819,7 @@ export async function runTimeline(args: string[]): Promise { const [at, output] = positional; if (!at) throw new Error('Usage: clip timeline frame []'); const atSec = num(at, 0); - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-frame', 'jpg'); const plan = buildFrameAtPlan( @@ -607,7 +833,7 @@ export async function runTimeline(args: string[]): Promise { } case 'undo': { - const { undone, doc, label } = await undoLastDocOp(); + const { undone, doc, label } = await undoLastDocOp(timeline); out( undone ? { undone: true, label, rev: doc.rev } @@ -617,7 +843,7 @@ export async function runTimeline(args: string[]): Promise { } case 'redo': { - const { redone, doc, label } = await redoDocOp(); + const { redone, doc, label } = await redoDocOp(timeline); out( redone ? { redone: true, label, rev: doc.rev } @@ -627,7 +853,7 @@ export async function runTimeline(args: string[]): Promise { } case 'log': { - const log = await readDocOpLog(); + const log = await readDocOpLog(timeline); out({ rev: log.rev, cursor: log.cursor, @@ -643,6 +869,25 @@ export async function runTimeline(args: string[]): Promise { } case 'export': { + if (flags.all === 'true') { + const [dir] = positional; + if (!dir) throw new Error('Usage: clip timeline export --all '); + if (flags.timeline !== undefined || (flags.format && flags.format !== 'mp4')) { + throw new Error( + '--all renders every short as mp4. Export one timeline with --timeline , or a project with --format.', + ); + } + const shorts = (await listTimelines()) + .map((t) => t.id) + .filter((id) => id !== MAIN_TIMELINE); + if (shorts.length === 0) { + throw new Error('No shorts to export. Make one with `clip timeline short`.'); + } + const exported = await exportTimelines(shorts, resolveInput(dir)); + out({ exported }); + if (exported.some((e) => 'error' in e)) process.exitCode = 1; + return; + } const [output] = positional; const format = flags.format || (output && interchangeFormatFor(output)) || 'mp4'; if (format !== 'mp4') { @@ -653,10 +898,14 @@ export async function runTimeline(args: string[]): Promise { const target = output ? resolveInput(output) : newOutputPath('timeline-export', interchangeExtension(parsed.data)); - const result = await exportInterchange(await readComposition(), await buildMediaRefs(), { - format: parsed.data, - output: target, - }); + const result = await exportInterchange( + await readComposition(timeline), + await buildMediaRefs(), + { + format: parsed.data, + output: target, + }, + ); out({ exported: target, format: result.format, @@ -666,7 +915,7 @@ export async function runTimeline(args: string[]): Promise { }); return; } - const comp = await readComposition(); + const comp = await readComposition(timeline); const media = await buildMediaMap(); const finalOutput = output ? resolveInput(output) : newOutputPath('timeline-export', 'mp4'); const plan = compileTimeline(comp, { media, dir: getWorkspace(), output: finalOutput }); diff --git a/src/ffmpeg/args/speech-levels.ts b/src/ffmpeg/args/speech-levels.ts new file mode 100644 index 0000000..cafb438 --- /dev/null +++ b/src/ffmpeg/args/speech-levels.ts @@ -0,0 +1,44 @@ +export interface SpeechLevelsArgs { + input: string; + /** Where to start measuring, in source seconds. */ + fromSec: number; + durationSec: number; +} + +/** Samples per measured frame at 16 kHz: 10 ms. */ +export const LEVEL_FRAME_SAMPLES = 160; +export const LEVEL_SAMPLE_RATE = 16000; + +/** + * The RMS level of every 10 ms of a source's first audio stream over a window, + * printed to stderr as `pts_time:…` / `lavfi.astats.Overall.RMS_level=…` lines. + * Mono at 16 kHz is plenty to tell speech from a pause, and the window is a few + * seconds, so it runs in well under a tenth of a second. + */ +export function buildSpeechLevelsArgs({ input, fromSec, durationSec }: SpeechLevelsArgs): string[] { + return [ + '-hide_banner', + '-nostats', + '-ss', + String(fromSec), + '-t', + String(durationSec), + '-i', + input, + '-map', + '0:a:0', + '-af', + [ + // A late start or a timestamp gap becomes silence, as for transcription, + // so frame n is always n x 10 ms after the seek point. + `aresample=${LEVEL_SAMPLE_RATE}:async=1:first_pts=0`, + 'aformat=channel_layouts=mono', + `asetnsamples=n=${LEVEL_FRAME_SAMPLES}:p=0`, + 'astats=metadata=1:reset=1', + 'ametadata=mode=print:key=lavfi.astats.Overall.RMS_level', + ].join(','), + '-f', + 'null', + '-', + ]; +} diff --git a/src/index.ts b/src/index.ts index 76eb806..d8e1a3f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -165,12 +165,17 @@ export { CompositionCorruptError, compositionOpsPath, compositionPath, + deleteTimeline, + listTimelines, mutateComposition, + NoTimelineError, overwriteComposition, readComposition, readDocOpLog, redoDocOp, resetComposition, + TimelineIdError, + type TimelineListing, undoLastDocOp, writeComposition, writeCompositionIfUnchanged, @@ -194,6 +199,17 @@ export { textClip, videoTrack, } from './timeline/ops.js'; +export { + endInPause, + findPauses, + measureSpeechLevels, + PAUSE_KEEP_SEC, + type Pause, + parseSpeechLevels, + pauseAt, + type SpeechLevels, + startInPause, +} from './timeline/pauses.js'; export { type Aspect, AspectSchema, @@ -222,9 +238,16 @@ export { renderCutSheet, reviewCuts, } from './timeline/review.js'; -export { type LoudnessResult, type RunPlanResult, runPlan } from './timeline/run-plan.js'; +export { + exportTimelines, + type LoudnessResult, + type RunPlanResult, + runPlan, + type TimelineExport, +} from './timeline/run-plan.js'; export { AudioStreamSchema, + MAIN_TIMELINE, type MediaId, MediaIdSchema, type MediaRef, @@ -232,8 +255,23 @@ export { makeMediaId, type Timecode, TimecodeSchema, + TimelineIdSchema, VideoStreamSchema, } from './timeline/schema.js'; +export { + checkShortTimelines, + DEFAULT_SHORT_LUFS, + planShort, + SHORT_LEAD_SEC, + ShortError, + type ShortPiece, + type ShortPlan, + type ShortRange, + type ShortRequest, + type ShortSources, + type ShortTranscriptLookup, + type SpeechLevelsLookup, +} from './timeline/shorts.js'; export { analyzeSilence, DEFAULT_SENSITIVITY, diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 7416f5d..a48c7d4 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -15,6 +15,8 @@ import { import type { Composition } from '../timeline/composition.js'; import { applyVerbs, + listTimelines, + NoTimelineError, readComposition, redoDocOp, undoLastDocOp, @@ -28,7 +30,9 @@ import { import { buildMediaMap } from '../timeline/media-registry.js'; import { reviewCuts } from '../timeline/review.js'; import { runPlan } from '../timeline/run-plan.js'; -import { transcribeClip } from '../timeline/transcript.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; +import { timelineOutline, transcribeClip } from '../timeline/transcript.js'; import { EditableVerbSchema } from '../timeline/verbs.js'; import { doctor } from '../tools/doctor.js'; import { WhisperLanguageSchema } from '../transcribe/whisper.js'; @@ -98,12 +102,30 @@ async function jpegContent( */ async function describeComposition( comp: Composition, -): Promise { + timeline: string, +): Promise { const media = await buildMediaMap(); const { exportable, blockers } = checkExportable(comp, media, getWorkspace()); - return { ...summarizeComposition(comp), exportable, blockers }; + return { timeline, ...summarizeComposition(comp), exportable, blockers }; } +/** Answer a missing timeline or a short that can't be made with `{ error }`: + * both are the agent's to fix, like an export it can't render. */ +async function answeringTimelineErrors( + run: () => Promise, +): Promise> { + try { + return await run(); + } catch (err) { + if (err instanceof NoTimelineError || err instanceof ShortError) { + return jsonResult({ error: err.message }); + } + throw err; + } +} + +const timelineInput = TimelineIdSchema.optional(); + /** * Build the MakeMyClip Editor MCP server — a second front door onto the SAME * op-aware, undoable CompositionDoc the CLI, the `clip ui`, and the Claude Code @@ -119,8 +141,9 @@ export function buildMcpServer(): McpServer { { title: 'Show the timeline', description: - 'Read the current timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', + 'Read a timeline document — tracks, clips, timings, and whether it can render (exportable + any blockers) — plus `timelines`, the ids of every timeline in the workspace (main and each short). Pass atSec to also get the rendered frame at that time as an image. Call this to ground yourself BEFORE editing and to VERIFY a change AFTER.', inputSchema: { + timeline: timelineInput, atSec: z .number() .nonnegative() @@ -130,35 +153,37 @@ export function buildMcpServer(): McpServer { ), }, }, - async ({ atSec }) => { - const comp = await readComposition(); - const summary = jsonResult(await describeComposition(comp)); - if (atSec === undefined) return summary; - const output = newOutputPath('mcp-frame', 'jpg'); - try { - const plan = buildFrameAtPlan( - comp, - { - media: await confinedMediaMap(comp), - dir: getWorkspace(), - output, - tag: `mcp${randomBytes(4).toString('hex')}`, - }, - atSec, - ); - await runPlan(plan); - return { content: [...summary.content, await jpegContent(output)] }; - } catch (err) { - if (!(err instanceof CompileError)) throw err; - const frameError = { - type: 'text' as const, - text: JSON.stringify({ frameError: err.message }), - }; - return { content: [...summary.content, frameError] }; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE, atSec }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const timelines = (await listTimelines()).map((t) => t.id); + const summary = jsonResult({ ...(await describeComposition(comp, timeline)), timelines }); + if (atSec === undefined) return summary; + const output = newOutputPath('mcp-frame', 'jpg'); + try { + const plan = buildFrameAtPlan( + comp, + { + media: await confinedMediaMap(comp), + dir: getWorkspace(), + output, + tag: `mcp${randomBytes(4).toString('hex')}`, + }, + atSec, + ); + await runPlan(plan); + return { content: [...summary.content, await jpegContent(output)] }; + } catch (err) { + if (!(err instanceof CompileError)) throw err; + const frameError = { + type: 'text' as const, + text: JSON.stringify({ frameError: err.message }), + }; + return { content: [...summary.content, frameError] }; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -167,22 +192,24 @@ export function buildMcpServer(): McpServer { title: 'Check the cuts', description: "Review every hard cut (abutting clips with no transition). Reports cuts whose audio is loud right at the edge, which usually means a clipped word or a click, and returns a contact sheet image: one row per cut, the outgoing clip's last frame then the incoming clip's first. Run it after cutting (e.g. remove_silence) and before exporting.", + inputSchema: { timeline: timelineInput }, }, - async () => { - const comp = await readComposition(); - const media = await confinedMediaMap(comp); - const output = newOutputPath('cut-sheet', 'jpg'); - try { - const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); - const summary = jsonResult({ - ...review, - sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, - }); - return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; - } finally { - await unlink(output).catch(() => undefined); - } - }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const media = await confinedMediaMap(comp); + const output = newOutputPath('cut-sheet', 'jpg'); + try { + const { sheet, ...review } = await reviewCuts(comp, media, output, getWorkspace()); + const summary = jsonResult({ + ...review, + sheet: sheet ? { cuts: sheet.cuts, truncated: sheet.truncated } : null, + }); + return sheet ? { content: [...summary.content, await jpegContent(sheet.path)] } : summary; + } finally { + await unlink(output).catch(() => undefined); + } + }), ); server.registerTool( @@ -190,50 +217,98 @@ export function buildMcpServer(): McpServer { { title: 'Edit the timeline', description: - 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', - inputSchema: { verbs: z.array(EditableVerbSchema).min(1) }, - }, - async ({ verbs }) => { - const { doc, ops } = await applyVerbs(verbs, makeVerbContext()); - return jsonResult({ applied: ops.length, document: await describeComposition(doc) }); + 'Apply one or more editing verbs to the document as ONE undoable change. Verbs: add_media, add_text, add_color, trim, move, split, remove, remove_silence, cut_words, remove_retakes, caption_speech, reframe, set_framing, set_loudness, transition. remove_silence cuts the pauses out of a media clip (sensitivity 0–1, default 0.5) and moves later clips on its track up; it applies nothing when no pause is found. cut_words cuts words fromIndex..toIndex out of a clip, sending the transcriptId from the timeline_transcript call they came from; remove_retakes cuts the earlier take of each phrase said twice in a row. caption_speech turns stored transcripts into captions on a track above the video (short phrases, or one word at a time with maxWords 1), replacing captions already there; export burns them in. Captions do not move with later cuts, so caption after cutting, or run it again after a cut. On a portrait canvas they sit above the bottom quarter and stay narrow enough to keep clear of the Shorts, Reels and TikTok overlays (margin sets the distance from the edge), and their width follows the font size. reframe changes the output shape (aspect 9:16 for Shorts/Reels/TikTok, 1:1, 4:5, 16:9) and fills it (fit cover, the default, crops; contain shows bars); x/y (0 = left/top, 0.5 = center, 1 = right/bottom) pick the part of every current media clip that shows. set_framing sets the fit, zoom (scale 0.1–4) and position of one media clip. Check the result with timeline_show atSec. set_loudness sets the integrated loudness the mp4 export normalizes to (targetLufs -14 for YouTube/Shorts/Reels/TikTok, -16 for podcasts, null for off). ' + + "make_short turns a long recording into a vertical short on a timeline of its own: send it with `timeline` set to a new id such as \"best-1\" (never main). It plays the given word ranges (read the source timeline's sentences with timeline_transcript and no clipId; the source defaults to main) with a little padding, cut from the source's clips so its silence and retake cuts carry in, on a 9:16 canvas filled edge to edge (x picks the part of the frame that shows), normalized to -14 LUFS, with captions. Pick whole sentences (15–60 s in all is typical); the result's warnings name a range that starts or ends mid-sentence. Look at a frame with timeline_show (same timeline, atSec), then adjust with reframe x. Undo on that timeline removes it; replace rebuilds it. Every other verb also takes `timeline`, so a short can be edited like main. " + + 'Batch related verbs into a single call. Media paths must be inside the workspace. Returns the updated document summary, including whether it can now render.', + inputSchema: { timeline: timelineInput, verbs: z.array(EditableVerbSchema).min(1) }, }, + async ({ timeline = MAIN_TIMELINE, verbs }) => + answeringTimelineErrors(async () => { + const { doc, ops, warnings } = await applyVerbs(verbs, makeVerbContext(), timeline); + return jsonResult({ + applied: ops.length, + ...(warnings.length > 0 ? { warnings } : {}), + document: await describeComposition(doc, timeline), + }); + }), ); server.registerTool( 'timeline_transcript', { - title: 'Transcribe a clip', + title: 'Transcribe the speech', description: - "Transcribe a media clip's speech with the local whisper.cpp and return its words with timeline times. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript. A word's `index` is stable for its source file.", + "Transcribe the timeline's speech with the local whisper.cpp. Without clipId it returns the timeline as sentences: each with its timeline start and end, the clip it starts in, its word indices fromIndex..toIndex and its text, one page of about 40 KB at a time. Pass nextFromSec as fromSec for the next page, or fromSec/toSec for a stretch of the timeline. Read it to pick a short's moments, then send whole sentences' indices to make_short. " + + "With clipId it returns that clip's words (view 'words'), each with its index and timeline time, plus phrases said twice in a row (retakes); view 'sentences' gives that clip's sentences instead. " + + "transcriptIds (by source; with more than one source each sentence names its mediaId) and transcriptId identify the transcript: send one back with an edit by index so it is refused, not misplaced, if the source was transcribed again. A word's `index` is stable for its source file. Optional feature: if whisper.cpp or a model isn't set up, the error says how. The first call per source file takes a while; later calls reuse the stored transcript.", inputSchema: { - clipId: z.string().describe('Media clip to transcribe.'), + timeline: timelineInput, + clipId: z + .string() + .optional() + .describe("Media clip to transcribe. Leave out for the whole timeline's sentences."), + view: z + .enum(['words', 'sentences']) + .optional() + .describe('words (default with a clipId) or sentences (default without one).'), + fromSec: z + .number() + .nonnegative() + .optional() + .describe("Sentences starting at or after this timeline time (a page's nextFromSec)."), + toSec: z + .number() + .positive() + .optional() + .describe('Sentences starting before this timeline time.'), language: WhisperLanguageSchema.optional(), }, }, - async ({ clipId, language }) => { - const comp = await readComposition(); - return jsonResult( - await transcribeClip(comp, clipId, await confinedMediaMap(comp), { language }), - ); - }, + async ({ timeline = MAIN_TIMELINE, clipId, view, fromSec, toSec, language }) => + answeringTimelineErrors(async () => { + const comp = await readComposition(timeline); + const media = await confinedMediaMap(comp); + if ((view ?? (clipId ? 'words' : 'sentences')) === 'words') { + if (!clipId) { + return jsonResult({ + error: "The words view needs a clipId. Leave view out for the timeline's sentences.", + }); + } + return jsonResult(await transcribeClip(comp, clipId, media, { language })); + } + const outline = await timelineOutline(comp, media, { clipId, fromSec, toSec, language }); + // Compact, as the page size is counted: a long recording's outline is + // what has to fit in one result. + return { content: [{ type: 'text' as const, text: JSON.stringify(outline) }] }; + }), ); server.registerTool( 'timeline_undo', - { title: 'Undo', description: 'Undo the most recent timeline edit.' }, - async () => { - const { undone, label } = await undoLastDocOp(); - return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + { + title: 'Undo', + description: 'Undo the most recent edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { undone, label } = await undoLastDocOp(timeline); + return jsonResult(undone ? { undone, label } : { undone, message: 'Nothing to undo.' }); + }), ); server.registerTool( 'timeline_redo', - { title: 'Redo', description: 'Redo the most recently undone timeline edit.' }, - async () => { - const { redone, label } = await redoDocOp(); - return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + { + title: 'Redo', + description: 'Redo the most recently undone edit of a timeline (default main).', + inputSchema: { timeline: timelineInput }, }, + async ({ timeline = MAIN_TIMELINE }) => + answeringTimelineErrors(async () => { + const { redone, label } = await redoDocOp(timeline); + return jsonResult(redone ? { redone, label } : { redone, message: 'Nothing to redo.' }); + }), ); server.registerTool( @@ -241,10 +316,11 @@ export function buildMcpServer(): McpServer { { title: 'Export the timeline', description: - 'Export the current timeline document to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + + 'Export a timeline (default main; pass `timeline` for a short) to a file in the workspace and return its path. format mp4 (default) renders the video, normalized to the loudness set with set_loudness (the result reports the measured loudness). ' + 'fcpxml writes an FCPXML project for Final Cut Pro and DaVinci Resolve; xmeml writes Final Cut Pro 7 XML for Premiere Pro (Resolve reads it too), with text as .srt caption files beside it. ' + 'Both reference the source media by absolute path, refuse what the editor file cannot hold with the fix to make, and return notes on what is carried only in part.', inputSchema: { + timeline: timelineInput, format: z .enum(['mp4', 'fcpxml', 'xmeml']) .optional() @@ -255,9 +331,9 @@ export function buildMcpServer(): McpServer { .describe('Output file inside the workspace (relative paths resolve against it).'), }, }, - async ({ format = 'mp4', output }) => { - const comp = await readComposition(); + async ({ timeline = MAIN_TIMELINE, format = 'mp4', output }) => { try { + const comp = await readComposition(timeline); if (format !== 'mp4') { const target = output ? resolveInWorkspace(output) @@ -290,7 +366,8 @@ export function buildMcpServer(): McpServer { if ( err instanceof CompileError || err instanceof InterchangeError || - err instanceof WorkspaceBoundaryError + err instanceof WorkspaceBoundaryError || + err instanceof NoTimelineError ) { return jsonResult({ error: err.message }); } diff --git a/src/timeline/captions.ts b/src/timeline/captions.ts index b16c45b..c74dc58 100644 --- a/src/timeline/captions.ts +++ b/src/timeline/captions.ts @@ -1,4 +1,4 @@ -import { videoLayers } from './compile.js'; +import { TEXT_EDGE_PX, videoLayers } from './compile.js'; import { type Anchor, type Clip, @@ -24,8 +24,14 @@ export const DEFAULT_CAPTION_WORDS = 7; /** Widest caption, in Latin characters (a CJK character counts as two). */ const MAX_CAPTION_WIDTH = 32; -/** A pause this long between words starts a new caption. */ -const PAUSE_SEC = 0.7; +/** The box drawtext draws behind a caption reaches this far past its text. */ +const CAPTION_BOX_PX = 12; +/** Share of a portrait canvas the vertical feeds' overlays cover: the caption, + * buttons and progress bar at the bottom, the action column at the sides. */ +const PORTRAIT_BOTTOM_SHARE = 0.25; +const PORTRAIT_SIDE_SHARE = 0.14; +/** A pause this long between words starts a new caption, or a new sentence. */ +export const PAUSE_SEC = 0.7; /** How long a caption stays up after its last word, unless the next one starts. */ const HOLD_SEC = 0.4; /** Shortest caption shown. */ @@ -108,8 +114,9 @@ export async function planCaptions( const removed = (track?.clips.length ?? 0) - kept.length; const style = { ...defaultCaptionStyle(comp), ...request.style }; + const maxWidth = captionWidthBudget(comp, style); const added: CompositionOp[] = sources.flatMap(({ clip, transcript }) => - groupCaptions(clipWords(transcript, clip), clipEndSec(clip), maxWords).map( + groupCaptions(clipWords(transcript, clip), clipEndSec(clip), maxWords, maxWidth).map( (cue): CompositionOp => ({ op: 'addClip', trackId, @@ -118,7 +125,7 @@ export async function planCaptions( text: cue.text, startSec: cue.startSec, durationSec: roundMs(cue.endSec - cue.startSec), - style, + style: fitCaptionStyle(cue.text, style, comp), }), }), ), @@ -196,23 +203,78 @@ async function everyTranscribedClip( return found; } -/** Captions sized to the canvas: white on a dark box, near the bottom. */ -export function defaultCaptionStyle(comp: Composition): TextStyle { +/** + * Where captions stay clear of what a platform draws over the video. On a + * portrait canvas (Shorts, Reels, TikTok) that is the bottom quarter and 14% + * at each side: none of them publishes pixel specs for organic posts and + * third-party estimates conflict, so one conservative box. Other shapes keep + * the usual 40 px edge. + */ +export function safeArea(comp: Composition): { side: number; bottom: number } { + if (comp.height <= comp.width) return { side: TEXT_EDGE_PX, bottom: TEXT_EDGE_PX }; return { + side: Math.round(PORTRAIT_SIDE_SHARE * comp.width), + bottom: Math.round(PORTRAIT_BOTTOM_SHARE * comp.height), + }; +} + +/** Captions sized to the canvas: white on a dark box, near the bottom, and on + * a portrait canvas inside its safe area. */ +export function defaultCaptionStyle(comp: Composition): TextStyle { + const style: TextStyle = { fontSize: Math.round(Math.min(comp.width, comp.height) / 20), color: 'white', background: 'black@0.6', anchor: 'bottom-center' satisfies Anchor, }; + if (comp.height <= comp.width) return style; + // The box drawtext draws behind the text reaches past it, so the text keeps + // that much further in for the box to stay inside the safe area too. + const { bottom } = safeArea(comp); + return { ...style, margin: { x: captionSideMargin(comp), y: bottom + CAPTION_BOX_PX } }; +} + +/** How far caption text keeps from the side edges: inside the safe area with + * its box on a portrait canvas, the usual 40 px otherwise. */ +export function captionSideMargin(comp: Composition): number { + return comp.height <= comp.width ? TEXT_EDGE_PX : safeArea(comp).side + CAPTION_BOX_PX; +} + +/** Pixels a caption line's text may take between the style's side margins. */ +function captionRoomPx(comp: Composition, style: TextStyle): number { + return Math.max(1, comp.width - 2 * (style.margin?.x ?? TEXT_EDGE_PX)); +} + +/** + * How wide a caption line may be, in em of its font: what fits between the + * style's side margins at its font size. drawtext doesn't wrap, so a wider + * line runs off the frame or into a platform's buttons. + */ +export function captionWidthBudget(comp: Composition, style: TextStyle): number { + return captionRoomPx(comp, style) / style.fontSize; +} + +/** The caption style a line is drawn with: a single word too wide for the + * room even on a line of its own gets a font size that fits it. */ +export function fitCaptionStyle(text: string, style: TextStyle, comp: Composition): TextStyle { + const width = textWidthEm(text); + const room = captionRoomPx(comp, style); + if (width * style.fontSize <= room) return style; + return { ...style, fontSize: Math.max(1, Math.floor(room / width)) }; } /** * Group a clip's words (timeline times) into captions: a new one starts after - * `maxWords` words, past the width limit, after a pause, or after a sentence - * ends. Each stays up a moment after its last word, but never into the next - * one or past `endSec`. + * `maxWords` words, past 32 characters or `maxWidthEm` em of estimated width, + * after a pause, or after a sentence ends. Each stays up a moment after its + * last word, but never into the next one or past `endSec`. */ -export function groupCaptions(words: ClipWord[], endSec: number, maxWords: number): CaptionCue[] { +export function groupCaptions( + words: ClipWord[], + endSec: number, + maxWords: number, + maxWidthEm: number = Number.POSITIVE_INFINITY, +): CaptionCue[] { const groups: ClipWord[][] = []; let current: ClipWord[] = []; for (const word of words) { @@ -222,7 +284,7 @@ export function groupCaptions(words: ClipWord[], endSec: number, maxWords: numbe (current.length >= maxWords || word.startSec - prev.endSec > PAUSE_SEC || endsSentence(prev.text) || - displayWidth(joinWords([...current, word].map((w) => w.text))) > MAX_CAPTION_WIDTH) + tooWide(joinWords([...current, word].map((w) => w.text)), maxWidthEm)) ) { groups.push(current); current = []; @@ -270,7 +332,9 @@ export function joinWords(texts: string[]): string { return out; } -function endsSentence(text: string): boolean { +/** Whether a word ends a sentence: it ends in . ! ? … (or 。!?), maybe + * followed by a closing quote or bracket. */ +export function endsSentence(text: string): boolean { return /[.!?。!?…]["')\]」』]?$/u.test(text); } @@ -283,6 +347,38 @@ function displayWidth(text: string): number { return width; } +function tooWide(text: string, maxWidthEm: number): boolean { + return displayWidth(text) > MAX_CAPTION_WIDTH || textWidthEm(text) > maxWidthEm; +} + +const WIDEST = /[WMmw@%#&ÆŒæœ]/u; +const NARROW = /[ijlItfr.,:;'’!|()[\]-]/u; +/** ASCII math signs: DejaVu draws them at about 0.84 em. */ +const SIGNS = /[+=<>~^]/u; +const ASCII = /^[\x20-\x7e]$/; + +/** + * A line's width in em, estimated from the characters' advances in common + * sans-serif caption fonts: for each class at least what Arial and DejaVu Sans + * draw (W and m are about 0.95–1.0 em), so the estimate errs wide and a line it + * lets through fits on screen. Accented Latin counts as its base letter; other + * scripts count as their widest letters (DejaVu's Щ and Ю are about 1.1 em). + */ +export function textWidthEm(text: string): number { + let width = 0; + for (const ch of text) { + const base = ch.normalize('NFD').charAt(0); + if (base === ' ') width += 0.32; + else if (WIDE.test(ch) || WIDEST.test(base)) width += 1; + else if (!ASCII.test(base)) width += base === base.toLowerCase() ? 0.95 : 1.1; + else if (SIGNS.test(base)) width += 0.85; + else if (NARROW.test(base)) width += 0.4; + else if (base !== base.toLowerCase()) width += 0.8; + else width += 0.64; + } + return width; +} + function roundMs(sec: number): number { return Math.round(sec * 1000) / 1000; } diff --git a/src/timeline/color.ts b/src/timeline/color.ts new file mode 100644 index 0000000..9e9c214 --- /dev/null +++ b/src/timeline/color.ts @@ -0,0 +1,58 @@ +import { z } from 'zod'; + +/** + * Timeline colors reach FFmpeg as bare option values (`color=c=…`, + * `pad=…:color=…`), where a comma, semicolon or bracket starts a new filter. So a + * color must be one FFmpeg reads as a color and nothing more: anything else + * could add filters such as `movie=`, which reads any file on disk. + */ + +/** FFmpeg's color names (`ffmpeg -colors`), which it matches in any case. */ +const COLOR_NAMES = new Set( + `aliceblue antiquewhite aqua aquamarine azure beige bisque black blanchedalmond blue + blueviolet brown burlywood cadetblue chartreuse chocolate coral cornflowerblue cornsilk + crimson cyan darkblue darkcyan darkgoldenrod darkgray darkgreen darkkhaki darkmagenta + darkolivegreen darkorange darkorchid darkred darksalmon darkseagreen darkslateblue + darkslategray darkturquoise darkviolet deeppink deepskyblue dimgray dodgerblue firebrick + floralwhite forestgreen fuchsia gainsboro ghostwhite gold goldenrod gray green greenyellow + honeydew hotpink indianred indigo ivory khaki lavender lavenderblush lawngreen lemonchiffon + lightblue lightcoral lightcyan lightgoldenrodyellow lightgreen lightgrey lightpink + lightsalmon lightseagreen lightskyblue lightslategray lightsteelblue lightyellow lime + limegreen linen magenta maroon mediumaquamarine mediumblue mediumorchid mediumpurple + mediumseagreen mediumslateblue mediumspringgreen mediumturquoise mediumvioletred + midnightblue mintcream mistyrose moccasin navajowhite navy oldlace olive olivedrab orange + orangered orchid palegoldenrod palegreen paleturquoise palevioletred papayawhip peachpuff + peru pink plum powderblue purple red rosybrown royalblue saddlebrown salmon sandybrown + seagreen seashell sienna silver skyblue slateblue slategray snow springgreen steelblue tan + teal thistle tomato turquoise violet wheat white whitesmoke yellow yellowgreen`.split(/\s+/), +); + +const HEX = /^(?:#|0x)?[0-9a-fA-F]{6}(?:[0-9a-fA-F]{2})?$/; +const HEX_ALPHA = /^0x[0-9a-fA-F]{1,2}$/; +const DECIMAL_ALPHA = /^(?:\d+(?:\.\d*)?|\.\d+)$/; + +/** Whether `value` is a color FFmpeg reads: a color name or hex (`#rrggbb`, + * `0xrrggbbaa`), with an optional `@alpha` from 0 to 1 (or 0x00 to 0xff). */ +export function isColor(value: string): boolean { + const [base = '', alpha, ...rest] = value.split('@'); + if (rest.length > 0) return false; + if (!HEX.test(base) && !COLOR_NAMES.has(base.toLowerCase())) return false; + return ( + alpha === undefined || + HEX_ALPHA.test(alpha) || + (DECIMAL_ALPHA.test(alpha) && Number(alpha) <= 1) + ); +} + +/** Why `value` was refused, with the forms that work. */ +export function invalidColorMessage(value: string): string { + return ( + `${JSON.stringify(value)} is not a color. Use a color name like black or white, ` + + 'or hex like #ffcc00, optionally with @0.5 for opacity.' + ); +} + +/** A color from an MCP client, the UI or the CLI (see `isColor`). */ +export const ColorSchema = z + .string() + .refine(isColor, { error: (issue) => invalidColorMessage(String(issue.input)) }); diff --git a/src/timeline/compile.ts b/src/timeline/compile.ts index 45dc785..5db459e 100644 --- a/src/timeline/compile.ts +++ b/src/timeline/compile.ts @@ -3,6 +3,7 @@ import { buildPreviewFrameArgs } from '../ffmpeg/args/preview.js'; import { buildTransitionArgs } from '../ffmpeg/args/transition.js'; import { quoteFilterArg } from '../ffmpeg/escape.js'; import { fontFileFor, MissingFontError } from '../ffmpeg/font.js'; +import { invalidColorMessage, isColor } from './color.js'; import { type Clip, type Composition, @@ -322,6 +323,7 @@ interface FadeSuppression { function effectFilters( clip: Clip, outDur: number, + fps: number, suppress: FadeSuppression = { in: false, out: false }, ): FilterChains { const video: string[] = []; @@ -343,7 +345,10 @@ function effectFilters( if (clip.kind === 'media') { const f = speedFactor(clip.effects); if (f !== 1) { - video.push(`setpts=PTS/${f}`); + // Back to the canvas rate right away: retimed frames otherwise reach the + // encoder off its frame grid, and which ones it drops or repeats varies, + // so a frame preview couldn't show the frame export plays. + video.push(`setpts=PTS/${f}`, `fps=${fps}`); audio.push(atempoChain(f)); } } @@ -399,6 +404,59 @@ function segPath(dir: string, index: number, clipId: string, tag?: string): stri return resolve(dir, `${SEG_PREFIX}-${tagInfix(tag)}${index}-${clipId}.mp4`); } +/** + * A clip's segment filters after its input: the video chain (framing onto the + * canvas, a text clip's own text, effects, then the text over it) and the audio + * effects. Shared by export and the frame preview, so a preview can't drift + * from what export draws. + */ +function clipChains( + comp: Composition, + clip: Clip, + index: number, + ctx: CompileContext, + overlays: TextClip[], + fadeSuppress: FadeSuppression = { in: false, out: false }, +): { chain: string; audio: string[]; textFiles: StepSideFile[] } { + const outDur = outputDuration(clip); + const { width, height, fps, background } = comp; + const { video: fx, audio } = effectFilters(clip, outDur, fps, fadeSuppress); + const textFiles: StepSideFile[] = []; + + if (clip.kind === 'text') { + const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); + textFiles.push({ path: textfile, content: clip.text }); + fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); + } + + // Overlay times are timeline seconds; the segment runs in output time, which + // a speed effect scales. + const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; + const end = clipEndSec(clip); + for (const overlay of overlays) { + const from = Math.max(overlay.startSec, clip.startSec); + const to = Math.min(clipEndSec(overlay), end); + if (to - from <= ABUT_EPSILON) continue; + const textfile = resolve( + ctx.dir, + `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, + ); + textFiles.push({ path: textfile, content: overlay.text }); + const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; + fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); + } + + // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish + // for generated sources that are already canvas-sized, but keeps the chain uniform.) + const geometry = [ + ...framingFilters(clipFraming(comp, clip), width, height, background), + 'setsar=1', + `fps=${fps}`, + 'format=yuv420p', + ]; + return { chain: [...geometry, ...fx].join(','), audio, textFiles }; +} + /** * Normalize one clip to a self-contained segment: canvas-sized video at the * composition fps, effects baked in, and a guaranteed stereo AAC audio track @@ -419,8 +477,11 @@ function buildSegmentStep( const outDur = outputDuration(clip); const { width, height, fps, background } = comp; const out = segPath(ctx.dir, index, clip.id, ctx.tag); - const { video: fx, audio: afx } = effectFilters(clip, outDur, fadeSuppress); - const textFiles: StepSideFile[] = []; + const { + chain: videoChain, + audio: afx, + textFiles, + } = clipChains(comp, clip, index, ctx, overlays, fadeSuppress); const inputArgs: string[] = []; let videoSource: string; // filtergraph input label feeding the video chain, e.g. [0:v] @@ -449,7 +510,7 @@ function buildSegmentStep( '-f', 'lavfi', '-i', - `color=c=${quoteColor(clip.color)}:s=${width}x${height}:r=${fps}`, + `color=c=${quoteColor(clip.color, `Color clip "${clip.id}" color`)}:s=${width}x${height}:r=${fps}`, ); inputArgs.push( '-f', @@ -465,7 +526,7 @@ function buildSegmentStep( '-f', 'lavfi', '-i', - `color=c=${quoteColor(background)}:s=${width}x${height}:r=${fps}`, + `color=c=${quoteColor(background, CANVAS_BACKGROUND)}:s=${width}x${height}:r=${fps}`, ); inputArgs.push( '-f', @@ -475,38 +536,8 @@ function buildSegmentStep( ); videoSource = '[0:v]'; audioSource = '1:a'; - const textfile = resolve(ctx.dir, `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}.txt`); - textFiles.push({ path: textfile, content: clip.text }); - fx.unshift(drawtextFilter(textfile, clip, fontFor(ctx, clip, false))); } - // Overlay times are timeline seconds; the segment runs in output time, which - // a speed effect scales. - const ratio = clipDuration(clip) > 0 ? outDur / clipDuration(clip) : 1; - const end = clipEndSec(clip); - for (const overlay of overlays) { - const from = Math.max(overlay.startSec, clip.startSec); - const to = Math.min(clipEndSec(overlay), end); - if (to - from <= ABUT_EPSILON) continue; - const textfile = resolve( - ctx.dir, - `${TEXT_PREFIX}-${tagInfix(ctx.tag)}${index}-${clip.id}-${overlay.id}.txt`, - ); - textFiles.push({ path: textfile, content: overlay.text }); - const window = `gte(t,${seconds((from - clip.startSec) * ratio)})*lt(t,${seconds((to - clip.startSec) * ratio)})`; - fx.push(drawtextFilter(textfile, overlay, fontFor(ctx, overlay, true), window)); - } - - // Geometry first: frame onto the canvas, square pixels, lock fps. (No-op-ish - // for generated sources that are already canvas-sized, but keeps the chain uniform.) - const geometry = [ - ...framingFilters(clipFraming(comp, clip), width, height, background), - 'setsar=1', - `fps=${fps}`, - 'format=yuv420p', - ]; - - const videoChain = [...geometry, ...fx].join(','); // Always normalize audio to a uniform stereo/48k layout BEFORE any effects, so // the fold's concat/acrossfade can't silently downmix when a source clip is // mono (and so every segment exposes a real [a] output). aformat is a harmless @@ -555,7 +586,7 @@ function framingFilters( height: number, background: string, ): string[] { - const pad = `color=${quoteColor(background)}`; + const pad = `color=${quoteColor(background, CANVAS_BACKGROUND)}`; if (isDefaultFraming(framing)) { return [ `scale=${width}:${height}:force_original_aspect_ratio=decrease`, @@ -577,13 +608,18 @@ function even(size: number): number { return Math.max(2, Math.round(size / 2) * 2); } -const CENTER_POS = { x: '(w-text_w)/2', y: '(h-text_h)/2' }; - /** Seconds for a filter expression: millisecond precision, no float noise. */ function seconds(sec: number): string { return String(Math.round(sec * 1000) / 1000); } +/** How much of the source a frame preview reads before the frame, at least. */ +const FRAME_PREROLL_SEC = 1; + +function floorUs(sec: number): number { + return Math.floor(sec * 1e6 + 1e-6) / 1e6; +} + /** The font file for a text clip, with a missing font reported against the clip. * Text over the picture always gets a file: see `ffmpeg/font.ts`. */ function fontFor(ctx: CompileContext, clip: TextClip, always: boolean): string | undefined { @@ -605,7 +641,7 @@ function drawtextFilter( enable?: string, ): string { const { style } = clip; - const pos = ANCHOR_EXPR[style.anchor] ?? CENTER_POS; + const pos = anchorPosition(style.anchor, style.margin); const opts = [`textfile=${quoteFilterArg(textfile)}`]; if (fontFile) opts.push(`fontfile=${quoteFilterArg(fontFile)}`); opts.push( @@ -622,21 +658,35 @@ function drawtextFilter( return `drawtext=${opts.join(':')}`; } -const EDGE = 40; -const ANCHOR_EXPR: Record = { - 'top-left': { x: `${EDGE}`, y: `${EDGE}` }, - 'top-center': { x: '(w-text_w)/2', y: `${EDGE}` }, - 'top-right': { x: `w-text_w-${EDGE}`, y: `${EDGE}` }, - 'center-left': { x: `${EDGE}`, y: '(h-text_h)/2' }, - center: { x: '(w-text_w)/2', y: '(h-text_h)/2' }, - 'center-right': { x: `w-text_w-${EDGE}`, y: '(h-text_h)/2' }, - 'bottom-left': { x: `${EDGE}`, y: `h-text_h-${EDGE}` }, - 'bottom-center': { x: '(w-text_w)/2', y: `h-text_h-${EDGE}` }, - 'bottom-right': { x: `w-text_w-${EDGE}`, y: `h-text_h-${EDGE}` }, -}; - -/** color filter `c=` rejects '#rrggbb'; normalize to 0x form, pass names through. */ -function quoteColor(color: string): string { +/** Inset of anchored text from the frame edges when its style sets none. */ +export const TEXT_EDGE_PX = 40; + +/** drawtext x/y for an anchor, kept `margin` pixels in from the edges it is + * anchored to. */ +function anchorPosition( + anchor: string, + margin: { x: number; y: number } = { x: TEXT_EDGE_PX, y: TEXT_EDGE_PX }, +): { x: string; y: string } { + const x = anchor.endsWith('left') + ? `${margin.x}` + : anchor.endsWith('right') + ? `w-text_w-${margin.x}` + : '(w-text_w)/2'; + const y = anchor.startsWith('top') + ? `${margin.y}` + : anchor.startsWith('bottom') + ? `h-text_h-${margin.y}` + : '(h-text_h)/2'; + return { x, y }; +} + +const CANVAS_BACKGROUND = 'Canvas background'; + +/** color filter `c=` rejects '#rrggbb'; normalize to 0x form, pass names through. + * The value lands unquoted in the graph, so anything but a color is refused: a + * stored document is only as safe as whatever wrote it. */ +function quoteColor(color: string, what: string): string { + if (!isColor(color)) throw new CompileError(`${what} ${invalidColorMessage(color)}`); return color.startsWith('#') ? `0x${color.slice(1)}` : color; } @@ -692,18 +742,18 @@ function assertAbutting(clips: Clip[]): void { /** * Render ONE frame at timeline time `atSec` — the agent's "eyes" on the doc. - * Goes through the REAL segment path (`buildSegmentStep`) so the preview can't - * diverge from what export would produce, then extracts a still from that - * normalized segment. Two steps: encode the active clip's segment, then grab the - * frame at the mapped offset. Runs the SAME abutting check as export, so it fails - * (rather than guessing a clip) on an unexportable doc; throws `CompileError` past - * the end, in a gap, or on an overlap, and inherits the v1 single-video-track / - * no-compositing guards. + * Uses the segment's own video chain (`clipChains`) so the preview can't diverge + * from what export would produce. A media clip is one step: seek the source to + * the frame and run the chain on a window starting there, with the timestamps + * offset so fades and text windows read the time they would in the segment, so + * the cost doesn't grow with the clip's length. A text or color card (generated, + * and short) is two: encode its segment, then grab the frame from it. Runs the + * SAME abutting check as export, so it fails (rather than guessing a clip) on an + * unexportable doc; throws `CompileError` past the end, in a gap, or on an + * overlap, and inherits the v1 single-video-track / no-compositing guards. * - * Known v1 limits (flag, don't fix here): a frame inside a transition window - * shows the underlying clip, not the xfade blend; `-ss` is keyframe-accurate - * (off by up to a GOP) — fine for a thumbnail; and a long clip re-encodes a whole - * segment to grab one frame. + * Known v1 limit (flag, don't fix here): a frame inside a transition window + * shows the underlying clip, not the xfade blend. */ export function buildFrameAtPlan( comp: Composition, @@ -728,9 +778,6 @@ export function buildFrameAtPlan( } assertCompilable(clip); - // Encode the clip's segment exactly as export would, then extract the frame. - const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); - // Map doc-local time to the post-speed SEGMENT timebase. The ratio is 1 unless // the clip carries a speed effect, which shortens the segment relative to the // document extent — so a frame still lands on the source content the document @@ -741,9 +788,59 @@ export function buildFrameAtPlan( const segTime = clipDur > 0 ? (localDoc * outDur) / clipDur : 0; const lastFrame = Math.max(0, outDur - 1 / comp.fps); const clamped = Math.max(0, Math.min(segTime, lastFrame)); + const label = `frame:${clip.id}@${atSec}`; + if (clip.kind === 'media') { + const info = ctx.media.get(clip.mediaId); + if (!info) { + throw new CompileError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + // Seconds into the clip's source. The seek lands a second before that and + // keeps every frame from the keyframe before it (`-noaccurate_seek`), so a + // frame that export repeats into this time (a source slower than the + // canvas, slow motion, or a screen recording holding a still frame) is + // still read; undoing the seek gives source time from the clip's in-point. + // Frames before the in-point are dropped, as export's seek drops them. The + // chain's speed `setpts` turns source time into segment time, which its + // fades and text read, and `trim` takes the frame at the frame's segment + // time. Rounded down, to the microsecond, so the seek never passes the + // clip's last frame or its out-point. + const local = floorUs(clamped * speedFactor(clip.effects)); + const seek = floorUs(clip.sourceInSec + Math.max(0, local - FRAME_PREROLL_SEC)); + // The first canvas frame at or after the frame's time (`trim` rounds an + // off-grid start to the nearest frame, which could be the one before), but + // no later than the segment's last frame: encoding to the clip's length + // keeps round(length x fps) frames. + const lastIndex = Math.max(0, Math.round(outDur * comp.fps) - 1); + const firstFrame = Math.min(Math.ceil(clamped * comp.fps - 1e-6), lastIndex) / comp.fps; + const { chain, textFiles } = clipChains(comp, clip, index, ctx, overlays); + const frameStep: FfmpegStep = { + label, + args: [ + '-y', + '-noaccurate_seek', + '-ss', + String(seek), + '-i', + info.path, + '-filter_complex', + // The seek shifted timestamps by (in-point + local) in whole stream + // ticks; shifting back by the same rounding leaves them as export's + // seek to the in-point does, tick for tick. + `[0:v]setpts=PTS+round(${seek}/TB)-round(${floorUs(clip.sourceInSec)}/TB),trim=start=0,${chain},trim=start=${firstFrame}[v]`, + ...['-map', '[v]', '-an', '-frames:v', '1', '-q:v', '2'], + ctx.output, + ], + output: ctx.output, + textFiles, + }; + return { steps: [frameStep], output: ctx.output, durationSec: 0 }; + } + + // Encode the card's segment exactly as export would, then extract the frame. + const segStep = buildSegmentStep(comp, clip, index, ctx, overlays); const frameStep: FfmpegStep = { - label: `frame:${clip.id}@${atSec}`, + label, args: buildPreviewFrameArgs({ input: segStep.output, output: ctx.output, atSec: clamped }), output: ctx.output, textFiles: [], diff --git a/src/timeline/composition.ts b/src/timeline/composition.ts index 1f9f1fc..b09127a 100644 --- a/src/timeline/composition.ts +++ b/src/timeline/composition.ts @@ -117,6 +117,11 @@ export const TextStyleSchema = z.object({ /** Box color behind the text, or null for none. */ background: z.string().nullable().default(null), anchor: AnchorSchema.default('center'), + /** How far, in pixels, anchored text keeps from the frame edges: `x` from + * the left or right edge, `y` from the top or bottom. Absent means 40. */ + margin: z + .object({ x: z.number().int().nonnegative(), y: z.number().int().nonnegative() }) + .optional(), }); export type TextStyle = z.infer; diff --git a/src/timeline/document-store.ts b/src/timeline/document-store.ts index d3f71c1..2eb18b4 100644 --- a/src/timeline/document-store.ts +++ b/src/timeline/document-store.ts @@ -1,10 +1,20 @@ import { randomBytes } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { readdir, readFile, rm } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; import { atomicWriteFile } from '../session/store.js'; -import { createRevisionedStore } from '../storage/revisioned-store.js'; -import { ensureWorkspace, getWorkspace } from '../workspace.js'; -import { type Composition, CompositionSchema, emptyComposition } from './composition.js'; +import { createRevisionedStore, type RevisionedStore } from '../storage/revisioned-store.js'; +import { + ensureWorkspace, + getWorkspace, + resolveInWorkspace, + WorkspaceBoundaryError, +} from '../workspace.js'; +import { + type Composition, + CompositionSchema, + compositionDuration, + emptyComposition, +} from './composition.js'; import { type DocOpLog, type DocOpLogEntry, @@ -17,17 +27,74 @@ import { undo as undoLog, } from './doc-op-log.js'; import { applyOps, applyOpsTracked, type CompositionOp } from './ops.js'; +import { MAIN_TIMELINE, TimelineIdSchema } from './schema.js'; import { type CompositionVerb, lowerVerbs, type VerbContext } from './verbs.js'; +/* + * A workspace holds several named timelines. `main` stays at the workspace root + * (`composition.json` + `composition-ops.json`), so existing workspaces and older + * `clip` versions keep working; every other timeline keeps the same two files in + * `timelines//`. Each timeline has its own rev, write lock and undo history. + */ const COMPOSITION_FILE = 'composition.json'; const COMPOSITION_OPS_FILE = 'composition-ops.json'; +const TIMELINES_DIR = 'timelines'; + +/** Thrown for a malformed timeline id, or for an operation `main` can't take. */ +export class TimelineIdError extends Error { + constructor(message: string) { + super(message); + this.name = 'TimelineIdError'; + } +} + +/** Thrown when a timeline other than `main` doesn't exist. Only `make_short` and + * `clip timeline new --timeline` create one, so a typo can't edit an empty document. */ +export class NoTimelineError extends Error { + readonly timeline: string; + constructor(timeline: string) { + super( + `No timeline "${timeline}" in the workspace. List them with \`clip timeline list\` ` + + '(timeline_show reports them too). A short gets its timeline from make_short (`clip timeline short`).', + ); + this.name = 'NoTimelineError'; + this.timeline = timeline; + } +} + +function checkedTimelineId(timeline: string): string { + const parsed = TimelineIdSchema.safeParse(timeline); + if (!parsed.success) { + throw new TimelineIdError( + `Invalid timeline id "${timeline}". ${parsed.error.issues[0]?.message ?? ''}`.trim(), + ); + } + return parsed.data; +} + +/** A non-main timeline's folder, checked against the workspace: `timelines/` + * or the folder could be a symlink out of it, and writes (temp files) and + * deletes act on the folder itself, not only on the files in it. */ +function timelineDir(timeline: string): string { + return resolveInWorkspace(join(TIMELINES_DIR, checkedTimelineId(timeline))); +} + +function timelineFile(timeline: string, file: string): string { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) return resolve(getWorkspace(), file); + timelineDir(timeline); + return resolveInWorkspace(join(TIMELINES_DIR, timeline, file)); +} + +export function compositionPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_FILE); +} -export function compositionPath(): string { - return resolve(getWorkspace(), COMPOSITION_FILE); +export function compositionOpsPath(timeline: string = MAIN_TIMELINE): string { + return timelineFile(timeline, COMPOSITION_OPS_FILE); } -export function compositionOpsPath(): string { - return resolve(getWorkspace(), COMPOSITION_OPS_FILE); +function isMissingFile(err: unknown): boolean { + return err instanceof Error && 'code' in err && (err.code === 'ENOENT' || err.code === 'ENOTDIR'); } /** Thrown when composition.json exists but cannot be parsed/validated — surfaced, @@ -65,31 +132,52 @@ export class CompositionConflictError extends Error { } } -export async function readComposition(): Promise { +/** The stored document, or null when the timeline has none yet. */ +async function readStoredComposition(timeline: string): Promise { await ensureWorkspace(); + const path = compositionPath(timeline); let raw: string; try { - raw = await readFile(compositionPath(), 'utf-8'); + raw = await readFile(path, 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyComposition(); + if (isMissingFile(err)) return null; throw err; } try { return CompositionSchema.parse(JSON.parse(raw)); } catch (err) { - throw new CompositionCorruptError(compositionPath(), { cause: err }); + throw new CompositionCorruptError(path, { cause: err }); } } -export async function writeComposition(comp: Composition): Promise { - await atomicWriteFile(compositionPath(), `${JSON.stringify(comp, null, 2)}\n`); +/** The document an edit starts from. A missing `main` is an empty document (a + * fresh workspace); any other missing timeline is an error unless the edit + * `create`s it. */ +async function readForEdit(timeline: string, create: boolean): Promise { + const comp = await readStoredComposition(timeline); + if (comp) return comp; + if (create || timeline === MAIN_TIMELINE) return emptyComposition(); + throw new NoTimelineError(timeline); } -/** On-disk rev, treating a corrupt file as rev 0 — a deliberate overwrite is - * about to discard it anyway. Non-corruption read errors still propagate. */ -async function readCompositionRevTolerant(): Promise { +/** Read a timeline's document (default `main`). Throws `NoTimelineError` for a + * timeline other than `main` that doesn't exist. */ +export async function readComposition(timeline: string = MAIN_TIMELINE): Promise { + return readForEdit(timeline, false); +} + +export async function writeComposition( + comp: Composition, + timeline: string = MAIN_TIMELINE, +): Promise { + await atomicWriteFile(compositionPath(timeline), `${JSON.stringify(comp, null, 2)}\n`); +} + +/** On-disk rev, treating a corrupt or missing file as rev 0 — a deliberate + * overwrite is about to replace it anyway. Other read errors still propagate. */ +async function readCompositionRevTolerant(timeline: string): Promise { try { - return (await readComposition()).rev; + return (await readStoredComposition(timeline))?.rev ?? 0; } catch (err) { if (err instanceof CompositionCorruptError) return 0; throw err; @@ -97,16 +185,26 @@ async function readCompositionRevTolerant(): Promise { } // Serialization + compare-and-swap, shared with the session store (see -// `storage/revisioned-store.ts`) so co-editing the doc (the agent + `clip ui`) -// can't silently lose updates. -const docStore = createRevisionedStore({ - read: readComposition, - write: writeComposition, - getRev: (comp) => comp.rev, - withRev: (comp, rev) => ({ ...comp, rev }), - readRevTolerant: readCompositionRevTolerant, - onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), -}); +// `storage/revisioned-store.ts`) so co-editing a timeline (the agent + `clip ui`) +// can't silently lose updates. One store per timeline, so each has its own lock. +const stores = new Map>(); + +function storeFor(timeline: string): RevisionedStore { + const id = checkedTimelineId(timeline); + let store = stores.get(id); + if (!store) { + store = createRevisionedStore({ + read: () => readComposition(id), + write: (comp) => writeComposition(comp, id), + getRev: (comp) => comp.rev, + withRev: (comp, rev) => ({ ...comp, rev }), + readRevTolerant: () => readCompositionRevTolerant(id), + onConflict: (expectedRev, actualRev) => new CompositionConflictError(expectedRev, actualRev), + }); + stores.set(id, store); + } + return store; +} // ─── op-log (undo/redo) ────────────────────────────────────────────────────── // @@ -130,13 +228,13 @@ function labelForOps(ops: CompositionOp[]): string { /** Read the raw op-log file. Missing OR corrupt → an empty log: undo history is a * convenience, never worth failing a read for. The caller reconciles it against * the authoritative document rev. */ -async function readDocOpLogFile(): Promise { +async function readDocOpLogFile(timeline: string): Promise { await ensureWorkspace(); let raw: string; try { - raw = await readFile(compositionOpsPath(), 'utf-8'); + raw = await readFile(compositionOpsPath(timeline), 'utf-8'); } catch (err: unknown) { - if (err instanceof Error && 'code' in err && err.code === 'ENOENT') return emptyDocOpLog(0); + if (isMissingFile(err)) return emptyDocOpLog(0); throw err; } try { @@ -146,8 +244,8 @@ async function readDocOpLogFile(): Promise { } } -async function writeDocOpLog(log: DocOpLog): Promise { - await atomicWriteFile(compositionOpsPath(), serializeDocOpLog(log)); +async function writeDocOpLog(log: DocOpLog, timeline: string): Promise { + await atomicWriteFile(compositionOpsPath(timeline), serializeDocOpLog(log)); } /** The op-log reconciled against the current document rev — the trustworthy view @@ -155,10 +253,10 @@ async function writeDocOpLog(log: DocOpLog): Promise { * inside the store's exclusive section so an interleaved IN-PROCESS commit can't * make a consistent log momentarily read as empty (cross-process reads stay * best-effort, as everywhere in this store). */ -export async function readDocOpLog(): Promise { - return docStore.runExclusive(async () => { - const docRev = (await readComposition()).rev; - return reconcile(await readDocOpLogFile(), docRev); +export async function readDocOpLog(timeline: string = MAIN_TIMELINE): Promise { + return storeFor(timeline).runExclusive(async () => { + const docRev = (await readComposition(timeline)).rev; + return reconcile(await readDocOpLogFile(timeline), docRev); }); } @@ -176,19 +274,21 @@ export async function readDocOpLog(): Promise { * shipping surface). */ async function commitDocAndLog( + timeline: string, + create: boolean, derive: ( doc: Composition, log: DocOpLog, ) => { doc: Composition; log: DocOpLog; result: T } | null, ): Promise<{ doc: Composition; result: T } | null> { - return docStore.runExclusive(async () => { + return storeFor(timeline).runExclusive(async () => { for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const log = reconcile(await readDocOpLogFile(), current.rev); + const current = await readForEdit(timeline, create); + const log = reconcile(await readDocOpLogFile(timeline), current.rev); const derived = derive(current, log); if (derived === null) return null; - const observed = (await readComposition()).rev; + const observed = (await readForEdit(timeline, create)).rev; if (observed !== current.rev) { if (attempt === MAX_COMMIT_ATTEMPTS) { throw new CompositionConflictError(current.rev, observed); @@ -199,8 +299,8 @@ async function commitDocAndLog( const nextRev = current.rev + 1; const committedDoc = { ...derived.doc, rev: nextRev }; const committedLog = { ...derived.log, rev: nextRev }; - await writeDocOpLog(committedLog); - await writeComposition(committedDoc); + await writeDocOpLog(committedLog, timeline); + await writeComposition(committedDoc, timeline); return { doc: committedDoc, result: derived.result }; } throw new CompositionConflictError(-1, -1); @@ -211,14 +311,18 @@ async function commitDocAndLog( * Read → apply ops → write, returning the new composition, and record the batch * (with its computed inverse) to the op-log so the edit is undoable. Serialized + * rev compare-and-swap via the shared revisioned store, so the agent and the UI - * can co-edit without lost updates. An empty batch is a true no-op. + * can co-edit without lost updates. An empty batch is a true no-op. `timeline` + * defaults to `main`; `create` lets the edit start a timeline that doesn't exist + * yet (from an empty document) instead of throwing `NoTimelineError`. */ export async function mutateComposition( ops: CompositionOp[], - opts?: { expectedBaseRev?: number }, + opts?: { expectedBaseRev?: number; timeline?: string; create?: boolean }, ): Promise { - if (ops.length === 0) return readComposition(); - const committed = await commitDocAndLog((current, log) => { + const timeline = opts?.timeline ?? MAIN_TIMELINE; + const create = opts?.create ?? false; + if (ops.length === 0) return readForEdit(timeline, create); + const committed = await commitDocAndLog(timeline, create, (current, log) => { // Optimistic-concurrency guard for callers whose ops were lowered against a // specific revision (e.g. `applyVerbs`, where a default append point is baked // from a snapshot read): if the doc moved under us, reject so the caller can @@ -235,7 +339,7 @@ export async function mutateComposition( }; return { doc, log: recordOps(log, entry), result: null }; }); - return committed ? committed.doc : readComposition(); + return committed ? committed.doc : readForEdit(timeline, create); } /** @@ -243,12 +347,21 @@ export async function mutateComposition( * op-aware mutation path the agent and `clip ui` share. Lowering (which ingests * files and mints ids) runs outside the write lock; `mutateComposition` then * validates, applies, and records the ops (so the edit lands in the undo stack). - * Returns the new document and the ops that were applied. + * Returns the new document and the ops that were applied. Edits `timeline` + * (default `main`), which must exist unless the batch starts with `make_short`. */ export async function applyVerbs( verbs: CompositionVerb[], ctx: VerbContext, -): Promise<{ doc: Composition; ops: CompositionOp[] }> { + timeline: string = MAIN_TIMELINE, +): Promise<{ doc: Composition; ops: CompositionOp[]; warnings: string[] }> { + const create = verbs[0]?.verb === 'make_short'; + let warnings: string[] = []; + const editCtx: VerbContext = { + ...ctx, + timeline, + warn: (message) => warnings.push(message), + }; // The default append point (`trackEnd`) is baked from the snapshot we lower // against, so if another in-process writer commits between the read and the // apply, re-lower against the fresh doc rather than land overlapping clips. A @@ -257,11 +370,16 @@ export async function applyVerbs( // a burst of N concurrent edits each re-lower and land instead of the (N-cap) // tail silently losing to a `CompositionConflictError`. for (let attempt = 1; attempt <= MAX_COMMIT_ATTEMPTS; attempt++) { - const current = await readComposition(); - const ops = await lowerVerbs(current, verbs, ctx); + const current = await readForEdit(timeline, create); + warnings = []; + const ops = await lowerVerbs(current, verbs, editCtx); try { - const doc = await mutateComposition(ops, { expectedBaseRev: current.rev }); - return { doc, ops }; + const doc = await mutateComposition(ops, { + expectedBaseRev: current.rev, + timeline, + create, + }); + return { doc, ops, warnings }; } catch (err) { if (err instanceof CompositionConflictError && attempt < MAX_COMMIT_ATTEMPTS) continue; throw err; @@ -273,33 +391,33 @@ export async function applyVerbs( /** Undo the most recent recorded edit (apply its inverse), moving the op-log * cursor back. `undone: false` when there is nothing to undo. */ -export async function undoLastDocOp(): Promise<{ +export async function undoLastDocOp(timeline: string = MAIN_TIMELINE): Promise<{ undone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = undoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.inverse), log: step.log, result: step.entry.label }; }); - if (!committed) return { undone: false, doc: await readComposition(), label: null }; + if (!committed) return { undone: false, doc: await readComposition(timeline), label: null }; return { undone: true, doc: committed.doc, label: committed.result }; } /** Redo the most recently undone edit (re-apply its forward ops), moving the * cursor forward. `redone: false` when there is nothing to redo. */ -export async function redoDocOp(): Promise<{ +export async function redoDocOp(timeline: string = MAIN_TIMELINE): Promise<{ redone: boolean; doc: Composition; label: string | null; }> { - const committed = await commitDocAndLog((current, log) => { + const committed = await commitDocAndLog(timeline, false, (current, log) => { const step = redoLog(log); if (!step) return null; return { doc: applyOps(current, step.entry.forward), log: step.log, result: step.entry.label }; }); - if (!committed) return { redone: false, doc: await readComposition(), label: null }; + if (!committed) return { redone: false, doc: await readComposition(timeline), label: null }; return { redone: true, doc: committed.doc, label: committed.result }; } @@ -318,9 +436,10 @@ export async function redoDocOp(): Promise<{ export async function writeCompositionIfUnchanged( comp: Composition, expectedRev: number, + timeline: string = MAIN_TIMELINE, ): Promise { - const committed = await docStore.writeIfUnchanged(comp, expectedRev); - await writeDocOpLog(emptyDocOpLog(committed.rev)); + const committed = await storeFor(timeline).writeIfUnchanged(comp, expectedRev); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); return committed; } @@ -332,12 +451,12 @@ export async function writeCompositionIfUnchanged( * replay an inverse against the wrong document. Log-first ordering (the seeded * log is empty anyway, so a crash resolves to the same empty history). */ -async function overwriteDocAndResetLog(comp: Composition): Promise { - return docStore.runExclusive(async () => { - const baseRev = await readCompositionRevTolerant(); +async function overwriteDocAndResetLog(comp: Composition, timeline: string): Promise { + return storeFor(timeline).runExclusive(async () => { + const baseRev = await readCompositionRevTolerant(timeline); const committed: Composition = { ...comp, rev: baseRev + 1 }; - await writeDocOpLog(emptyDocOpLog(committed.rev)); - await writeComposition(committed); + await writeDocOpLog(emptyDocOpLog(committed.rev), timeline); + await writeComposition(committed, timeline); return committed; }); } @@ -348,12 +467,85 @@ async function overwriteDocAndResetLog(comp: Composition): Promise * recovered). Resets undo history — a recovered document has none — and advances * `rev` so a stale log can never collide with the new revision. */ -export function overwriteComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); +export function overwriteComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); } /** Start a fresh timeline (`clip timeline new`): replace the document, clear undo - * history, and advance `rev` past the previous one (never reuse a rev). */ -export function resetComposition(comp: Composition = emptyComposition()): Promise { - return overwriteDocAndResetLog(comp); + * history, and advance `rev` past the previous one (never reuse a rev). Creates + * `timeline` when it doesn't exist yet. */ +export function resetComposition( + comp: Composition = emptyComposition(), + timeline: string = MAIN_TIMELINE, +): Promise { + return overwriteDocAndResetLog(comp, timeline); +} + +// ─── the workspace's timelines ─────────────────────────────────────────────── + +/** A timeline in the workspace, or one whose document can't be read. */ +export type TimelineListing = + | { id: string; width: number; height: number; durationSec: number; rev: number } + | { id: string; error: string }; + +/** `main` first, then every `timelines//` holding a document, by id. A + * document that can't be read is listed with its error instead of failing the + * list; a `timelines/` folder that leads out of the workspace is refused. */ +export async function listTimelines(): Promise { + await ensureWorkspace(); + const listed: TimelineListing[] = []; + for (const id of [MAIN_TIMELINE, ...(await timelineFolders())]) { + try { + const comp = await readStoredComposition(id); + if (!comp && id !== MAIN_TIMELINE) continue; + const doc = comp ?? emptyComposition(); + listed.push({ + id, + width: doc.width, + height: doc.height, + durationSec: Math.round(compositionDuration(doc) * 1000) / 1000, + rev: doc.rev, + }); + } catch (err) { + if (!(err instanceof CompositionCorruptError || err instanceof WorkspaceBoundaryError)) { + throw err; + } + listed.push({ id, error: err.message }); + } + } + return listed; +} + +async function timelineFolders(): Promise { + let names: string[]; + try { + names = await readdir(resolveInWorkspace(TIMELINES_DIR)); + } catch (err) { + if (isMissingFile(err)) return []; + throw err; + } + return names + .filter((name) => name !== MAIN_TIMELINE && TimelineIdSchema.safeParse(name).success) + .sort(); +} + +/** Remove a timeline and its undo history. `main` can't be deleted. */ +export async function deleteTimeline(timeline: string): Promise { + if (checkedTimelineId(timeline) === MAIN_TIMELINE) { + throw new TimelineIdError( + "The main timeline can't be deleted. Start it over with `clip timeline new`.", + ); + } + await storeFor(timeline).runExclusive(async () => { + try { + if (!(await readStoredComposition(timeline))) throw new NoTimelineError(timeline); + } catch (err) { + // A corrupt document is deleted like any other. + if (!(err instanceof CompositionCorruptError)) throw err; + } + await rm(timelineDir(timeline), { recursive: true, force: true }); + }); } diff --git a/src/timeline/interchange-fcpxml.ts b/src/timeline/interchange-fcpxml.ts index 8afe048..35e9bed 100644 --- a/src/timeline/interchange-fcpxml.ts +++ b/src/timeline/interchange-fcpxml.ts @@ -111,14 +111,19 @@ function titlePlacement( height: number, ): { position: string; alignment: 'left' | 'center' | 'right' } { const { anchor, fontSize } = title.style; + const margin = title.style.margin ?? { x: EDGE, y: EDGE }; const alignment = anchor.endsWith('left') ? 'left' : anchor.endsWith('right') ? 'right' : 'center'; const x = - alignment === 'left' ? -(width / 2 - EDGE) : alignment === 'right' ? width / 2 - EDGE : 0; - const rowOffset = height / 2 - EDGE - fontSize / 2; + alignment === 'left' + ? -(width / 2 - margin.x) + : alignment === 'right' + ? width / 2 - margin.x + : 0; + const rowOffset = height / 2 - margin.y - fontSize / 2; const y = anchor.startsWith('top') ? rowOffset : anchor.startsWith('bottom') ? -rowOffset : 0; return { position: `${num(x)} ${num(y)}`, alignment }; } diff --git a/src/timeline/pauses.ts b/src/timeline/pauses.ts new file mode 100644 index 0000000..ebda2a4 --- /dev/null +++ b/src/timeline/pauses.ts @@ -0,0 +1,146 @@ +import { + buildSpeechLevelsArgs, + LEVEL_FRAME_SAMPLES, + LEVEL_SAMPLE_RATE, +} from '../ffmpeg/args/speech-levels.js'; +import { runFfmpeg } from '../ffmpeg/run.js'; + +/** + * Pauses in speech, found in the audio itself. whisper.cpp's word edges run + * late at sentence ends: it gives each pause to the word before it, and the + * next word can start a third of a second into its own speech. A cut placed by + * the transcript alone then keeps the start of the next sentence. Measured + * levels show where the voice actually stops, so an edge is moved into the + * pause the transcript points at. + */ + +/** RMS level of each 10 ms of a source's audio, dBFS, from `fromSec` on. */ +export interface SpeechLevels { + fromSec: number; + frameSec: number; + db: number[]; +} + +/** A stretch of a source quiet enough to be a pause, in source seconds. */ +export interface Pause { + startSec: number; + endSec: number; +} + +/** Room tone kept between a cut and the speech beside it. */ +export const PAUSE_KEEP_SEC = 0.1; +/** How far before the transcript's boundary the pause may lie: whisper runs + * up to about 0.75 s late there. */ +const SEARCH_BEFORE_SEC = 0.9; +/** …and after it, for a boundary whisper put early. */ +const SEARCH_AFTER_SEC = 0.05; +/** Measured around a boundary, for a noise floor and speech level of its own. */ +const MEASURE_BEFORE_SEC = 1.5; +const MEASURE_AFTER_SEC = 1; +/** Quiet means this far over the window's floor (its 5th percentile)… */ +const ABOVE_FLOOR_DB = 10; +/** …and this far under its speech (its 95th percentile). */ +const BELOW_SPEECH_DB = 20; +/** The shortest quiet stretch counted as a pause. */ +const MIN_PAUSE_SEC = 0.08; +/** Sound shorter than this inside a pause (a click, a lip noise) doesn't end it. */ +const MAX_BLIP_SEC = 0.04; +/** Two pauses at least this long in reach make the boundary ambiguous. */ +const COMPETING_PAUSE_SEC = 0.12; +const SILENT_DB = -120; +/** At or below this, a frame is digital silence, not a room. */ +const DIGITAL_SILENCE_DB = -100; + +/** Read the per-frame levels `buildSpeechLevelsArgs` prints. */ +export function parseSpeechLevels(stderr: string, fromSec: number): SpeechLevels { + const db: number[] = []; + for (const match of stderr.matchAll(/lavfi\.astats\.Overall\.RMS_level=(\S+)/g)) { + const value = Number(match[1]); + db.push(Number.isFinite(value) ? Math.max(value, SILENT_DB) : SILENT_DB); + } + return { fromSec, frameSec: LEVEL_FRAME_SAMPLES / LEVEL_SAMPLE_RATE, db }; +} + +/** Measure a source's levels around a boundary (runs FFmpeg). */ +export async function measureSpeechLevels( + path: string, + boundarySec: number, +): Promise { + const fromSec = Math.max(0, boundarySec - MEASURE_BEFORE_SEC); + const durationSec = boundarySec + MEASURE_AFTER_SEC - fromSec; + const { stderr } = await runFfmpeg(buildSpeechLevelsArgs({ input: path, fromSec, durationSec })); + return parseSpeechLevels(stderr, fromSec); +} + +/** Every pause in the levels, by a threshold set from the levels themselves. */ +export function findPauses(levels: SpeechLevels): Pause[] { + // Digital silence (padding where the audio starts late, or a muted stretch) + // is quiet, but it isn't the room's floor: leave it out of the estimate, or + // the room tone between sentences would count as sound. + const sorted = levels.db.filter((level) => level > DIGITAL_SILENCE_DB).sort((a, b) => a - b); + const at = (q: number): number => sorted[Math.floor(q * (sorted.length - 1))] ?? SILENT_DB; + const threshold = Math.min(at(0.05) + ABOVE_FLOOR_DB, at(0.95) - BELOW_SPEECH_DB); + const quiet = levels.db.map((level) => level <= DIGITAL_SILENCE_DB || level <= threshold); + const maxBlip = Math.round(MAX_BLIP_SEC / levels.frameSec); + // A short burst of sound between two quiet stretches is part of the pause. + for (let i = 0; i < quiet.length; ) { + if (quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && !quiet[j]) j++; + if (i > 0 && j < quiet.length && j - i <= maxBlip) quiet.fill(true, i, j); + i = j; + } + const pauses: Pause[] = []; + for (let i = 0; i < quiet.length; ) { + if (!quiet[i]) { + i++; + continue; + } + let j = i; + while (j < quiet.length && quiet[j]) j++; + if ((j - i) * levels.frameSec >= MIN_PAUSE_SEC - 1e-9) { + pauses.push({ + startSec: levels.fromSec + i * levels.frameSec, + endSec: levels.fromSec + j * levels.frameSec, + }); + } + i = j; + } + return pauses; +} + +/** + * The pause at a transcript boundary: the longest pause reaching into + * [boundary − 0.9 s, boundary + 0.05 s] that is still going after the word + * the boundary ends has started (not the pause before that word). `none` when there is none (speech runs on, or music under + * it); `ambiguous` when two pauses of 0.12 s or more are in reach, since the + * transcript can't say which one it means. + */ +export function pauseAt( + levels: SpeechLevels, + boundarySec: number, + wordBeforeStartSec: number, +): Pause | 'none' | 'ambiguous' { + // A pause that ends by the time that word starts is the one before it. + const from = Math.max(boundarySec - SEARCH_BEFORE_SEC, wordBeforeStartSec + SEARCH_AFTER_SEC); + const to = boundarySec + SEARCH_AFTER_SEC; + const inReach = findPauses(levels).filter((p) => p.endSec > from && p.startSec < to); + if (inReach.length === 0) return 'none'; + if (inReach.filter((p) => p.endSec - p.startSec >= COMPETING_PAUSE_SEC - 1e-9).length > 1) { + return 'ambiguous'; + } + return inReach.reduce((a, b) => (b.endSec - b.startSec > a.endSec - a.startSec ? b : a)); +} + +/** Where a short ends in a pause: a little room tone after the voice stops. */ +export function endInPause(pause: Pause): number { + return pause.startSec + Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} + +/** Where a short starts in a pause: a little room tone before the voice. */ +export function startInPause(pause: Pause): number { + return pause.endSec - Math.min(PAUSE_KEEP_SEC, (pause.endSec - pause.startSec) / 2); +} diff --git a/src/timeline/run-plan.ts b/src/timeline/run-plan.ts index 69f64b3..e89b3af 100644 --- a/src/timeline/run-plan.ts +++ b/src/timeline/run-plan.ts @@ -1,5 +1,5 @@ import { randomBytes } from 'node:crypto'; -import { rename, unlink, writeFile } from 'node:fs/promises'; +import { mkdir, rename, unlink, writeFile } from 'node:fs/promises'; import { dirname, extname, join } from 'node:path'; import { buildLoudnormApplyArgs, @@ -7,7 +7,12 @@ import { parseLoudnormJson, } from '../ffmpeg/args/loudnorm.js'; import { filterGraphFileOption, requireDrawtext, runFfmpeg } from '../ffmpeg/run.js'; -import type { FfmpegPlan } from './compile.js'; +import { getWorkspace } from '../workspace.js'; +import { compileTimeline, type FfmpegPlan, type MediaInfo } from './compile.js'; +import type { Composition } from './composition.js'; +import { readComposition } from './document-store.js'; +import { buildMediaMap } from './media-registry.js'; +import type { MediaId } from './schema.js'; export interface LoudnessResult { targetLufs: number; @@ -150,3 +155,56 @@ async function normalizeLoudness( throw err; } } + +/** A timeline `exportTimelines` rendered, or why it couldn't. */ +export type TimelineExport = + | { + timeline: string; + rev: number; + output: string; + durationSec: number; + durationMs: number; + loudness?: LoudnessResult; + } + | { timeline: string; error: string }; + +/** + * Render each timeline to `/.mp4`, one at a time. A timeline that + * can't render is reported with its error, and the rest still render. `media` + * resolves each document's files (default: the registry as ingested); callers + * acting for the UI pass one that re-checks them against the workspace. + */ +export async function exportTimelines( + timelines: readonly string[], + dir: string, + media: (comp: Composition) => Promise> = () => buildMediaMap(), +): Promise { + await mkdir(dir, { recursive: true }); + // Intermediates are named per run, so a concurrent export or frame preview + // in the same workspace can't overwrite them. + const run = randomBytes(4).toString('hex'); + const results: TimelineExport[] = []; + for (const timeline of timelines) { + try { + const comp = await readComposition(timeline); + const plan = compileTimeline(comp, { + media: await media(comp), + dir: getWorkspace(), + output: join(dir, `${timeline}.mp4`), + tag: `${run}-${timeline}`, + }); + const result = await runPlan(plan); + results.push({ + timeline, + rev: comp.rev, + output: result.output, + durationSec: plan.durationSec, + durationMs: result.durationMs, + ...(result.loudness ? { loudness: result.loudness } : {}), + }); + } catch (err) { + results.push({ timeline, error: err instanceof Error ? err.message : String(err) }); + } + } + return results; +} diff --git a/src/timeline/schema.ts b/src/timeline/schema.ts index 88b07c5..76881ae 100644 --- a/src/timeline/schema.ts +++ b/src/timeline/schema.ts @@ -9,6 +9,20 @@ export const MediaIdSchema = z.string().regex(/^m_[a-f0-9]{12}$/, { message: 'MediaId must be of the form m_<12 hex chars>', }); +/** The timeline every workspace starts with, kept at `composition.json`. */ +export const MAIN_TIMELINE = 'main'; + +/** A named timeline in the workspace: `main`, or a short such as `best-1`. */ +export const TimelineIdSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9-]{0,47}$/, { + message: + 'A timeline id is 1–48 lowercase letters, digits or hyphens, starting with a letter or digit (e.g. "best-1").', + }) + .describe( + 'Timeline to work on (default "main"). A short made with make_short is a timeline of its own.', + ); + export const VideoStreamSchema = z.object({ codec: z.string(), width: z.number().int().positive(), diff --git a/src/timeline/shorts.ts b/src/timeline/shorts.ts new file mode 100644 index 0000000..7ba17bc --- /dev/null +++ b/src/timeline/shorts.ts @@ -0,0 +1,609 @@ +import { + captionWidthBudget, + DEFAULT_CAPTION_TRACK, + DEFAULT_CAPTION_WORDS, + defaultCaptionStyle, + endsSentence, + fitCaptionStyle, + groupCaptions, + PAUSE_SEC, +} from './captions.js'; +import { speedFactor, videoLayers } from './compile.js'; +import { + type Clip, + type Composition, + clipEndSec, + type Effect, + type MediaClip, + makeClipId, +} from './composition.js'; +import { applyOps, type CompositionOp, mediaClip, textClip, videoTrack } from './ops.js'; +import { endInPause, pauseAt, type SpeechLevels, startInPause } from './pauses.js'; +import { type Aspect, canvasFor, planReframe } from './reframe.js'; +import { MAIN_TIMELINE, type MediaId } from './schema.js'; +import { clipWords, type Transcript } from './transcript.js'; + +/** + * Shorts: a vertical clip cut out of a longer timeline by transcript word ranges, + * built on a timeline of its own. It is cut from the source timeline's clips, not + * from the raw file, so silence and retake cuts already made there carry in. It + * is a snapshot: later edits to the source don't flow into it. The agent or the + * person picks the words; nothing here scores highlights. + */ + +/** + * Kept before a range's first word. whisper.cpp hands each pause to the word + * before it, so the next word tends to start late: on test speech its first + * words started up to 0.35 s after the voice did. 0.25 s left 3 of 29 sentence + * starts short by at most 0.1 s. A range ends where its last word does, with no + * padding: that word already runs on through the pause after it. + */ +export const SHORT_LEAD_SEC = 0.25; +/** Loudness a short is normalized to unless asked otherwise: the vertical platforms' -14 LUFS. */ +export const DEFAULT_SHORT_LUFS = -14; + +const SHORT_TRACK = 'v0'; +/** A range whose snapped edges leave less than this keeps the transcript's. */ +const MIN_SNAPPED_SEC = 0.2; +/** How long a word pulled inside a cut lasts. */ +const PULLED_WORD_SEC = 0.25; +/** The least a word starts after the one before it. */ +const WORD_ORDER_SEC = 0.01; +/** A piece shorter than this is dropped: too short to hear, just a flash. */ +const MIN_PIECE_SEC = 0.05; +const SAME_SEC = 1e-6; + +/** Words `fromIndex..toIndex` (inclusive) of the transcript behind a media clip + * on the source timeline. */ +export interface ShortRange { + clipId: string; + fromIndex: number; + toIndex: number; + /** The `transcriptId` the indices were read from; refused if it changed. */ + transcriptId?: string; +} + +export interface ShortRequest { + /** Played in this order, so a later moment can open the short as its hook. */ + ranges: ShortRange[]; + aspect?: Aspect; + /** Horizontal position of the picture in the frame: 0 left, 0.5 center, 1 right. */ + x?: number; + y?: number; + /** Burn in captions (default true). */ + captions?: boolean; + maxWords?: number; + /** Default -14; null leaves loudness off. */ + targetLufs?: number | null; + /** Rebuild a timeline that already holds a short. */ + replace?: boolean; +} + +/** A window of a source file the short plays, in source seconds. */ +export interface ShortPiece { + mediaId: MediaId; + sourceInSec: number; + sourceOutSec: number; +} + +export interface ShortPlan { + ops: CompositionOp[]; + pieces: ShortPiece[]; + durationSec: number; + /** Captions written (0 with captions off). */ + captions: number; + warnings: string[]; +} + +/** Thrown when a short can't be made from the request. */ +export class ShortError extends Error { + constructor(message: string) { + super(message); + this.name = 'ShortError'; + } +} + +/** A media clip of `comp` and its source's stored transcript (never transcribed here). */ +export type ShortTranscriptLookup = ( + comp: Composition, + clipId: string, + transcriptId?: string, +) => Promise<{ clip: MediaClip; transcript: Transcript }>; + +/** The audio levels of a media clip's source around a time, in source seconds + * (runs FFmpeg); null when the source has no sound. */ +export type SpeechLevelsLookup = ( + comp: Composition, + clipId: string, + aroundSec: number, +) => Promise; + +/** What `planShort` reads beyond the documents. */ +export interface ShortSources { + transcript: ShortTranscriptLookup; + /** Moves each edge into the pause the audio shows there. Without it, edges + * follow the transcript. */ + levels?: SpeechLevelsLookup; + mintId?: () => string; +} + +/** Refuse a short on `main` or on the timeline it is cut from. */ +export function checkShortTimelines(target: string, source: string = MAIN_TIMELINE): void { + if (target === MAIN_TIMELINE) { + throw new ShortError( + 'A short is built on a timeline of its own: pick a new timeline id such as "best-1", not main.', + ); + } + if (source === target) { + throw new ShortError( + `A short can't be cut from its own timeline "${target}". Cut it from main or another timeline.`, + ); + } +} + +interface WordWindow { + mediaId: MediaId; + transcript: Transcript; + /** The range's words, by index into its transcript. */ + fromIndex: number; + toIndex: number; + words: { startSec: number; endSec: number }[]; + /** Padded edges, source seconds. */ + start: number; + end: number; + /** First word's start and last word's end. */ + speechStart: number; + speechEnd: number; +} + +interface Piece extends ShortPiece { + effects: Effect[]; + /** The source clip's speed, refused only if the piece is played. */ + speed: number; + sourceClipId: string; + transcript: Transcript; + /** The word ranges the piece plays, which are what it captions, with the + * source window each was cut to. */ + asked: { fromIndex: number; toIndex: number; start: number; end: number }[]; +} + +/** + * Plan the edit that builds a short on `target` from `source`: for each range, + * the source timeline's clips under its words (padded), laid end to end on a + * fresh picture track, on a 9:16 canvas filled edge to edge, normalized to + * -14 LUFS and captioned. One batch, so one undo takes the whole short away. + */ +export async function planShort( + target: Composition, + source: Composition, + request: ShortRequest, + sources: ShortSources, +): Promise { + const mintId = sources.mintId ?? makeClipId; + if (request.ranges.length === 0) throw new ShortError('Give at least one word range.'); + if (!request.replace && target.tracks.some((t) => t.clips.length > 0)) { + throw new ShortError( + 'That timeline already holds a short. Pass replace (`--replace`) to rebuild it, or pick a new timeline id.', + ); + } + const picture = videoLayers(source).picture; + if (!picture) { + throw new ShortError( + 'The source timeline has no picture track to cut from: it is empty or has more than one.', + ); + } + + const windows: WordWindow[] = []; + const warnings: string[] = []; + for (const [i, range] of request.ranges.entries()) { + const { clip, transcript } = await sources.transcript(source, range.clipId, range.transcriptId); + const window = wordWindow(transcript, range, clip); + warnings.push(...sentenceEdgeWarnings(window, i + 1)); + windows.push( + sources.levels + ? await snapToPauses(window, source, clip.id, sources.levels, i + 1, warnings) + : window, + ); + } + const pieces: Piece[] = []; + windows.forEach((window, i) => { + const range = request.ranges[i]; + const found = piecesUnder(picture.clips, withoutReplay(window, windows[i - 1])); + if (found.length === 0) { + throw new ShortError( + `Words ${range?.fromIndex}..${range?.toIndex} were cut out of the source timeline, so there is nothing to play. Pick words that are still on it.`, + ); + } + pieces.push(...found); + }); + const sped = pieces.find((p) => p.speed !== 1); + if (sped) { + throw new ShortError( + `Clip "${sped.sourceClipId}" on the source timeline plays at ${sped.speed}x, and a short can't include a speed change yet. ` + + 'Remove its speed effect or pick words outside it.', + ); + } + const joined = joinContinuous(pieces); + + const ops: CompositionOp[] = target.tracks.map( + (t): CompositionOp => ({ op: 'removeTrack', trackId: t.id }), + ); + const canvas = canvasFor(source, request.aspect ?? '9:16'); + ops.push( + { + op: 'setCanvas', + width: canvas.width, + height: canvas.height, + fps: source.fps, + background: source.background, + fit: 'cover', + }, + { op: 'addTrack', track: videoTrack({ id: SHORT_TRACK }) }, + ); + let at = 0; + const placed = joined.map((piece) => { + const clip = mediaClip({ + id: mintId(), + mediaId: piece.mediaId, + sourceInSec: piece.sourceInSec, + sourceOutSec: piece.sourceOutSec, + // Rounded to the microsecond: clean numbers, far inside the export's + // abutting tolerance. + startSec: Math.round(at * 1e6) / 1e6, + effects: piece.effects, + }); + ops.push({ op: 'addClip', trackId: SHORT_TRACK, clip }); + at += piece.sourceOutSec - piece.sourceInSec; + return { piece, clip }; + }); + + if (request.x !== undefined || request.y !== undefined) { + const framed = planReframe(applyOps(target, ops), { + width: canvas.width, + height: canvas.height, + fit: 'cover', + x: request.x, + y: request.y, + }); + ops.push(...framed.ops); + } + + const targetLufs = request.targetLufs === undefined ? DEFAULT_SHORT_LUFS : request.targetLufs; + if ((target.loudness?.targetLufs ?? null) !== targetLufs) { + ops.push({ op: 'setLoudness', targetLufs }); + } + + let captions = 0; + if (request.captions !== false) { + // Each piece captions only its own range's words: the lead reaches into the + // word before, which whisper.cpp stretches over the pause, and that word + // isn't part of the short even when another range asks for it. + const built = applyOps(target, ops); + const style = defaultCaptionStyle(built); + const maxWidth = captionWidthBudget(built, style); + const cues = placed.flatMap(({ piece, clip }) => { + const words = piece.asked.flatMap((r) => + wordsInside(piece.transcript.words.slice(r.fromIndex, r.toIndex + 1), r.start, r.end), + ); + return groupCaptions( + clipWords({ ...piece.transcript, words }, clip), + clipEndSec(clip), + request.maxWords ?? DEFAULT_CAPTION_WORDS, + maxWidth, + ); + }); + if (cues.length > 0) { + ops.push({ op: 'addTrack', track: videoTrack({ id: DEFAULT_CAPTION_TRACK }) }); + for (const cue of cues) { + ops.push({ + op: 'addClip', + trackId: DEFAULT_CAPTION_TRACK, + clip: textClip({ + id: mintId(), + text: cue.text, + startSec: cue.startSec, + durationSec: roundMs(cue.endSec - cue.startSec), + style: fitCaptionStyle(cue.text, style, built), + }), + }); + } + } + captions = cues.length; + } + + return { + ops, + pieces: joined.map(({ mediaId, sourceInSec, sourceOutSec }) => ({ + mediaId, + sourceInSec, + sourceOutSec, + })), + durationSec: roundMs(at), + captions, + warnings, + }; +} + +/** + * The source window a range plays: its first word's start minus the lead, to + * its last word's end. The lead stops halfway into the word before (whisper.cpp + * words abut, so that word also holds the pause) and at the start of the file. + */ +function wordWindow(transcript: Transcript, range: ShortRange, clip: MediaClip): WordWindow { + const first = transcript.words[range.fromIndex]; + const last = transcript.words[range.toIndex]; + if (!first || !last || range.toIndex < range.fromIndex) { + throw new ShortError( + `Words ${range.fromIndex}..${range.toIndex} aren't in the transcript of clip "${clip.id}" ` + + `(it has ${transcript.words.length} words, numbered from 0).`, + ); + } + const before = transcript.words[range.fromIndex - 1]; + const start = Math.min( + first.startSec, + Math.max( + first.startSec - SHORT_LEAD_SEC, + before ? (before.startSec + before.endSec) / 2 : 0, + 0, + ), + ); + return { + mediaId: clip.mediaId, + transcript, + fromIndex: range.fromIndex, + toIndex: range.toIndex, + words: transcript.words.slice(range.fromIndex, range.toIndex + 1), + start: roundMs(start), + end: last.endSec, + speechStart: first.startSec, + speechEnd: last.endSec, + }; +} + +/** + * Move a range's edges into the pauses its source's audio shows at its + * transcript boundaries: a little room tone before the first word and after + * the last, never the next sentence's first syllable. An edge with no clear + * pause (speech runs on, music under it, or two pauses the transcript can't + * tell apart) keeps the transcript rule, with a warning unless the range was + * cut mid-sentence there anyway. + */ +async function snapToPauses( + window: WordWindow, + source: Composition, + clipId: string, + levelsFor: SpeechLevelsLookup, + rangeNumber: number, + warnings: string[], +): Promise { + const words = window.transcript.words; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (!first || !last) return window; + + let { start, end } = window; + let startLevels: SpeechLevels | null; + let endLevels: SpeechLevels | null; + try { + startLevels = await levelsFor(source, clipId, first.startSec); + endLevels = startLevels && (await levelsFor(source, clipId, last.endSec)); + } catch { + // FFmpeg couldn't read the audio; the short is still made, by the transcript. + warnings.push( + `Range ${rangeNumber}: its audio couldn't be measured, so its edges follow the transcript. Look at it or listen.`, + ); + return window; + } + if (!startLevels) return window; // no sound to measure + const startPause = pauseAt(startLevels, first.startSec, before?.startSec ?? 0); + if (typeof startPause === 'object') start = startInPause(startPause); + else if (before && !midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber}: no clear pause before "${first.text}", so its start follows the transcript and may clip that word or keep the end of "${before.text}". Look at it or listen.`, + ); + } + const endPause = endLevels ? pauseAt(endLevels, last.endSec, last.startSec) : 'none'; + if (typeof endPause === 'object') end = endInPause(endPause); + else if (after && !midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber}: no clear pause after "${last.text}", so its end follows the transcript and may keep the start of "${after.text}". Look at it or listen.`, + ); + } + // A pause found on the wrong side of a short range would leave nothing. + if (end - start < MIN_SNAPPED_SEC) return window; + start = roundMs(start); + end = roundMs(end); + return { ...window, start, end, words: wordsInside(window.words, start, end) }; +} + +/** + * A range's words kept inside the window it was cut to, in order. A word + * whisper places past an edge (it runs late at a sentence end) is pulled just + * inside, so the range still holds its speech and the word keeps its caption; + * no word starts before the one ahead of it, or a caption shown a word at a + * time would lose that one. + */ +function wordsInside( + words: W[], + start: number, + end: number, +): W[] { + const starts = words.map((w) => { + if (w.startSec >= end) return Math.max(start, end - PULLED_WORD_SEC); + if (w.endSec <= start) return start; + return w.startSec; + }); + // In order going forward, then back from the cut: a word pushed up against + // the end makes room by moving the ones before it earlier. + for (let i = 1; i < starts.length; i++) { + starts[i] = Math.max(starts[i] ?? 0, (starts[i - 1] ?? 0) + WORD_ORDER_SEC); + } + for (let i = starts.length - 1; i >= 0; i--) { + const limit = i === starts.length - 1 ? end : (starts[i + 1] ?? end); + starts[i] = Math.max(start, Math.min(starts[i] ?? 0, limit - WORD_ORDER_SEC)); + } + return words.map((w, i) => { + const startSec = starts[i] ?? w.startSec; + const endSec = + w.startSec >= end + ? end + : w.endSec <= start + ? Math.min(end, start + PULLED_WORD_SEC) + : Math.max(w.endSec, startSec + WORD_ORDER_SEC); + return startSec === w.startSec && endSec === w.endSec ? w : { ...w, startSec, endSec }; + }); +} + +/** Whether two neighboring words belong to one sentence: no sentence end and + * no real pause between them. */ +function midSentence(a: { text: string; endSec: number }, b: { startSec: number }): boolean { + return !endsSentence(a.text) && b.startSec - a.endSec < PAUSE_SEC; +} + +/** + * Where a range starts or ends inside a sentence: right after a word that + * doesn't end one (with no real pause between), or right before the next word + * of the same sentence. Named by the words on either side; the short is still + * made, since a cut mid-sentence can be meant. + */ +function sentenceEdgeWarnings(window: WordWindow, rangeNumber: number): string[] { + const words = window.transcript.words; + const warnings: string[] = []; + const before = words[window.fromIndex - 1]; + const first = words[window.fromIndex]; + if (before && first && midSentence(before, first)) { + warnings.push( + `Range ${rangeNumber} starts mid-sentence, between "${before.text}" and "${first.text}" (word ${window.fromIndex}).`, + ); + } + const last = words[window.toIndex]; + const after = words[window.toIndex + 1]; + if (last && after && midSentence(last, after)) { + warnings.push( + `Range ${rangeNumber} ends mid-sentence, between "${last.text}" and "${after.text}" (word ${window.toIndex}).`, + ); + } + return warnings; +} + +/** + * A window that picks up later in the same file than the one before it starts + * where that one's padding ended, so two ranges of neighboring words play + * straight through instead of replaying the padding they share. A range that + * repeats or goes back to earlier words is left as asked. + */ +function withoutReplay(window: WordWindow, before: WordWindow | undefined): WordWindow { + if ( + !before || + before.mediaId !== window.mediaId || + window.speechStart < before.speechEnd || + window.start >= before.end + ) { + return window; + } + return { ...window, start: Math.min(before.end, window.speechStart) }; +} + +/** + * The parts of a window that the source timeline's picture clips of its file + * play, in timeline order. A moment the timeline plays twice is taken once. + * Parts a split left continuous are joined first; then a part holding none of + * the range's words (only padding) is left out, since it would only add a jump + * cut, as is a sliver too short to see. + */ +function piecesUnder(clips: readonly Clip[], window: WordWindow): Piece[] { + const taken: [number, number][] = []; + const parts: Piece[] = []; + for (const clip of [...clips].sort((a, b) => a.startSec - b.startSec)) { + if (clip.kind !== 'media' || clip.mediaId !== window.mediaId) continue; + const lo = Math.max(window.start, clip.sourceInSec); + const hi = Math.min(window.end, clip.sourceOutSec); + if (hi <= lo) continue; + for (const [a, b] of subtract([lo, hi], taken)) { + parts.push({ + mediaId: window.mediaId, + sourceInSec: a, + sourceOutSec: b, + // Fades belonged to the source clip's edges, not the short's. + effects: clip.effects.filter((e) => e.type === 'adjust' || e.type === 'volume'), + speed: speedFactor(clip.effects), + sourceClipId: clip.id, + transcript: window.transcript, + asked: [ + { + fromIndex: window.fromIndex, + toIndex: window.toIndex, + start: window.start, + end: window.end, + }, + ], + }); + taken.push([a, b]); + } + } + return joinContinuous(parts).filter( + (p) => p.sourceOutSec - p.sourceInSec >= MIN_PIECE_SEC && holdsSpeech(p, window), + ); +} + +/** Whether a piece plays any of the range's words, not just touches one's edge + * or the gap between two that were cut. */ +function holdsSpeech(piece: Piece, window: WordWindow): boolean { + return window.words.some((w) => + w.endSec > w.startSec + ? piece.sourceOutSec > w.startSec && piece.sourceInSec < w.endSec + : piece.sourceInSec <= w.startSec && piece.sourceOutSec >= w.endSec, + ); +} + +/** `[lo, hi]` minus the `taken` intervals, in order. */ +function subtract([lo, hi]: [number, number], taken: [number, number][]): [number, number][] { + let parts: [number, number][] = [[lo, hi]]; + for (const [a, b] of taken) { + parts = parts.flatMap(([p, q]): [number, number][] => { + if (b <= p || a >= q) return [[p, q]]; + const rest: [number, number][] = []; + if (a > p) rest.push([p, a]); + if (b < q) rest.push([b, q]); + return rest; + }); + } + return parts; +} + +/** Merge pieces that continue one another in the same file with the same + * effects (a split, not a cut, on the source), so the short has no cut there. */ +function joinContinuous(pieces: Piece[]): Piece[] { + const joined: Piece[] = []; + for (const piece of pieces) { + const prev = joined[joined.length - 1]; + if ( + prev && + prev.mediaId === piece.mediaId && + Math.abs(prev.sourceOutSec - piece.sourceInSec) < SAME_SEC && + prev.speed === piece.speed && + JSON.stringify(prev.effects) === JSON.stringify(piece.effects) + ) { + joined[joined.length - 1] = { + ...prev, + sourceOutSec: piece.sourceOutSec, + // One range across a split comes back once, or its captions would repeat. + asked: [ + ...prev.asked, + ...piece.asked.filter( + (r) => !prev.asked.some((q) => q.fromIndex === r.fromIndex && q.toIndex === r.toIndex), + ), + ], + }; + } else { + joined.push(piece); + } + } + return joined; +} + +function roundMs(sec: number): number { + return Math.round(sec * 1000) / 1000; +} diff --git a/src/timeline/transcript.ts b/src/timeline/transcript.ts index 5452d55..cd2d742 100644 --- a/src/timeline/transcript.ts +++ b/src/timeline/transcript.ts @@ -13,7 +13,8 @@ import { WhisperLanguageSchema, } from '../transcribe/whisper.js'; import { getWorkspace, resolveInWorkspace } from '../workspace.js'; -import { checkExportable, type MediaInfo, withoutTextOverlays } from './compile.js'; +import { endsSentence, joinWords, PAUSE_SEC } from './captions.js'; +import { checkExportable, type MediaInfo, videoLayers, withoutTextOverlays } from './compile.js'; import { type Composition, findClip, type MediaClip, makeClipId } from './composition.js'; import { applyOp, type CompositionOp, type SourceRange } from './ops.js'; import { type MediaId, MediaIdSchema } from './schema.js'; @@ -305,6 +306,222 @@ export async function transcribeClip( }; } +/** Longest sentence an outline line holds; whisper sometimes leaves out the + * punctuation, and a line has to end somewhere. */ +const MAX_SENTENCE_WORDS = 40; +/** Default size of one outline page, in bytes of compact JSON: about 10k + * tokens, which an MCP client takes in one result. */ +export const OUTLINE_PAGE_BYTES = 40_000; + +/** A word in timeline order, with the media clip it plays in. */ +export interface OutlineWord extends ClipWord { + clipId: string; +} + +/** One sentence of the timeline's speech. `fromIndex..toIndex` are word indices + * into its source's transcript, as `cut_words` and `make_short` take them, and + * `clipId` is the clip its first word plays in. */ +export interface OutlineSentence { + startSec: number; + endSec: number; + clipId: string; + /** The sentence's source, named only when the timeline has more than one, + * so its `transcriptIds` entry can be found. */ + mediaId?: MediaId; + fromIndex: number; + toIndex: number; + text: string; +} + +export interface TimelineOutline { + sentences: OutlineSentence[]; + /** The `transcriptId` of each source the sentences come from. */ + transcriptIds: Record; + /** Where the next page starts (pass it as `fromSec`); absent on the last page. */ + nextFromSec?: number; +} + +/** + * Group words (one source, timeline order) into sentences: a sentence ends at + * sentence-ending punctuation, before a pause longer than `PAUSE_SEC`, after + * `MAX_SENTENCE_WORDS` words, where the timeline goes back in the source, or + * where it skips ahead over words `played` (by index) somewhere else on it, so + * a sentence's index range never takes in words it doesn't show. + */ +export function outlineSentences( + words: OutlineWord[], + played: ReadonlySet = new Set(), +): OutlineSentence[] { + const groups: OutlineWord[][] = []; + let current: OutlineWord[] = []; + for (const word of words) { + const prev = current[current.length - 1]; + if ( + prev && + (endsSentence(prev.text) || + word.startSec - prev.endSec > PAUSE_SEC || + word.index <= prev.index || + skipsPlayed(prev.index, word.index, played) || + current.length >= MAX_SENTENCE_WORDS) + ) { + groups.push(current); + current = []; + } + current.push(word); + } + if (current.length > 0) groups.push(current); + return groups.flatMap((group) => { + const first = group[0]; + const last = group[group.length - 1]; + if (!first || !last) return []; + return [ + { + startSec: first.startSec, + endSec: last.endSec, + clipId: first.clipId, + fromIndex: first.index, + toIndex: last.index, + text: joinWords(group.map((w) => w.text)), + }, + ]; + }); +} + +function skipsPlayed(from: number, to: number, played: ReadonlySet): boolean { + for (let i = from + 1; i < to; i++) if (played.has(i)) return true; + return false; +} + +export interface OutlineOptions extends TranscribeOptions { + /** Only this media clip's words. */ + clipId?: string; + /** Sentences starting at or after this timeline time. */ + fromSec?: number; + /** Sentences starting before this timeline time. */ + toSec?: number; + /** Page size in bytes of compact JSON (default `OUTLINE_PAGE_BYTES`). */ + maxBytes?: number; + /** How a source's transcript is had (default: `transcribeMedia`, cached). */ + transcribe?: (mediaId: MediaId, info: MediaInfo) => Promise; +} + +/** + * The timeline's speech as sentences, in timeline order: every media clip on + * the picture track, each source transcribed once (or read from its stored + * transcript). A sentence runs on across cuts between clips of the same source + * (a silence cut splits a clip mid-sentence), and a word a cut splits in two is + * listed once. A page holds at most `maxBytes` of compact JSON; `nextFromSec` + * says where the next one starts. + */ +export async function timelineOutline( + comp: Composition, + media: Map, + opts: OutlineOptions = {}, +): Promise { + const transcribe = opts.transcribe ?? ((mediaId, info) => transcribeMedia(mediaId, info, opts)); + const clips = + opts.clipId !== undefined + ? [locateMediaClip(comp, opts.clipId, media).clip] + : (videoLayers(comp).picture?.clips ?? []) + .filter((c): c is MediaClip => c.kind === 'media') + .sort((a, b) => a.startSec - b.startSec); + + const transcripts = new Map(); + const placed: { clip: MediaClip; words: ClipWord[] }[] = []; + /** Word indices each source plays anywhere in the outlined clips. */ + const played = new Map>(); + for (const clip of clips) { + let transcript = transcripts.get(clip.mediaId); + if (transcript === undefined) { + const info = media.get(clip.mediaId); + if (!info) { + throw new TranscriptError(`No media registered for "${clip.mediaId}" (clip "${clip.id}").`); + } + transcript = info.hasAudio ? await transcribe(clip.mediaId, info) : null; + transcripts.set(clip.mediaId, transcript); + } + if (!transcript) continue; + const words = clipWords(transcript, clip); + placed.push({ clip, words }); + const indices = played.get(clip.mediaId) ?? new Set(); + for (const w of words) indices.add(w.index); + played.set(clip.mediaId, indices); + } + + const bySource: { mediaId: MediaId; sentences: OutlineSentence[] }[] = []; + let run: OutlineWord[] = []; + let runMedia: MediaId | undefined; + let prevClip: MediaClip | undefined; + const flush = (): void => { + if (runMedia) { + bySource.push({ mediaId: runMedia, sentences: outlineSentences(run, played.get(runMedia)) }); + } + }; + for (const { clip, words } of placed) { + if (clip.mediaId !== runMedia) { + flush(); + run = []; + runMedia = clip.mediaId; + } + // A clip that carries on later in the source than the one before it (a cut + // between them), rather than playing part of it again. + const continues = + prevClip?.mediaId === clip.mediaId && clip.sourceInSec >= prevClip.sourceOutSec - 1e-6; + for (const word of words) { + // A word a cut runs through shows in the clips on both sides of it: one + // word, from its start in the first to its end in the last. + const last = run[run.length - 1]; + if (continues && last?.index === word.index) { + run[run.length - 1] = { ...last, endSec: word.endSec }; + } else { + run.push({ ...word, clipId: clip.id }); + } + } + prevClip = clip; + } + flush(); + + const transcriptIds: Record = {}; + for (const [mediaId, transcript] of transcripts) { + if (transcript) transcriptIds[mediaId] = transcriptIdOf(transcript); + } + const named = Object.keys(transcriptIds).length > 1; + const sentences = bySource.flatMap(({ mediaId, sentences }) => + named ? sentences.map((s) => ({ ...s, mediaId })) : sentences, + ); + const fromSec = opts.fromSec ?? 0; + const toSec = opts.toSec ?? Number.POSITIVE_INFINITY; + const inRange = sentences.filter((s) => s.startSec >= fromSec && s.startSec < toSec); + + const maxBytes = opts.maxBytes ?? OUTLINE_PAGE_BYTES; + // The page's own fields, with room for the largest nextFromSec. + let bytes = Buffer.byteLength( + JSON.stringify({ sentences: [], transcriptIds, nextFromSec: 99999.999 }), + ); + const page: OutlineSentence[] = []; + for (const [i, sentence] of inRange.entries()) { + const size = Buffer.byteLength(JSON.stringify(sentence)) + 1; + if (bytes + size > maxBytes && page.length > 0) { + // The next page starts at a time, so sentences that start together stay + // on one page: split before them, or, when they fill this page from its + // first line, keep all of them here. + const tied = sentence.startSec; + const kept = page.filter((s) => s.startSec !== tied); + if (kept.length > 0) return { sentences: kept, transcriptIds, nextFromSec: tied }; + const rest = inRange.slice(i).findIndex((s) => s.startSec !== tied); + if (rest === -1) return { sentences: inRange, transcriptIds }; + return { + sentences: inRange.slice(0, i + rest), + transcriptIds, + nextFromSec: inRange[i + rest]?.startSec, + }; + } + page.push(sentence); + bytes += size; + } + return { sentences: page, transcriptIds }; +} + /** An inclusive range of transcript word indices. */ export interface WordRange { fromIndex: number; diff --git a/src/timeline/verbs.ts b/src/timeline/verbs.ts index b72e9ec..b0ad625 100644 --- a/src/timeline/verbs.ts +++ b/src/timeline/verbs.ts @@ -1,5 +1,11 @@ import { z } from 'zod'; -import { DEFAULT_CAPTION_TRACK, DEFAULT_CAPTION_WORDS, planCaptions } from './captions.js'; +import { + captionSideMargin, + DEFAULT_CAPTION_TRACK, + DEFAULT_CAPTION_WORDS, + planCaptions, +} from './captions.js'; +import { ColorSchema } from './color.js'; import type { MediaInfo } from './compile.js'; import { AnchorSchema, @@ -21,7 +27,8 @@ import { planFraming, planReframe, } from './reframe.js'; -import type { MediaId } from './schema.js'; +import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from './schema.js'; +import { checkShortTimelines, planShort, type SpeechLevelsLookup } from './shorts.js'; import { SensitivitySchema, type SilenceCut } from './silence.js'; import { clipWords, @@ -109,7 +116,9 @@ const EDITABLE_VERB_SCHEMAS = [ 'Optional clip id — supply one to reference this clip in a later verb of the SAME batch.', ), durationSec: z.number().positive(), - color: z.string().optional().describe('Card color (default black).'), + color: ColorSchema.optional().describe( + 'Card color: a name like black or white, or hex like #ffcc00 (default black).', + ), track: z.string().optional(), startSec: z.number().nonnegative().optional(), }), @@ -206,6 +215,14 @@ const EDITABLE_VERB_SCHEMAS = [ .optional() .describe('Box color behind the text (default black@0.6), or null for none.'), anchor: AnchorSchema.optional().describe('Where captions sit (default bottom-center).'), + margin: z + .number() + .int() + .nonnegative() + .optional() + .describe( + 'How far, in pixels, captions keep from the bottom (or top) edge. Default: a quarter of the height on a portrait canvas, clear of the Shorts, Reels and TikTok overlays; 40 otherwise.', + ), }), z.object({ verb: z.literal('reframe'), @@ -277,6 +294,71 @@ const EDITABLE_VERB_SCHEMAS = [ 'Normalize the exported mp4 to this integrated loudness in LUFS: -14 for YouTube, Shorts, Reels and TikTok, -16 for podcasts. null turns it off. Applies to mp4 export only.', ), }), + z.object({ + verb: z.literal('make_short'), + ranges: z + .array( + z.object({ + clipId: z + .string() + .describe( + 'A media clip on the source timeline whose transcript the indices come from.', + ), + fromIndex: z + .number() + .int() + .nonnegative() + .describe('First word of the range: its `index` from timeline_transcript.'), + toIndex: z.number().int().nonnegative().describe('Last word of the range (inclusive).'), + transcriptId: z + .string() + .optional() + .describe( + 'The `transcriptId` the indices were read from; refused if the clip was transcribed again since.', + ), + }), + ) + .min(1) + .max(20) + .describe( + 'Word ranges, played in this order (a later moment can open the short as its hook). Pick whole sentences from the timeline_transcript sentence outline; 15–60 s in all is typical. A range that starts or ends mid-sentence is made anyway, with a warning.', + ), + source: TimelineIdSchema.optional().describe( + 'Timeline to cut from (default "main"). Its silence and retake cuts carry into the short.', + ), + aspect: AspectSchema.optional().describe('Output shape (default 9:16).'), + x: z + .number() + .min(0) + .max(1) + .optional() + .describe( + 'Horizontal position of the picture: 0 shows the left edge of the source, 0.5 the center, 1 the right edge. Put the speaker in frame.', + ), + y: z.number().min(0).max(1).optional().describe('Vertical position: 0 top, 1 bottom.'), + captions: z + .boolean() + .optional() + .describe('Burn in captions from the transcript (default true).'), + maxWords: z + .number() + .int() + .min(1) + .max(12) + .optional() + .describe( + `Most words per caption (default ${DEFAULT_CAPTION_WORDS}); 1 shows one word at a time.`, + ), + targetLufs: TargetLufsSchema.nullable() + .optional() + .describe( + 'Export loudness in LUFS (default -14, for Shorts, Reels and TikTok); null for off.', + ), + replace: z + .boolean() + .optional() + .describe('Rebuild this timeline when it already holds a short (default: refuse).'), + }), z.object({ verb: z.literal('transition'), afterClipId: z @@ -331,8 +413,18 @@ export interface VerbContext { clipId: string, transcriptId?: string, ) => Promise<{ clip: MediaClip; transcript: Transcript; media: Map }>; + /** Read another timeline of the workspace (make_short's source). */ + readTimeline: (timeline: string) => Promise; + /** A media clip's source audio levels around a time (runs FFmpeg), for + * placing a short's edges in pauses; null when the source has no sound. */ + speechLevels: SpeechLevelsLookup; /** Track a clip is placed on when the verb doesn't say. */ defaultTrack: string; + /** The timeline being edited (default `main`); `applyVerbs` sets it. */ + timeline?: string; + /** Hears what a verb did that the caller may not have meant (e.g. a short + * cut mid-sentence); `applyVerbs` collects them. */ + warn?: (message: string) => void; } const DEFAULT_TRACK = 'v0'; @@ -472,6 +564,7 @@ export async function lowerVerb( if (verb.color !== undefined) style.color = verb.color; if (verb.background !== undefined) style.background = verb.background; if (verb.anchor !== undefined) style.anchor = verb.anchor; + if (verb.margin !== undefined) style.margin = { x: captionSideMargin(comp), y: verb.margin }; const plan = await planCaptions( comp, { @@ -493,6 +586,27 @@ export async function lowerVerb( return (comp.loudness?.targetLufs ?? null) === verb.targetLufs ? [] : [{ op: 'setLoudness', targetLufs: verb.targetLufs }]; + case 'make_short': { + const sourceId = verb.source ?? MAIN_TIMELINE; + checkShortTimelines(ctx.timeline ?? MAIN_TIMELINE, sourceId); + const plan = await planShort( + comp, + await ctx.readTimeline(sourceId), + { + ranges: verb.ranges, + aspect: verb.aspect, + x: verb.x, + y: verb.y, + captions: verb.captions, + maxWords: verb.maxWords, + targetLufs: verb.targetLufs, + replace: verb.replace, + }, + { transcript: ctx.transcript, levels: ctx.speechLevels }, + ); + for (const warning of plan.warnings) ctx.warn?.(warning); + return plan.ops; + } case 'transition': return [ { diff --git a/src/ui/server.ts b/src/ui/server.ts index 735a0d4..319f47c 100644 --- a/src/ui/server.ts +++ b/src/ui/server.ts @@ -8,7 +8,7 @@ import { fileURLToPath } from 'node:url'; import { serve } from '@hono/node-server'; import { serveStatic } from '@hono/node-server/serve-static'; import getPort from 'get-port'; -import { Hono } from 'hono'; +import { type Context, Hono } from 'hono'; import { csrf } from 'hono/csrf'; import open from 'open'; import { ZodError, z } from 'zod'; @@ -26,9 +26,13 @@ import { import { applyVerbs, CompositionConflictError, + deleteTimeline, + listTimelines, + NoTimelineError, readComposition, readDocOpLog, redoDocOp, + TimelineIdError, undoLastDocOp, } from '../timeline/document-store.js'; import { @@ -40,8 +44,9 @@ import { import { buildMediaMap } from '../timeline/media-registry.js'; import { CompositionOpError } from '../timeline/ops.js'; import { ReframeError } from '../timeline/reframe.js'; -import { runPlan } from '../timeline/run-plan.js'; -import type { MediaId } from '../timeline/schema.js'; +import { exportTimelines, runPlan } from '../timeline/run-plan.js'; +import { MAIN_TIMELINE, type MediaId, TimelineIdSchema } from '../timeline/schema.js'; +import { ShortError } from '../timeline/shorts.js'; import { SilenceAnalysisError } from '../timeline/silence.js'; import { TranscriptError, transcribeClip } from '../timeline/transcript.js'; import { CompositionVerbSchema } from '../timeline/verbs.js'; @@ -57,14 +62,20 @@ import { resolveInWorkspace, WorkspaceBoundaryError, } from '../workspace.js'; -import { clipConfinedMedia, confinedMediaRefs, makeVerbContext } from './timeline-tools.js'; +import { + clipConfinedMedia, + confinedMediaMap, + confinedMediaRefs, + makeVerbContext, +} from './timeline-tools.js'; /** Errors about the clip or its setup, not the server: silence removal or - * transcription can't work on this clip, whisper.cpp isn't set up, or a - * reframe asks for a size or position that can't be. */ + * transcription can't work on this clip, whisper.cpp isn't set up, a reframe + * asks for a size or position that can't be, or a short can't be made. */ function isClipConditionError(err: unknown): boolean { return ( err instanceof ReframeError || + err instanceof ShortError || err instanceof SilenceAnalysisError || err instanceof TranscriptError || err instanceof TranscriptionUnavailableError @@ -73,6 +84,17 @@ function isClipConditionError(err: unknown): boolean { import { isRegisteredTool, TOOL_REGISTRY } from './tool-registry.js'; +/** The timeline a `/api/timeline*` request names with `?timeline=` (default + * `main`); the middleware has already refused a malformed id. */ +function timelineOf(c: Context): string { + return c.req.query('timeline') ?? MAIN_TIMELINE; +} + +/** A timeline that doesn't exist, or a malformed id: `onError` answers 404 or 400. */ +function isTimelineError(err: unknown): boolean { + return err instanceof NoTimelineError || err instanceof TimelineIdError; +} + const HERE = dirname(fileURLToPath(import.meta.url)); // `HERE` resolves to different paths depending on whether we're running // the bundled `dist/cli.js` (everything is flattened so `HERE === dist/`) @@ -159,6 +181,18 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { + if (err instanceof NoTimelineError) return c.json({ error: err.message }, 404); + if (err instanceof TimelineIdError) return c.json({ error: err.message }, 400); + // Otherwise answer as Hono does by default. + if ('getResponse' in err && typeof err.getResponse === 'function') { + const res = (err.getResponse as () => Response)(); + return c.newResponse(res.body, res); + } + console.error(err); + return c.text('Internal Server Error', 500); + }); + app.get('/api/workspace', async (c) => { return c.json({ path: getWorkspace() }); }); @@ -226,9 +260,19 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { + const id = c.req.query('timeline') ?? MAIN_TIMELINE; + if (!TimelineIdSchema.safeParse(id).success) { + return c.json({ error: `Invalid timeline id "${id}".` }, 400); + } + await next(); + }); + + /** GET /api/timeline — the timeline's composition document. */ app.get('/api/timeline', async (c) => { - return c.json({ document: await readComposition() }); + return c.json({ document: await readComposition(timelineOf(c)) }); }); /** @@ -248,9 +292,10 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { try { - const comp = await readComposition(); + const comp = await readComposition(timelineOf(c)); const clipId = c.req.param('clipId'); return c.json(await transcribeClip(comp, clipId, await clipConfinedMedia(comp, clipId))); } catch (err) { + if (isTimelineError(err)) throw err; if (err instanceof WorkspaceBoundaryError) return c.json({ error: err.message }, 403); if (isClipConditionError(err)) return c.json({ error: (err as Error).message }, 422); const message = err instanceof Error ? err.message : String(err); @@ -290,7 +336,7 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { - const comp = await readComposition(); + const comp = await readComposition(timelineOf(c)); const media = await buildMediaMap(); return c.json(checkExportable(comp, media, getWorkspace())); }); @@ -309,7 +355,7 @@ export async function startUiServer(options: UiServerOptions = {}): Promise {}); + if (isTimelineError(err)) throw err; if (err instanceof CompileError) return c.json({ error: err.message }, 422); // Not a document problem — a transient render failure (e.g. an FFmpeg // hiccup). Surface it as retryable instead of leaking a raw 500. @@ -389,12 +436,13 @@ export async function startUiServer(options: UiServerOptions = {}): Promise { - const { undone, label } = await undoLastDocOp(); + const { undone, label } = await undoLastDocOp(timelineOf(c)); return c.json({ undone, label: label ?? null }); }); /** POST /api/timeline/redo — redo the most recently undone document edit. */ app.post('/api/timeline/redo', async (c) => { - const { redone, label } = await redoDocOp(); + const { redone, label } = await redoDocOp(timelineOf(c)); return c.json({ redone, label: label ?? null }); }); /** GET /api/timeline/history — undo/redo availability for the doc op-log. */ app.get('/api/timeline/history', async (c) => { - const log = await readDocOpLog(); + const log = await readDocOpLog(timelineOf(c)); return c.json({ canUndo: log.cursor > 0, canRedo: log.cursor < log.entries.length, }); }); + // ─── The workspace's timelines: main and every short ────────────── + + /** GET /api/timelines — main first, then every short, with its shape. */ + app.get('/api/timelines', async (c) => { + try { + return c.json({ timelines: await listTimelines() }); + } catch (err) { + if (err instanceof WorkspaceBoundaryError) return c.json({ error: err.message }, 403); + throw err; + } + }); + + /** DELETE /api/timelines/:id — remove a short and its undo history. Main can't go. */ + app.delete('/api/timelines/:id', async (c) => { + const id = c.req.param('id'); + try { + await deleteTimeline(id); + return c.json({ deleted: id }); + } catch (err) { + if (err instanceof WorkspaceBoundaryError) return c.json({ error: err.message }, 403); + throw err; + } + }); + + /** + * POST /api/timelines/export — render every short, one at a time, to + * `shorts-/.mp4` in the workspace, and log each to the session so it + * shows in Outputs. A short that can't render is reported with its error. + */ + app.post('/api/timelines/export', async (c) => { + try { + const shorts = (await listTimelines()).map((t) => t.id).filter((id) => id !== MAIN_TIMELINE); + if (shorts.length === 0) { + return c.json({ error: 'No shorts to export yet. Make one from the transcript.' }, 422); + } + const dir = resolve(getWorkspace(), `shorts-${randomBytes(4).toString('hex')}`); + const exported = []; + for (const result of await exportTimelines(shorts, dir, confinedMediaMap)) { + if ('error' in result) { + exported.push(result); + continue; + } + const { timeline, rev, output, durationSec, loudness } = result; + const entry = await appendOp({ + tool: 'timeline_export', + args: { timeline, rev }, + result: { path: output, durationSec, ...(loudness ? { loudness } : {}) }, + }); + exported.push({ ...result, id: entry.id }); + } + return c.json({ dir, exported }); + } catch (err) { + if (err instanceof WorkspaceBoundaryError) return c.json({ error: err.message }, 403); + throw err; + } + }); + /** * Session safety endpoints. These intentionally live outside the * /api/tools/:name registry because snapshot/undo are meta-operations on diff --git a/src/ui/timeline-tools.ts b/src/ui/timeline-tools.ts index 15235b6..881628b 100644 --- a/src/ui/timeline-tools.ts +++ b/src/ui/timeline-tools.ts @@ -7,7 +7,9 @@ import { type Fit, findClip, } from '../timeline/composition.js'; +import { readComposition } from '../timeline/document-store.js'; import { buildMediaMap, buildMediaRefs } from '../timeline/media-registry.js'; +import { measureSpeechLevels } from '../timeline/pauses.js'; import { ownFraming } from '../timeline/reframe.js'; import type { MediaId, MediaRef } from '../timeline/schema.js'; import { analyzeSilence } from '../timeline/silence.js'; @@ -33,6 +35,13 @@ export function makeVerbContext(): VerbContext { }); return { mediaId: result.mediaId, durationSec: result.ref.durationSec }; }, + readTimeline: (timeline) => readComposition(timeline), + speechLevels: async (comp, clipId, aroundSec) => { + const clip = findClip(comp, clipId)?.clip; + if (clip?.kind !== 'media') return null; + const info = (await clipConfinedMedia(comp, clipId)).get(clip.mediaId); + return info?.hasAudio ? measureSpeechLevels(info.path, aroundSec) : null; + }, analyzeSilence: async (comp, clipId, sensitivity) => analyzeSilence(comp, clipId, { media: await clipConfinedMedia(comp, clipId), sensitivity }), transcript: async (comp, clipId, transcriptId) => { diff --git a/src/ui/web/src/App.tsx b/src/ui/web/src/App.tsx index 8092dda..b2ef559 100644 --- a/src/ui/web/src/App.tsx +++ b/src/ui/web/src/App.tsx @@ -1,4 +1,4 @@ -import { useState } from 'react'; +import { useEffect, useMemo, useState } from 'react'; import { ClipInspector } from './components/ClipInspector.js'; import { DetailPane } from './components/DetailPane.js'; import { DocTimeline } from './components/DocTimeline.js'; @@ -17,18 +17,32 @@ import { Header } from './components/Header.js'; import { ImportZone } from './components/ImportZone.js'; import { OpList } from './components/OpList.js'; import { Timeline } from './components/Timeline.js'; +import { type TimelineNotice, TimelineTabs } from './components/TimelineTabs.js'; import { ToolPickerModal } from './components/ToolPickerModal.js'; import { Viewer } from './components/Viewer.js'; import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts.js'; import { useSession } from './hooks/useSession.js'; import { useSessionSafety } from './hooks/useSessionSafety.js'; import { useTimeline } from './hooks/useTimeline.js'; -import { findClip, nextClipOnTrack } from './lib/composition.js'; +import { useTimelines } from './hooks/useTimelines.js'; +import { findClip, mediaNames, nextClipOnTrack } from './lib/composition.js'; +import { MAIN_TIMELINE, nextShortId, withTimeline } from './lib/timelines.js'; import { applyTimelineVerbs, type Verb } from './lib/verbs.js'; +/** A short `POST /api/timelines/export` rendered, or why it couldn't. */ +type ExportedShort = + | { timeline: string; output: string; loudness?: { outputLufs: number | null } } + | { timeline: string; error: string }; + export function App() { const { session, loading, error, refresh } = useSession(); - const { composition, refresh: refreshTimeline } = useTimeline(); + const { + timelines, + current: timeline, + select: selectTimeline, + refresh: refreshTimelines, + } = useTimelines(); + const { composition, missing, refresh: refreshTimeline } = useTimeline(timeline); const safety = useSessionSafety(); const [selectedId, setSelectedId] = useState(null); const [selectedClipId, setSelectedClipId] = useState(null); @@ -38,10 +52,31 @@ export function App() { const [safetyError, setSafetyError] = useState(null); const [verbBusy, setVerbBusy] = useState(false); const [verbError, setVerbError] = useState(null); + const [exportingAll, setExportingAll] = useState(false); + const [notice, setNotice] = useState(null); const selectedEntry = selectedId === null ? null : (session.entries.find((e) => e.id === selectedId) ?? null); const selected = selectedClipId === null ? null : findClip(composition, selectedClipId); + const names = useMemo(() => mediaNames(session.entries), [session.entries]); + const freeShortId = nextShortId(timelines.map((t) => t.id)); + + // Each timeline has its own clips and length, so switching starts clean. + function openTimeline(id: string) { + selectTimeline(id); + setSelectedClipId(null); + setPlayheadSec(0); + setVerbError(null); + } + + // A short deleted elsewhere (the agent, another tab) falls back to main. + useEffect(() => { + if (missing && timeline !== MAIN_TIMELINE) { + selectTimeline(MAIN_TIMELINE); + setSelectedClipId(null); + setPlayheadSec(0); + } + }, [missing, timeline, selectTimeline]); // Selecting an op and selecting a doc-clip are mutually exclusive — each owns // the inspector pane, so picking one clears the other. @@ -62,7 +97,7 @@ export function App() { setVerbBusy(true); setVerbError(null); try { - const { applied } = await applyTimelineVerbs(verbs); + const { applied } = await applyTimelineVerbs(verbs, timeline); await Promise.all([refreshTimeline(), refresh()]); return applied; } catch (err) { @@ -73,10 +108,100 @@ export function App() { } } + /** Make a short on timeline `id` and open it; false when it couldn't be made. */ + async function handleMakeShort(id: string, verb: Verb): Promise { + setVerbBusy(true); + setVerbError(null); + try { + const { warnings } = await applyTimelineVerbs([verb], id); + await Promise.all([refreshTimelines(), refresh()]); + openTimeline(id); + setNotice({ + kind: warnings.length > 0 ? 'warn' : 'info', + lines: [`Made ${id}.`, ...warnings], + }); + return true; + } catch (err) { + setVerbError(err instanceof Error ? err.message : String(err)); + return false; + } finally { + setVerbBusy(false); + } + } + + async function handleDeleteTimeline(id: string): Promise { + if ( + !window.confirm( + `Delete the short "${id}"? Its edits and undo history go with it; files you exported stay.`, + ) + ) { + return; + } + setSafetyError(null); + try { + const res = await fetch(`/api/timelines/${encodeURIComponent(id)}`, { method: 'DELETE' }); + if (!res.ok) { + const json = (await res.json().catch(() => ({}))) as { error?: string }; + throw new Error(json.error ?? `HTTP ${res.status}`); + } + await refreshTimelines(); + if (id === timeline) openTimeline(MAIN_TIMELINE); + } catch (err) { + setSafetyError(`Delete failed: ${err instanceof Error ? err.message : String(err)}`); + } + } + + async function handleExportAll(): Promise { + setExportingAll(true); + setSafetyError(null); + setNotice(null); + try { + const res = await fetch('/api/timelines/export', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: '{}', + }); + const json = (await res.json()) as { + dir?: string; + exported?: ExportedShort[]; + error?: string; + }; + if (!res.ok || !json.exported) throw new Error(json.error ?? `HTTP ${res.status}`); + await refresh(); // each short now shows in Outputs + const saved = json.exported.flatMap((e) => + 'error' in e + ? [] + : [ + e.loudness?.outputLufs != null + ? `${e.timeline} (${e.loudness.outputLufs.toFixed(1)} LUFS)` + : e.timeline, + ], + ); + const failed = json.exported.flatMap((e) => + 'error' in e ? [`${e.timeline} didn't render: ${e.error}`] : [], + ); + setNotice({ + kind: failed.length > 0 ? 'warn' : 'info', + lines: [ + ...(saved.length > 0 + ? [`Saved ${saved.length} in ${json.dir ?? 'the workspace'}: ${saved.join(', ')}.`] + : []), + ...failed, + ], + }); + } catch (err) { + setSafetyError(`Export failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + setExportingAll(false); + } + } + async function handleTimelineHistory(direction: 'undo' | 'redo'): Promise { setVerbError(null); try { - const res = await fetch(`/api/timeline/${direction}`, { method: 'POST' }); + const res = await fetch(withTimeline(`/api/timeline/${direction}`, timeline), { + method: 'POST', + }); if (!res.ok) throw new Error(`HTTP ${res.status}`); await Promise.all([refreshTimeline(), refresh()]); } catch (err) { @@ -90,7 +215,7 @@ export function App() { setVerbBusy(true); setSafetyError(null); try { - const res = await fetch('/api/timeline/export', { + const res = await fetch(withTimeline('/api/timeline/export', timeline), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: '{}', @@ -201,7 +326,7 @@ export function App() {
- +
{error ? (
@@ -223,7 +348,9 @@ export function App() { ) : selected ? ( c.id)} busy={verbBusy} error={verbError} + nextShortId={freeShortId} onApply={handleApplyVerbs} + onMakeShort={handleMakeShort} onSeek={setPlayheadSec} onClose={() => setSelectedClipId(null)} /> @@ -242,8 +371,20 @@ export function App() {
+ void handleDeleteTimeline(id)} + onExportAll={() => void handleExportAll()} + onDismissNotice={() => setNotice(null)} + /> ; playheadSec: number; hasNext: boolean; /** The timeline's fit, which a clip without its own follows. */ @@ -61,8 +69,12 @@ export function ClipInspector({ trackMediaIds: string[]; busy: boolean; error: string | null; + /** A name no timeline uses yet, offered for a new short. */ + nextShortId: string; /** Resolves to the number of ops applied, or null if the edit failed. */ onApply: (verbs: Verb[]) => Promise; + /** Make a short on timeline `id` and open it; resolves false when that failed. */ + onMakeShort: (id: string, verb: Verb) => Promise; /** Move the playhead to a timeline time. */ onSeek: (sec: number) => void; onClose: () => void; @@ -84,7 +96,7 @@ export function ClipInspector({
-

{clipLabel(clip)}

+

{clipLabel(clip, mediaNames)}

{clip.kind} · {clip.id} · {start.toFixed(2)}–{end.toFixed(2)}s ( {clipDuration(clip).toFixed(2)}s) @@ -191,7 +203,15 @@ export function ClipInspector({ ) : null} {clip.kind === 'media' ? ( - + ) : null}
diff --git a/src/ui/web/src/components/DocTimeline.tsx b/src/ui/web/src/components/DocTimeline.tsx index fe1f4dd..7d67328 100644 --- a/src/ui/web/src/components/DocTimeline.tsx +++ b/src/ui/web/src/components/DocTimeline.tsx @@ -1,5 +1,6 @@ import { useEffect, useRef, useState } from 'react'; import { clipDuration, clipEndSec, clipLabel, compositionDuration } from '../lib/composition.js'; +import { withTimeline } from '../lib/timelines.js'; import type { Verb } from '../lib/verbs.js'; import type { Clip, Composition, MediaClip } from '../types.js'; import { CaptionsPanel } from './CaptionsPanel.js'; @@ -20,7 +21,9 @@ const MAX_THUMBS = 16; // cap thumbnails per clip to bound requests * playhead is controlled by the parent so the clip inspector can split at it. */ export function DocTimeline({ + timeline, composition, + mediaNames, selectedClipId, onSelectClip, playheadSec, @@ -32,7 +35,11 @@ export function DocTimeline({ onApply, verbError, }: { + /** The timeline shown: `main` or a short. */ + timeline: string; composition: Composition; + /** File name of each media id, for labeling media clips. */ + mediaNames: ReadonlyMap; selectedClipId: string | null; onSelectClip: (clipId: string | null) => void; playheadSec: number; @@ -71,7 +78,7 @@ export function DocTimeline({ }); useEffect(() => { let alive = true; - fetch(`/api/timeline/history?rev=${composition.rev}`) + fetch(withTimeline(`/api/timeline/history?rev=${composition.rev}`, timeline)) .then((r) => r.json() as Promise<{ canUndo: boolean; canRedo: boolean }>) .then((h) => { if (alive) setHistory(h); @@ -80,7 +87,7 @@ export function DocTimeline({ return () => { alive = false; }; - }, [composition.rev]); + }, [timeline, composition.rev]); return (
@@ -116,7 +123,12 @@ export function DocTimeline({ > ↷ - +
{hasClips ? ( @@ -156,6 +168,7 @@ export function DocTimeline({ @@ -205,10 +218,12 @@ export function DocTimeline({ function ClipBlock({ clip, + label, selected, onSelect, }: { clip: Clip; + label: string; selected: boolean; onSelect: (clipId: string | null) => void; }) { @@ -220,10 +235,10 @@ function ClipBlock({ className={`doc-tl-clip doc-tl-clip-${clip.kind}${selected ? ' selected' : ''}`} style={{ left, width }} onClick={() => onSelect(selected ? null : clip.id)} - title={`${clipLabel(clip)} · ${clip.startSec.toFixed(2)}–${clipEndSec(clip).toFixed(2)}s`} + title={`${label} · ${clip.startSec.toFixed(2)}–${clipEndSec(clip).toFixed(2)}s`} > {clip.kind === 'media' ? : null} - {clipLabel(clip)} + {label} {clipDuration(clip).toFixed(1)}s ); diff --git a/src/ui/web/src/components/ExportButton.tsx b/src/ui/web/src/components/ExportButton.tsx index 32d78c4..827aba6 100644 --- a/src/ui/web/src/components/ExportButton.tsx +++ b/src/ui/web/src/components/ExportButton.tsx @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; type ProjectFormat = 'fcpxml' | 'xmeml'; @@ -15,8 +16,8 @@ const PROJECTS: readonly { format: ProjectFormat; label: string; detail: string /** POST the export route for an editor project; throws with the server's * message (e.g. the 422 naming what the project can't hold). */ -async function exportProject(format: ProjectFormat): Promise { - const res = await fetch('/api/timeline/export', { +async function exportProject(format: ProjectFormat, timeline: string): Promise { + const res = await fetch(withTimeline('/api/timeline/export', timeline), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ format }), @@ -35,10 +36,12 @@ const fileName = (path: string): string => path.split(/[\\/]/).pop() ?? path; * carries only in part. */ export function ExportButton({ + timeline, busy, disabled, onExportVideo, }: { + timeline: string; busy: boolean; disabled: boolean; onExportVideo: () => void; @@ -72,7 +75,7 @@ export function ExportButton({ setError(null); setCopied(false); try { - setResult(await exportProject(format)); + setResult(await exportProject(format, timeline)); } catch (err) { setError(err instanceof Error ? err.message : String(err)); } finally { diff --git a/src/ui/web/src/components/FramingSection.tsx b/src/ui/web/src/components/FramingSection.tsx index 62d39d2..2576e34 100644 --- a/src/ui/web/src/components/FramingSection.tsx +++ b/src/ui/web/src/components/FramingSection.tsx @@ -110,7 +110,7 @@ export function FramingSection({ ); } -function Slider({ +export function Slider({ label, value, min, diff --git a/src/ui/web/src/components/TimelineTabs.tsx b/src/ui/web/src/components/TimelineTabs.tsx new file mode 100644 index 0000000..0d7057d --- /dev/null +++ b/src/ui/web/src/components/TimelineTabs.tsx @@ -0,0 +1,104 @@ +import { MAIN_TIMELINE, shapeLabel, type TimelineListing } from '../lib/timelines.js'; + +/** What the last short-making or export-all did, shown under the tabs. */ +export interface TimelineNotice { + kind: 'info' | 'warn'; + lines: string[]; +} + +/** + * One tab per timeline in the workspace, main and every short, each with its + * shape. A short's ✕ deletes it (the caller confirms), and "Export all shorts" + * renders each one. Shown once there is a short. + */ +export function TimelineTabs({ + timelines, + current, + exportingAll, + notice, + onSelect, + onDelete, + onExportAll, + onDismissNotice, +}: { + timelines: TimelineListing[]; + current: string; + exportingAll: boolean; + notice: TimelineNotice | null; + onSelect: (id: string) => void; + onDelete: (id: string) => void; + onExportAll: () => void; + onDismissNotice: () => void; +}) { + const shorts = timelines.filter((t) => t.id !== MAIN_TIMELINE); + if (shorts.length === 0 && notice === null) return null; + + return ( +
+
+
+ {timelines.map((t) => { + const isCurrent = t.id === current; + const shape = 'error' in t ? 'unreadable' : shapeLabel(t.width, t.height); + return ( +
+ + {t.id === MAIN_TIMELINE ? null : ( + + )} +
+ ); + })} +
+ +
+ {notice ? ( +
+
+ {notice.lines.map((line) => ( +
{line}
+ ))} +
+ +
+ ) : null} +
+ ); +} diff --git a/src/ui/web/src/components/TranscriptSection.tsx b/src/ui/web/src/components/TranscriptSection.tsx index e3f648e..307d3f9 100644 --- a/src/ui/web/src/components/TranscriptSection.tsx +++ b/src/ui/web/src/components/TranscriptSection.tsx @@ -1,6 +1,8 @@ import { useCallback, useEffect, useRef, useState } from 'react'; +import { isTimelineId, MAIN_TIMELINE, withTimeline } from '../lib/timelines.js'; import type { Verb } from '../lib/verbs.js'; import type { MediaClip } from '../types.js'; +import { Slider } from './FramingSection.js'; /** Mirrors ClipTranscript in src/timeline/transcript.ts. */ interface ClipWord { @@ -19,19 +21,28 @@ interface ClipTranscript { /** * Edit a media clip by its words: transcribe on request (the local whisper.cpp * is optional, so its setup message shows here when it's missing), click a word - * to move the playhead, shift-click to select a range, then cut it. Phrases - * said twice in a row are marked and can all be cut at once. Every cut is one - * undoable edit through the verb layer. + * to move the playhead, shift-click to select a range, then cut it or make a + * short of it. Phrases said twice in a row are marked and can all be cut at + * once. Every cut is one undoable edit through the verb layer. */ export function TranscriptSection({ + timeline, clip, busy, + nextShortId, onApply, + onMakeShort, onSeek, }: { + /** The timeline the clip is on, which a short is cut from. */ + timeline: string; clip: MediaClip; busy: boolean; + /** A name no timeline uses yet, offered for a new short. */ + nextShortId: string; onApply: (verbs: Verb[]) => Promise; + /** Make a short on timeline `id` and open it; resolves false when that failed. */ + onMakeShort: (id: string, verb: Verb) => Promise; onSeek: (sec: number) => void; }) { const [transcript, setTranscript] = useState(null); @@ -39,6 +50,10 @@ export function TranscriptSection({ const [error, setError] = useState(null); const [anchor, setAnchor] = useState(null); const [focus, setFocus] = useState(null); + // null until the name is edited, so the offered name follows the timelines. + const [shortName, setShortName] = useState(null); + const [shortX, setShortX] = useState(0.5); + const [nameError, setNameError] = useState(null); // Set once the transcript has been asked for; `latest` numbers requests so a // response for an older clip window is dropped instead of shown. const wanted = useRef(false); @@ -50,7 +65,9 @@ export function TranscriptSection({ setLoading(true); setError(null); try { - const res = await fetch(`/api/timeline/transcript/${encodeURIComponent(clip.id)}`); + const res = await fetch( + withTimeline(`/api/timeline/transcript/${encodeURIComponent(clip.id)}`, timeline), + ); const json = (await res.json()) as ClipTranscript & { error?: string }; if (request !== latest.current) return; if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`); @@ -60,7 +77,7 @@ export function TranscriptSection({ } finally { if (request === latest.current) setLoading(false); } - }, [clip.id]); + }, [clip.id, timeline]); // A cut or trim changes the clip's window: reload a transcript that was asked // for, even one still loading (its response is then dropped as outdated). The @@ -83,6 +100,33 @@ export function TranscriptSection({ from === null || to === null ? 0 : words.filter((w) => w.index >= from && w.index <= to).length; const inRetake = (index: number): boolean => retakes.some((r) => index >= r.fromIndex && index <= r.toIndex); + const name = shortName ?? nextShortId; + + const makeShort = async (): Promise => { + if (from === null || to === null || !transcript) return; + const id = name.trim(); + if (!isTimelineId(id)) { + setNameError('Name it with lowercase letters, digits and dashes, like best-1.'); + return; + } + if (id === MAIN_TIMELINE || id === timeline) { + setNameError(`A short needs a name of its own, not ${id}.`); + return; + } + const made = await onMakeShort(id, { + verb: 'make_short', + ranges: [ + { clipId: clip.id, fromIndex: from, toIndex: to, transcriptId: transcript.transcriptId }, + ], + source: timeline, + x: shortX, + }); + if (made) { + setAnchor(null); + setFocus(null); + setShortName(null); + } + }; return (
@@ -182,6 +226,45 @@ export function TranscriptSection({ ) : null}
+ {selectedCount > 0 ? ( +
+ + + {nameError ?
{nameError}
: null} + +

+ A 9:16 short of these words with captions, at -14 LUFS, its edges in the pauses. It + opens when made; Undo there takes it apart. +

+
+ ) : null}

Click a word to move the playhead; shift-click to select a range. Undo restores.

diff --git a/src/ui/web/src/components/Viewer.tsx b/src/ui/web/src/components/Viewer.tsx index 5782193..ddcbf55 100644 --- a/src/ui/web/src/components/Viewer.tsx +++ b/src/ui/web/src/components/Viewer.tsx @@ -1,4 +1,5 @@ import { useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; interface Exportable { exportable: boolean; @@ -16,7 +17,7 @@ type FrameState = * from the export. The playhead is debounced so scrubbing doesn't fire an FFmpeg * render per pixel, and a banner surfaces whether the document can render at all. */ -export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { +export function Viewer({ timeline, atSec, rev }: { timeline: string; atSec: number; rev: number }) { const [debouncedAt, setDebouncedAt] = useState(atSec); const [status, setStatus] = useState(null); const [frame, setFrame] = useState({ kind: 'loading' }); @@ -30,7 +31,7 @@ export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { // Re-check exportability whenever the document changes (rev cache-busts). useEffect(() => { let alive = true; - fetch(`/api/timeline/exportable?rev=${rev}`) + fetch(withTimeline(`/api/timeline/exportable?rev=${rev}`, timeline)) .then((r) => r.json() as Promise) .then((s) => { if (alive) setStatus(s); @@ -41,13 +42,13 @@ export function Viewer({ atSec, rev }: { atSec: number; rev: number }) { return () => { alive = false; }; - }, [rev]); + }, [timeline, rev]); // Fetch the composited frame as a blob (not via ) so a superseded // scrub is ABORTED instead of left rendering, and a transient 503 reads as a // clean message rather than a broken image. The previous frame stays on screen // until the next one decodes — no blank flash while FFmpeg works. - const src = `/api/timeline/frame?at=${debouncedAt.toFixed(2)}&rev=${rev}`; + const src = withTimeline(`/api/timeline/frame?at=${debouncedAt.toFixed(2)}&rev=${rev}`, timeline); useEffect(() => { const controller = new AbortController(); fetch(src, { signal: controller.signal }) diff --git a/src/ui/web/src/hooks/useTimeline.ts b/src/ui/web/src/hooks/useTimeline.ts index e815df4..8fe93fa 100644 --- a/src/ui/web/src/hooks/useTimeline.ts +++ b/src/ui/web/src/hooks/useTimeline.ts @@ -1,4 +1,5 @@ -import { useCallback, useEffect, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; +import { withTimeline } from '../lib/timelines.js'; import type { Composition } from '../types.js'; const EMPTY: Composition = { @@ -14,34 +15,49 @@ const EMPTY: Composition = { }; /** - * Polls `/api/timeline` (the CompositionDoc — the source of truth for assembled - * edits) every `intervalMs`. Mirrors `useSession`: returns the latest doc plus - * loading/error state and a `refresh()` for callers that just mutated it (e.g. - * after a verb POST) and want the view to update without waiting for the poll. + * Polls `/api/timeline` for `timeline` (the CompositionDoc — the source of truth + * for assembled edits) every `intervalMs`. Mirrors `useSession`: returns the + * latest doc plus loading/error state and a `refresh()` for callers that just + * mutated it (e.g. after a verb POST) and want the view to update without + * waiting for the poll. `missing` is set when the timeline no longer exists. */ -export function useTimeline(intervalMs = 2000): { +export function useTimeline( + timeline: string, + intervalMs = 2000, +): { composition: Composition; loading: boolean; error: string | null; + missing: boolean; refresh: () => Promise; } { - const [composition, setComposition] = useState(EMPTY); + // Kept with the timeline it belongs to, so a switch shows nothing of the last one. + const [loaded, setLoaded] = useState<{ timeline: string; doc: Composition } | null>(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); + // The timeline whose fetch last answered 404, so another one never inherits it. + const [missingTimeline, setMissingTimeline] = useState(null); + // A response for a timeline switched away from is dropped, not shown. + const wanted = useRef(timeline); + wanted.current = timeline; const load = useCallback(async () => { try { - const res = await fetch('/api/timeline'); + const res = await fetch(withTimeline('/api/timeline', timeline)); + if (wanted.current !== timeline) return; + setMissingTimeline(res.status === 404 ? timeline : null); if (!res.ok) throw new Error(`HTTP ${res.status}`); const json = (await res.json()) as { document: Composition }; - setComposition(json.document); + if (wanted.current !== timeline) return; + setLoaded({ timeline, doc: json.document }); setError(null); setLoading(false); } catch (err) { + if (wanted.current !== timeline) return; setError(err instanceof Error ? err.message : String(err)); setLoading(false); } - }, []); + }, [timeline]); useEffect(() => { void load(); @@ -49,5 +65,6 @@ export function useTimeline(intervalMs = 2000): { return () => window.clearInterval(timer); }, [intervalMs, load]); - return { composition, loading, error, refresh: load }; + const composition = loaded?.timeline === timeline ? loaded.doc : EMPTY; + return { composition, loading, error, missing: missingTimeline === timeline, refresh: load }; } diff --git a/src/ui/web/src/hooks/useTimelines.ts b/src/ui/web/src/hooks/useTimelines.ts new file mode 100644 index 0000000..179d8e8 --- /dev/null +++ b/src/ui/web/src/hooks/useTimelines.ts @@ -0,0 +1,48 @@ +import { useCallback, useEffect, useState } from 'react'; +import { isTimelineId, MAIN_TIMELINE, type TimelineListing } from '../lib/timelines.js'; + +/** The timeline the page's `?timeline=` names, so a reload keeps it. */ +function timelineInUrl(): string { + const id = new URLSearchParams(window.location.search).get('timeline'); + return id && isTimelineId(id) ? id : MAIN_TIMELINE; +} + +/** + * The workspace's timelines (main and every short), polled like the document, + * and the one being edited, kept in the page URL. + */ +export function useTimelines(intervalMs = 2000): { + timelines: TimelineListing[]; + current: string; + select: (id: string) => void; + refresh: () => Promise; +} { + const [timelines, setTimelines] = useState([]); + const [current, setCurrent] = useState(timelineInUrl); + + const load = useCallback(async () => { + try { + const res = await fetch('/api/timelines'); + if (!res.ok) return; + setTimelines(((await res.json()) as { timelines: TimelineListing[] }).timelines); + } catch { + // The next poll tries again; the tabs keep the last list meanwhile. + } + }, []); + + useEffect(() => { + void load(); + const timer = window.setInterval(load, intervalMs); + return () => window.clearInterval(timer); + }, [intervalMs, load]); + + const select = useCallback((id: string) => { + setCurrent(id); + const url = new URL(window.location.href); + if (id === MAIN_TIMELINE) url.searchParams.delete('timeline'); + else url.searchParams.set('timeline', id); + window.history.replaceState(null, '', url); + }, []); + + return { timelines, current, select, refresh: load }; +} diff --git a/src/ui/web/src/lib/composition.ts b/src/ui/web/src/lib/composition.ts index fe0f2f3..9576d7a 100644 --- a/src/ui/web/src/lib/composition.ts +++ b/src/ui/web/src/lib/composition.ts @@ -1,4 +1,5 @@ -import type { Clip, Composition, Track } from '../types.js'; +import type { Clip, Composition, SessionEntry, Track } from '../types.js'; +import { shortName } from './playable-clips.js'; // Frontend mirror of the doc geometry helpers in src/timeline/composition.ts. // The timeline view positions clips by their DOCUMENT extent (startSec + @@ -23,9 +24,25 @@ export function compositionDuration(comp: Composition): number { return end; } -/** Short, human-readable label for a clip on the timeline. */ -export function clipLabel(clip: Clip): string { - if (clip.kind === 'media') return clip.mediaId; +/** File name of each ingested media id, read from the session log's `ingest` + * entries (`{ mediaId, ref: { path } }`) — the same registry export resolves + * media through. */ +export function mediaNames(entries: SessionEntry[]): Map { + const names = new Map(); + for (const entry of entries) { + if (entry.tool !== 'ingest') continue; + const { mediaId, ref } = entry.result as { mediaId?: unknown; ref?: { path?: unknown } }; + if (typeof mediaId === 'string' && typeof ref?.path === 'string') { + names.set(mediaId, shortName(ref.path)); + } + } + return names; +} + +/** Short, human-readable label for a clip on the timeline. A media clip shows its + * file name, or its media id when the session log doesn't name it. */ +export function clipLabel(clip: Clip, names: ReadonlyMap): string { + if (clip.kind === 'media') return names.get(clip.mediaId) ?? clip.mediaId; if (clip.kind === 'text') return `“${clip.text.slice(0, 20)}”`; return `color · ${clip.color}`; } diff --git a/src/ui/web/src/lib/playable-clips.ts b/src/ui/web/src/lib/playable-clips.ts index e34955b..a9452f5 100644 --- a/src/ui/web/src/lib/playable-clips.ts +++ b/src/ui/web/src/lib/playable-clips.ts @@ -26,7 +26,7 @@ function isVideoPath(path: string): boolean { return VIDEO_EXT.has(ext); } -function shortName(path: string): string { +export function shortName(path: string): string { const base = path.split('/').pop() ?? path; // Strip the 8-char random prefix (`xxxxxxxx-`) that we add to import // filenames and tool outputs, so cards show "demo.mp4" instead of diff --git a/src/ui/web/src/lib/timelines.ts b/src/ui/web/src/lib/timelines.ts new file mode 100644 index 0000000..683591a --- /dev/null +++ b/src/ui/web/src/lib/timelines.ts @@ -0,0 +1,34 @@ +/** The timeline at `composition.json`; every other timeline is a short. */ +export const MAIN_TIMELINE = 'main'; + +/** Mirrors TimelineIdSchema in src/timeline/schema.ts. */ +const TIMELINE_ID = /^[a-z0-9][a-z0-9-]{0,47}$/; + +/** Mirrors TimelineListing in src/timeline/document-store.ts. */ +export type TimelineListing = + | { id: string; width: number; height: number; durationSec: number; rev: number } + | { id: string; error: string }; + +export function isTimelineId(id: string): boolean { + return TIMELINE_ID.test(id); +} + +/** An API path for the given timeline: every /api/timeline route reads `?timeline=`. */ +export function withTimeline(path: string, timeline: string): string { + return `${path}${path.includes('?') ? '&' : '?'}timeline=${encodeURIComponent(timeline)}`; +} + +/** `16:9`, `9:16`, `1:1`, `4:5`… from the canvas size. */ +export function shapeLabel(width: number, height: number): string { + const gcd = (a: number, b: number): number => (b === 0 ? a : gcd(b, a % b)); + const d = gcd(width, height); + const [w, h] = [width / d, height / d]; + return w <= 32 && h <= 32 ? `${w}:${h}` : `${width}×${height}`; +} + +/** The first `short-N` no timeline uses yet. */ +export function nextShortId(ids: readonly string[]): string { + let n = 1; + while (ids.includes(`short-${n}`)) n++; + return `short-${n}`; +} diff --git a/src/ui/web/src/lib/verbs.ts b/src/ui/web/src/lib/verbs.ts index e528cf5..e3a0ecd 100644 --- a/src/ui/web/src/lib/verbs.ts +++ b/src/ui/web/src/lib/verbs.ts @@ -1,4 +1,5 @@ import type { Composition, Fit } from '../types.js'; +import { withTimeline } from './timelines.js'; /** A timeline editing verb (mirrors CompositionVerb in src/timeline/verbs.ts). * Loosely typed at the field level — the server validates with Zod, which is the @@ -44,26 +45,39 @@ export type Verb = kind?: string; durationSec?: number; track?: string; + } + | { + verb: 'make_short'; + ranges: { clipId: string; fromIndex: number; toIndex: number; transcriptId?: string }[]; + source?: string; + x?: number; }; /** - * POST editing verbs to /api/timeline/verbs — the same op-aware, undoable path - * the agent and CLI use. Throws with the server's message on a 4xx/5xx (e.g. a - * 422 for a bad clip reference or a 403 for a workspace-boundary violation) so - * the caller can surface it. `applied` is the op count; 0 means the verbs changed - * nothing (e.g. remove_silence found no pause). + * POST editing verbs to /api/timeline/verbs for `timeline` — the same op-aware, + * undoable path the agent and CLI use. Throws with the server's message on a + * 4xx/5xx (e.g. a 422 for a bad clip reference or a 403 for a workspace-boundary + * violation) so the caller can surface it. `applied` is the op count; 0 means the + * verbs changed nothing (e.g. remove_silence found no pause). `warnings` name what + * a verb did that may not have been meant (a short cut mid-sentence). */ export async function applyTimelineVerbs( verbs: Verb[], -): Promise<{ document: Composition; applied: number }> { - const res = await fetch('/api/timeline/verbs', { + timeline: string, +): Promise<{ document: Composition; applied: number; warnings: string[] }> { + const res = await fetch(withTimeline('/api/timeline/verbs', timeline), { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ verbs }), }); - const json = (await res.json()) as { document?: Composition; applied?: number; error?: string }; + const json = (await res.json()) as { + document?: Composition; + applied?: number; + warnings?: string[]; + error?: string; + }; if (!res.ok || !json.document) { throw new Error(json.error ?? `HTTP ${res.status}`); } - return { document: json.document, applied: json.applied ?? 0 }; + return { document: json.document, applied: json.applied ?? 0, warnings: json.warnings ?? [] }; } diff --git a/src/ui/web/src/styles.css b/src/ui/web/src/styles.css index 2c1767f..4bbb062 100644 --- a/src/ui/web/src/styles.css +++ b/src/ui/web/src/styles.css @@ -186,6 +186,14 @@ body, } .op-list .op { + /* Each op is a