From 680ec35f88a2d9b437b7b502298fcd499f876e30 Mon Sep 17 00:00:00 2001 From: PeterTakahashi Date: Tue, 28 Jul 2026 03:04:04 +0900 Subject: [PATCH 01/10] feat(cli): headless record/export/info commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a CLI so scripts, CI pipelines, and AI coding agents can drive OpenScreen end-to-end without a visible window: openscreen record --duration 20 --project demo.openscreen --json openscreen export demo.openscreen -o demo.mp4 --audio voice.m4a --json openscreen info demo.openscreen --json Design: CLI mode boots Electron without HUD/tray/menu and drives a hidden runner window (windowType=cli-export|cli-record) that reuses the existing pipelines unchanged — VideoExporter/GifExporter for export (the approach already proven by the HEADLESS=true e2e test) and useScreenRecorder for recording (native SCK/WGC helpers, browser fallback). registerIpcHandlers is reused with inert window callbacks, so the 2900-line handler module is untouched. - electron/cli/args.ts: pure argv parser + unit tests; skips leading Chromium switches (AppImage --no-sandbox) before subcommand detection - electron/cli/cliMain.ts: stdio protocol (NDJSON with --json, TTY-aware progress otherwise), SIGINT/SIGTERM/stdin stop, EPIPE-safe writes, exit codes 0/1/2, console rerouted to stderr, SwiftShader fallback - src/cli/CliExportRunner.tsx: loads .openscreen via the native bridge, rebuilds the editor's exporter config deterministically (1280x720 reference preview box, overridable via --preview-size), optional voiceover mix (--audio/--audio-mode/--audio-offset) that copies video packets and re-renders audio offline via OfflineAudioContext+mediabunny - src/cli/CliRecordRunner.tsx: source pick by display index or window title, mic device by label, --duration/signal/stdin stop, optional ready-to-export --project output - useScreenRecorder: expose startRecordingImmediately() (skips countdown) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- README.md | 13 + docs/cli.md | 177 +++++++++++++ electron/cli/args.test.ts | 153 +++++++++++ electron/cli/args.ts | 346 ++++++++++++++++++++++++ electron/cli/cliMain.ts | 442 +++++++++++++++++++++++++++++++ electron/electron-env.d.ts | 6 + electron/main.ts | 28 +- electron/preload.ts | 19 ++ electron/windows.ts | 2 +- package.json | 1 + src/App.tsx | 14 + src/cli/CliExportRunner.tsx | 323 ++++++++++++++++++++++ src/cli/CliRecordRunner.tsx | 274 +++++++++++++++++++ src/hooks/useScreenRecorder.ts | 3 + src/lib/cliContracts.ts | 83 ++++++ src/lib/exporter/voiceoverMix.ts | 136 ++++++++++ 16 files changed, 2012 insertions(+), 8 deletions(-) create mode 100644 docs/cli.md create mode 100644 electron/cli/args.test.ts create mode 100644 electron/cli/args.ts create mode 100644 electron/cli/cliMain.ts create mode 100644 src/cli/CliExportRunner.tsx create mode 100644 src/cli/CliRecordRunner.tsx create mode 100644 src/lib/cliContracts.ts create mode 100644 src/lib/exporter/voiceoverMix.ts diff --git a/README.md b/README.md index 6ce5cc87c5..3e0fc586e2 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,19 @@ The goal of this continuation is to keep OpenScreen alive as a fully open-source - Export to MP4 or GIF in multiple aspect ratios and resolutions, rendered and encoded on the GPU (Metal on macOS, D3D11 on Windows, Vulkan on Linux) with an automatic CPU fallback. - Languages supported: Arabic, English, Spanish, French, Italian, Japanese, Korean, Portuguese (Brazil), Russian, Turkish, Vietnamese, Simplified Chinese, and Traditional Chinese. +## Command-line interface (headless) + +OpenScreen ships a CLI for scripts, CI, and AI coding agents: record the screen +headlessly, edit the `.openscreen` project JSON programmatically (zooms, +annotations, trims), and render MP4/GIF with the full export pipeline — no +visible windows, NDJSON output with `--json`. + +```bash +openscreen record --duration 20 --project demo.openscreen --json +openscreen export demo.openscreen -o demo.mp4 --json +``` + +See [docs/cli.md](./docs/cli.md). ## Installation diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000000..8ebb310803 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,177 @@ +# OpenScreen CLI + +Headless command-line interface for recording the screen and exporting `.openscreen` +projects — no visible windows, machine-readable output. Designed so scripts, CI +pipelines, and AI coding agents can produce polished product demos automatically: + +``` +record → edit the project JSON programmatically → export → MP4/GIF +``` + +## Running + +Development (after `npm run build-vite` and, for recording, the native helper build): + +```bash +npm run cli -- [options] +# or directly: +./node_modules/.bin/electron . [options] +``` + +Packaged app (the CLI ships inside the normal binary): + +```bash +# macOS +/Applications/Openscreen.app/Contents/MacOS/Openscreen export demo.openscreen -o demo.mp4 +# Windows +"C:\Program Files\Openscreen\Openscreen.exe" export demo.openscreen -o demo.mp4 +``` + +CLI runs skip the single-instance lock, so they work while the GUI app is open. + +## Commands + +### `openscreen record` + +Records headlessly through the same pipeline as the GUI: the native +ScreenCaptureKit helper on macOS, the WGC helper on Windows (browser capture as +fallback). Recordings land in the app's recordings directory +(`/recordings/`), exactly like GUI recordings — including the +`.cursor.json` cursor-telemetry sidecar used for editable cursors and auto-zoom. + +```bash +openscreen record --duration 30 --project demo.openscreen --json +openscreen record --window "My App" --mic --system-audio +openscreen record --display 1 --cursor system +``` + +| Option | Meaning | +|---|---| +| `--display ` | Screen index to record (default 0) | +| `--window ` | Record the first window whose title contains `<title>` | +| `--mic` / `--mic-device <name>` | Capture microphone (optionally by device-label substring) | +| `--system-audio` | Capture system audio | +| `--cursor <editable-overlay\|system>` | Hide the system cursor and record telemetry (default), or bake it into the video | +| `--duration <seconds>` | Stop automatically | +| `--project <out.openscreen>` | Write a ready-to-export project file when done | +| `--json` | NDJSON events on stdout | + +Stopping without `--duration`: send SIGINT/SIGTERM to the process, or type +`stop` + Enter on its stdin. + +Platform notes: + +- **macOS**: requires the Swift helper (`npm run build:native:mac`, needs Xcode) + and the Screen Recording permission for whatever binary hosts Electron + (your terminal during development). Webcam capture is not available in CLI + recording on macOS (same limitation as the native helper). +- **Windows**: requires the WGC helper (`npm run build:native:win`). SIGTERM + does not exist on Windows — stop recordings with Ctrl+C, stdin `stop`, or + `--duration` (a hard `taskkill` loses the recording). Microphone access is + gated by Settings → Privacy → Microphone; there is no programmatic prompt. +- **Linux**: uses the browser capture pipeline; cursor options are limited, + matching the GUI. On Wayland, capture goes through the PipeWire portal, + which may show a system picker dialog and requires a desktop session + (headless/SSH sessions without a portal cannot record). + +### `openscreen export` + +Renders a project to MP4 or GIF using the app's real export pipeline (WebCodecs + +PixiJS, faster than realtime) in a hidden window. Falls back to SwiftShader when +no GPU is available (CI), and applies everything the editor would: zooms, trims, +speed regions, wallpaper/padding, annotations, cursor rendering, webcam layouts. + +```bash +openscreen export demo.openscreen # format/quality from the project +openscreen export demo.openscreen -o out.mp4 --quality source +openscreen export demo.openscreen -o out.gif --gif-fps 20 --gif-size large +openscreen export demo.openscreen --json | while read line; do ...; done +``` + +| Option | Meaning | +|---|---| +| `-o, --out <path>` | Output file; extension picks the format. Default: next to the project | +| `--format <mp4\|gif>` | Override the project's stored format | +| `--quality <medium\|good\|source>` | MP4 quality | +| `--gif-fps <15\|20\|25\|30>`, `--gif-size <medium\|large\|original>` | GIF settings | +| `--preview-size <WxH>` | Reference preview box (default `1280x720`), see below | +| `--audio <file>` | Mix a voiceover file into the MP4 (mp3/wav/m4a — anything Chromium can decode; AIFF is not supported) | +| `--audio-mode <mix\|replace>` | Layer the voiceover over the recording's audio (default `mix`) or replace it | +| `--audio-offset <seconds>` | Delay before the voiceover starts (default 0) | +| `--json` | NDJSON progress + result on stdout | + +`--audio` re-muxes after the render: video packets are copied untouched, and the +audio is mixed offline (OfflineAudioContext) and re-encoded to AAC. MP4 only. + +**Media path rule**: for safety, a project's referenced media is only auto-approved +when it lives in the app's recordings directory or **next to the project file**. +Keep `.openscreen` files beside their media (or record via the CLI, which uses +the recordings directory). + +**`--preview-size`**: annotation font sizes and border radii are stored in +preview-pixel space and scaled by `export size / preview size` — in the GUI the +"preview" is the editor window, so results depend on window size. The CLI uses a +deterministic reference box instead (the composition fitted into 1280×720). +Authoring tip for scripts: treat annotation `fontSize` as "pixels in a +1280-wide preview". + +### `openscreen info` + +Prints a project summary (referenced media and whether it exists, format, +region counts). Exits non-zero if the referenced video is missing. + +```bash +openscreen info demo.openscreen --json +``` + +## Machine-readable output (`--json`) + +One JSON object per line on stdout (NDJSON). stderr carries diagnostics only. + +```jsonl +{"event":"started","command":"export"} +{"event":"progress","percentage":42,"currentFrame":50,"totalFrames":120,"estimatedTimeRemaining":3} +{"event":"done","success":true,"outputPath":"/path/out.mp4","format":"mp4","width":1920,"height":1080} +``` + +Record emits `log` events (`Recording started`, …), `stopping`, and a final +`done` carrying `screenVideoPath`, `cursorDataPath`, `durationMs`, and +`projectPath` when `--project` was used. Exit code is 0 on success, 1 on +failure, 2 on bad arguments. + +## Example: automated product demo (for scripts/agents) + +```bash +# 1. Record 20 seconds of the running app +openscreen record --window "MyProduct" --duration 20 --project demo.openscreen --json + +# 2. Edit the project: add a zoom and a caption (plain JSON) +node -e ' + const fs = require("fs"); + const p = JSON.parse(fs.readFileSync("demo.openscreen", "utf8")); + p.editor.zoomRegions.push({ id: "z1", startMs: 2000, endMs: 6000, depth: 3, + focus: { cx: 0.5, cy: 0.4 }, focusMode: "manual", source: "manual" }); + p.editor.annotationRegions.push({ id: "a1", startMs: 500, endMs: 4000, + type: "text", content: "One-click setup", textContent: "One-click setup", + position: { x: 8, y: 6 }, size: { width: 40, height: 12 }, + style: { fontSize: 24, color: "#fff" }, zIndex: 1 }); + fs.writeFileSync("demo.openscreen", JSON.stringify(p, null, 2)); +' + +# 3. Narrate with any TTS (macOS `say` shown; any engine producing mp3/wav/m4a works) +say -o voice.m4a --file-format=m4af "Welcome to my product. Here's a quick tour." + +# 4. Render with the voiceover mixed in +openscreen export demo.openscreen -o demo.mp4 --audio voice.m4a --audio-mode replace --json +``` + +## Architecture + +- `electron/cli/args.ts` — pure argv parser (unit-tested in `args.test.ts`). +- `electron/cli/cliMain.ts` — headless boot: no HUD/tray/menu/dock, stdio + protocol, signal handling, exit codes. Registers the same IPC surface as the + GUI (`registerIpcHandlers`) with inert window callbacks. +- `src/cli/CliExportRunner.tsx` / `src/cli/CliRecordRunner.tsx` — hidden-window + runners (`?windowType=cli-export|cli-record`) that drive the existing + exporter classes and `useScreenRecorder` hook. +- Contracts shared between main and renderer: `src/lib/cliContracts.ts`. diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts new file mode 100644 index 0000000000..bf3bab8a7b --- /dev/null +++ b/electron/cli/args.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest"; +import { parseCliArgs } from "./args"; + +const CWD = "/work"; + +function parse(args: string[]) { + return parseCliArgs(["electron", "app-path", ...args], 2, CWD); +} + +describe("parseCliArgs", () => { + it("returns null when no subcommand is present (GUI launch)", () => { + expect(parse([])).toBeNull(); + expect(parse(["--some-chromium-flag"])).toBeNull(); + }); + + it("skips leading Chromium switches before the subcommand (AppImage --no-sandbox)", () => { + expect(parse(["--no-sandbox", "export", "demo.openscreen"])).toMatchObject({ + kind: "export", + projectPath: "/work/demo.openscreen", + }); + expect(parse(["--no-sandbox", "--enable-unsafe-swiftshader", "record"])).toMatchObject({ + kind: "record", + }); + expect(parse(["--no-sandbox", "--help"])).toMatchObject({ kind: "help" }); + expect(parse(["--no-sandbox"])).toBeNull(); + }); + + it("parses a minimal export command and resolves relative paths", () => { + const cmd = parse(["export", "demo.openscreen"]); + expect(cmd).toMatchObject({ + kind: "export", + projectPath: "/work/demo.openscreen", + outPath: null, + format: null, + }); + }); + + it("parses export options and infers format from --out extension", () => { + const cmd = parse([ + "export", + "/p/demo.openscreen", + "-o", + "out.gif", + "--gif-fps", + "20", + "--preview-size", + "1600x900", + "--json", + ]); + expect(cmd).toMatchObject({ + kind: "export", + outPath: "/work/out.gif", + format: "gif", + gifFrameRate: 20, + previewWidth: 1600, + previewHeight: 900, + json: true, + }); + }); + + it("rejects a --format that conflicts with the --out extension", () => { + const cmd = parse(["export", "a.openscreen", "-o", "x.mp4", "--format", "gif"]); + expect(cmd).toMatchObject({ kind: "error" }); + }); + + it("rejects export without a project path", () => { + expect(parse(["export"])).toMatchObject({ kind: "error" }); + }); + + it("parses voiceover audio options", () => { + const cmd = parse([ + "export", + "a.openscreen", + "--audio", + "voice.mp3", + "--audio-mode", + "replace", + "--audio-offset", + "1.5", + ]); + expect(cmd).toMatchObject({ + kind: "export", + audioPath: "/work/voice.mp3", + audioMode: "replace", + audioOffsetSec: 1.5, + }); + }); + + it("defaults audio mode to mix and rejects --audio with gif", () => { + expect(parse(["export", "a.openscreen", "--audio", "v.mp3"])).toMatchObject({ + audioMode: "mix", + audioOffsetSec: 0, + }); + expect(parse(["export", "a.openscreen", "--audio", "v.mp3", "--format", "gif"])).toMatchObject({ + kind: "error", + }); + expect(parse(["export", "a.openscreen", "--audio-offset", "-1"])).toMatchObject({ + kind: "error", + }); + }); + + it("parses record defaults", () => { + expect(parse(["record"])).toMatchObject({ + kind: "record", + displayIndex: 0, + windowTitle: null, + mic: false, + systemAudio: false, + cursorMode: "editable-overlay", + durationMs: null, + }); + }); + + it("parses record options", () => { + const cmd = parse([ + "record", + "--display", + "1", + "--mic-device", + "MacBook", + "--system-audio", + "--duration", + "12.5", + "--project", + "demo.openscreen", + ]); + expect(cmd).toMatchObject({ + kind: "record", + displayIndex: 1, + mic: true, + micDevice: "MacBook", + systemAudio: true, + durationMs: 12500, + projectOut: "/work/demo.openscreen", + }); + }); + + it("rejects invalid record values", () => { + expect(parse(["record", "--duration", "0"])).toMatchObject({ kind: "error" }); + expect(parse(["record", "--cursor", "off"])).toMatchObject({ kind: "error" }); + expect(parse(["record", "--project", "demo.json"])).toMatchObject({ kind: "error" }); + }); + + it("parses info and help", () => { + expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({ + kind: "info", + projectPath: "/work/demo.openscreen", + json: true, + }); + expect(parse(["help"])).toMatchObject({ kind: "help" }); + expect(parse(["--help"])).toMatchObject({ kind: "help" }); + }); +}); diff --git a/electron/cli/args.ts b/electron/cli/args.ts new file mode 100644 index 0000000000..0ed95c8586 --- /dev/null +++ b/electron/cli/args.ts @@ -0,0 +1,346 @@ +// Pure argv parser for the OpenScreen CLI. No Electron imports so it can be +// unit-tested under plain vitest. + +import path from "node:path"; +import type { CliExportRequest, CliRecordRequest, CliRequest } from "../../src/lib/cliContracts"; + +export interface CliInfoCommand { + kind: "info"; + projectPath: string; +} + +export interface CliHelpCommand { + kind: "help"; +} + +export interface CliErrorCommand { + kind: "error"; + message: string; +} + +export type CliCommand = (CliRequest | CliInfoCommand | CliHelpCommand | CliErrorCommand) & { + /** Machine-readable NDJSON output on stdout instead of human progress. */ + json?: boolean; +}; + +const SUBCOMMANDS = new Set(["export", "record", "info", "help", "--help", "-h"]); + +export const CLI_USAGE = `OpenScreen CLI + +Usage: + openscreen export <project.openscreen> [options] Render a project to MP4/GIF + openscreen record [options] Record the screen headlessly + openscreen info <project.openscreen> [--json] Inspect a project file + openscreen help Show this help + +Export options: + -o, --out <path> Output file (.mp4 or .gif). Default: next to the project file + --format <mp4|gif> Override the format stored in the project + --quality <medium|good|source> + MP4 quality (default: from project) + --gif-fps <15|20|25|30> GIF frame rate (default: from project) + --gif-size <medium|large|original> + GIF size preset (default: from project) + --preview-size <WxH> Reference preview box for annotation scaling (default 1280x720) + --audio <file> Mix a voiceover audio file into the MP4 (mp3/wav/m4a) + --audio-mode <mix|replace> + Layer over the recording's audio (default) or replace it + --audio-offset <seconds> Delay before the voiceover starts (default 0) + --json NDJSON progress/result on stdout + +Record options (recording is saved into the app's recordings directory): + --display <n> Screen index to record (default 0) + --window <title> Record the first window whose title contains <title> + --mic Capture the default microphone + --mic-device <name> Microphone device name (implies --mic) + --system-audio Capture system audio + --cursor <editable-overlay|system> + Cursor capture mode (default editable-overlay) + --duration <seconds> Stop automatically after this long + --project <out.openscreen> + Write a ready-to-export project file when done + --json NDJSON events on stdout + +Stopping a recording: send SIGINT/SIGTERM, type "stop" + Enter on stdin, +or pass --duration. +`; + +function takeValue(argv: string[], i: number, flag: string): [string, number] { + const next = argv[i + 1]; + if (next === undefined || next.startsWith("--")) { + throw new Error(`${flag} requires a value`); + } + return [next, i + 1]; +} + +function resolvePath(p: string, cwd: string): string { + return path.isAbsolute(p) ? p : path.resolve(cwd, p); +} + +/** + * Extracts CLI arguments from process.argv. Returns null when no subcommand is + * present (normal GUI launch). `firstArgIndex` should be 1 for packaged builds + * and 2 when running via `electron <app-path> ...`. + */ +export function parseCliArgs( + argv: string[], + firstArgIndex: number, + cwd: string = process.cwd(), +): CliCommand | null { + const rawArgs = argv.slice(firstArgIndex).filter((a) => !a.startsWith("--inspect")); + + // Skip *leading* Chromium/Electron switches (e.g. the AppImage's required + // `--no-sandbox`) so `Openscreen --no-sandbox export demo.openscreen` still + // enters CLI mode. Only leading dash-tokens are skipped — everything after + // the subcommand belongs to the subcommand parser. `--help`/`-h` are ours. + let subIndex = 0; + while ( + subIndex < rawArgs.length && + rawArgs[subIndex].startsWith("-") && + rawArgs[subIndex] !== "--help" && + rawArgs[subIndex] !== "-h" + ) { + subIndex++; + } + + const args = rawArgs.slice(subIndex); + const sub = args[0]; + if (!sub || !SUBCOMMANDS.has(sub)) return null; + if (sub === "help" || sub === "--help" || sub === "-h") return { kind: "help" }; + + try { + if (sub === "export") return parseExport(args.slice(1), cwd); + if (sub === "record") return parseRecord(args.slice(1), cwd); + return parseInfo(args.slice(1), cwd); + } catch (error) { + return { kind: "error", message: error instanceof Error ? error.message : String(error) }; + } +} + +function parseExport(args: string[], cwd: string): CliCommand { + const request: CliExportRequest & { json?: boolean } = { + kind: "export", + projectPath: "", + outPath: null, + format: null, + quality: null, + gifFrameRate: null, + gifSizePreset: null, + previewWidth: null, + previewHeight: null, + audioPath: null, + audioMode: "mix", + audioOffsetSec: 0, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case "-o": + case "--out": { + const [value, next] = takeValue(args, i, arg); + request.outPath = resolvePath(value, cwd); + i = next; + break; + } + case "--format": { + const [value, next] = takeValue(args, i, arg); + if (value !== "mp4" && value !== "gif") { + throw new Error(`--format must be mp4 or gif, got "${value}"`); + } + request.format = value; + i = next; + break; + } + case "--quality": { + const [value, next] = takeValue(args, i, arg); + if (value !== "medium" && value !== "good" && value !== "source") { + throw new Error(`--quality must be medium, good or source, got "${value}"`); + } + request.quality = value; + i = next; + break; + } + case "--gif-fps": { + const [value, next] = takeValue(args, i, arg); + const fps = Number(value); + if (fps !== 15 && fps !== 20 && fps !== 25 && fps !== 30) { + throw new Error(`--gif-fps must be 15, 20, 25 or 30, got "${value}"`); + } + request.gifFrameRate = fps; + i = next; + break; + } + case "--gif-size": { + const [value, next] = takeValue(args, i, arg); + if (value !== "medium" && value !== "large" && value !== "original") { + throw new Error(`--gif-size must be medium, large or original, got "${value}"`); + } + request.gifSizePreset = value; + i = next; + break; + } + case "--preview-size": { + const [value, next] = takeValue(args, i, arg); + const match = /^(\d+)x(\d+)$/.exec(value); + if (!match) throw new Error(`--preview-size must look like 1280x720, got "${value}"`); + request.previewWidth = Number(match[1]); + request.previewHeight = Number(match[2]); + i = next; + break; + } + case "--audio": { + const [value, next] = takeValue(args, i, arg); + request.audioPath = resolvePath(value, cwd); + i = next; + break; + } + case "--audio-mode": { + const [value, next] = takeValue(args, i, arg); + if (value !== "mix" && value !== "replace") { + throw new Error(`--audio-mode must be mix or replace, got "${value}"`); + } + request.audioMode = value; + i = next; + break; + } + case "--audio-offset": { + const [value, next] = takeValue(args, i, arg); + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds < 0) { + throw new Error( + `--audio-offset must be a non-negative number of seconds, got "${value}"`, + ); + } + request.audioOffsetSec = seconds; + i = next; + break; + } + case "--json": + request.json = true; + break; + default: + if (arg.startsWith("-")) throw new Error(`Unknown export option: ${arg}`); + if (request.projectPath) throw new Error(`Unexpected extra argument: ${arg}`); + request.projectPath = resolvePath(arg, cwd); + } + } + + if (!request.projectPath) throw new Error("export requires a <project.openscreen> path"); + if (request.audioPath && request.format === "gif") { + throw new Error("--audio is only supported for MP4 exports"); + } + if (request.outPath) { + const ext = path.extname(request.outPath).toLowerCase(); + if (ext !== ".mp4" && ext !== ".gif") { + throw new Error(`--out must end in .mp4 or .gif, got "${request.outPath}"`); + } + const extFormat = ext === ".gif" ? "gif" : "mp4"; + if (request.format && request.format !== extFormat) { + throw new Error(`--format ${request.format} conflicts with --out extension ${ext}`); + } + request.format = extFormat; + } + return request; +} + +function parseRecord(args: string[], cwd: string): CliCommand { + const request: CliRecordRequest & { json?: boolean } = { + kind: "record", + displayIndex: 0, + windowTitle: null, + mic: false, + micDevice: null, + systemAudio: false, + cursorMode: "editable-overlay", + durationMs: null, + projectOut: null, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + switch (arg) { + case "--display": { + const [value, next] = takeValue(args, i, arg); + const index = Number(value); + if (!Number.isInteger(index) || index < 0) { + throw new Error(`--display must be a non-negative integer, got "${value}"`); + } + request.displayIndex = index; + i = next; + break; + } + case "--window": { + const [value, next] = takeValue(args, i, arg); + request.windowTitle = value; + i = next; + break; + } + case "--mic": + request.mic = true; + break; + case "--mic-device": { + const [value, next] = takeValue(args, i, arg); + request.mic = true; + request.micDevice = value; + i = next; + break; + } + case "--system-audio": + request.systemAudio = true; + break; + case "--cursor": { + const [value, next] = takeValue(args, i, arg); + if (value !== "editable-overlay" && value !== "system") { + throw new Error(`--cursor must be editable-overlay or system, got "${value}"`); + } + request.cursorMode = value; + i = next; + break; + } + case "--duration": { + const [value, next] = takeValue(args, i, arg); + const seconds = Number(value); + if (!Number.isFinite(seconds) || seconds <= 0) { + throw new Error(`--duration must be a positive number of seconds, got "${value}"`); + } + request.durationMs = Math.round(seconds * 1000); + i = next; + break; + } + case "--project": { + const [value, next] = takeValue(args, i, arg); + if (!value.endsWith(".openscreen")) { + throw new Error(`--project must end in .openscreen, got "${value}"`); + } + request.projectOut = resolvePath(value, cwd); + i = next; + break; + } + case "--json": + request.json = true; + break; + default: + throw new Error(`Unknown record option: ${arg}`); + } + } + return request; +} + +function parseInfo(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let json = false; + for (const arg of args) { + if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown info option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("info requires a <project.openscreen> path"); + return { kind: "info", projectPath, json }; +} diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts new file mode 100644 index 0000000000..ed3d487fbb --- /dev/null +++ b/electron/cli/cliMain.ts @@ -0,0 +1,442 @@ +// Headless CLI mode: boots Electron without HUD/tray/menu, drives a hidden +// renderer window (windowType=cli-export | cli-record) that reuses the app's +// existing export and recording pipelines, and reports progress on stdio. + +import fs from "node:fs/promises"; +import path from "node:path"; +import readline from "node:readline"; +import { fileURLToPath } from "node:url"; +import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron"; +import type { CliDoneResult, CliProgressEvent, CliRequest } from "../../src/lib/cliContracts"; +import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; +import { ASSET_BASE_URL_ARG } from "../windows"; +import { CLI_USAGE, type CliCommand } from "./args"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"]; +const RENDERER_DIST = path.join(__dirname, "..", "dist"); + +interface CliOutput { + json: boolean; + event(event: string, data?: Record<string, unknown>): void; + info(message: string): void; + error(message: string): void; + progress(p: CliProgressEvent): void; +} + +// stdout/stderr may be a closed pipe (`openscreen export | head`); writes must +// never take the process down — Electron would show a GUI error dialog. +function safeWrite(stream: NodeJS.WriteStream, text: string): void { + try { + stream.write(text); + } catch { + // EPIPE or closed stream; drop the output. + } +} + +function createOutput(json: boolean): CliOutput { + const isTty = process.stdout.isTTY === true; + let progressLineActive = false; + let lastProgressText = ""; + const clearProgressLine = () => { + if (progressLineActive) { + safeWrite(process.stdout, "\n"); + progressLineActive = false; + } + }; + return { + json, + event(event, data = {}) { + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event, ...data })}\n`); + } + }, + info(message) { + if (json) return; + clearProgressLine(); + safeWrite(process.stdout, `${message}\n`); + }, + error(message) { + clearProgressLine(); + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event: "error", message })}\n`); + } else { + safeWrite(process.stderr, `Error: ${message}\n`); + } + }, + progress(p) { + if (json) { + safeWrite(process.stdout, `${JSON.stringify({ event: "progress", ...p })}\n`); + return; + } + const frames = + p.currentFrame !== undefined && p.totalFrames + ? ` frame ${p.currentFrame}/${p.totalFrames}` + : ""; + const eta = + p.estimatedTimeRemaining !== undefined && Number.isFinite(p.estimatedTimeRemaining) + ? ` ETA ${Math.max(0, Math.round(p.estimatedTimeRemaining))}s` + : ""; + const phase = p.phase ? ` [${p.phase}]` : ""; + const text = `Exporting ${Math.round(p.percentage)}%${frames}${eta}${phase}`; + if (isTty) { + if (text === lastProgressText) return; + lastProgressText = text; + safeWrite(process.stdout, `\r\x1b[2K${text}`); + progressLineActive = true; + } else { + // Piped/non-TTY consumers get one line per whole-percent (or phase) change. + const coarse = `${Math.round(p.percentage)}%${phase}`; + if (coarse === lastProgressText) return; + lastProgressText = coarse; + safeWrite(process.stdout, `${text}\n`); + } + }, + }; +} + +function loadRunnerWindow(windowType: string): BrowserWindow { + const win = new BrowserWindow({ + width: 1280, + height: 720, + show: false, + webPreferences: { + preload: path.join(__dirname, "preload.mjs"), + additionalArguments: [ASSET_BASE_URL_ARG], + nodeIntegration: false, + contextIsolation: true, + // Same relaxation as the editor window: exporters load recording media + // via file:// URLs. + webSecurity: false, + backgroundThrottling: false, + }, + }); + + if (VITE_DEV_SERVER_URL) { + win.loadURL(`${VITE_DEV_SERVER_URL}?windowType=${windowType}`); + } else { + win.loadFile(path.join(RENDERER_DIST, "index.html"), { query: { windowType } }); + } + return win; +} + +function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) { + // The recording/export pipelines are driven through the same IPC surface the + // GUI uses. Window-management callbacks become no-ops; "switch-to-editor" + // (fired by the recorder hook after it stores a finished session) is handled + // by the runner itself, so an inert factory is enough. + const noop = () => { + // Intentionally empty: CLI mode has no HUD/tray/editor windows to manage. + }; + const notAvailable = () => { + throw new Error("Window not available in CLI mode"); + }; + registerIpcHandlers( + noop, // createEditorWindow: recording finished; runner drives completion + notAvailable, // createSourceSelectorWindow + notAvailable, // createCountdownOverlayWindow + notAvailable, // createNotesWindow + cliWindowRef, + () => null, + () => null, + () => null, + noop, // onRecordingStateChange: no tray to update + noop, // switchToHud + ); +} + +async function writeProjectFile(projectOut: string, projectData: unknown): Promise<void> { + await fs.mkdir(path.dirname(projectOut), { recursive: true }); + await fs.writeFile(projectOut, JSON.stringify(projectData, null, 2), "utf8"); +} + +function setupRecordStopSignals(stop: (reason: string) => void): void { + // SIGINT covers Ctrl+C everywhere; SIGTERM never fires on Windows, where + // stdin "stop" or --duration are the graceful alternatives (see docs/cli.md). + process.on("SIGINT", () => stop("SIGINT")); + process.on("SIGTERM", () => stop("SIGTERM")); + try { + // Touching process.stdin can throw on Windows GUI-subsystem builds when + // spawned with stdio "ignore"; signals and --duration still work then. + const rl = readline.createInterface({ input: process.stdin }); + rl.on("line", (line) => { + const trimmed = line.trim().toLowerCase(); + if (trimmed === "stop" || trimmed === "q" || trimmed === "quit") { + stop("stdin"); + } + }); + rl.on("close", () => { + // stdin EOF is not a stop signal: agents may spawn the CLI with + // stdin closed and stop it via SIGINT/--duration instead. + }); + } catch { + // stdin unavailable; signals and --duration still work. + } +} + +async function runInfoCommand(projectPath: string, json: boolean): Promise<number> { + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as { + version?: number; + media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; + videoPath?: string; + editor?: Record<string, unknown>; + }; + const editor = data.editor ?? {}; + const count = (key: string) => + Array.isArray(editor[key]) ? (editor[key] as unknown[]).length : 0; + const screenVideoPath = data.media?.screenVideoPath ?? data.videoPath ?? null; + const mediaExists = screenVideoPath + ? await fs + .access(screenVideoPath) + .then(() => true) + .catch(() => false) + : false; + + const summary = { + projectPath, + version: data.version ?? null, + screenVideoPath, + screenVideoExists: mediaExists, + webcamVideoPath: data.media?.webcamVideoPath ?? null, + cursorCaptureMode: data.media?.cursorCaptureMode ?? null, + exportFormat: (editor.exportFormat as string) ?? null, + exportQuality: (editor.exportQuality as string) ?? null, + aspectRatio: (editor.aspectRatio as string) ?? null, + zoomRegions: count("zoomRegions"), + trimRegions: count("trimRegions"), + speedRegions: count("speedRegions"), + annotationRegions: count("annotationRegions"), + }; + + if (json) { + process.stdout.write(`${JSON.stringify(summary)}\n`); + } else { + process.stdout.write( + [ + `Project: ${summary.projectPath} (version ${summary.version ?? "?"})`, + `Video: ${summary.screenVideoPath ?? "(none)"}${mediaExists ? "" : " [MISSING]"}`, + `Webcam: ${summary.webcamVideoPath ?? "(none)"}`, + `Cursor: ${summary.cursorCaptureMode ?? "(unknown)"}`, + `Export: ${summary.exportFormat ?? "?"} / ${summary.exportQuality ?? "?"} / ${summary.aspectRatio ?? "?"}`, + `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations`, + ].join("\n") + "\n", + ); + } + return summary.screenVideoPath && !mediaExists ? 1 : 0; +} + +export function runCli(command: CliCommand): void { + if (command.kind === "help") { + process.stdout.write(CLI_USAGE); + app.exit(0); + return; + } + if (command.kind === "error") { + process.stderr.write(`Error: ${command.message}\n\n${CLI_USAGE}`); + app.exit(2); + return; + } + + const output = createOutput(command.json === true); + + // stdout belongs to the CLI protocol (NDJSON / progress); reroute the app's + // own console chatter (e.g. "[native-sck] starting…") to stderr. + for (const level of ["log", "info", "warn", "error", "debug"] as const) { + console[level] = (...args: unknown[]) => { + safeWrite( + process.stderr, + `${args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ")}\n`, + ); + }; + } + + // A consumer closing the pipe (`openscreen export | head`) must not crash the + // process, and main-process exceptions must never surface as Electron's GUI + // error dialog — report on stderr and exit non-zero instead. + const ignoreStreamError = () => { + // Intentionally empty: EPIPE from a closed consumer is not an error here. + }; + process.stdout.on("error", ignoreStreamError); + process.stderr.on("error", ignoreStreamError); + process.on("uncaughtException", (error) => { + safeWrite(process.stderr, `Fatal: ${error?.stack ?? String(error)}\n`); + app.exit(1); + }); + process.on("unhandledRejection", (reason) => { + safeWrite( + process.stderr, + `Fatal (unhandled rejection): ${reason instanceof Error ? (reason.stack ?? reason.message) : String(reason)}\n`, + ); + app.exit(1); + }); + + // GPU may be unavailable in CI/servers; let Chromium fall back to SwiftShader + // so the WebGL-based export renderer still works. + app.commandLine.appendSwitch("enable-unsafe-swiftshader"); + + // Never show the dock icon for CLI runs. + if (process.platform === "darwin") { + app.dock?.hide(); + } + + app.on("window-all-closed", () => { + // Completion is signalled via cli-done; a vanished window is a failure. + output.error("Renderer window closed unexpectedly"); + app.exit(1); + }); + + void app + .whenReady() + .then(async () => { + if (command.kind === "info") { + const code = await runInfoCommand(command.projectPath, command.json === true); + app.exit(code); + return; + } + + await fs.mkdir(path.join(app.getPath("userData"), "recordings"), { recursive: true }); + + // Media/screen permissions for the renderer (mic metering, future browser + // capture paths). Mirrors the GUI allowlist. + const allowed = [ + "media", + "audioCapture", + "microphone", + "videoCapture", + "camera", + "screen", + "display-capture", + ]; + session.defaultSession.setPermissionCheckHandler((_wc, permission) => + allowed.includes(permission), + ); + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => + callback(allowed.includes(permission)), + ); + + // Browser-pipeline recording fallback (e.g. Linux, missing native helper) + // resolves the pre-selected source exactly like the GUI does. + session.defaultSession.setDisplayMediaRequestHandler( + (request, callback) => { + const source = getSelectedDesktopSource(); + if (!request.videoRequested || !source) { + callback({}); + return; + } + callback({ + video: source, + ...(request.audioRequested && process.platform === "win32" + ? { audio: "loopback" as const } + : {}), + }); + }, + { useSystemPicker: false }, + ); + + if (command.kind === "record" && command.mic && process.platform === "darwin") { + const micStatus = systemPreferences.getMediaAccessStatus("microphone"); + if (micStatus !== "granted") { + await systemPreferences.askForMediaAccess("microphone"); + } + } + + let cliWindow: BrowserWindow | null = null; + registerAppHandlersForCli(() => cliWindow); + + // Registered by the GUI boot path (main.ts) rather than registerIpcHandlers; + // the renderer's i18n init invokes it unconditionally. + ipcMain.handle("set-locale", () => { + // Locale only affects GUI menus/tray, which do not exist in CLI mode. + }); + ipcMain.handle("update-global-shortcut", () => ({ success: false })); + + const request: CliRequest = command; + ipcMain.handle("cli-get-request", () => request); + ipcMain.on("cli-log", (_event, level: string, message: string) => { + if (level === "error") { + output.error(message); + } else { + output.info(message); + output.event("log", { message }); + } + }); + ipcMain.on("cli-progress", (_event, progress: CliProgressEvent) => { + output.progress(progress); + }); + + let finished = false; + ipcMain.handle("cli-done", async (_event, result: CliDoneResult) => { + if (finished) return; + finished = true; + + try { + if (result.success && command.kind === "record" && command.projectOut) { + if (result.projectData !== undefined) { + await writeProjectFile(command.projectOut, result.projectData); + result.projectPath = command.projectOut; + } + } + } catch (error) { + result.success = false; + result.error = `Recording succeeded but writing the project file failed: ${String(error)}`; + } + + if (result.success) { + for (const warning of result.warnings ?? []) { + output.info(`Warning: ${warning}`); + output.event("warning", { message: warning }); + } + if (command.kind === "export") { + output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); + } else { + output.info(`Recording saved → ${result.screenVideoPath}`); + if (result.cursorDataPath) output.info(`Cursor data → ${result.cursorDataPath}`); + if (result.projectPath) output.info(`Project → ${result.projectPath}`); + } + output.event("done", { ...result }); + } else { + output.error(result.error ?? "Unknown failure"); + output.event("done", { success: false, error: result.error }); + } + + // Give the renderer a beat to resolve the invoke before exiting. + setTimeout(() => app.exit(result.success ? 0 : 1), 50); + }); + + if (command.kind === "record") { + const stop = (reason: string) => { + output.info(`Stopping recording (${reason})…`); + output.event("stopping", { reason }); + cliWindow?.webContents.send("cli-stop-recording"); + }; + setupRecordStopSignals(stop); + } + + const windowType = command.kind === "export" ? "cli-export" : "cli-record"; + cliWindow = loadRunnerWindow(windowType); + + // Surface renderer console errors/warnings on stderr — the hidden window + // has no other way to show what went wrong (toasts are invisible). + cliWindow.webContents.on("console-message", (details) => { + if (details.level === "error" || details.level === "warning") { + safeWrite(process.stderr, `[renderer] ${details.message}\n`); + } + }); + + cliWindow.webContents.on("did-fail-load", (_e, code, description) => { + output.error(`Failed to load runner window: ${description} (${code})`); + app.exit(1); + }); + cliWindow.webContents.on("render-process-gone", (_e, details) => { + output.error(`Renderer crashed: ${details.reason}`); + app.exit(1); + }); + + output.event("started", { command: command.kind }); + }) + .catch((error) => { + output.error(error instanceof Error ? (error.stack ?? error.message) : String(error)); + app.exit(1); + }); +} diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index dd761b0b05..bbac97658f 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -394,6 +394,12 @@ interface Window { callback: (event: import("./stt/transcriptionContract").SttStatusEvent) => void, ) => () => void; }; + // CLI mode (hidden runner windows; see electron/cli/) + cliGetRequest: () => Promise<import("../src/lib/cliContracts").CliRequest>; + cliProgress: (progress: import("../src/lib/cliContracts").CliProgressEvent) => void; + cliLog: (level: "info" | "error", message: string) => void; + cliDone: (result: import("../src/lib/cliContracts").CliDoneResult) => Promise<void>; + onCliStopRecording: (callback: () => void) => () => void; setLocale: (locale: string) => Promise<void>; saveDiagnostic: (payload: { error: string; diff --git a/electron/main.ts b/electron/main.ts index 1fa013935f..7549121f3f 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -12,6 +12,8 @@ import { Tray, } from "electron"; import { ShortcutBinding } from "../src/lib/shortcuts"; +import { parseCliArgs } from "./cli/args"; +import { runCli } from "./cli/cliMain"; import { isDiagnosticModeEnabled, mainLogBuffer } from "./diagnostics/main-log-buffer"; import { loadAndRegisterGlobalShortcut, @@ -33,6 +35,10 @@ import { const __dirname = path.dirname(fileURLToPath(import.meta.url)); +// CLI mode: `openscreen export|record|info|help ...` runs headless without +// HUD/tray/menu. Parsed before any GUI side effects; see electron/cli/. +const cliCommand = parseCliArgs(process.argv, app.isPackaged ? 1 : 2); + // Use Screen & System Audio Recording permissions instead of the CoreAudio Tap API on macOS. // Tap needs NSAudioCaptureUsageDescription in the parent app's Info.plist, which breaks when // running from a terminal/IDE during dev. @@ -126,11 +132,15 @@ function showMainWindow() { createWindow(); } -const stableInstanceLock = acquireStableInstanceLock(); -const hasElectronSingleInstanceLock = app.requestSingleInstanceLock(); +// CLI runs skip the single-instance lock so `openscreen export/record` works +// while the GUI app is open (they share nothing but the recordings directory). +const stableInstanceLock = cliCommand ? null : acquireStableInstanceLock(); +const hasElectronSingleInstanceLock = cliCommand ? false : app.requestSingleInstanceLock(); const hasSingleInstanceLock = Boolean(stableInstanceLock && hasElectronSingleInstanceLock); -if (hasSingleInstanceLock) { +if (cliCommand) { + runCli(cliCommand); +} else if (hasSingleInstanceLock) { app.on("second-instance", () => { showMainWindow(); }); @@ -482,11 +492,15 @@ function createCountdownOverlayWindowWrapper() { // Closing every window quits the app (tray goes too). The in-app "Return to Recorder" // button covers the editor-to-HUD round-trip, so closing the last window means "I'm done". -app.on("window-all-closed", () => { - app.quit(); -}); +// CLI mode owns its own lifecycle (see electron/cli/cliMain.ts). +if (!cliCommand) { + app.on("window-all-closed", () => { + app.quit(); + }); +} app.on("activate", () => { + if (cliCommand) return; // On macOS, re-open a window when the dock icon is clicked and none are open. const hasVisibleWindow = BrowserWindow.getAllWindows().some((window) => { if (window.isDestroyed() || !window.isVisible()) { @@ -507,7 +521,7 @@ app.on("will-quit", () => { stableInstanceLock?.release(); }); -const appReady = hasSingleInstanceLock ? app.whenReady() : null; +const appReady = !cliCommand && hasSingleInstanceLock ? app.whenReady() : null; appReady?.then(async () => { if (isDiagnosticModeEnabled()) { diff --git a/electron/preload.ts b/electron/preload.ts index dfc9db84f8..bc1ac803ed 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -407,6 +407,7 @@ contextBridge.exposeInMainWorld("electronAPI", { sendCloseConfirmResponse: (choice: "save" | "discard" | "cancel") => { ipcRenderer.send("close-confirm-response", choice); }, +<<<<<<< HEAD // ponytail: forward renderer console output to main-process stdout so // recorder diagnostics land next to the main-process logs in dev output. // One-way fire-and-forget; we deliberately don't await the IPC. @@ -428,4 +429,22 @@ contextBridge.exposeInMainWorld("electronAPI", { return () => ipcRenderer.removeListener("stt:status", listener); }, }, + // --- CLI mode (hidden runner windows; see electron/cli/) --- + cliGetRequest: () => { + return ipcRenderer.invoke("cli-get-request"); + }, + cliProgress: (progress: unknown) => { + ipcRenderer.send("cli-progress", progress); + }, + cliLog: (level: "info" | "error", message: string) => { + ipcRenderer.send("cli-log", level, message); + }, + cliDone: (result: unknown) => { + return ipcRenderer.invoke("cli-done", result); + }, + onCliStopRecording: (callback: () => void) => { + const listener = () => callback(); + ipcRenderer.on("cli-stop-recording", listener); + return () => ipcRenderer.removeListener("cli-stop-recording", listener); + }, }); diff --git a/electron/windows.ts b/electron/windows.ts index bf219ca9c3..687a3a175b 100644 --- a/electron/windows.ts +++ b/electron/windows.ts @@ -88,7 +88,7 @@ function applyContentProtection(win: BrowserWindow, label: string) { const ASSET_BASE_DIR = process.defaultApp ? path.join(__dirname, "..", "public") : process.resourcesPath; -const ASSET_BASE_URL_ARG = `--asset-base-url=${pathToFileURL(`${ASSET_BASE_DIR}${path.sep}`).toString()}`; +export const ASSET_BASE_URL_ARG = `--asset-base-url=${pathToFileURL(`${ASSET_BASE_DIR}${path.sep}`).toString()}`; let hudOverlayWindow: BrowserWindow | null = null; diff --git a/package.json b/package.json index f15735ddf6..66f2f88905 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "diagnostic:run": "node scripts/diagnostic-tool/diagnostic.mjs", "diagnostic:smoke:win": "node scripts/diagnostic-tool/diagnostic.mjs --duration 3", "build-vite": "tsc && vite build", + "cli": "electron .", "test:e2e": "playwright test", "test:e2e:windows-native-checklist": "playwright test tests/e2e/windows-native-checklist.spec.ts", "prepare": "husky", diff --git a/src/App.tsx b/src/App.tsx index f0e4d1154f..8bbed307ec 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -20,6 +20,8 @@ const VideoEditorEntry = lazy(() => default: module.default, })), ); +const CliExportRunner = lazy(() => import("./cli/CliExportRunner")); +const CliRecordRunner = lazy(() => import("./cli/CliRecordRunner")); const ShortcutsConfigDialog = lazy(() => import("./components/video-editor/ShortcutsConfigDialog").then((module) => ({ default: module.ShortcutsConfigDialog, @@ -76,6 +78,18 @@ export default function App() { return <SourceSelector />; case "countdown-overlay": return <CountdownOverlay />; + case "cli-export": + return ( + <Suspense fallback={null}> + <CliExportRunner /> + </Suspense> + ); + case "cli-record": + return ( + <Suspense fallback={null}> + <CliRecordRunner /> + </Suspense> + ); case "editor": return ( <ShortcutsProvider> diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx new file mode 100644 index 0000000000..b8b0ea51ce --- /dev/null +++ b/src/cli/CliExportRunner.tsx @@ -0,0 +1,323 @@ +// Hidden-window runner for `openscreen export`. Loads an .openscreen project, +// rebuilds the same exporter configuration the editor's export dialog would, +// and streams progress back to the CLI controller in the main process. + +import { useEffect, useRef, useState } from "react"; +import { + DEFAULT_CURSOR_SETTINGS, + DEFAULT_SOURCE_DIMENSIONS, +} from "@/components/video-editor/editorDefaults"; +import { + normalizeProjectEditor, + resolveProjectMedia, + toFileUrl, + validateProjectData, +} from "@/components/video-editor/projectPersistence"; +import type { CursorTelemetryPoint } from "@/components/video-editor/types"; +import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; +import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; +import { calculateOutputDimensions, GifExporter } from "@/lib/exporter/gifExporter"; +import { + calculateEffectiveSourceDimensions, + calculateMp4ExportSettings, +} from "@/lib/exporter/mp4ExportSettings"; +import type { ExportProgress } from "@/lib/exporter/types"; +import { GIF_SIZE_PRESETS } from "@/lib/exporter/types"; +import { VideoExporter } from "@/lib/exporter/videoExporter"; +import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; +import { nativeBridgeClient } from "@/native"; +import type { CursorRecordingData, NativePlatform } from "@/native/contracts"; +import { getAspectRatioValue, getNativeAspectRatioValue } from "@/utils/aspectRatioUtils"; + +// Mirrors the private helper in VideoEditor.tsx. +function isClickInteractionType(interactionType: string | null | undefined) { + return ( + interactionType === "click" || + interactionType === "double-click" || + interactionType === "right-click" || + interactionType === "middle-click" + ); +} + +function probeVideoDimensions(url: string): Promise<{ width: number; height: number }> { + return new Promise((resolve, reject) => { + const video = document.createElement("video"); + video.preload = "metadata"; + video.muted = true; + video.onloadedmetadata = () => { + const width = video.videoWidth; + const height = video.videoHeight; + video.removeAttribute("src"); + video.load(); + resolve({ width, height }); + }; + video.onerror = () => reject(new Error(`Failed to load video metadata: ${url}`)); + video.src = url; + }); +} + +/** Fit the composition aspect ratio into the reference preview box, mirroring + * how the editor sizes its on-screen preview container. */ +function fitPreviewBox(aspectRatioValue: number, boxWidth: number, boxHeight: number) { + let width = boxWidth; + let height = boxWidth / aspectRatioValue; + if (height > boxHeight) { + height = boxHeight; + width = boxHeight * aspectRatioValue; + } + return { width: Math.round(width), height: Math.round(height) }; +} + +function replaceExtension(filePath: string, newExtension: string): string { + return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; +} + +async function runExport(request: CliExportRequest): Promise<CliDoneResult> { + const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); + if (!loaded.success || loaded.project === undefined) { + throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + } + if (!validateProjectData(loaded.project)) { + throw new Error("Project file is not a valid .openscreen project"); + } + const project = loaded.project; + const media = resolveProjectMedia(project); + if (!media) { + throw new Error("Project file does not reference any recorded media"); + } + const editor = normalizeProjectEditor(project.editor ?? {}); + + const format = request.format ?? editor.exportFormat; + if (request.audioPath && format === "gif") { + throw new Error( + "--audio is only supported for MP4 exports (this project's stored format is gif; pass --format mp4)", + ); + } + const quality = request.quality ?? editor.exportQuality; + const gifFrameRate = request.gifFrameRate ?? editor.gifFrameRate; + const gifSizePreset = request.gifSizePreset ?? editor.gifSizePreset; + const outPath = + request.outPath ?? replaceExtension(request.projectPath, format === "gif" ? ".gif" : ".mp4"); + + const videoUrl = toFileUrl(media.screenVideoPath); + const webcamVideoUrl = media.webcamVideoPath ? toFileUrl(media.webcamVideoPath) : undefined; + + // Cursor sidecar data (native recordings). Both lookups tolerate missing files. + let cursorTelemetry: CursorTelemetryPoint[] = []; + let cursorRecordingData: CursorRecordingData | null = null; + try { + cursorTelemetry = await nativeBridgeClient.cursor.getTelemetry(media.screenVideoPath); + } catch { + cursorTelemetry = []; + } + try { + cursorRecordingData = await nativeBridgeClient.cursor.getRecordingData(media.screenVideoPath); + } catch { + cursorRecordingData = null; + } + + const recordingClicks = + cursorRecordingData?.samples + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs) ?? []; + const cursorClickTimestamps = + recordingClicks.length > 0 + ? recordingClicks + : cursorTelemetry + .filter((sample) => isClickInteractionType(sample.interactionType)) + .map((sample) => sample.timeMs); + + let platform: NativePlatform | null = null; + try { + platform = await nativeBridgeClient.system.getPlatform(); + } catch { + platform = null; + } + const hasEditableCursorRecording = + (media.cursorCaptureMode ?? "editable-overlay") === "editable-overlay" && + (platform === "win32" || platform === "darwin") && + hasNativeCursorRecordingData(cursorRecordingData); + const effectiveShowCursor = DEFAULT_CURSOR_SETTINGS.show && hasEditableCursorRecording; + + const probed = await probeVideoDimensions(videoUrl); + const sourceWidth = probed.width || DEFAULT_SOURCE_DIMENSIONS.width; + const sourceHeight = probed.height || DEFAULT_SOURCE_DIMENSIONS.height; + const effectiveSourceDimensions = calculateEffectiveSourceDimensions( + sourceWidth, + sourceHeight, + editor.cropRegion, + ); + const aspectRatioValue = + editor.aspectRatio === "native" + ? getNativeAspectRatioValue(sourceWidth, sourceHeight, editor.cropRegion) + : getAspectRatioValue(editor.aspectRatio); + + const preview = fitPreviewBox( + aspectRatioValue, + request.previewWidth ?? 1280, + request.previewHeight ?? 720, + ); + + const onProgress = (progress: ExportProgress) => { + window.electronAPI.cliProgress({ + percentage: progress.percentage, + currentFrame: progress.currentFrame, + totalFrames: progress.totalFrames, + estimatedTimeRemaining: progress.estimatedTimeRemaining, + phase: progress.phase, + }); + }; + + const sharedConfig = { + videoUrl, + webcamVideoUrl, + wallpaper: editor.wallpaper, + zoomRegions: editor.zoomRegions, + cameraFullscreenRegions: editor.cameraFullscreenRegions, + trimRegions: editor.trimRegions, + speedRegions: editor.speedRegions, + showShadow: editor.shadowIntensity > 0, + shadowIntensity: editor.shadowIntensity, + showBlur: editor.showBlur, + motionBlurAmount: editor.motionBlurAmount, + borderRadius: editor.borderRadius, + padding: editor.padding, + cropRegion: editor.cropRegion, + cursorRecordingData, + cursorScale: effectiveShowCursor ? DEFAULT_CURSOR_SETTINGS.size : 0, + cursorSmoothing: DEFAULT_CURSOR_SETTINGS.smoothing, + cursorMotionBlur: DEFAULT_CURSOR_SETTINGS.motionBlur, + cursorClickBounce: DEFAULT_CURSOR_SETTINGS.clickBounce, + cursorClipToBounds: DEFAULT_CURSOR_SETTINGS.clipToBounds, + cursorTheme: editor.cursorTheme, + annotationRegions: editor.annotationRegions, + webcamLayoutPreset: editor.webcamLayoutPreset, + webcamMaskShape: editor.webcamMaskShape, + webcamMirrored: editor.webcamMirrored, + webcamReactiveZoom: editor.webcamReactiveZoom, + webcamSizePreset: editor.webcamSizePreset, + webcamPosition: editor.webcamPosition, + previewWidth: preview.width, + previewHeight: preview.height, + cursorTelemetry, + cursorClickTimestamps, + onProgress, + }; + + let blob: Blob; + let warnings: string[] | undefined; + let outWidth: number; + let outHeight: number; + + if (format === "gif") { + const gifDimensions = calculateOutputDimensions( + effectiveSourceDimensions.width, + effectiveSourceDimensions.height, + gifSizePreset, + GIF_SIZE_PRESETS, + aspectRatioValue, + ); + outWidth = gifDimensions.width; + outHeight = gifDimensions.height; + const gifExporter = new GifExporter({ + ...sharedConfig, + width: gifDimensions.width, + height: gifDimensions.height, + frameRate: gifFrameRate, + loop: editor.gifLoop, + sizePreset: gifSizePreset, + videoPadding: editor.padding, + }); + const result = await gifExporter.export(); + if (!result.success || !result.blob) { + throw new Error(result.error ?? "GIF export failed"); + } + blob = result.blob; + warnings = result.warnings; + } else { + const mp4Settings = calculateMp4ExportSettings({ + quality, + sourceWidth: effectiveSourceDimensions.width, + sourceHeight: effectiveSourceDimensions.height, + aspectRatioValue, + }); + outWidth = mp4Settings.width; + outHeight = mp4Settings.height; + const exporter = new VideoExporter({ + ...sharedConfig, + width: mp4Settings.width, + height: mp4Settings.height, + frameRate: 60, + bitrate: mp4Settings.bitrate, + codec: "avc1.640033", + }); + const result = await exporter.export(); + if (!result.success || !result.blob) { + throw new Error(result.error ?? "MP4 export failed"); + } + blob = result.blob; + warnings = result.warnings; + } + + if (request.audioPath && format === "mp4") { + window.electronAPI.cliProgress({ percentage: 100, phase: "mixing-voiceover" }); + const audioResponse = await fetch(toFileUrl(request.audioPath)); + if (!audioResponse.ok) { + throw new Error(`Failed to read voiceover file: ${request.audioPath}`); + } + const voiceoverData = await audioResponse.arrayBuffer(); + blob = await mixVoiceoverIntoVideo(blob, { + voiceoverData, + mode: request.audioMode, + offsetSec: request.audioOffsetSec, + }); + } + + const arrayBuffer = await blob.arrayBuffer(); + const saveResult = await window.electronAPI.writeExportToPath(arrayBuffer, outPath); + if (!saveResult.success || !saveResult.path) { + throw new Error(saveResult.message ?? `Failed to write output to ${outPath}`); + } + + return { + success: true, + outputPath: saveResult.path, + format, + width: outWidth, + height: outHeight, + warnings, + }; +} + +export function CliExportRunner() { + const startedRef = useRef(false); + const [status, setStatus] = useState("Starting export…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = (await window.electronAPI.cliGetRequest()) as CliExportRequest; + if (request.kind !== "export") { + throw new Error(`cli-export window received a ${request.kind} request`); + } + setStatus(`Exporting ${request.projectPath}…`); + const result = await runExport(request); + await window.electronAPI.cliDone(result); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliExportRunner; diff --git a/src/cli/CliRecordRunner.tsx b/src/cli/CliRecordRunner.tsx new file mode 100644 index 0000000000..7bf6c7afd3 --- /dev/null +++ b/src/cli/CliRecordRunner.tsx @@ -0,0 +1,274 @@ +// Hidden-window runner for `openscreen record`. Reuses the full recording +// pipeline via useScreenRecorder (native macOS/Windows helpers with browser +// fallback), driven by the CLI controller instead of the HUD. + +import { useEffect, useRef, useState } from "react"; +import { + normalizeProjectEditor, + PROJECT_VERSION, +} from "@/components/video-editor/projectPersistence"; +import { useScreenRecorder } from "@/hooks/useScreenRecorder"; +import type { CliRecordRequest } from "@/lib/cliContracts"; + +type Phase = "init" | "recording" | "stopping" | "done"; + +async function pickSource(request: CliRecordRequest): Promise<ProcessedDesktopSource> { + const sources = await window.electronAPI.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 32, height: 18 }, + }); + + if (request.windowTitle) { + const needle = request.windowTitle.toLowerCase(); + const match = sources.find( + (source) => source.id.startsWith("window:") && source.name.toLowerCase().includes(needle), + ); + if (!match) { + const windows = sources + .filter((s) => s.id.startsWith("window:")) + .map((s) => ` - ${s.name}`) + .join("\n"); + throw new Error( + `No window title contains "${request.windowTitle}". Open windows:\n${windows}`, + ); + } + return match; + } + + const screens = sources.filter((source) => source.id.startsWith("screen:")); + const screen = screens[request.displayIndex]; + if (!screen) { + throw new Error( + `Display index ${request.displayIndex} not found (${screens.length} screen(s) available)`, + ); + } + return screen; +} + +async function resolveMicDeviceId(deviceNameFilter: string | null): Promise<{ + deviceId: string | undefined; + deviceName: string | undefined; +}> { + if (!deviceNameFilter) return { deviceId: undefined, deviceName: undefined }; + + // Labels require an active permission grant; a short-lived stream unlocks them. + let probeStream: MediaStream | null = null; + try { + probeStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + } catch { + // Enumeration below may still work with empty labels. + } + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + const needle = deviceNameFilter.toLowerCase(); + const match = devices.find( + (device) => device.kind === "audioinput" && device.label.toLowerCase().includes(needle), + ); + if (!match) { + const labels = devices + .filter((d) => d.kind === "audioinput" && d.label) + .map((d) => ` - ${d.label}`) + .join("\n"); + throw new Error(`No microphone label contains "${deviceNameFilter}". Devices:\n${labels}`); + } + return { deviceId: match.deviceId, deviceName: match.label }; + } finally { + probeStream?.getTracks().forEach((track) => track.stop()); + } +} + +/** A minimal .openscreen project referencing the finished recording, with all + * editor settings at their defaults — ready for `openscreen export` or the GUI. */ +function buildDefaultProject(session: { + screenVideoPath: string; + webcamVideoPath?: string; + cursorCaptureMode?: string; +}) { + return { + version: PROJECT_VERSION, + media: { + screenVideoPath: session.screenVideoPath, + ...(session.webcamVideoPath ? { webcamVideoPath: session.webcamVideoPath } : {}), + ...(session.cursorCaptureMode ? { cursorCaptureMode: session.cursorCaptureMode } : {}), + }, + editor: normalizeProjectEditor({}), + }; +} + +export function CliRecordRunner() { + const recorder = useScreenRecorder(); + const startedRef = useRef(false); + const requestRef = useRef<CliRecordRequest | null>(null); + // Re-render trigger once the bootstrap has applied all recorder settings; + // the start effect below cannot rely on recorder state deps alone because + // default-valued settings (no mic, no system audio) never change. + const [requestReady, setRequestReady] = useState<CliRecordRequest | null>(null); + const phaseRef = useRef<Phase>("init"); + const recordingStartedAtRef = useRef<number | null>(null); + const [status, setStatus] = useState("Preparing recording…"); + + const { + recording, + saving, + startRecordingImmediately, + toggleRecording, + setMicrophoneEnabled, + setMicrophoneDeviceId, + setMicrophoneDeviceName, + setSystemAudioEnabled, + setCursorCaptureMode, + } = recorder; + + // Keep latest values in refs for the stop/finish effects. + const toggleRecordingRef = useRef(toggleRecording); + toggleRecordingRef.current = toggleRecording; + const recordingRef = useRef(recording); + recordingRef.current = recording; + + const fail = async (error: unknown) => { + phaseRef.current = "done"; + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + }; + + // Bootstrap: pick source, configure recorder, start. + // biome-ignore lint/correctness/useExhaustiveDependencies: intentional run-once bootstrap; startedRef guards re-entry + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + // The recorder hook surfaces some failures via blocking alert(); a + // hidden window must never show (or hang on) a modal. + window.alert = (message?: unknown) => { + window.electronAPI.cliLog("error", `Recorder: ${String(message)}`); + }; + + const request = (await window.electronAPI.cliGetRequest()) as CliRecordRequest; + if (request.kind !== "record") { + throw new Error(`cli-record window received a ${request.kind} request`); + } + requestRef.current = request; + + const source = await pickSource(request); + await window.electronAPI.selectSource(source); + window.electronAPI.cliLog("info", `Recording source: ${source.name}`); + + if (request.mic) { + const mic = await resolveMicDeviceId(request.micDevice); + setMicrophoneEnabled(true); + setMicrophoneDeviceId(mic.deviceId); + setMicrophoneDeviceName(mic.deviceName); + if (mic.deviceName) { + window.electronAPI.cliLog("info", `Microphone: ${mic.deviceName}`); + } + } + setSystemAudioEnabled(request.systemAudio); + setCursorCaptureMode(request.cursorMode); + setStatus("Starting recording…"); + setRequestReady(request); + } catch (error) { + await fail(error); + } + })(); + }, []); + + // The setters above land on the *next* render; start only once they have. + const configuredRef = useRef(false); + // biome-ignore lint/correctness/useExhaustiveDependencies: fires when recorder settings match the request; other referenced values are stable refs/callbacks + useEffect(() => { + const request = requestReady; + if (!request || configuredRef.current || phaseRef.current !== "init") return; + const micReady = !request.mic || recorder.microphoneEnabled; + const systemAudioReady = recorder.systemAudioEnabled === request.systemAudio; + const cursorReady = recorder.cursorCaptureMode === request.cursorMode; + if (!micReady || !systemAudioReady || !cursorReady) return; + + configuredRef.current = true; + phaseRef.current = "recording"; + void (async () => { + try { + await startRecordingImmediately(); + } catch (error) { + await fail(error); + } + })(); + }, [ + requestReady, + recorder.microphoneEnabled, + recorder.systemAudioEnabled, + recorder.cursorCaptureMode, + ]); + + // Recording state transitions: report start and arm the duration timer. + useEffect(() => { + const request = requestRef.current; + if (recording && recordingStartedAtRef.current === null) { + recordingStartedAtRef.current = Date.now(); + setStatus("Recording…"); + window.electronAPI.cliLog("info", "Recording started"); + + if (request?.durationMs) { + const timer = setTimeout(() => { + if (recordingRef.current && phaseRef.current === "recording") { + phaseRef.current = "stopping"; + window.electronAPI.cliLog("info", `Duration reached (${request.durationMs}ms)`); + toggleRecordingRef.current(); + } + }, request.durationMs); + return () => clearTimeout(timer); + } + } + }, [recording]); + + // External stop (SIGINT / stdin via main process). + useEffect(() => { + return window.electronAPI.onCliStopRecording(() => { + if (recordingRef.current && phaseRef.current === "recording") { + phaseRef.current = "stopping"; + setStatus("Stopping…"); + toggleRecordingRef.current(); + } + }); + }, []); + + // Completion: recording flipped off and the session finished saving. + // biome-ignore lint/correctness/useExhaustiveDependencies: completion is keyed on recording/saving; fail is stable + useEffect(() => { + if (recordingStartedAtRef.current === null) return; + if (recording || saving) return; + if (phaseRef.current === "done") return; + phaseRef.current = "done"; + + void (async () => { + try { + const sessionResult = await window.electronAPI.getCurrentRecordingSession(); + const session = sessionResult?.session; + if (!session?.screenVideoPath) { + throw new Error("Recording finished but no session manifest was stored"); + } + const durationMs = Date.now() - (recordingStartedAtRef.current ?? Date.now()); + const request = requestRef.current; + await window.electronAPI.cliDone({ + success: true, + screenVideoPath: session.screenVideoPath, + webcamVideoPath: session.webcamVideoPath, + cursorDataPath: `${session.screenVideoPath}.cursor.json`, + durationMs, + ...(request?.projectOut ? { projectData: buildDefaultProject(session) } : {}), + }); + } catch (error) { + await fail(error); + } + })(); + }, [recording, saving]); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliRecordRunner; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 19afd52f5b..1da548b148 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -55,6 +55,8 @@ type UseScreenRecorderReturn = { saving: boolean; elapsedSeconds: number; toggleRecording: () => void; + /** Starts recording with no countdown overlay. Used by the headless CLI runner. */ + startRecordingImmediately: () => Promise<void>; togglePaused: () => void; canPauseRecording: boolean; restartRecording: () => void; @@ -2062,6 +2064,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { saving, elapsedSeconds, toggleRecording, + startRecordingImmediately: () => startRecording(), togglePaused, canPauseRecording, restartRecording, diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts new file mode 100644 index 0000000000..cdd7a45063 --- /dev/null +++ b/src/lib/cliContracts.ts @@ -0,0 +1,83 @@ +// Shared request/response contracts between the CLI entry in the Electron main +// process (electron/cli/) and the hidden renderer runners (src/cli/). Keep this +// file dependency-free so both build targets can import it. + +import type { ExportQuality, GifFrameRate, GifSizePreset } from "./exporter/types"; + +export type CliCursorCaptureMode = "editable-overlay" | "system"; + +export interface CliExportRequest { + kind: "export"; + /** Absolute path to the .openscreen project file. */ + projectPath: string; + /** Absolute output path; null = derive from projectPath + format. */ + outPath: string | null; + /** null = use the format stored in the project. */ + format: "mp4" | "gif" | null; + /** null = use the quality stored in the project. */ + quality: ExportQuality | null; + gifFrameRate: GifFrameRate | null; + gifSizePreset: GifSizePreset | null; + /** + * Reference preview box used to scale annotation text and border radii the + * same way the editor's on-screen preview does. The composition is fitted + * into this box, mirroring the editor layout. Defaults to 1280x720. + */ + previewWidth: number | null; + previewHeight: number | null; + /** Absolute path to a voiceover audio file to mix into the export (MP4 only). */ + audioPath: string | null; + /** "mix" layers the voiceover over the recording's audio; "replace" drops the original. */ + audioMode: "mix" | "replace"; + /** Delay before the voiceover starts, in seconds. */ + audioOffsetSec: number; +} + +export interface CliRecordRequest { + kind: "record"; + /** Index into the available screen sources (0 = primary display). */ + displayIndex: number; + /** Case-insensitive substring match against window titles; overrides displayIndex. */ + windowTitle: string | null; + mic: boolean; + /** Microphone device label substring; null = system default. */ + micDevice: string | null; + systemAudio: boolean; + cursorMode: CliCursorCaptureMode; + /** Auto-stop after this many milliseconds; null = stop via signal/stdin. */ + durationMs: number | null; + /** When set, write a ready-to-export .openscreen project here after recording. */ + projectOut: string | null; +} + +export type CliRequest = CliExportRequest | CliRecordRequest; + +export interface CliProgressEvent { + percentage: number; + currentFrame?: number; + totalFrames?: number; + estimatedTimeRemaining?: number; + phase?: string; +} + +export interface CliDoneResult { + success: boolean; + error?: string; + warnings?: string[]; + /** Export: the written output file. */ + outputPath?: string; + format?: string; + width?: number; + height?: number; + /** Record: produced artifacts. */ + screenVideoPath?: string; + webcamVideoPath?: string; + cursorDataPath?: string; + projectPath?: string; + durationMs?: number; + /** + * Record: a ready-to-save .openscreen project object built by the runner. + * The main process writes it to the --project path (renderer has no fs). + */ + projectData?: unknown; +} diff --git a/src/lib/exporter/voiceoverMix.ts b/src/lib/exporter/voiceoverMix.ts new file mode 100644 index 0000000000..2694105370 --- /dev/null +++ b/src/lib/exporter/voiceoverMix.ts @@ -0,0 +1,136 @@ +// Post-export voiceover mixing for the CLI (`openscreen export --audio`). +// +// Takes the finished MP4 blob, copies its video packets untouched (no +// re-encode), renders a new audio track with OfflineAudioContext — the +// original audio and the voiceover mixed, or the voiceover alone — and +// re-muxes both into a new MP4 via mediabunny. + +import { + ALL_FORMATS, + AudioBufferSource, + BlobSource, + BufferTarget, + EncodedPacketSink, + EncodedVideoPacketSource, + Input, + Mp4OutputFormat, + Output, +} from "mediabunny"; + +export type VoiceoverMixMode = "mix" | "replace"; + +export interface VoiceoverMixOptions { + /** Encoded audio file bytes (mp3/wav/m4a/aiff — anything decodeAudioData accepts). */ + voiceoverData: ArrayBuffer; + mode: VoiceoverMixMode; + /** Delay before the voiceover starts, in seconds. */ + offsetSec: number; + /** Gain applied to the original track in "mix" mode (0..1). */ + originalGain?: number; +} + +const OUTPUT_SAMPLE_RATE = 48_000; +const OUTPUT_CHANNELS = 2; +const VOICEOVER_AUDIO_BITRATE = 192_000; + +async function decodeToBuffer( + context: OfflineAudioContext, + data: ArrayBuffer, +): Promise<AudioBuffer> { + // decodeAudioData detaches the buffer, so hand it a copy. + return context.decodeAudioData(data.slice(0)); +} + +/** Renders the final audio track: original bed (optional) + offset voiceover. */ +async function renderMixedAudio( + videoData: ArrayBuffer, + durationSec: number, + options: VoiceoverMixOptions, +): Promise<AudioBuffer> { + const frameCount = Math.max(1, Math.ceil(durationSec * OUTPUT_SAMPLE_RATE)); + const context = new OfflineAudioContext(OUTPUT_CHANNELS, frameCount, OUTPUT_SAMPLE_RATE); + + const voiceover = await decodeToBuffer(context, options.voiceoverData); + const voiceoverNode = context.createBufferSource(); + voiceoverNode.buffer = voiceover; + voiceoverNode.connect(context.destination); + voiceoverNode.start(Math.max(0, options.offsetSec)); + + if (options.mode === "mix") { + try { + const original = await decodeToBuffer(context, videoData); + const originalNode = context.createBufferSource(); + originalNode.buffer = original; + const gainNode = context.createGain(); + gainNode.gain.value = options.originalGain ?? 1; + originalNode.connect(gainNode); + gainNode.connect(context.destination); + originalNode.start(0); + } catch { + // The exported video has no decodable audio track; the voiceover + // becomes the only audio, same as "replace". + } + } + + return context.startRendering(); +} + +/** + * Returns a new MP4 blob with the same video stream and the mixed audio track. + * The video packets are copied without re-encoding. + */ +export async function mixVoiceoverIntoVideo( + videoBlob: Blob, + options: VoiceoverMixOptions, +): Promise<Blob> { + const input = new Input({ source: new BlobSource(videoBlob), formats: ALL_FORMATS }); + try { + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("Exported file has no video track to remux"); + } + const codec = videoTrack.codec; + if (!codec) { + throw new Error("Exported file's video codec was not recognized"); + } + const decoderConfig = await videoTrack.getDecoderConfig(); + if (!decoderConfig) { + throw new Error("Exported file's video decoder config could not be read"); + } + const durationSec = await input.computeDuration(); + + const videoData = await videoBlob.arrayBuffer(); + const mixedAudio = await renderMixedAudio(videoData, durationSec, options); + + const target = new BufferTarget(); + const output = new Output({ + format: new Mp4OutputFormat({ fastStart: "in-memory" }), + target, + }); + const videoSource = new EncodedVideoPacketSource(codec); + output.addVideoTrack(videoSource); + const audioSource = new AudioBufferSource({ + codec: "aac", + bitrate: VOICEOVER_AUDIO_BITRATE, + }); + output.addAudioTrack(audioSource); + await output.start(); + + const sink = new EncodedPacketSink(videoTrack); + let isFirstPacket = true; + for await (const packet of sink.packets()) { + await videoSource.add(packet, isFirstPacket ? { decoderConfig } : undefined); + isFirstPacket = false; + } + await audioSource.add(mixedAudio); + + await output.finalize(); + const buffer = target.buffer; + if (!buffer) { + throw new Error("Voiceover remux produced no output"); + } + return new Blob([buffer], { type: "video/mp4" }); + } finally { + input.dispose(); + } +} From 502e9d2ddbcfc32cada92bec2af2b2ca0a574af6 Mon Sep 17 00:00:00 2001 From: PeterTakahashi <seiya4@icloud.com> Date: Tue, 28 Jul 2026 03:28:56 +0900 Subject: [PATCH 02/10] fix(cli): address CodeRabbit review on #176 Actionable findings: - voiceoverMix: duck the original bed to 0.4 gain by default in "mix" mode so the unity-gain sum cannot hard-clip; skip buffering the whole MP4 in "replace" mode; cancel the mediabunny Output on remux failure - CliRecordRunner: a stop signal arriving before capture starts is now remembered and applied the moment recording begins; add a 30s watchdog so a silently failed start no longer hangs the CLI; move ref sync out of render into an effect - cliMain: guard window-all-closed with the finished flag so teardown after a successful run can't report a false failure; route info-command output through safeWrite; harden console rerouting against circular args - args: reject zero dimensions in --preview-size - CliExportRunner: 30s timeout on the video metadata probe - preload: type the CLI bridge methods against cliContracts - args.test: platform-agnostic path assertions (win32-safe) - docs: fence language, document mix-mode ducking Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- docs/cli.md | 4 ++- electron/cli/args.test.ts | 16 +++++----- electron/cli/args.ts | 9 ++++-- electron/cli/cliMain.ts | 27 +++++++++++----- electron/preload.ts | 6 ++-- src/cli/CliExportRunner.tsx | 18 +++++++++-- src/cli/CliRecordRunner.tsx | 31 +++++++++++++++++-- src/lib/exporter/voiceoverMix.ts | 53 +++++++++++++++++++------------- 8 files changed, 117 insertions(+), 47 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 8ebb310803..f546442d45 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -4,7 +4,7 @@ Headless command-line interface for recording the screen and exporting `.openscr projects — no visible windows, machine-readable output. Designed so scripts, CI pipelines, and AI coding agents can produce polished product demos automatically: -``` +```text record → edit the project JSON programmatically → export → MP4/GIF ``` @@ -102,6 +102,8 @@ openscreen export demo.openscreen --json | while read line; do ...; done `--audio` re-muxes after the render: video packets are copied untouched, and the audio is mixed offline (OfflineAudioContext) and re-encoded to AAC. MP4 only. +In `mix` mode the original audio is ducked to 40% under the voiceover so the +sum cannot clip; use `replace` to drop the original entirely. **Media path rule**: for safety, a project's referenced media is only auto-approved when it lives in the app's recordings directory or **next to the project file**. diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index bf3bab8a7b..f8886bf7ec 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -1,7 +1,9 @@ +import path from "node:path"; import { describe, expect, it } from "vitest"; import { parseCliArgs } from "./args"; -const CWD = "/work"; +const CWD = path.resolve("/work"); +const inCwd = (name: string) => path.resolve(CWD, name); function parse(args: string[]) { return parseCliArgs(["electron", "app-path", ...args], 2, CWD); @@ -16,7 +18,7 @@ describe("parseCliArgs", () => { it("skips leading Chromium switches before the subcommand (AppImage --no-sandbox)", () => { expect(parse(["--no-sandbox", "export", "demo.openscreen"])).toMatchObject({ kind: "export", - projectPath: "/work/demo.openscreen", + projectPath: inCwd("demo.openscreen"), }); expect(parse(["--no-sandbox", "--enable-unsafe-swiftshader", "record"])).toMatchObject({ kind: "record", @@ -29,7 +31,7 @@ describe("parseCliArgs", () => { const cmd = parse(["export", "demo.openscreen"]); expect(cmd).toMatchObject({ kind: "export", - projectPath: "/work/demo.openscreen", + projectPath: inCwd("demo.openscreen"), outPath: null, format: null, }); @@ -49,7 +51,7 @@ describe("parseCliArgs", () => { ]); expect(cmd).toMatchObject({ kind: "export", - outPath: "/work/out.gif", + outPath: inCwd("out.gif"), format: "gif", gifFrameRate: 20, previewWidth: 1600, @@ -80,7 +82,7 @@ describe("parseCliArgs", () => { ]); expect(cmd).toMatchObject({ kind: "export", - audioPath: "/work/voice.mp3", + audioPath: inCwd("voice.mp3"), audioMode: "replace", audioOffsetSec: 1.5, }); @@ -131,7 +133,7 @@ describe("parseCliArgs", () => { micDevice: "MacBook", systemAudio: true, durationMs: 12500, - projectOut: "/work/demo.openscreen", + projectOut: inCwd("demo.openscreen"), }); }); @@ -144,7 +146,7 @@ describe("parseCliArgs", () => { it("parses info and help", () => { expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({ kind: "info", - projectPath: "/work/demo.openscreen", + projectPath: inCwd("demo.openscreen"), json: true, }); expect(parse(["help"])).toMatchObject({ kind: "help" }); diff --git a/electron/cli/args.ts b/electron/cli/args.ts index 0ed95c8586..69578198b4 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -184,8 +184,13 @@ function parseExport(args: string[], cwd: string): CliCommand { const [value, next] = takeValue(args, i, arg); const match = /^(\d+)x(\d+)$/.exec(value); if (!match) throw new Error(`--preview-size must look like 1280x720, got "${value}"`); - request.previewWidth = Number(match[1]); - request.previewHeight = Number(match[2]); + const previewWidth = Number(match[1]); + const previewHeight = Number(match[2]); + if (previewWidth <= 0 || previewHeight <= 0) { + throw new Error(`--preview-size dimensions must be positive, got "${value}"`); + } + request.previewWidth = previewWidth; + request.previewHeight = previewHeight; i = next; break; } diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index ed3d487fbb..02fe23585d 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -210,9 +210,10 @@ async function runInfoCommand(projectPath: string, json: boolean): Promise<numbe }; if (json) { - process.stdout.write(`${JSON.stringify(summary)}\n`); + safeWrite(process.stdout, `${JSON.stringify(summary)}\n`); } else { - process.stdout.write( + safeWrite( + process.stdout, [ `Project: ${summary.projectPath} (version ${summary.version ?? "?"})`, `Video: ${summary.screenVideoPath ?? "(none)"}${mediaExists ? "" : " [MISSING]"}`, @@ -242,12 +243,17 @@ export function runCli(command: CliCommand): void { // stdout belongs to the CLI protocol (NDJSON / progress); reroute the app's // own console chatter (e.g. "[native-sck] starting…") to stderr. + const stringifyArg = (value: unknown): string => { + if (typeof value === "string") return value; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } + }; for (const level of ["log", "info", "warn", "error", "debug"] as const) { console[level] = (...args: unknown[]) => { - safeWrite( - process.stderr, - `${args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ")}\n`, - ); + safeWrite(process.stderr, `${args.map(stringifyArg).join(" ")}\n`); }; } @@ -271,6 +277,10 @@ export function runCli(command: CliCommand): void { app.exit(1); }); + // Set once cli-done has been received; suppresses the window-all-closed + // failure path during the normal teardown race after a successful run. + let finished = false; + // GPU may be unavailable in CI/servers; let Chromium fall back to SwiftShader // so the WebGL-based export renderer still works. app.commandLine.appendSwitch("enable-unsafe-swiftshader"); @@ -281,7 +291,9 @@ export function runCli(command: CliCommand): void { } app.on("window-all-closed", () => { - // Completion is signalled via cli-done; a vanished window is a failure. + // Completion is signalled via cli-done; a vanished window is a failure + // only while the run is still in flight. + if (finished) return; output.error("Renderer window closed unexpectedly"); app.exit(1); }); @@ -365,7 +377,6 @@ export function runCli(command: CliCommand): void { output.progress(progress); }); - let finished = false; ipcMain.handle("cli-done", async (_event, result: CliDoneResult) => { if (finished) return; finished = true; diff --git a/electron/preload.ts b/electron/preload.ts index bc1ac803ed..6aa0a886c7 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -430,16 +430,16 @@ contextBridge.exposeInMainWorld("electronAPI", { }, }, // --- CLI mode (hidden runner windows; see electron/cli/) --- - cliGetRequest: () => { + cliGetRequest: (): Promise<import("../src/lib/cliContracts").CliRequest> => { return ipcRenderer.invoke("cli-get-request"); }, - cliProgress: (progress: unknown) => { + cliProgress: (progress: import("../src/lib/cliContracts").CliProgressEvent) => { ipcRenderer.send("cli-progress", progress); }, cliLog: (level: "info" | "error", message: string) => { ipcRenderer.send("cli-log", level, message); }, - cliDone: (result: unknown) => { + cliDone: (result: import("../src/lib/cliContracts").CliDoneResult) => { return ipcRenderer.invoke("cli-done", result); }, onCliStopRecording: (callback: () => void) => { diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index b8b0ea51ce..56f60581e2 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -44,14 +44,26 @@ function probeVideoDimensions(url: string): Promise<{ width: number; height: num const video = document.createElement("video"); video.preload = "metadata"; video.muted = true; + const cleanup = () => { + clearTimeout(timer); + video.removeAttribute("src"); + video.load(); + }; + // A stalled load fires neither event; without a deadline the CLI hangs. + const timer = setTimeout(() => { + cleanup(); + reject(new Error(`Timed out reading video metadata: ${url}`)); + }, 30_000); video.onloadedmetadata = () => { const width = video.videoWidth; const height = video.videoHeight; - video.removeAttribute("src"); - video.load(); + cleanup(); resolve({ width, height }); }; - video.onerror = () => reject(new Error(`Failed to load video metadata: ${url}`)); + video.onerror = () => { + cleanup(); + reject(new Error(`Failed to load video metadata: ${url}`)); + }; video.src = url; }); } diff --git a/src/cli/CliRecordRunner.tsx b/src/cli/CliRecordRunner.tsx index 7bf6c7afd3..e19c451f9f 100644 --- a/src/cli/CliRecordRunner.tsx +++ b/src/cli/CliRecordRunner.tsx @@ -105,6 +105,9 @@ export function CliRecordRunner() { const [requestReady, setRequestReady] = useState<CliRecordRequest | null>(null); const phaseRef = useRef<Phase>("init"); const recordingStartedAtRef = useRef<number | null>(null); + // A stop (SIGINT/stdin) can land while the capture helper is still starting; + // remember it and apply as soon as recording flips on. + const stopRequestedRef = useRef(false); const [status, setStatus] = useState("Preparing recording…"); const { @@ -121,9 +124,11 @@ export function CliRecordRunner() { // Keep latest values in refs for the stop/finish effects. const toggleRecordingRef = useRef(toggleRecording); - toggleRecordingRef.current = toggleRecording; const recordingRef = useRef(recording); - recordingRef.current = recording; + useEffect(() => { + toggleRecordingRef.current = toggleRecording; + recordingRef.current = recording; + }); const fail = async (error: unknown) => { phaseRef.current = "done"; @@ -190,6 +195,18 @@ export function CliRecordRunner() { void (async () => { try { await startRecordingImmediately(); + // The hook reports start failures via toast/console, not by + // rejecting — without a deadline a failed start would hang the + // CLI forever. + setTimeout(() => { + if (recordingStartedAtRef.current === null && phaseRef.current === "recording") { + void fail( + new Error( + "Recording did not start within 30s — see stderr for the underlying capture error", + ), + ); + } + }, 30_000); } catch (error) { await fail(error); } @@ -209,6 +226,13 @@ export function CliRecordRunner() { setStatus("Recording…"); window.electronAPI.cliLog("info", "Recording started"); + if (stopRequestedRef.current) { + phaseRef.current = "stopping"; + setStatus("Stopping…"); + toggleRecordingRef.current(); + return; + } + if (request?.durationMs) { const timer = setTimeout(() => { if (recordingRef.current && phaseRef.current === "recording") { @@ -229,6 +253,9 @@ export function CliRecordRunner() { phaseRef.current = "stopping"; setStatus("Stopping…"); toggleRecordingRef.current(); + } else { + // Capture is still starting; stop as soon as it comes up. + stopRequestedRef.current = true; } }); }, []); diff --git a/src/lib/exporter/voiceoverMix.ts b/src/lib/exporter/voiceoverMix.ts index 2694105370..0c63056277 100644 --- a/src/lib/exporter/voiceoverMix.ts +++ b/src/lib/exporter/voiceoverMix.ts @@ -20,7 +20,7 @@ import { export type VoiceoverMixMode = "mix" | "replace"; export interface VoiceoverMixOptions { - /** Encoded audio file bytes (mp3/wav/m4a/aiff — anything decodeAudioData accepts). */ + /** Encoded audio file bytes (mp3/wav/m4a — anything decodeAudioData accepts). */ voiceoverData: ArrayBuffer; mode: VoiceoverMixMode; /** Delay before the voiceover starts, in seconds. */ @@ -29,6 +29,10 @@ export interface VoiceoverMixOptions { originalGain?: number; } +// Duck the original bed under the voiceover by default so the unity-gain sum +// of two loud sources doesn't hard-clip. +const DEFAULT_ORIGINAL_GAIN = 0.4; + const OUTPUT_SAMPLE_RATE = 48_000; const OUTPUT_CHANNELS = 2; const VOICEOVER_AUDIO_BITRATE = 192_000; @@ -43,7 +47,7 @@ async function decodeToBuffer( /** Renders the final audio track: original bed (optional) + offset voiceover. */ async function renderMixedAudio( - videoData: ArrayBuffer, + videoData: ArrayBuffer | null, durationSec: number, options: VoiceoverMixOptions, ): Promise<AudioBuffer> { @@ -56,13 +60,13 @@ async function renderMixedAudio( voiceoverNode.connect(context.destination); voiceoverNode.start(Math.max(0, options.offsetSec)); - if (options.mode === "mix") { + if (options.mode === "mix" && videoData) { try { const original = await decodeToBuffer(context, videoData); const originalNode = context.createBufferSource(); originalNode.buffer = original; const gainNode = context.createGain(); - gainNode.gain.value = options.originalGain ?? 1; + gainNode.gain.value = options.originalGain ?? DEFAULT_ORIGINAL_GAIN; originalNode.connect(gainNode); gainNode.connect(context.destination); originalNode.start(0); @@ -99,7 +103,9 @@ export async function mixVoiceoverIntoVideo( } const durationSec = await input.computeDuration(); - const videoData = await videoBlob.arrayBuffer(); + // The full-file bytes are only needed to decode the original bed in + // "mix" mode; "replace" skips the copy entirely. + const videoData = options.mode === "mix" ? await videoBlob.arrayBuffer() : null; const mixedAudio = await renderMixedAudio(videoData, durationSec, options); const target = new BufferTarget(); @@ -107,24 +113,29 @@ export async function mixVoiceoverIntoVideo( format: new Mp4OutputFormat({ fastStart: "in-memory" }), target, }); - const videoSource = new EncodedVideoPacketSource(codec); - output.addVideoTrack(videoSource); - const audioSource = new AudioBufferSource({ - codec: "aac", - bitrate: VOICEOVER_AUDIO_BITRATE, - }); - output.addAudioTrack(audioSource); - await output.start(); + try { + const videoSource = new EncodedVideoPacketSource(codec); + output.addVideoTrack(videoSource); + const audioSource = new AudioBufferSource({ + codec: "aac", + bitrate: VOICEOVER_AUDIO_BITRATE, + }); + output.addAudioTrack(audioSource); + await output.start(); - const sink = new EncodedPacketSink(videoTrack); - let isFirstPacket = true; - for await (const packet of sink.packets()) { - await videoSource.add(packet, isFirstPacket ? { decoderConfig } : undefined); - isFirstPacket = false; - } - await audioSource.add(mixedAudio); + const sink = new EncodedPacketSink(videoTrack); + let isFirstPacket = true; + for await (const packet of sink.packets()) { + await videoSource.add(packet, isFirstPacket ? { decoderConfig } : undefined); + isFirstPacket = false; + } + await audioSource.add(mixedAudio); - await output.finalize(); + await output.finalize(); + } catch (error) { + await output.cancel().catch(() => undefined); + throw error; + } const buffer = target.buffer; if (!buffer) { throw new Error("Voiceover remux produced no output"); From ea0ba5967564e6e13afed64dc63eb3c8be7b8e3b Mon Sep 17 00:00:00 2001 From: PeterTakahashi <seiya4@icloud.com> Date: Tue, 28 Jul 2026 22:03:08 +0900 Subject: [PATCH 03/10] feat(cli): add auto-zoom, sources, pack and captions commands Four additions that close the gap between CLI and GUI workflows: - export --auto-zoom: runs the editor's cursor-dwell suggestion engine (timeline/zoomSuggestionUtils) over the recording's telemetry and adds focus-following zoom regions before rendering; never overlaps regions already in the project - sources: lists capturable displays, windows and microphones (same enumeration as the GUI picker) so scripts can pick --display/--window/ --mic-device values without trial and error - pack <project> --out <dir>: copies the project plus referenced media and the cursor-telemetry sidecar into one portable folder; the media approval path (getApprovedProjectSession) and the export runner gain a same-basename sibling fallback so a packed folder keeps working after being moved to another location or machine - captions <project>: transcribes the project's audio with the bundled on-device Whisper worker (mirrors VideoEditor.generateAutoCaptions: leading-silence trim, retry on empty, word grouping) and writes auto-caption annotations into the project; re-running replaces earlier auto-captions and preserves manual annotations Also: console rerouting now serializes Error objects properly instead of printing "{}". Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- docs/cli.md | 46 ++++++++- electron/cli/args.test.ts | 40 ++++++++ electron/cli/args.ts | 108 ++++++++++++++++++++- electron/cli/cliMain.ts | 172 +++++++++++++++++++++++++++++++++- electron/ipc/handlers.ts | 31 +++++- src/App.tsx | 14 +++ src/cli/CliCaptionsRunner.tsx | 168 +++++++++++++++++++++++++++++++++ src/cli/CliExportRunner.tsx | 70 +++++++++++++- src/cli/CliSourcesRunner.tsx | 87 +++++++++++++++++ src/lib/cliContracts.ts | 36 ++++++- 10 files changed, 758 insertions(+), 14 deletions(-) create mode 100644 src/cli/CliCaptionsRunner.tsx create mode 100644 src/cli/CliSourcesRunner.tsx diff --git a/docs/cli.md b/docs/cli.md index f546442d45..046e5696d0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -74,6 +74,17 @@ Platform notes: which may show a system picker dialog and requires a desktop session (headless/SSH sessions without a portal cannot record). +### `openscreen sources` + +Lists capturable displays, windows, and microphones — the same enumeration the +GUI picker uses — so scripts and agents can choose `--display`, `--window`, and +`--mic-device` values without guesswork. + +```bash +openscreen sources # human-readable +openscreen sources --json # {"event":"done","sources":{displays,windows,microphones,...}} +``` + ### `openscreen export` Renders a project to MP4 or GIF using the app's real export pipeline (WebCodecs + @@ -95,6 +106,7 @@ openscreen export demo.openscreen --json | while read line; do ...; done | `--quality <medium\|good\|source>` | MP4 quality | | `--gif-fps <15\|20\|25\|30>`, `--gif-size <medium\|large\|original>` | GIF settings | | `--preview-size <WxH>` | Reference preview box (default `1280x720`), see below | +| `--auto-zoom` | Add automatic zooms from cursor telemetry before rendering — the same dwell-detection engine as the editor's magic wand. Existing zoom regions are kept; suggestions never overlap them | | `--audio <file>` | Mix a voiceover file into the MP4 (mp3/wav/m4a — anything Chromium can decode; AIFF is not supported) | | `--audio-mode <mix\|replace>` | Layer the voiceover over the recording's audio (default `mix`) or replace it | | `--audio-offset <seconds>` | Delay before the voiceover starts (default 0) | @@ -117,6 +129,36 @@ deterministic reference box instead (the composition fitted into 1280×720). Authoring tip for scripts: treat annotation `fontSize` as "pixels in a 1280-wide preview". +### `openscreen pack` + +Copies a project and everything it references (screen/webcam video, cursor +telemetry sidecar) into one portable folder and rewrites the project's media +paths: + +```bash +openscreen pack demo.openscreen --out bundle/ +``` + +The folder survives being moved or shipped as a CI artifact: when the stored +absolute paths go stale, the loader falls back to files with the same basename +next to the project file. + +### `openscreen captions` + +Transcribes the project's audio with the app's on-device Whisper model (no +upload; language auto-detected) and writes the resulting caption annotations +into the project. Re-running replaces earlier auto-captions; manual annotations +are preserved. + +```bash +openscreen captions demo.openscreen --min-words 2 --max-words 7 +openscreen export demo.openscreen -o demo.mp4 # subtitles are burned in +``` + +Requires an audio track in the project's video (e.g. `record --mic`, or a +voiceover mixed in with a re-recorded source). Accuracy reflects the bundled +whisper-tiny model: strong for English, rough for other languages. + ### `openscreen info` Prints a project summary (referenced media and whether it exists, format, @@ -163,8 +205,8 @@ node -e ' # 3. Narrate with any TTS (macOS `say` shown; any engine producing mp3/wav/m4a works) say -o voice.m4a --file-format=m4af "Welcome to my product. Here's a quick tour." -# 4. Render with the voiceover mixed in -openscreen export demo.openscreen -o demo.mp4 --audio voice.m4a --audio-mode replace --json +# 4. Render with auto-zooms and the voiceover mixed in +openscreen export demo.openscreen -o demo.mp4 --auto-zoom --audio voice.m4a --audio-mode replace --json ``` ## Architecture diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index f8886bf7ec..575e0f7c5d 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -143,6 +143,46 @@ describe("parseCliArgs", () => { expect(parse(["record", "--project", "demo.json"])).toMatchObject({ kind: "error" }); }); + it("parses --auto-zoom", () => { + expect(parse(["export", "a.openscreen", "--auto-zoom"])).toMatchObject({ + kind: "export", + autoZoom: true, + }); + expect(parse(["export", "a.openscreen"])).toMatchObject({ autoZoom: false }); + }); + + it("parses sources", () => { + expect(parse(["sources", "--json"])).toMatchObject({ kind: "sources", json: true }); + expect(parse(["sources", "--bogus"])).toMatchObject({ kind: "error" }); + }); + + it("parses pack", () => { + expect(parse(["pack", "demo.openscreen", "--out", "bundle"])).toMatchObject({ + kind: "pack", + projectPath: inCwd("demo.openscreen"), + outDir: inCwd("bundle"), + }); + expect(parse(["pack", "demo.openscreen"])).toMatchObject({ kind: "error" }); + }); + + it("parses captions", () => { + expect( + parse(["captions", "demo.openscreen", "--min-words", "1", "--max-words", "5"]), + ).toMatchObject({ + kind: "captions", + projectPath: inCwd("demo.openscreen"), + minWordsPerCaption: 1, + maxWordsPerCaption: 5, + }); + expect(parse(["captions", "demo.openscreen"])).toMatchObject({ + minWordsPerCaption: 2, + maxWordsPerCaption: 7, + }); + expect( + parse(["captions", "a.openscreen", "--min-words", "9", "--max-words", "3"]), + ).toMatchObject({ kind: "error" }); + }); + it("parses info and help", () => { expect(parse(["info", "demo.openscreen", "--json"])).toMatchObject({ kind: "info", diff --git a/electron/cli/args.ts b/electron/cli/args.ts index 69578198b4..a14d74b066 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -18,18 +18,38 @@ export interface CliErrorCommand { message: string; } -export type CliCommand = (CliRequest | CliInfoCommand | CliHelpCommand | CliErrorCommand) & { +export type CliCommand = ( + | CliRequest + | CliInfoCommand + | CliPackCommand + | CliHelpCommand + | CliErrorCommand +) & { /** Machine-readable NDJSON output on stdout instead of human progress. */ json?: boolean; }; -const SUBCOMMANDS = new Set(["export", "record", "info", "help", "--help", "-h"]); +const SUBCOMMANDS = new Set([ + "export", + "record", + "sources", + "pack", + "captions", + "info", + "help", + "--help", + "-h", +]); export const CLI_USAGE = `OpenScreen CLI Usage: openscreen export <project.openscreen> [options] Render a project to MP4/GIF openscreen record [options] Record the screen headlessly + openscreen sources [--json] List displays, windows and microphones + openscreen pack <project.openscreen> --out <dir> Copy project + media into one portable folder + openscreen captions <project.openscreen> Add auto-captions (on-device Whisper) to a project + [--min-words <n>] [--max-words <n>] openscreen info <project.openscreen> [--json] Inspect a project file openscreen help Show this help @@ -42,6 +62,7 @@ Export options: --gif-size <medium|large|original> GIF size preset (default: from project) --preview-size <WxH> Reference preview box for annotation scaling (default 1280x720) + --auto-zoom Add automatic zooms from cursor telemetry (editor's magic wand) --audio <file> Mix a voiceover audio file into the MP4 (mp3/wav/m4a) --audio-mode <mix|replace> Layer over the recording's audio (default) or replace it @@ -111,6 +132,9 @@ export function parseCliArgs( try { if (sub === "export") return parseExport(args.slice(1), cwd); if (sub === "record") return parseRecord(args.slice(1), cwd); + if (sub === "sources") return parseSources(args.slice(1)); + if (sub === "pack") return parsePack(args.slice(1), cwd); + if (sub === "captions") return parseCaptions(args.slice(1), cwd); return parseInfo(args.slice(1), cwd); } catch (error) { return { kind: "error", message: error instanceof Error ? error.message : String(error) }; @@ -128,6 +152,7 @@ function parseExport(args: string[], cwd: string): CliCommand { gifSizePreset: null, previewWidth: null, previewHeight: null, + autoZoom: false, audioPath: null, audioMode: "mix", audioOffsetSec: 0, @@ -194,6 +219,9 @@ function parseExport(args: string[], cwd: string): CliCommand { i = next; break; } + case "--auto-zoom": + request.autoZoom = true; + break; case "--audio": { const [value, next] = takeValue(args, i, arg); request.audioPath = resolvePath(value, cwd); @@ -332,6 +360,82 @@ function parseRecord(args: string[], cwd: string): CliCommand { return request; } +function parseSources(args: string[]): CliCommand { + let json = false; + for (const arg of args) { + if (arg === "--json") { + json = true; + } else { + throw new Error(`Unknown sources option: ${arg}`); + } + } + return { kind: "sources", json }; +} + +export interface CliPackCommand { + kind: "pack"; + projectPath: string; + outDir: string; +} + +function parsePack(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let outDir = ""; + let json = false; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "-o" || arg === "--out") { + const [value, next] = takeValue(args, i, arg); + outDir = resolvePath(value, cwd); + i = next; + } else if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown pack option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("pack requires a <project.openscreen> path"); + if (!outDir) throw new Error("pack requires --out <directory>"); + return { kind: "pack", projectPath, outDir, json }; +} + +function parseCaptions(args: string[], cwd: string): CliCommand { + let projectPath = ""; + let minWordsPerCaption = 2; + let maxWordsPerCaption = 7; + let json = false; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === "--min-words" || arg === "--max-words") { + const [value, next] = takeValue(args, i, arg); + const count = Number(value); + if (!Number.isInteger(count) || count < 1) { + throw new Error(`${arg} must be a positive integer, got "${value}"`); + } + if (arg === "--min-words") minWordsPerCaption = count; + else maxWordsPerCaption = count; + i = next; + } else if (arg === "--json") { + json = true; + } else if (arg.startsWith("-")) { + throw new Error(`Unknown captions option: ${arg}`); + } else if (projectPath) { + throw new Error(`Unexpected extra argument: ${arg}`); + } else { + projectPath = resolvePath(arg, cwd); + } + } + if (!projectPath) throw new Error("captions requires a <project.openscreen> path"); + if (minWordsPerCaption > maxWordsPerCaption) { + throw new Error("--min-words cannot exceed --max-words"); + } + return { kind: "captions", projectPath, minWordsPerCaption, maxWordsPerCaption, json }; +} + function parseInfo(args: string[], cwd: string): CliCommand { let projectPath = ""; let json = false; diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index 02fe23585d..dbf8035ab3 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -7,7 +7,12 @@ import path from "node:path"; import readline from "node:readline"; import { fileURLToPath } from "node:url"; import { app, BrowserWindow, ipcMain, session, systemPreferences } from "electron"; -import type { CliDoneResult, CliProgressEvent, CliRequest } from "../../src/lib/cliContracts"; +import type { + CliDoneResult, + CliProgressEvent, + CliRequest, + CliSourcesResult, +} from "../../src/lib/cliContracts"; import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; import { ASSET_BASE_URL_ARG } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; @@ -145,6 +150,35 @@ function registerAppHandlersForCli(cliWindowRef: () => BrowserWindow | null) { ); } +function printSources(output: CliOutput, sources: CliSourcesResult): void { + if (output.json) { + // The "done" event already carries the payload; nothing extra to print. + return; + } + const lines: string[] = []; + lines.push("Displays:"); + for (const display of sources.displays) { + lines.push(` ${display.index} ${display.name} (${display.id})`); + } + lines.push("Windows:"); + if (sources.windows.length === 0) { + lines.push(" (none)"); + } + for (const win of sources.windows) { + lines.push(` - ${win.name}`); + } + lines.push("Microphones:"); + if (sources.microphoneLabelsUnavailable) { + lines.push(" (labels unavailable — grant microphone permission to see device names)"); + } else if (sources.microphones.length === 0) { + lines.push(" (none)"); + } + for (const mic of sources.microphones) { + lines.push(` - ${mic.label}`); + } + output.info(lines.join("\n")); +} + async function writeProjectFile(projectOut: string, projectData: unknown): Promise<void> { await fs.mkdir(path.dirname(projectOut), { recursive: true }); await fs.writeFile(projectOut, JSON.stringify(projectData, null, 2), "utf8"); @@ -174,6 +208,111 @@ function setupRecordStopSignals(stop: (reason: string) => void): void { } } +interface PackedProjectData { + version?: number; + media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; + videoPath?: string; + editor?: Record<string, unknown>; +} + +/** Copies a project and everything it references into one portable folder. */ +async function runPackCommand(projectPath: string, outDir: string, json: boolean): Promise<number> { + const emit = (message: string) => { + if (!json) safeWrite(process.stdout, `${message}\n`); + }; + + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as PackedProjectData; + const media = data.media ?? (data.videoPath ? { screenVideoPath: data.videoPath } : undefined); + const screenVideoPath = media?.screenVideoPath; + if (!screenVideoPath) { + throw new Error("Project file does not reference a screen video"); + } + + const projectDir = path.dirname(path.resolve(projectPath)); + const resolveSource = async (mediaPath: string): Promise<string> => { + const exists = await fs + .stat(mediaPath) + .then((stats) => stats.isFile()) + .catch(() => false); + if (exists) return mediaPath; + const sibling = path.join(projectDir, path.basename(mediaPath)); + const siblingExists = await fs + .stat(sibling) + .then((stats) => stats.isFile()) + .catch(() => false); + if (siblingExists) return sibling; + throw new Error(`Referenced media not found: ${mediaPath}`); + }; + + await fs.mkdir(outDir, { recursive: true }); + + const copied: string[] = []; + const copyIn = async (sourcePath: string): Promise<string> => { + const destination = path.join(outDir, path.basename(sourcePath)); + if (path.resolve(sourcePath) !== path.resolve(destination)) { + await fs.copyFile(sourcePath, destination); + } + copied.push(destination); + return destination; + }; + + const screenSource = await resolveSource(screenVideoPath); + const newScreenPath = await copyIn(screenSource); + + let newWebcamPath: string | undefined; + if (media.webcamVideoPath) { + newWebcamPath = await copyIn(await resolveSource(media.webcamVideoPath)); + } + + // Cursor telemetry sidecar sits at "<video path>.cursor.json". + const cursorSidecar = `${screenSource}.cursor.json`; + const hasCursorData = await fs + .stat(cursorSidecar) + .then((stats) => stats.isFile()) + .catch(() => false); + if (hasCursorData) { + await copyIn(cursorSidecar); + } + + const packedProject: PackedProjectData = { + ...data, + media: { + ...media, + screenVideoPath: newScreenPath, + ...(newWebcamPath ? { webcamVideoPath: newWebcamPath } : {}), + }, + }; + delete packedProject.videoPath; + const packedProjectPath = path.join(outDir, path.basename(projectPath)); + await fs.writeFile(packedProjectPath, JSON.stringify(packedProject, null, 2), "utf8"); + + if (json) { + safeWrite( + process.stdout, + `${JSON.stringify({ + event: "done", + success: true, + projectPath: packedProjectPath, + files: [packedProjectPath, ...copied], + cursorData: hasCursorData, + })}\n`, + ); + } else { + emit(`Packed project → ${packedProjectPath}`); + for (const file of copied) { + emit(` + ${path.basename(file)}`); + } + if (!hasCursorData) { + emit(" (no cursor telemetry sidecar found)"); + } + emit( + "The folder is self-contained: if the stored paths go stale after moving it, the loader falls back to files next to the project.", + ); + } + return 0; +} + async function runInfoCommand(projectPath: string, json: boolean): Promise<number> { const raw = await fs.readFile(projectPath, "utf8"); const data = JSON.parse(raw) as { @@ -245,6 +384,7 @@ export function runCli(command: CliCommand): void { // own console chatter (e.g. "[native-sck] starting…") to stderr. const stringifyArg = (value: unknown): string => { if (typeof value === "string") return value; + if (value instanceof Error) return value.stack ?? value.message; try { return JSON.stringify(value) ?? String(value); } catch { @@ -307,6 +447,16 @@ export function runCli(command: CliCommand): void { return; } + if (command.kind === "pack") { + const code = await runPackCommand( + command.projectPath, + command.outDir, + command.json === true, + ); + app.exit(code); + return; + } + await fs.mkdir(path.join(app.getPath("userData"), "recordings"), { recursive: true }); // Media/screen permissions for the renderer (mic metering, future browser @@ -388,9 +538,12 @@ export function runCli(command: CliCommand): void { result.projectPath = command.projectOut; } } + if (result.success && command.kind === "captions" && result.projectData !== undefined) { + await writeProjectFile(command.projectPath, result.projectData); + } } catch (error) { result.success = false; - result.error = `Recording succeeded but writing the project file failed: ${String(error)}`; + result.error = `Run succeeded but writing the project file failed: ${String(error)}`; } if (result.success) { @@ -398,7 +551,13 @@ export function runCli(command: CliCommand): void { output.info(`Warning: ${warning}`); output.event("warning", { message: warning }); } - if (command.kind === "export") { + if (command.kind === "sources" && result.sources) { + printSources(output, result.sources); + } else if (command.kind === "captions") { + output.info( + `Added ${result.captionCount ?? 0} caption annotation(s) → ${result.projectPath}`, + ); + } else if (command.kind === "export") { output.info(`Exported ${result.format ?? ""} → ${result.outputPath}`); } else { output.info(`Recording saved → ${result.screenVideoPath}`); @@ -424,7 +583,12 @@ export function runCli(command: CliCommand): void { setupRecordStopSignals(stop); } - const windowType = command.kind === "export" ? "cli-export" : "cli-record"; + const windowType = { + export: "cli-export", + record: "cli-record", + sources: "cli-sources", + captions: "cli-captions", + }[command.kind]; cliWindow = loadRunnerWindow(windowType); // Surface renderer console errors/warnings on stderr — the hidden window diff --git a/electron/ipc/handlers.ts b/electron/ipc/handlers.ts index 77d2cac1bd..23085f20db 100644 --- a/electron/ipc/handlers.ts +++ b/electron/ipc/handlers.ts @@ -378,13 +378,40 @@ async function getApprovedProjectSession( trustedDirs.push(path.dirname(path.resolve(projectFilePath))); } - const screenVideoPath = await approveReadableVideoPath(media.screenVideoPath, trustedDirs); + // Packed/portable projects: when the stored absolute path no longer exists + // (project moved to another machine or directory), fall back to a file with + // the same basename next to the project file (see `openscreen pack`). + const resolveWithSiblingFallback = async (mediaPath: string): Promise<string> => { + if (!projectFilePath) return mediaPath; + const exists = await fs + .stat(mediaPath) + .then((stats) => stats.isFile()) + .catch(() => false); + if (exists) return mediaPath; + const sibling = path.join( + path.dirname(path.resolve(projectFilePath)), + path.basename(mediaPath), + ); + const siblingExists = await fs + .stat(sibling) + .then((stats) => stats.isFile()) + .catch(() => false); + return siblingExists ? sibling : mediaPath; + }; + + const screenVideoPath = await approveReadableVideoPath( + await resolveWithSiblingFallback(media.screenVideoPath), + trustedDirs, + ); if (!screenVideoPath) { throw new Error("Project references an invalid or unsupported screen video path"); } const webcamVideoPath = media.webcamVideoPath - ? await approveReadableVideoPath(media.webcamVideoPath, trustedDirs) + ? await approveReadableVideoPath( + await resolveWithSiblingFallback(media.webcamVideoPath), + trustedDirs, + ) : undefined; if (media.webcamVideoPath && !webcamVideoPath) { throw new Error("Project references an invalid or unsupported webcam video path"); diff --git a/src/App.tsx b/src/App.tsx index 8bbed307ec..53d52c742d 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -22,6 +22,8 @@ const VideoEditorEntry = lazy(() => ); const CliExportRunner = lazy(() => import("./cli/CliExportRunner")); const CliRecordRunner = lazy(() => import("./cli/CliRecordRunner")); +const CliSourcesRunner = lazy(() => import("./cli/CliSourcesRunner")); +const CliCaptionsRunner = lazy(() => import("./cli/CliCaptionsRunner")); const ShortcutsConfigDialog = lazy(() => import("./components/video-editor/ShortcutsConfigDialog").then((module) => ({ default: module.ShortcutsConfigDialog, @@ -90,6 +92,18 @@ export default function App() { <CliRecordRunner /> </Suspense> ); + case "cli-sources": + return ( + <Suspense fallback={null}> + <CliSourcesRunner /> + </Suspense> + ); + case "cli-captions": + return ( + <Suspense fallback={null}> + <CliCaptionsRunner /> + </Suspense> + ); case "editor": return ( <ShortcutsProvider> diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx new file mode 100644 index 0000000000..32d01ca119 --- /dev/null +++ b/src/cli/CliCaptionsRunner.tsx @@ -0,0 +1,168 @@ +// Hidden-window runner for `openscreen captions`: transcribes the project's +// audio with the on-device Whisper worker and writes the resulting caption +// annotations back into the project. Mirrors VideoEditor.generateAutoCaptions. + +import { useEffect, useRef, useState } from "react"; +import { + normalizeProjectEditor, + resolveProjectMedia, + toFileUrl, + validateProjectData, +} from "@/components/video-editor/projectPersistence"; +import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types"; +import { captionSegmentsToAnnotationRegions } from "@/lib/captioning/annotationsFromCaptions"; +import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; +import { + shiftTrimRegionsMsForCaptionBuffer, + trimLeadingSilenceMono16k, +} from "@/lib/captioning/leadingSilence"; +import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; +import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; +import { nativeBridgeClient } from "@/native"; + +/** Highest trailing number across existing region ids, so new ids never collide. */ +function nextNumericIdFrom(regions: { id: string }[]): number { + let max = 0; + for (const region of regions) { + const match = /(\d+)$/.exec(region.id); + if (match) max = Math.max(max, Number(match[1])); + } + return max + 1; +} + +async function runCaptions(request: CliCaptionsRequest): Promise<CliDoneResult> { + const loaded = await nativeBridgeClient.project.loadProjectFileFromPath(request.projectPath); + if (!loaded.success || loaded.project === undefined) { + throw new Error(loaded.error ?? loaded.message ?? "Failed to load project file"); + } + if (!validateProjectData(loaded.project)) { + throw new Error("Project file is not a valid .openscreen project"); + } + const project = loaded.project; + const media = resolveProjectMedia(project); + if (!media) { + throw new Error("Project file does not reference any recorded media"); + } + const editor = normalizeProjectEditor(project.editor ?? {}); + const trimRegions: TrimRegion[] = editor.trimRegions; + + window.electronAPI.cliLog("info", "Extracting audio…"); + const videoUrl = toFileUrl(media.screenVideoPath); + const { samples, durationSec } = await extractMono16kFromVideoUrl(videoUrl); + if (!Number.isFinite(durationSec) || durationSec <= 0 || samples.length < 800) { + throw new Error("The project's video has no usable audio track to transcribe"); + } + + const { samples: speechSamples, trimSec } = trimLeadingSilenceMono16k(samples); + if (speechSamples.length < 800) { + throw new Error("No speech detected in the project's audio"); + } + + const trimMs = Math.round(trimSec * 1000); + const trimRegionsForTranscribe = shiftTrimRegionsMsForCaptionBuffer(trimRegions, trimMs); + + const transcribeOptions = { + onStatus: (phase: "model" | "transcribe") => { + window.electronAPI.cliLog( + "info", + phase === "model" ? "Loading caption model…" : "Transcribing…", + ); + }, + }; + + let { segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(speechSamples, { + trimRegions: trimRegionsForTranscribe, + ...transcribeOptions, + }); + let transcribedFromTrimmedBuffer = true; + + // Leading-silence trimming can return empty even when the full source has + // speech. Retry once against the untrimmed buffer before giving up. + if (segmentsRaw.length === 0 && trimSec > 0) { + ({ segments: segmentsRaw, granularity } = await transcribeMono16kToSegments(samples, { + trimRegions, + ...transcribeOptions, + })); + transcribedFromTrimmedBuffer = false; + } + + const segments = + transcribedFromTrimmedBuffer && trimSec > 0 + ? segmentsRaw.map((segment) => ({ + ...segment, + startSec: segment.startSec + trimSec, + endSec: segment.endSec + trimSec, + })) + : segmentsRaw; + + // Re-running the command replaces earlier auto-captions instead of stacking + // duplicates; manually added annotations are preserved. + const manualAnnotations: AnnotationRegion[] = editor.annotationRegions.filter( + (annotation) => annotation.annotationSource !== "auto-caption", + ); + const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]); + const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1); + + let { regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + minWordsPerCaption: request.minWordsPerCaption, + maxWordsPerCaption: request.maxWordsPerCaption, + timestampGranularity: granularity, + }); + if (regions.length === 0 && segments.length > 0) { + ({ regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + minWordsPerCaption: 1, + maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + timestampGranularity: granularity, + })); + } + if (regions.length === 0) { + throw new Error("Transcription produced no caption segments"); + } + + const updatedProject = { + ...project, + editor: { + ...editor, + annotationRegions: [...manualAnnotations, ...regions], + }, + }; + + return { + success: true, + projectPath: request.projectPath, + projectData: updatedProject, + captionCount: regions.length, + }; +} + +export function CliCaptionsRunner() { + const startedRef = useRef(false); + const [status] = useState("Generating captions…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = await window.electronAPI.cliGetRequest(); + if (request.kind !== "captions") { + throw new Error(`cli-captions window received a ${request.kind} request`); + } + const result = await runCaptions(request); + await window.electronAPI.cliDone(result); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliCaptionsRunner; diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 56f60581e2..08e600b343 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -13,7 +13,13 @@ import { toFileUrl, validateProjectData, } from "@/components/video-editor/projectPersistence"; -import type { CursorTelemetryPoint } from "@/components/video-editor/types"; +import { buildAutoZoomSuggestions } from "@/components/video-editor/timeline/zoomSuggestionUtils"; +import type { CursorTelemetryPoint, ZoomRegion } from "@/components/video-editor/types"; +import { + clampFocusToDepth, + DEFAULT_ZOOM_DEPTH, + ZOOM_DEPTH_SCALES, +} from "@/components/video-editor/types"; import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; import { calculateOutputDimensions, GifExporter } from "@/lib/exporter/gifExporter"; @@ -39,7 +45,9 @@ function isClickInteractionType(interactionType: string | null | undefined) { ); } -function probeVideoDimensions(url: string): Promise<{ width: number; height: number }> { +function probeVideoDimensions( + url: string, +): Promise<{ width: number; height: number; durationMs: number }> { return new Promise((resolve, reject) => { const video = document.createElement("video"); video.preload = "metadata"; @@ -57,8 +65,9 @@ function probeVideoDimensions(url: string): Promise<{ width: number; height: num video.onloadedmetadata = () => { const width = video.videoWidth; const height = video.videoHeight; + const durationMs = Number.isFinite(video.duration) ? Math.round(video.duration * 1000) : 0; cleanup(); - resolve({ width, height }); + resolve({ width, height, durationMs }); }; video.onerror = () => { cleanup(); @@ -80,6 +89,32 @@ function fitPreviewBox(aspectRatioValue: number, boxWidth: number, boxHeight: nu return { width: Math.round(width), height: Math.round(height) }; } +/** Mirrors the editor's buildAutoZoomRegions: cursor-dwell suggestions that + * follow the cursor (focusMode "auto") and never overlap existing regions. */ +function buildAutoZoomRegions( + cursorTelemetry: CursorTelemetryPoint[], + totalMs: number, + existingRegions: ZoomRegion[], +): ZoomRegion[] { + const suggestions = buildAutoZoomSuggestions({ + cursorTelemetry, + totalMs, + existingRegions, + defaultDurationMs: Math.max(1000, Math.round(totalMs * 0.05)), + }); + let nextId = 1; + return suggestions.map((suggestion) => ({ + id: `cli-auto-zoom-${nextId++}`, + startMs: Math.round(suggestion.span.start), + endMs: Math.round(suggestion.span.end), + depth: DEFAULT_ZOOM_DEPTH, + customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], + focus: clampFocusToDepth(suggestion.focus, DEFAULT_ZOOM_DEPTH), + focusMode: "auto" as const, + source: "auto" as const, + })); +} + function replaceExtension(filePath: string, newExtension: string): string { return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; } @@ -97,6 +132,20 @@ async function runExport(request: CliExportRequest): Promise<CliDoneResult> { if (!media) { throw new Error("Project file does not reference any recorded media"); } + // Prefer the main process's approved session paths: they carry the + // packed-project sibling fallback when the stored absolute paths are stale. + try { + const sessionResult = await window.electronAPI.getCurrentRecordingSession(); + const session = sessionResult?.session; + if (session?.screenVideoPath) { + media.screenVideoPath = session.screenVideoPath; + if (media.webcamVideoPath && session.webcamVideoPath) { + media.webcamVideoPath = session.webcamVideoPath; + } + } + } catch { + // Fall back to the paths stored in the project file. + } const editor = normalizeProjectEditor(project.editor ?? {}); const format = request.format ?? editor.exportFormat; @@ -154,6 +203,21 @@ async function runExport(request: CliExportRequest): Promise<CliDoneResult> { const probed = await probeVideoDimensions(videoUrl); const sourceWidth = probed.width || DEFAULT_SOURCE_DIMENSIONS.width; const sourceHeight = probed.height || DEFAULT_SOURCE_DIMENSIONS.height; + + if (request.autoZoom) { + const autoRegions = buildAutoZoomRegions( + cursorTelemetry, + probed.durationMs, + editor.zoomRegions, + ); + if (autoRegions.length > 0) { + editor.zoomRegions = [...editor.zoomRegions, ...autoRegions]; + } + window.electronAPI.cliLog( + "info", + `Auto-zoom: added ${autoRegions.length} region(s) from cursor telemetry`, + ); + } const effectiveSourceDimensions = calculateEffectiveSourceDimensions( sourceWidth, sourceHeight, diff --git a/src/cli/CliSourcesRunner.tsx b/src/cli/CliSourcesRunner.tsx new file mode 100644 index 0000000000..a1e464e232 --- /dev/null +++ b/src/cli/CliSourcesRunner.tsx @@ -0,0 +1,87 @@ +// Hidden-window runner for `openscreen sources`: enumerates capturable +// displays/windows (via the same get-sources IPC the GUI picker uses) and +// microphone inputs, then hands the payload to the CLI controller to print. + +import { useEffect, useRef, useState } from "react"; +import type { CliSourcesResult } from "@/lib/cliContracts"; + +async function enumerateMicrophones(): Promise<{ + microphones: { label: string }[]; + microphoneLabelsUnavailable: boolean; +}> { + const listInputs = async () => + (await navigator.mediaDevices.enumerateDevices()).filter( + (device) => device.kind === "audioinput", + ); + + let inputs = await listInputs(); + + // Labels are blank until a getUserMedia grant exists; a short-lived probe + // stream unlocks them without leaving anything recording. + if (inputs.length > 0 && inputs.every((device) => !device.label)) { + let probeStream: MediaStream | null = null; + try { + probeStream = await navigator.mediaDevices.getUserMedia({ audio: true }); + inputs = await listInputs(); + } catch { + // Permission denied — report devices without labels. + } finally { + probeStream?.getTracks().forEach((track) => track.stop()); + } + } + + const labeled = inputs.filter((device) => device.label); + return { + microphones: labeled.map((device) => ({ label: device.label })), + microphoneLabelsUnavailable: inputs.length > 0 && labeled.length === 0, + }; +} + +async function enumerateSources(): Promise<CliSourcesResult> { + const sources = await window.electronAPI.getSources({ + types: ["screen", "window"], + thumbnailSize: { width: 32, height: 18 }, + }); + + const displays = sources + .filter((source) => source.id.startsWith("screen:")) + .map((source, index) => ({ index, id: source.id, name: source.name })); + const windows = sources + .filter((source) => source.id.startsWith("window:")) + .map((source) => ({ id: source.id, name: source.name })); + + const { microphones, microphoneLabelsUnavailable } = await enumerateMicrophones(); + return { displays, windows, microphones, microphoneLabelsUnavailable }; +} + +export function CliSourcesRunner() { + const startedRef = useRef(false); + const [status] = useState("Enumerating sources…"); + + useEffect(() => { + if (startedRef.current) return; + startedRef.current = true; + + void (async () => { + try { + const request = await window.electronAPI.cliGetRequest(); + if (request.kind !== "sources") { + throw new Error(`cli-sources window received a ${request.kind} request`); + } + const sources = await enumerateSources(); + await window.electronAPI.cliDone({ success: true, sources }); + } catch (error) { + const message = error instanceof Error ? (error.stack ?? error.message) : String(error); + await window.electronAPI.cliDone({ success: false, error: message }); + } + })(); + }, []); + + return ( + <div className="flex h-screen items-center justify-center bg-[#09090b] text-white/60 text-sm"> + {status} + </div> + ); +} + +export default CliSourcesRunner; diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts index cdd7a45063..111f2ed2a5 100644 --- a/src/lib/cliContracts.ts +++ b/src/lib/cliContracts.ts @@ -25,6 +25,12 @@ export interface CliExportRequest { */ previewWidth: number | null; previewHeight: number | null; + /** + * Add automatic zoom regions derived from cursor-dwell telemetry (same + * suggestion engine as the editor's magic wand) before rendering. Existing + * zoom regions are preserved; suggestions never overlap them. + */ + autoZoom: boolean; /** Absolute path to a voiceover audio file to mix into the export (MP4 only). */ audioPath: string | null; /** "mix" layers the voiceover over the recording's audio; "replace" drops the original. */ @@ -50,7 +56,31 @@ export interface CliRecordRequest { projectOut: string | null; } -export type CliRequest = CliExportRequest | CliRecordRequest; +export interface CliSourcesRequest { + kind: "sources"; +} + +export interface CliCaptionsRequest { + kind: "captions"; + /** Absolute path to the .openscreen project file (updated in place). */ + projectPath: string; + minWordsPerCaption: number; + maxWordsPerCaption: number; +} + +export type CliRequest = + | CliExportRequest + | CliRecordRequest + | CliSourcesRequest + | CliCaptionsRequest; + +export interface CliSourcesResult { + displays: { index: number; id: string; name: string }[]; + windows: { id: string; name: string }[]; + microphones: { label: string }[]; + /** True when microphone labels required a permission the user hasn't granted. */ + microphoneLabelsUnavailable: boolean; +} export interface CliProgressEvent { percentage: number; @@ -80,4 +110,8 @@ export interface CliDoneResult { * The main process writes it to the --project path (renderer has no fs). */ projectData?: unknown; + /** Sources: enumeration payload printed by the main process. */ + sources?: CliSourcesResult; + /** Captions: number of caption annotations generated. */ + captionCount?: number; } From 2fd52b7c0ce9b6073b530b3d2827260f51c1cf4c Mon Sep 17 00:00:00 2001 From: PeterTakahashi <seiya4@icloud.com> Date: Tue, 28 Jul 2026 22:23:19 +0900 Subject: [PATCH 04/10] fix(cli): address second CodeRabbit review round - docs: real JSON payload example for `sources --json`; align the demo narration wording with --audio-mode replace - args.test: edge-case coverage for sources/pack/captions parsers (missing paths, unknown options, extra positionals, non-integer and zero word counts) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- docs/cli.md | 20 ++++++++++++++++++-- electron/cli/args.test.ts | 8 ++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 046e5696d0..b9a95b51c5 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -82,7 +82,22 @@ GUI picker uses — so scripts and agents can choose `--display`, `--window`, an ```bash openscreen sources # human-readable -openscreen sources --json # {"event":"done","sources":{displays,windows,microphones,...}} +openscreen sources --json +``` + +`--json` emits the payload on the final `done` event: + +```json +{ + "event": "done", + "success": true, + "sources": { + "displays": [{ "index": 0, "id": "screen:1:0", "name": "Entire screen" }], + "windows": [{ "id": "window:210:0", "name": "My App" }], + "microphones": [{ "label": "MacBook Pro Microphone (Built-in)" }], + "microphoneLabelsUnavailable": false + } +} ``` ### `openscreen export` @@ -205,7 +220,8 @@ node -e ' # 3. Narrate with any TTS (macOS `say` shown; any engine producing mp3/wav/m4a works) say -o voice.m4a --file-format=m4af "Welcome to my product. Here's a quick tour." -# 4. Render with auto-zooms and the voiceover mixed in +# 4. Render with auto-zooms; the voiceover replaces the recording's own audio +# (drop --audio-mode replace to duck the original under the narration instead) openscreen export demo.openscreen -o demo.mp4 --auto-zoom --audio voice.m4a --audio-mode replace --json ``` diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index 575e0f7c5d..3c00bc2002 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -154,6 +154,7 @@ describe("parseCliArgs", () => { it("parses sources", () => { expect(parse(["sources", "--json"])).toMatchObject({ kind: "sources", json: true }); expect(parse(["sources", "--bogus"])).toMatchObject({ kind: "error" }); + expect(parse(["sources", "extra-arg"])).toMatchObject({ kind: "error" }); }); it("parses pack", () => { @@ -163,6 +164,13 @@ describe("parseCliArgs", () => { outDir: inCwd("bundle"), }); expect(parse(["pack", "demo.openscreen"])).toMatchObject({ kind: "error" }); + expect(parse(["pack", "--out", "bundle"])).toMatchObject({ kind: "error" }); + expect(parse(["pack", "demo.openscreen", "--out", "bundle", "--bogus"])).toMatchObject({ + kind: "error", + }); + expect(parse(["pack", "a.openscreen", "b.openscreen", "--out", "bundle"])).toMatchObject({ + kind: "error", + }); }); it("parses captions", () => { From 6aeee9845583fab9d558122fc4a4263ff0498337 Mon Sep 17 00:00:00 2001 From: Etienne Lescot <etiennelescot@gmail.com> Date: Sat, 1 Aug 2026 13:42:47 +0200 Subject: [PATCH 05/10] fix(cli): drop a conflict marker the rebase left in preload.ts Resolving the preload conflict kept both sides but missed the opening marker, so the file didn't parse. Both the stt block and the CLI bridge members are intended -- they collided only because each side appended to the same object. --- electron/preload.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/electron/preload.ts b/electron/preload.ts index 6aa0a886c7..f3b44d57d6 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -407,7 +407,6 @@ contextBridge.exposeInMainWorld("electronAPI", { sendCloseConfirmResponse: (choice: "save" | "discard" | "cancel") => { ipcRenderer.send("close-confirm-response", choice); }, -<<<<<<< HEAD // ponytail: forward renderer console output to main-process stdout so // recorder diagnostics land next to the main-process logs in dev output. // One-way fire-and-forget; we deliberately don't await the IPC. From eb996863128e10797634bf6d1f5a2aea7b58de97 Mon Sep 17 00:00:00 2001 From: PeterTakahashi <seiya4@icloud.com> Date: Sat, 1 Aug 2026 21:57:28 +0900 Subject: [PATCH 06/10] fix(cli): port the export runner and captions onto the 1.8 native pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the six-symbol port on top of contrib/pr-176-rebased: - CliExportRunner now migrates the .openscreen project to an AxcutDocument (migrateProjectDataToAxcutDocument + applyProbedDuration — without the probe the clip has no duration and the export is a single frame) and drives exportMultiNative/exportGifNative, mirroring the v4 ExportDialog: same clip list, same scene JSON, same crop-aware smallest-clip sizing, same gif dimension capping. Progress is synthesized from the native frame-count push (export:native-progress) so the CLI's NDJSON contract (percentage/currentFrame/totalFrames/ETA + done{outputPath,...}) is unchanged. --preview-size becomes an accepted no-op (annotation geometry is percentage-based in the scene); --audio now reads the natively written file back, mixes the voiceover, and overwrites outPath. - Auto-zoom repointed to @/lib/ai-edition/timeline/zoom-suggestions and zoom-scale; suggestions append straight to document.zoomRanges. - Deleted helpers vendored under src/cli/vendor/ with provenance notes: captionSegmentsToAnnotationRegions (the CLI still writes caption annotations into projects — the v2 route the migration preserves), trimLeadingSilenceMono16k, clampZoomFocus, hasNativeCursorRecordingData. - Cursor plumbing dropped from the runner: the native compositor discovers the <video>.cursor.json sidecar by filename convention. Known limitation carried over: the native pipeline has no cancel and no extra-audio-track concept; kill mid-export leaves the worker running until process exit, and --audio costs one extra read/write of the output. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- src/cli/CliCaptionsRunner.tsx | 4 +- src/cli/CliExportRunner.tsx | 459 +++++++++++------------ src/cli/vendor/captionRegions.ts | 606 +++++++++++++++++++++++++++++++ src/cli/vendor/leadingSilence.ts | 80 ++++ src/cli/vendor/zoomHelpers.ts | 30 ++ 5 files changed, 932 insertions(+), 247 deletions(-) create mode 100644 src/cli/vendor/captionRegions.ts create mode 100644 src/cli/vendor/leadingSilence.ts create mode 100644 src/cli/vendor/zoomHelpers.ts diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx index 32d01ca119..7bcf213911 100644 --- a/src/cli/CliCaptionsRunner.tsx +++ b/src/cli/CliCaptionsRunner.tsx @@ -10,12 +10,12 @@ import { validateProjectData, } from "@/components/video-editor/projectPersistence"; import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types"; -import { captionSegmentsToAnnotationRegions } from "@/lib/captioning/annotationsFromCaptions"; +import { captionSegmentsToAnnotationRegions } from "./vendor/captionRegions"; import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; import { shiftTrimRegionsMsForCaptionBuffer, trimLeadingSilenceMono16k, -} from "@/lib/captioning/leadingSilence"; +} from "./vendor/leadingSilence"; import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; import { nativeBridgeClient } from "@/native"; diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 08e600b343..2b42da58ee 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -1,49 +1,41 @@ // Hidden-window runner for `openscreen export`. Loads an .openscreen project, -// rebuilds the same exporter configuration the editor's export dialog would, -// and streams progress back to the CLI controller in the main process. +// migrates it to the AxcutDocument the native Rust compositor consumes, and +// drives exportMultiNative/exportGifNative — mirroring the v4 ExportDialog so +// CLI exports and GUI exports stay pixel-identical. The hidden window does no +// compositing itself: the render runs in the main process; this runner only +// builds the clip list + scene JSON and relays progress. import { useEffect, useRef, useState } from "react"; -import { - DEFAULT_CURSOR_SETTINGS, - DEFAULT_SOURCE_DIMENSIONS, -} from "@/components/video-editor/editorDefaults"; import { normalizeProjectEditor, resolveProjectMedia, toFileUrl, validateProjectData, } from "@/components/video-editor/projectPersistence"; -import { buildAutoZoomSuggestions } from "@/components/video-editor/timeline/zoomSuggestionUtils"; -import type { CursorTelemetryPoint, ZoomRegion } from "@/components/video-editor/types"; +import type { CursorTelemetryPoint } from "@/components/video-editor/types"; +import { migrateProjectDataToAxcutDocument } from "@/lib/ai-edition/document/migrate"; import { - clampFocusToDepth, - DEFAULT_ZOOM_DEPTH, - ZOOM_DEPTH_SCALES, -} from "@/components/video-editor/types"; + collectEffectiveClipDims, + type Dims, + pickExtremeDims, + resolveAspectRatioValue, +} from "@/lib/ai-edition/document/outputFormat"; +import { applyProbedDuration } from "@/lib/ai-edition/document/timeline"; +import type { AxcutDocument } from "@/lib/ai-edition/schema"; +import { getEditorSettings } from "@/lib/ai-edition/store/editorSettings"; +import { resolveClipSourceEndSec } from "@/lib/ai-edition/timeline/clipDuration"; +import { DEFAULT_ZOOM_DEPTH, ZOOM_DEPTH_SCALES } from "@/lib/ai-edition/timeline/zoom-scale"; +import { buildAutoZoomSuggestions } from "@/lib/ai-edition/timeline/zoom-suggestions"; import type { CliDoneResult, CliExportRequest } from "@/lib/cliContracts"; -import { hasNativeCursorRecordingData } from "@/lib/cursor/nativeCursor"; -import { calculateOutputDimensions, GifExporter } from "@/lib/exporter/gifExporter"; -import { - calculateEffectiveSourceDimensions, - calculateMp4ExportSettings, -} from "@/lib/exporter/mp4ExportSettings"; -import type { ExportProgress } from "@/lib/exporter/types"; -import { GIF_SIZE_PRESETS } from "@/lib/exporter/types"; -import { VideoExporter } from "@/lib/exporter/videoExporter"; +import { GIF_SIZE_PRESETS, type GifSizePreset } from "@/lib/exporter"; +import { calculateMp4ExportSettings } from "@/lib/exporter/mp4ExportSettings"; import { mixVoiceoverIntoVideo } from "@/lib/exporter/voiceoverMix"; -import { nativeBridgeClient } from "@/native"; -import type { CursorRecordingData, NativePlatform } from "@/native/contracts"; -import { getAspectRatioValue, getNativeAspectRatioValue } from "@/utils/aspectRatioUtils"; +import { exportGifNative, exportMultiNative, nativeBridgeClient } from "@/native"; +import type { CompositorClipInput } from "@/native/contracts"; +import { buildSceneDescription, resolveVisibleClips } from "@/native/sceneDescription"; +import { clampZoomFocus } from "./vendor/zoomHelpers"; -// Mirrors the private helper in VideoEditor.tsx. -function isClickInteractionType(interactionType: string | null | undefined) { - return ( - interactionType === "click" || - interactionType === "double-click" || - interactionType === "right-click" || - interactionType === "middle-click" - ); -} +const MP4_EXPORT_FPS = 60; function probeVideoDimensions( url: string, @@ -77,46 +69,75 @@ function probeVideoDimensions( }); } -/** Fit the composition aspect ratio into the reference preview box, mirroring - * how the editor sizes its on-screen preview container. */ -function fitPreviewBox(aspectRatioValue: number, boxWidth: number, boxHeight: number) { - let width = boxWidth; - let height = boxWidth / aspectRatioValue; - if (height > boxHeight) { - height = boxHeight; - width = boxHeight * aspectRatioValue; +function replaceExtension(filePath: string, newExtension: string): string { + return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; +} + +/** Mirrors ExportDialog.buildNativeClipList: trim-narrowed visible clips mapped + * onto the native multiclip contract. Kept in lock-step with + * buildSceneDescription so export and scene agree on the clip stream. */ +function buildNativeClipList(axcutDocument: AxcutDocument): CompositorClipInput[] { + const assetById = new Map(axcutDocument.assets.map((asset) => [asset.id, asset])); + return resolveVisibleClips(axcutDocument).flatMap((clip) => { + const asset = assetById.get(clip.assetId); + if (!asset?.originalPath) { + return []; + } + const cam = asset.cameraTrack; + const sourceEndSec = resolveClipSourceEndSec(clip, asset); + return [ + { + screenPath: asset.originalPath, + webcamPath: cam?.sourcePath ?? asset.originalPath, + sourceStartSec: clip.sourceStartSec, + sourceEndSec, + webcamOffsetSec: cam ? (cam.startMs + cam.offsetMs) / 1000 : 0, + hasAudio: true, + }, + ]; + }); +} + +/** Mirrors ExportDialog.gifOutputDims: cap height at the preset, keep even. */ +function gifOutputDims( + preset: GifSizePreset, + tierDims: { width: number; height: number } | null, +): { width?: number; height?: number } { + if (!tierDims) return {}; + const maxHeight = GIF_SIZE_PRESETS[preset].maxHeight; + if (!Number.isFinite(maxHeight) || tierDims.height <= maxHeight) { + return { width: tierDims.width, height: tierDims.height }; } - return { width: Math.round(width), height: Math.round(height) }; + const scale = maxHeight / tierDims.height; + const even = (n: number) => Math.max(2, Math.round(n * scale) & ~1); + return { width: even(tierDims.width), height: even(tierDims.height) }; } -/** Mirrors the editor's buildAutoZoomRegions: cursor-dwell suggestions that - * follow the cursor (focusMode "auto") and never overlap existing regions. */ -function buildAutoZoomRegions( +function appendAutoZoomRanges( + axcutDocument: AxcutDocument, cursorTelemetry: CursorTelemetryPoint[], totalMs: number, - existingRegions: ZoomRegion[], -): ZoomRegion[] { +): number { const suggestions = buildAutoZoomSuggestions({ cursorTelemetry, totalMs, - existingRegions, + existingRegions: axcutDocument.zoomRanges, defaultDurationMs: Math.max(1000, Math.round(totalMs * 0.05)), }); let nextId = 1; - return suggestions.map((suggestion) => ({ - id: `cli-auto-zoom-${nextId++}`, - startMs: Math.round(suggestion.span.start), - endMs: Math.round(suggestion.span.end), - depth: DEFAULT_ZOOM_DEPTH, - customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], - focus: clampFocusToDepth(suggestion.focus, DEFAULT_ZOOM_DEPTH), - focusMode: "auto" as const, - source: "auto" as const, - })); -} - -function replaceExtension(filePath: string, newExtension: string): string { - return filePath.replace(/\.(openscreen|json)$/i, "") + newExtension; + for (const suggestion of suggestions) { + axcutDocument.zoomRanges.push({ + id: `cli-auto-zoom-${nextId++}`, + startMs: Math.round(suggestion.span.start), + endMs: Math.round(suggestion.span.end), + depth: DEFAULT_ZOOM_DEPTH, + customScale: ZOOM_DEPTH_SCALES[DEFAULT_ZOOM_DEPTH], + focus: clampZoomFocus(suggestion.focus), + focusMode: "auto", + source: "auto", + }); + } + return suggestions.length; } async function runExport(request: CliExportRequest): Promise<CliDoneResult> { @@ -159,210 +180,158 @@ async function runExport(request: CliExportRequest): Promise<CliDoneResult> { const gifSizePreset = request.gifSizePreset ?? editor.gifSizePreset; const outPath = request.outPath ?? replaceExtension(request.projectPath, format === "gif" ? ".gif" : ".mp4"); + if (request.previewWidth !== null || request.previewHeight !== null) { + window.electronAPI.cliLog( + "info", + "--preview-size is a no-op on the native pipeline (annotation geometry is percentage-based) and kept only for CLI compatibility", + ); + } - const videoUrl = toFileUrl(media.screenVideoPath); - const webcamVideoUrl = media.webcamVideoPath ? toFileUrl(media.webcamVideoPath) : undefined; - - // Cursor sidecar data (native recordings). Both lookups tolerate missing files. + // Cursor telemetry: only needed to compute --auto-zoom suggestions. The + // native compositor discovers the `<video>.cursor.json` sidecar itself. let cursorTelemetry: CursorTelemetryPoint[] = []; - let cursorRecordingData: CursorRecordingData | null = null; - try { - cursorTelemetry = await nativeBridgeClient.cursor.getTelemetry(media.screenVideoPath); - } catch { - cursorTelemetry = []; - } - try { - cursorRecordingData = await nativeBridgeClient.cursor.getRecordingData(media.screenVideoPath); - } catch { - cursorRecordingData = null; + if (request.autoZoom) { + try { + cursorTelemetry = await nativeBridgeClient.cursor.getTelemetry(media.screenVideoPath); + } catch { + cursorTelemetry = []; + } } - const recordingClicks = - cursorRecordingData?.samples - .filter((sample) => isClickInteractionType(sample.interactionType)) - .map((sample) => sample.timeMs) ?? []; - const cursorClickTimestamps = - recordingClicks.length > 0 - ? recordingClicks - : cursorTelemetry - .filter((sample) => isClickInteractionType(sample.interactionType)) - .map((sample) => sample.timeMs); + const probed = await probeVideoDimensions(toFileUrl(media.screenVideoPath)); - let platform: NativePlatform | null = null; - try { - platform = await nativeBridgeClient.system.getPlatform(); - } catch { - platform = null; + // Migrate the .openscreen project onto the AxcutDocument the native + // compositor consumes. The migration is pure and carries zooms, annotations, + // trims and the legacy editor settings; the clip's duration is unknown until + // probed, so applyProbedDuration must run or the export is a single frame. + let axcutDocument = migrateProjectDataToAxcutDocument({ + ...project, + media, + editor, + }); + const primaryAssetId = axcutDocument.project.primaryAssetId ?? axcutDocument.assets[0]?.id; + if (!primaryAssetId) { + throw new Error("Project migration produced no media asset"); + } + if (probed.durationMs > 0) { + axcutDocument = applyProbedDuration(axcutDocument, primaryAssetId, probed.durationMs / 1000); } - const hasEditableCursorRecording = - (media.cursorCaptureMode ?? "editable-overlay") === "editable-overlay" && - (platform === "win32" || platform === "darwin") && - hasNativeCursorRecordingData(cursorRecordingData); - const effectiveShowCursor = DEFAULT_CURSOR_SETTINGS.show && hasEditableCursorRecording; - - const probed = await probeVideoDimensions(videoUrl); - const sourceWidth = probed.width || DEFAULT_SOURCE_DIMENSIONS.width; - const sourceHeight = probed.height || DEFAULT_SOURCE_DIMENSIONS.height; if (request.autoZoom) { - const autoRegions = buildAutoZoomRegions( - cursorTelemetry, - probed.durationMs, - editor.zoomRegions, - ); - if (autoRegions.length > 0) { - editor.zoomRegions = [...editor.zoomRegions, ...autoRegions]; - } - window.electronAPI.cliLog( - "info", - `Auto-zoom: added ${autoRegions.length} region(s) from cursor telemetry`, - ); + const added = appendAutoZoomRanges(axcutDocument, cursorTelemetry, probed.durationMs); + window.electronAPI.cliLog("info", `Auto-zoom: added ${added} region(s) from cursor telemetry`); } - const effectiveSourceDimensions = calculateEffectiveSourceDimensions( - sourceWidth, - sourceHeight, - editor.cropRegion, - ); - const aspectRatioValue = - editor.aspectRatio === "native" - ? getNativeAspectRatioValue(sourceWidth, sourceHeight, editor.cropRegion) - : getAspectRatioValue(editor.aspectRatio); - const preview = fitPreviewBox( - aspectRatioValue, - request.previewWidth ?? 1280, - request.previewHeight ?? 720, + // Output sizing mirrors the ExportDialog: crop-aware smallest clip on the + // timeline, normalized to the document's aspect ratio. + const probedAssetDims: Record<string, Dims> = { + [primaryAssetId]: { width: probed.width, height: probed.height }, + }; + const smallestSource = + pickExtremeDims(collectEffectiveClipDims(axcutDocument, probedAssetDims), "smallest") ?? + ({ width: probed.width, height: probed.height } as Dims); + const aspectRatioValue = resolveAspectRatioValue( + axcutDocument, + getEditorSettings(axcutDocument).aspectRatio, ); + const outDims = calculateMp4ExportSettings({ + quality, + sourceWidth: smallestSource.width, + sourceHeight: smallestSource.height, + aspectRatioValue, + }); - const onProgress = (progress: ExportProgress) => { + const clips = buildNativeClipList(axcutDocument); + if (clips.length === 0) { + throw new Error("The project's timeline has no visible clips to export"); + } + const sceneJson = JSON.stringify(buildSceneDescription(axcutDocument)); + + // Progress: native pushes raw encoded-frame counts; totals and pacing are + // computed here, mirroring the ExportDialog. + const outFps = format === "gif" ? gifFrameRate : MP4_EXPORT_FPS; + const totalFrames = Math.max( + 1, + Math.round( + clips.reduce((sum, clip) => sum + Math.max(0, clip.sourceEndSec - clip.sourceStartSec), 0) * + outFps, + ), + ); + const exportStartedAt = Date.now(); + const unsubscribeProgress = window.electronAPI.onNativeExportProgress?.((frames: number) => { + const elapsedSec = (Date.now() - exportStartedAt) / 1000; + const rate = frames > 0 ? frames / Math.max(elapsedSec, 0.001) : 0; window.electronAPI.cliProgress({ - percentage: progress.percentage, - currentFrame: progress.currentFrame, - totalFrames: progress.totalFrames, - estimatedTimeRemaining: progress.estimatedTimeRemaining, - phase: progress.phase, + percentage: Math.min(100, (frames / totalFrames) * 100), + currentFrame: frames, + totalFrames, + estimatedTimeRemaining: rate > 0 ? Math.max(0, (totalFrames - frames) / rate) : 0, }); - }; - - const sharedConfig = { - videoUrl, - webcamVideoUrl, - wallpaper: editor.wallpaper, - zoomRegions: editor.zoomRegions, - cameraFullscreenRegions: editor.cameraFullscreenRegions, - trimRegions: editor.trimRegions, - speedRegions: editor.speedRegions, - showShadow: editor.shadowIntensity > 0, - shadowIntensity: editor.shadowIntensity, - showBlur: editor.showBlur, - motionBlurAmount: editor.motionBlurAmount, - borderRadius: editor.borderRadius, - padding: editor.padding, - cropRegion: editor.cropRegion, - cursorRecordingData, - cursorScale: effectiveShowCursor ? DEFAULT_CURSOR_SETTINGS.size : 0, - cursorSmoothing: DEFAULT_CURSOR_SETTINGS.smoothing, - cursorMotionBlur: DEFAULT_CURSOR_SETTINGS.motionBlur, - cursorClickBounce: DEFAULT_CURSOR_SETTINGS.clickBounce, - cursorClipToBounds: DEFAULT_CURSOR_SETTINGS.clipToBounds, - cursorTheme: editor.cursorTheme, - annotationRegions: editor.annotationRegions, - webcamLayoutPreset: editor.webcamLayoutPreset, - webcamMaskShape: editor.webcamMaskShape, - webcamMirrored: editor.webcamMirrored, - webcamReactiveZoom: editor.webcamReactiveZoom, - webcamSizePreset: editor.webcamSizePreset, - webcamPosition: editor.webcamPosition, - previewWidth: preview.width, - previewHeight: preview.height, - cursorTelemetry, - cursorClickTimestamps, - onProgress, - }; - - let blob: Blob; - let warnings: string[] | undefined; - let outWidth: number; - let outHeight: number; + }); - if (format === "gif") { - const gifDimensions = calculateOutputDimensions( - effectiveSourceDimensions.width, - effectiveSourceDimensions.height, - gifSizePreset, - GIF_SIZE_PRESETS, - aspectRatioValue, - ); - outWidth = gifDimensions.width; - outHeight = gifDimensions.height; - const gifExporter = new GifExporter({ - ...sharedConfig, - width: gifDimensions.width, - height: gifDimensions.height, - frameRate: gifFrameRate, - loop: editor.gifLoop, - sizePreset: gifSizePreset, - videoPadding: editor.padding, - }); - const result = await gifExporter.export(); - if (!result.success || !result.blob) { - throw new Error(result.error ?? "GIF export failed"); + try { + if (format === "gif") { + const dims = gifOutputDims(gifSizePreset, outDims); + await exportGifNative(clips, outPath, sceneJson, { + ...dims, + fps: gifFrameRate, + loopCount: editor.gifLoop ? 0 : 1, + }); + return { + success: true, + outputPath: outPath, + format, + width: dims.width, + height: dims.height, + }; } - blob = result.blob; - warnings = result.warnings; - } else { - const mp4Settings = calculateMp4ExportSettings({ - quality, - sourceWidth: effectiveSourceDimensions.width, - sourceHeight: effectiveSourceDimensions.height, - aspectRatioValue, - }); - outWidth = mp4Settings.width; - outHeight = mp4Settings.height; - const exporter = new VideoExporter({ - ...sharedConfig, - width: mp4Settings.width, - height: mp4Settings.height, - frameRate: 60, - bitrate: mp4Settings.bitrate, - codec: "avc1.640033", + + // MP4: native writes outPath directly. When a voiceover is requested, mix + // it afterwards (the native pipeline has no extra-audio-track concept) and + // overwrite the same file. + await exportMultiNative(clips, outPath, sceneJson, { + width: outDims.width, + height: outDims.height, + fps: MP4_EXPORT_FPS, + codec: "h264", }); - const result = await exporter.export(); - if (!result.success || !result.blob) { - throw new Error(result.error ?? "MP4 export failed"); - } - blob = result.blob; - warnings = result.warnings; - } - if (request.audioPath && format === "mp4") { - window.electronAPI.cliProgress({ percentage: 100, phase: "mixing-voiceover" }); - const audioResponse = await fetch(toFileUrl(request.audioPath)); - if (!audioResponse.ok) { - throw new Error(`Failed to read voiceover file: ${request.audioPath}`); + if (request.audioPath) { + window.electronAPI.cliProgress({ percentage: 100, phase: "mixing-voiceover" }); + const [videoResponse, audioResponse] = await Promise.all([ + fetch(toFileUrl(outPath)), + fetch(toFileUrl(request.audioPath)), + ]); + if (!videoResponse.ok) { + throw new Error(`Failed to read the exported video back for mixing: ${outPath}`); + } + if (!audioResponse.ok) { + throw new Error(`Failed to read voiceover file: ${request.audioPath}`); + } + const mixed = await mixVoiceoverIntoVideo(await videoResponse.blob(), { + voiceoverData: await audioResponse.arrayBuffer(), + mode: request.audioMode, + offsetSec: request.audioOffsetSec, + }); + const saveResult = await window.electronAPI.writeExportToPath( + await mixed.arrayBuffer(), + outPath, + ); + if (!saveResult.success) { + throw new Error(saveResult.message ?? `Failed to write mixed output to ${outPath}`); + } } - const voiceoverData = await audioResponse.arrayBuffer(); - blob = await mixVoiceoverIntoVideo(blob, { - voiceoverData, - mode: request.audioMode, - offsetSec: request.audioOffsetSec, - }); - } - const arrayBuffer = await blob.arrayBuffer(); - const saveResult = await window.electronAPI.writeExportToPath(arrayBuffer, outPath); - if (!saveResult.success || !saveResult.path) { - throw new Error(saveResult.message ?? `Failed to write output to ${outPath}`); + return { + success: true, + outputPath: outPath, + format, + width: outDims.width, + height: outDims.height, + }; + } finally { + unsubscribeProgress?.(); } - - return { - success: true, - outputPath: saveResult.path, - format, - width: outWidth, - height: outHeight, - warnings, - }; } export function CliExportRunner() { diff --git a/src/cli/vendor/captionRegions.ts b/src/cli/vendor/captionRegions.ts new file mode 100644 index 0000000000..f8ce2726d2 --- /dev/null +++ b/src/cli/vendor/captionRegions.ts @@ -0,0 +1,606 @@ +// Vendored for the CLI: captionSegmentsToAnnotationRegions was removed from @/lib/captioning/annotationsFromCaptions in the 1.8 line (the v4 editor renders captions natively). The CLI still writes caption *annotations* into .openscreen projects, so the converter lives on here verbatim. + +import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types"; + +import type { CaptionSegment } from "@/lib/captioning/transcribe"; + +/** Wide lower-third bar; `position.x` is top-left as % of container, so center with (100 - width) / 2. */ +const CAPTION_WIDTH = 92; +const CAPTION_HEIGHT = 12; +const CAPTION_BOTTOM_MARGIN = 2; + +const CAPTION_POSITION = { + x: (100 - CAPTION_WIDTH) / 2, + y: 100 - CAPTION_HEIGHT - CAPTION_BOTTOM_MARGIN, +}; + +const CAPTION_SIZE = { width: CAPTION_WIDTH, height: CAPTION_HEIGHT }; + +const CAPTION_STYLE: AnnotationTextStyle = { + color: "#ffffff", + backgroundColor: "rgba(255, 255, 255, 0)", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", +}; + +/** Nudge caption starts earlier (seconds); Whisper onsets run slightly late. Do not offset ends too, that pulls lines off-screen early. */ +const AUTO_CAPTION_START_BIAS_SEC = 0; + +/** Extra hold after Whisper's segment end (seconds); model end times run early vs trailing vowels. Separate from the start bias. */ +const AUTO_CAPTION_END_HOLD_SEC = 0; + +/** Inside one Whisper phrase, sub-lines can be shorter (do not steal time from neighbors). */ +const WORD_SPLIT_MIN_SPAN_SEC = 0.02; + +/** Brief linger after the last word in a line (seconds); trimmed if it would overlap the next line. */ +const CAPTION_LINE_END_TAIL_SEC = 0; + +/** A real silence between word-level timestamps should start a new caption run. */ +const WORD_RUN_BREAK_GAP_SEC = 0.24; + +/** Min time between consecutive caption regions (seconds); keeps a visible gap so blocks don't read as one clip. Small so short pauses survive. */ +const MIN_CAPTION_TIMELINE_GAP_SEC = 0; + +/** Same text again with almost no gap or overlap; common Whisper/chunk artifact. */ +const DEDUPE_SAME_TEXT_MAX_GAP_SEC = 0.55; + +export const SAME_CONTENT_ECHO_MAX_GAP_SEC = 1.15; + +function normalizeCaptionKey(text: string): string { + return text + .trim() + .replace(/\s+/g, " ") + .replace(/[\u2018\u2019]/g, "'") + .replace(/[\u201C\u201D]/g, '"') + .toLowerCase() + .replace(/[.!?,;:]+$/g, ""); +} + +/** Legacy echo-collapse helper kept for reference while phrase timing uses raw model spans. */ +export function collapseSameContentEchoes(segments: CaptionSegment[]): CaptionSegment[] { + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + const out: CaptionSegment[] = []; + const lastIndexByKey = new Map<string, number>(); + + for (const seg of sorted) { + const key = normalizeCaptionKey(seg.text); + const hit = lastIndexByKey.get(key); + if (hit !== undefined) { + const prev = out[hit]!; + if (seg.startSec < prev.endSec + SAME_CONTENT_ECHO_MAX_GAP_SEC) { + prev.startSec = Math.min(prev.startSec, seg.startSec); + prev.endSec = Math.max(prev.endSec, seg.endSec); + continue; + } + } + out.push({ + startSec: seg.startSec, + endSec: seg.endSec, + text: seg.text.trim(), + }); + lastIndexByKey.set(key, out.length - 1); + } + return out; +} + +/** + * Collapse adjacent duplicate lines (overlapping or tiny gap). Does not merge the same phrase + * repeated later in the video when separated by real silence. + */ +function dedupeAdjacentCaptionRepeats(segments: CaptionSegment[]): CaptionSegment[] { + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + const out: CaptionSegment[] = []; + for (const seg of sorted) { + const t = seg.text.trim(); + const prev = out[out.length - 1]; + if (prev && normalizeCaptionKey(prev.text) === normalizeCaptionKey(t)) { + const overlap = prev.endSec - seg.startSec; + const gap = seg.startSec - prev.endSec; + if (overlap > 0.015 || gap < DEDUPE_SAME_TEXT_MAX_GAP_SEC) { + prev.startSec = Math.min(prev.startSec, seg.startSec); + prev.endSec = Math.max(prev.endSec, seg.endSec); + continue; + } + } + out.push({ startSec: seg.startSec, endSec: seg.endSec, text: t }); + } + return out; +} + +/** Trim only real overlaps. Avoid synthetic lead/lag so caption timing matches model output. */ +function finalizeCaptionSegmentsForPlayback(segments: CaptionSegment[]): CaptionSegment[] { + const OVERLAP_TRIM_SEC = 0.002; + + const sortedRaw = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + + const a = sortedRaw.map((seg) => { + let s = seg.startSec + AUTO_CAPTION_START_BIAS_SEC; + let e = seg.endSec + AUTO_CAPTION_END_HOLD_SEC; + s = Math.max(0, s); + if (e <= s) e = s + 0.02; + return { startSec: s, endSec: e, text: seg.text.trim() }; + }); + + for (let i = 1; i < a.length; i++) { + if (a[i].startSec < a[i - 1].endSec - OVERLAP_TRIM_SEC) { + a[i - 1].endSec = Math.max(a[i - 1].startSec + 1e-4, a[i].startSec); + } + } + + return a; +} + +/** Default min gap between auto-caption blocks on the timeline (ms); matches `MIN_CAPTION_TIMELINE_GAP_SEC`. */ +export const DEFAULT_AUTO_CAPTION_MIN_GAP_MS = Math.round(MIN_CAPTION_TIMELINE_GAP_SEC * 1000); + +/** + * Enforce a min gap between consecutive `auto-caption` regions (by start time). Shortens the previous + * region's end when possible, else shifts the following region later so blocks can't sit completely flush. + */ +export function reconcileAutoCaptionTimelineGaps( + regions: AnnotationRegion[], + minGapMs: number = DEFAULT_AUTO_CAPTION_MIN_GAP_MS, +): AnnotationRegion[] { + const gap = Math.max(0, Math.round(minGapMs)); + if (regions.length === 0 || gap === 0) return regions; + + const autoCandidates = regions.filter((r) => r.annotationSource === "auto-caption"); + if (autoCandidates.length <= 1) return regions; + + const sorted = [...autoCandidates].sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); + const fixed: AnnotationRegion[] = []; + let prev = { ...sorted[0]! }; + fixed.push(prev); + + for (let i = 1; i < sorted.length; i++) { + let cur = { ...sorted[i]! }; + const minStart = prev.endMs + gap; + + if (cur.startMs < minStart) { + const newPrevEnd = cur.startMs - gap; + if (newPrevEnd >= prev.startMs + 1) { + prev = { ...prev, endMs: newPrevEnd }; + fixed[fixed.length - 1] = prev; + } else { + const dur = Math.max(1, cur.endMs - cur.startMs); + cur = { ...cur, startMs: minStart, endMs: minStart + dur }; + } + } + + fixed.push(cur); + prev = cur; + } + + const fixedById = new Map(fixed.map((r) => [r.id, r])); + return regions.map((r) => fixedById.get(r.id) ?? r); +} + +/** Join phrases that are close in time so the editor does not create dozens of separate overlays. */ +export function mergeAdjacentCaptionSegments( + segments: CaptionSegment[], + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[] { + const maxGapSec = options?.maxGapSec ?? 1.35; + const maxChars = options?.maxChars ?? 320; + const maxBlockDurationSec = options?.maxBlockDurationSec ?? 12; + + const sorted = [...segments].sort((a, b) => a.startSec - b.startSec); + const out: CaptionSegment[] = []; + + for (const seg of sorted) { + const text = seg.text.trim(); + if (!text) continue; + + const prev = out[out.length - 1]; + if (!prev) { + out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); + continue; + } + + const gap = seg.startSec - prev.endSec; + const mergedText = `${prev.text} ${text}`.trim(); + const mergedEnd = Math.max(prev.endSec, seg.endSec); + const wouldSpan = mergedEnd - prev.startSec; + if (gap <= maxGapSec && mergedText.length <= maxChars && wouldSpan <= maxBlockDurationSec) { + prev.endSec = mergedEnd; + prev.text = mergedText; + } else { + out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); + } + } + + return out; +} + +function partitionPhraseCaptionSegments( + segments: CaptionSegment[], + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[][] { + const maxGapSec = options?.maxGapSec ?? 0; + const maxChars = options?.maxChars ?? Number.POSITIVE_INFINITY; + const maxBlockDurationSec = options?.maxBlockDurationSec ?? Number.POSITIVE_INFINITY; + + const sorted = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + if (sorted.length === 0) return []; + + const groups: CaptionSegment[][] = []; + let current: CaptionSegment[] = []; + + for (const seg of sorted) { + const text = seg.text.trim(); + if (!text) continue; + + if (current.length === 0) { + current.push({ ...seg, text }); + continue; + } + + const prev = current[current.length - 1]!; + const groupStart = current[0]!.startSec; + const gap = seg.startSec - prev.endSec; + const currentChars = current.reduce((sum, item) => sum + item.text.length, 0); + const wouldChars = currentChars + 1 + text.length; + const wouldSpan = Math.max(prev.endSec, seg.endSec) - groupStart; + + if (gap <= maxGapSec && wouldChars <= maxChars && wouldSpan <= maxBlockDurationSec) { + current.push({ ...seg, text }); + continue; + } + + groups.push(current); + current = [{ ...seg, text }]; + } + + if (current.length > 0) { + groups.push(current); + } + + return groups; +} + +export interface CaptionSegmentLayoutOptions { + /** Lower bound on words per on-screen caption (default 2). */ + minWordsPerCaption?: number; + /** Upper bound on words per on-screen caption (default 7). */ + maxWordsPerCaption?: number; + /** + * `word`: each `CaptionSegment` is a single token with Whisper word timestamps (default). + * `phrase`: merged phrase spans; use proportional line splitting inside each span. + */ + timestampGranularity?: "word" | "phrase"; +} + +function computeCaptionLineIndexRanges( + wordCount: number, + minWords: number, + maxWords: number, +): Array<{ from: number; to: number }> { + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const sliceRanges: Array<{ from: number; to: number }> = []; + let i = 0; + while (i < wordCount) { + const remaining = wordCount - i; + if (remaining <= maxW) { + if (sliceRanges.length > 0 && remaining < minW) { + sliceRanges[sliceRanges.length - 1]!.to = wordCount; + } else { + sliceRanges.push({ from: i, to: wordCount }); + } + break; + } + + let take = maxW; + const after = remaining - take; + if (after > 0 && after < minW) { + take = remaining - minW; + if (take < minW) { + sliceRanges.push({ from: i, to: wordCount }); + break; + } + if (take > maxW) { + take = maxW; + } + } + sliceRanges.push({ from: i, to: i + take }); + i += take; + } + return sliceRanges; +} + +/** + * Groups per-word segments into on-screen lines using each token's Whisper timestamps + * (no proportional stretching across a long phrase span). + */ +export function groupTimedCaptionWordsIntoLines( + segments: CaptionSegment[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const words = [...segments] + .filter((s) => s.text.trim()) + .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); + if (words.length === 0) return []; + + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const out: CaptionSegment[] = []; + + let runStart = 0; + const flushRun = (runEndExclusive: number) => { + const run = words.slice(runStart, runEndExclusive); + if (run.length === 0) return; + const ranges = computeCaptionLineIndexRanges(run.length, minW, maxW); + for (const { from, to } of ranges) { + const slice = run.slice(from, to); + const s = slice[0]!.startSec; + const rawEnd = slice[slice.length - 1]!.endSec; + const e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, rawEnd + CAPTION_LINE_END_TAIL_SEC); + out.push({ + startSec: s, + endSec: e, + text: slice.map((w) => w.text.trim()).join(" "), + }); + } + }; + + for (let i = 1; i < words.length; i++) { + const prev = words[i - 1]!; + const cur = words[i]!; + const gap = cur.startSec - prev.endSec; + if (gap >= WORD_RUN_BREAK_GAP_SEC) { + flushRun(i); + runStart = i; + } + } + flushRun(words.length); + + for (let i = 0; i < out.length - 1; i++) { + if (out[i]!.endSec > out[i + 1]!.startSec + 1e-3) { + out[i]!.endSec = Math.max( + out[i]!.startSec + WORD_SPLIT_MIN_SPAN_SEC, + out[i + 1]!.startSec - 1e-4, + ); + } + } + return out; +} + +/** + * Splits each merged transcription span into shorter captions with about + * `minWords`-`maxWords` words. Times are interpolated by character weight inside the span. + */ +export function splitMergedCaptionsByWordBounds( + merged: CaptionSegment[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const out: CaptionSegment[] = []; + + for (const seg of merged) { + const words = seg.text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) continue; + + if (words.length <= maxW) { + out.push({ + startSec: seg.startSec, + endSec: seg.endSec, + text: words.join(" "), + }); + continue; + } + + out.push(...splitOneSegmentByWordBounds(seg.startSec, seg.endSec, words, minW, maxW)); + } + + return out; +} + +function wrapCaptionTextByWordBounds(text: string, minWords: number, maxWords: number): string { + const words = text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) return ""; + const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); + const maxW = Math.max(minW, Math.floor(maxWords)); + const ranges = computeCaptionLineIndexRanges(words.length, minW, maxW); + return ranges.map(({ from, to }) => words.slice(from, to).join(" ")).join("\n"); +} + +function expandPhraseSegmentToPseudoWords(segment: CaptionSegment): CaptionSegment[] { + const words = segment.text.trim().split(/\s+/).filter(Boolean); + if (words.length === 0) return []; + if (words.length === 1) { + return [ + { + startSec: segment.startSec, + endSec: segment.endSec, + text: words[0]!, + }, + ]; + } + + return splitOneSegmentByWordBounds(segment.startSec, segment.endSec, words, 1, 1); +} + +export function groupPhraseCaptionSegmentsIntoLines( + segments: CaptionSegment[], + minWords: number, + maxWords: number, + options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, +): CaptionSegment[] { + const groups = partitionPhraseCaptionSegments(segments, options); + const out: CaptionSegment[] = []; + + for (const group of groups) { + if (group.length === 1) { + const only = group[0]!; + const wrapped = wrapCaptionTextByWordBounds(only.text, minWords, maxWords).trim(); + if (!wrapped) continue; + const lineTexts = wrapped + .split("\n") + .map((t) => t.trim()) + .filter(Boolean); + const n = lineTexts.length; + const rawDur = only.endSec - only.startSec; + if (n > 1 && rawDur < n * WORD_SPLIT_MIN_SPAN_SEC) { + out.push({ + startSec: only.startSec, + endSec: only.endSec, + text: lineTexts.join(" "), + }); + continue; + } + const dur = Math.max(rawDur, WORD_SPLIT_MIN_SPAN_SEC * n); + if (n <= 1) { + out.push({ + startSec: only.startSec, + endSec: only.endSec, + text: lineTexts[0] ?? wrapped, + }); + continue; + } + for (let i = 0; i < n; i++) { + const startSec = only.startSec + (dur * i) / n; + const boundary = only.startSec + (dur * (i + 1)) / n; + const endSec = + i === n - 1 ? only.endSec : Math.max(startSec + WORD_SPLIT_MIN_SPAN_SEC, boundary); + out.push({ + startSec, + endSec, + text: lineTexts[i]!, + }); + } + continue; + } + + const pseudoWords = group.flatMap(expandPhraseSegmentToPseudoWords); + out.push(...groupTimedCaptionWordsIntoLines(pseudoWords, minWords, maxWords)); + } + + return out; +} + +function splitOneSegmentByWordBounds( + startSec: number, + endSec: number, + words: string[], + minWords: number, + maxWords: number, +): CaptionSegment[] { + const sliceRanges = computeCaptionLineIndexRanges(words.length, minWords, maxWords); + + const dur = Math.max(endSec - startSec, 0.05); + const weights = words.map((w) => Math.max(1, w.length)); + const totalW = weights.reduce((a, b) => a + b, 0); + + const weightSum = (from: number, to: number) => { + let s = 0; + for (let k = from; k < to; k++) s += weights[k] ?? 0; + return s; + }; + + const result: CaptionSegment[] = []; + let prevEnd = startSec; + for (const { from, to } of sliceRanges) { + const wb = weightSum(0, from); + const ws = weightSum(from, to); + let s = startSec + (wb / totalW) * dur; + let e = startSec + ((wb + ws) / totalW) * dur; + s = Math.max(s, prevEnd); + e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, e); + e = Math.min(e, endSec); + if (e <= s) { + e = Math.min(endSec, s + WORD_SPLIT_MIN_SPAN_SEC); + } + prevEnd = e; + result.push({ + startSec: s, + endSec: e, + text: words.slice(from, to).join(" "), + }); + } + if (result.length > 0) { + result[result.length - 1].endSec = endSec; + for (let i = 0; i < result.length - 1; i++) { + if (result[i].endSec > result[i + 1].startSec + 0.002) { + result[i].endSec = Math.max(result[i].startSec + 1e-4, result[i + 1].startSec); + } + } + } + return result; +} + +export function captionSegmentsToAnnotationRegions( + segments: CaptionSegment[], + startNumericId: number, + startZIndex: number, + layout?: CaptionSegmentLayoutOptions, +): { regions: AnnotationRegion[]; nextNumericId: number; nextZIndex: number } { + // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a + // normalized key and would merge spans while keeping only the first token's text. + const minW = layout?.minWordsPerCaption ?? 2; + const maxW = layout?.maxWordsPerCaption ?? 7; + const granularity = layout?.timestampGranularity ?? "word"; + + const grouped = + granularity === "phrase" + ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW) + : groupTimedCaptionWordsIntoLines(segments, minW, maxW); + + const dedupedOut = dedupeAdjacentCaptionRepeats(grouped); + const finalized = finalizeCaptionSegmentsForPlayback(dedupedOut); + + let nid = startNumericId; + let z = startZIndex; + const regions: AnnotationRegion[] = []; + + for (const seg of finalized) { + const startMs = Math.round(seg.startSec * 1000); + const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1); + regions.push({ + id: `annotation-${nid++}`, + startMs, + endMs, + type: "text", + content: seg.text, + annotationSource: "auto-caption", + position: { ...CAPTION_POSITION }, + size: { ...CAPTION_SIZE }, + style: { ...CAPTION_STYLE }, + zIndex: z++, + }); + } + + return { + regions: reconcileAutoCaptionTimelineGaps(regions), + nextNumericId: nid, + nextZIndex: z, + }; +} + +export function maxAnnotationNumericId(regions: AnnotationRegion[]): number { + let max = 0; + for (const r of regions) { + const m = /^annotation-(\d+)$/.exec(r.id); + if (m) max = Math.max(max, Number.parseInt(m[1], 10)); + } + return max; +} + +export function maxAnnotationZIndex(regions: AnnotationRegion[]): number { + if (regions.length === 0) return 0; + return Math.max(...regions.map((r) => r.zIndex)); +} diff --git a/src/cli/vendor/leadingSilence.ts b/src/cli/vendor/leadingSilence.ts new file mode 100644 index 0000000000..1f6b1b8469 --- /dev/null +++ b/src/cli/vendor/leadingSilence.ts @@ -0,0 +1,80 @@ +// Vendored for the CLI: @/lib/captioning/leadingSilence was deleted in the 1.8 line. The CLI's captions command still trims leading silence before transcription, so the module lives on here verbatim. + +/** Caption path is always mono 16 kHz after `extractMono16kFromVideoUrl`. */ +import type { TrimRegion } from "@/components/video-editor/types"; + +const SAMPLE_RATE = 16_000; + +/** Window length for peak detection (~50 ms). */ +const WINDOW_SAMPLES = 800; + +/** Coarse hop so long intros scan quickly (~50 ms steps). */ +const HOP_SAMPLES = 800; + +/** Max |sample| in a window below this counts as silence (float PCM ~[-1, 1]). */ +const PEAK_THRESHOLD = 0.012; + +/** Keep a little audio before the first peak so word onsets are not clipped. */ +const PRE_ROLL_SEC = 0.12; + +/** Do not scan more than this much audio for leading silence (performance + pathological files). */ +const MAX_LEADING_SCAN_SEC = 15 * 60; + +/** + * Drops quiet audio at the beginning so Whisper is not fed a long silent prefix (which can skew + * the first phrase and wastes work). Returned `trimSec` must be added back to every segment time. + */ +export function trimLeadingSilenceMono16k(samples: Float32Array): { + samples: Float32Array; + trimSec: number; +} { + if (samples.length < WINDOW_SAMPLES) { + return { samples, trimSec: 0 }; + } + + const maxIndex = Math.min( + samples.length - WINDOW_SAMPLES, + Math.floor(MAX_LEADING_SCAN_SEC * SAMPLE_RATE), + ); + + let firstSpeechSample = -1; + for (let i = 0; i <= maxIndex; i += HOP_SAMPLES) { + let peak = 0; + for (let j = 0; j < WINDOW_SAMPLES; j++) { + peak = Math.max(peak, Math.abs(samples[i + j]!)); + } + if (peak > PEAK_THRESHOLD) { + firstSpeechSample = i; + break; + } + } + + if (firstSpeechSample <= 0) { + return { samples, trimSec: 0 }; + } + + const preRollSamples = Math.round(PRE_ROLL_SEC * SAMPLE_RATE); + const start = Math.max(0, firstSpeechSample - preRollSamples); + return { + samples: samples.subarray(start), + trimSec: start / SAMPLE_RATE, + }; +} + +/** + * When audio is trimmed from the front, Whisper times are relative to the shortened buffer. + * Shift trim regions by the same offset so `segmentOverlapsTrim` still uses consistent coordinates. + */ +export function shiftTrimRegionsMsForCaptionBuffer( + regions: TrimRegion[], + trimMs: number, +): TrimRegion[] { + if (trimMs <= 0) return regions; + return regions + .map((r) => ({ + ...r, + startMs: Math.max(0, r.startMs - trimMs), + endMs: Math.max(0, r.endMs - trimMs), + })) + .filter((r) => r.endMs > r.startMs); +} diff --git a/src/cli/vendor/zoomHelpers.ts b/src/cli/vendor/zoomHelpers.ts new file mode 100644 index 0000000000..151b080df9 --- /dev/null +++ b/src/cli/vendor/zoomHelpers.ts @@ -0,0 +1,30 @@ +// Vendored for the CLI: clampFocusToDepth was deleted from +// @/components/video-editor/types in the 1.8 line with no successor, and +// hasNativeCursorRecordingData left with @/lib/cursor/nativeCursor. Both are +// tiny pure predicates the CLI export runner still needs. + +import type { CursorRecordingData } from "@/native/contracts"; + +export interface ZoomFocusPoint { + cx: number; + cy: number; +} + +function clamp(value: number, min: number, max: number): number { + if (Number.isNaN(value)) return (min + max) / 2; + return Math.min(max, Math.max(min, value)); +} + +export function clampZoomFocus(focus: ZoomFocusPoint): ZoomFocusPoint { + return { cx: clamp(focus.cx, 0, 1), cy: clamp(focus.cy, 0, 1) }; +} + +export function hasNativeCursorRecordingData( + recordingData: CursorRecordingData | null | undefined, +): recordingData is CursorRecordingData { + return Boolean( + recordingData && + recordingData.samples.length > 0 && + (recordingData.assets.length > 0 || recordingData.provider === "none"), + ); +} From a297cee17030c023bd0dc9205bf475a8ade37615 Mon Sep 17 00:00:00 2001 From: PeterTakahashi <seiya4@icloud.com> Date: Sat, 1 Aug 2026 22:03:53 +0900 Subject: [PATCH 07/10] fix(cli): register STT for headless captions; update CLI docs for 1.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - captions: the whisper.cpp stt:transcribe handler is registered in the GUI boot path, so CLI mode registered nothing and the renderer adapter failed with "No handler registered" — register registerSttIpc in runCli like set-locale. - docs/cli.md: one-shot dev build block (renderer, capture helpers, ffmpeg + Rust compositor, whisper STT server), --audio read-back behavior, --preview-size no-op, no-cancel caveat, whisper.cpp model auto-download. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ceTJjoPJYa5BdmXZdDc5J --- docs/cli.md | 34 +++++++++++++++++++++++----------- electron/cli/cliMain.ts | 5 +++++ src/cli/CliCaptionsRunner.tsx | 8 ++++---- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index b9a95b51c5..0d0f8a598d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -10,7 +10,16 @@ record → edit the project JSON programmatically → export → MP4/GIF ## Running -Development (after `npm run build-vite` and, for recording, the native helper build): +Development — build once per checkout: + +```bash +npm run build-vite # renderer + main +npm run build:native:mac # capture helpers (recording) +npm run fetch:ffmpeg:mac && npm run build:native:compositor:mac # Rust compositor (export) +bash scripts/build-whisper-stt.sh # STT server (captions; model downloads on first run) +``` + +Then: ```bash npm run cli -- <command> [options] @@ -127,8 +136,10 @@ openscreen export demo.openscreen --json | while read line; do ...; done | `--audio-offset <seconds>` | Delay before the voiceover starts (default 0) | | `--json` | NDJSON progress + result on stdout | -`--audio` re-muxes after the render: video packets are copied untouched, and the -audio is mixed offline (OfflineAudioContext) and re-encoded to AAC. MP4 only. +`--audio` mixes after the native render: the exported file is read back, video +packets are copied untouched, and the audio is mixed offline +(OfflineAudioContext) and re-encoded to AAC before overwriting the output. +MP4 only. In `mix` mode the original audio is ducked to 40% under the voiceover so the sum cannot clip; use `replace` to drop the original entirely. @@ -137,12 +148,12 @@ when it lives in the app's recordings directory or **next to the project file**. Keep `.openscreen` files beside their media (or record via the CLI, which uses the recordings directory). -**`--preview-size`**: annotation font sizes and border radii are stored in -preview-pixel space and scaled by `export size / preview size` — in the GUI the -"preview" is the editor window, so results depend on window size. The CLI uses a -deterministic reference box instead (the composition fitted into 1280×720). -Authoring tip for scripts: treat annotation `fontSize` as "pixels in a -1280-wide preview". +**`--preview-size`** is accepted for compatibility but is a no-op on the +native pipeline: annotation geometry is percentage-based in the scene +description, so exports no longer depend on any preview box. + +**No cancel**: the native compositor has no abort mechanism — killing the CLI +mid-export stops output but the render worker runs until process exit. ### `openscreen pack` @@ -171,8 +182,9 @@ openscreen export demo.openscreen -o demo.mp4 # subtitles are burned in ``` Requires an audio track in the project's video (e.g. `record --mic`, or a -voiceover mixed in with a re-recorded source). Accuracy reflects the bundled -whisper-tiny model: strong for English, rough for other languages. +voiceover mixed in with a re-recorded source). Transcription runs on the +native whisper.cpp engine (ggml-small); the model downloads automatically on +first use. ### `openscreen info` diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index dbf8035ab3..9410131085 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -14,6 +14,7 @@ import type { CliSourcesResult, } from "../../src/lib/cliContracts"; import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; +import { registerSttIpc } from "../stt"; import { ASSET_BASE_URL_ARG } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; @@ -506,6 +507,10 @@ export function runCli(command: CliCommand): void { let cliWindow: BrowserWindow | null = null; registerAppHandlersForCli(() => cliWindow); + // Speech-to-text backs the captions command; registered by the GUI boot + // path (main.ts) rather than registerIpcHandlers. + registerSttIpc(ipcMain); + // Registered by the GUI boot path (main.ts) rather than registerIpcHandlers; // the renderer's i18n init invokes it unconditionally. ipcMain.handle("set-locale", () => { diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx index 7bcf213911..beee6a46d4 100644 --- a/src/cli/CliCaptionsRunner.tsx +++ b/src/cli/CliCaptionsRunner.tsx @@ -10,15 +10,15 @@ import { validateProjectData, } from "@/components/video-editor/projectPersistence"; import type { AnnotationRegion, TrimRegion } from "@/components/video-editor/types"; -import { captionSegmentsToAnnotationRegions } from "./vendor/captionRegions"; import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; +import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; +import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; +import { nativeBridgeClient } from "@/native"; +import { captionSegmentsToAnnotationRegions } from "./vendor/captionRegions"; import { shiftTrimRegionsMsForCaptionBuffer, trimLeadingSilenceMono16k, } from "./vendor/leadingSilence"; -import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; -import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; -import { nativeBridgeClient } from "@/native"; /** Highest trailing number across existing region ids, so new ids never collide. */ function nextNumericIdFrom(regions: { id: string }[]): number { From 17fde1af1b235ae5181cb71377287284fdfb9756 Mon Sep 17 00:00:00 2001 From: Etienne Lescot <etiennelescot@gmail.com> Date: Sun, 2 Aug 2026 13:36:08 +0200 Subject: [PATCH 08/10] =?UTF-8?q?fix(cli):=20three=20review=20findings=20?= =?UTF-8?q?=E2=80=94=20gif+audio,=20pack=20collisions,=20piped=20help?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - parseExport validated --audio against a format that --out had not resolved yet, so `--audio v.m4a -o out.gif` slipped through; check after inference. - pack derived the destination from the basename alone, so a webcam video sharing a name with the screen video overwrote it and both project paths pointed at one file. - help/error output bypassed safeWrite, so `openscreen help | head` could throw before the EPIPE guards are installed. --- electron/cli/args.test.ts | 4 ++++ electron/cli/args.ts | 7 ++++--- electron/cli/cliMain.ts | 12 +++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index 3c00bc2002..e33acb6724 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -96,6 +96,10 @@ describe("parseCliArgs", () => { expect(parse(["export", "a.openscreen", "--audio", "v.mp3", "--format", "gif"])).toMatchObject({ kind: "error", }); + // gif inferred from --out, with no explicit --format + expect(parse(["export", "a.openscreen", "--audio", "v.mp3", "-o", "out.gif"])).toMatchObject({ + kind: "error", + }); expect(parse(["export", "a.openscreen", "--audio-offset", "-1"])).toMatchObject({ kind: "error", }); diff --git a/electron/cli/args.ts b/electron/cli/args.ts index a14d74b066..56fea1ee5f 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -260,9 +260,6 @@ function parseExport(args: string[], cwd: string): CliCommand { } if (!request.projectPath) throw new Error("export requires a <project.openscreen> path"); - if (request.audioPath && request.format === "gif") { - throw new Error("--audio is only supported for MP4 exports"); - } if (request.outPath) { const ext = path.extname(request.outPath).toLowerCase(); if (ext !== ".mp4" && ext !== ".gif") { @@ -274,6 +271,10 @@ function parseExport(args: string[], cwd: string): CliCommand { } request.format = extFormat; } + // After --out resolves the format, so `--audio … -o out.gif` is caught too. + if (request.audioPath && request.format === "gif") { + throw new Error("--audio is only supported for MP4 exports"); + } return request; } diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index 9410131085..611c3cae2b 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -250,7 +250,13 @@ async function runPackCommand(projectPath: string, outDir: string, json: boolean const copied: string[] = []; const copyIn = async (sourcePath: string): Promise<string> => { - const destination = path.join(outDir, path.basename(sourcePath)); + const ext = path.extname(sourcePath); + const stem = path.basename(sourcePath, ext); + let destination = path.join(outDir, stem + ext); + // Screen and webcam can share a basename across directories; don't overwrite. + for (let n = 1; copied.includes(destination); n++) { + destination = path.join(outDir, `${stem}-${n}${ext}`); + } if (path.resolve(sourcePath) !== path.resolve(destination)) { await fs.copyFile(sourcePath, destination); } @@ -369,12 +375,12 @@ async function runInfoCommand(projectPath: string, json: boolean): Promise<numbe export function runCli(command: CliCommand): void { if (command.kind === "help") { - process.stdout.write(CLI_USAGE); + safeWrite(process.stdout, CLI_USAGE); app.exit(0); return; } if (command.kind === "error") { - process.stderr.write(`Error: ${command.message}\n\n${CLI_USAGE}`); + safeWrite(process.stderr, `Error: ${command.message}\n\n${CLI_USAGE}`); app.exit(2); return; } From c8b832b9837be6741fc0b3039196b3fe05d19318 Mon Sep 17 00:00:00 2001 From: Etienne Lescot <etiennelescot@gmail.com> Date: Mon, 3 Aug 2026 08:17:02 +0200 Subject: [PATCH 09/10] refactor(cli): stop forking the live captioning module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/cli/vendor/captionRegions.ts was annotationsFromCaptions.ts copied whole (~460 identical lines) for the sake of the annotation converter that the 1.8 line did delete. The copy had already lost dedupeAdjacentCaptionRepeats and finalizeCaptionSegmentsForPlayback, so CLI captions were drifting from the editor's. Only the converter is CLI-specific, so only it stays — in src/cli, not in a vendor folder, importing the grouping/dedupe helpers from the live module. Output is unchanged: the shared code was byte-identical, and the dropped reconcileAutoCaptionTimelineGaps was a no-op (its gap constant is 0). Also drops the unused hasNativeCursorRecordingData from vendor/zoomHelpers. --- src/cli/CliCaptionsRunner.tsx | 8 +- src/cli/captionAnnotations.test.ts | 35 ++ src/cli/captionAnnotations.ts | 76 ++++ src/cli/vendor/captionRegions.ts | 606 ----------------------------- src/cli/vendor/zoomHelpers.ts | 17 +- 5 files changed, 117 insertions(+), 625 deletions(-) create mode 100644 src/cli/captionAnnotations.test.ts create mode 100644 src/cli/captionAnnotations.ts delete mode 100644 src/cli/vendor/captionRegions.ts diff --git a/src/cli/CliCaptionsRunner.tsx b/src/cli/CliCaptionsRunner.tsx index beee6a46d4..620210ee2e 100644 --- a/src/cli/CliCaptionsRunner.tsx +++ b/src/cli/CliCaptionsRunner.tsx @@ -14,7 +14,7 @@ import { extractMono16kFromVideoUrl } from "@/lib/captioning/extractMono16k"; import { transcribeMono16kToSegments } from "@/lib/captioning/transcribe"; import type { CliCaptionsRequest, CliDoneResult } from "@/lib/cliContracts"; import { nativeBridgeClient } from "@/native"; -import { captionSegmentsToAnnotationRegions } from "./vendor/captionRegions"; +import { captionSegmentsToAnnotationRegions } from "./captionAnnotations"; import { shiftTrimRegionsMsForCaptionBuffer, trimLeadingSilenceMono16k, @@ -103,17 +103,17 @@ async function runCaptions(request: CliCaptionsRequest): Promise<CliDoneResult> const startNumericId = nextNumericIdFrom([...editor.annotationRegions, ...editor.zoomRegions]); const startZIndex = manualAnnotations.reduce((max, a) => Math.max(max, a.zIndex + 1), 1); - let { regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + let regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { minWordsPerCaption: request.minWordsPerCaption, maxWordsPerCaption: request.maxWordsPerCaption, timestampGranularity: granularity, }); if (regions.length === 0 && segments.length > 0) { - ({ regions } = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { + regions = captionSegmentsToAnnotationRegions(segments, startNumericId, startZIndex, { minWordsPerCaption: 1, maxWordsPerCaption: Number.MAX_SAFE_INTEGER, timestampGranularity: granularity, - })); + }); } if (regions.length === 0) { throw new Error("Transcription produced no caption segments"); diff --git a/src/cli/captionAnnotations.test.ts b/src/cli/captionAnnotations.test.ts new file mode 100644 index 0000000000..34cee80d9b --- /dev/null +++ b/src/cli/captionAnnotations.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import type { CaptionSegment } from "@/lib/captioning/transcribe"; +import { captionSegmentsToAnnotationRegions } from "./captionAnnotations"; + +const words = (...texts: string[]): CaptionSegment[] => + texts.map((text, i) => ({ text, startSec: i * 0.5, endSec: i * 0.5 + 0.4 })); + +describe("captionSegmentsToAnnotationRegions", () => { + it("numbers ids and z-indexes from the given start, with non-empty spans", () => { + const regions = captionSegmentsToAnnotationRegions(words("one", "two", "three", "four"), 7, 3); + + expect(regions.length).toBeGreaterThan(0); + expect(regions.map((r) => r.id)).toEqual(regions.map((_, i) => `annotation-${7 + i}`)); + expect(regions.map((r) => r.zIndex)).toEqual(regions.map((_, i) => 3 + i)); + for (const region of regions) { + expect(region.endMs).toBeGreaterThan(region.startMs); + expect(region.annotationSource).toBe("auto-caption"); + expect(region.content.trim()).not.toBe(""); + } + }); + + it("groups phrase-granularity segments one line at a time", () => { + const regions = captionSegmentsToAnnotationRegions( + [ + { text: "hello there", startSec: 0, endSec: 1 }, + { text: "second line", startSec: 1.2, endSec: 2 }, + ], + 1, + 1, + { timestampGranularity: "phrase" }, + ); + + expect(regions.map((r) => r.content)).toEqual(["hello there", "second line"]); + }); +}); diff --git a/src/cli/captionAnnotations.ts b/src/cli/captionAnnotations.ts new file mode 100644 index 0000000000..890f54b2ba --- /dev/null +++ b/src/cli/captionAnnotations.ts @@ -0,0 +1,76 @@ +// CLI-only: the v4 editor renders captions natively, but `openscreen captions` +// still writes caption *annotations* into .openscreen projects. Only the +// annotation conversion lives here — the segment grouping/dedupe helpers come +// from the live captioning module so the CLI can't drift from the editor. + +import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types"; +import { + type CaptionSegmentLayoutOptions, + dedupeAdjacentCaptionRepeats, + finalizeCaptionSegmentsForPlayback, + groupPhraseCaptionSegmentsIntoLines, + groupTimedCaptionWordsIntoLines, +} from "@/lib/captioning/annotationsFromCaptions"; +import type { CaptionSegment } from "@/lib/captioning/transcribe"; + +/** Wide lower-third bar; `position.x` is top-left as % of container, so center with (100 - width) / 2. */ +const CAPTION_WIDTH = 92; +const CAPTION_HEIGHT = 12; +const CAPTION_BOTTOM_MARGIN = 2; + +const CAPTION_POSITION = { + x: (100 - CAPTION_WIDTH) / 2, + y: 100 - CAPTION_HEIGHT - CAPTION_BOTTOM_MARGIN, +}; + +const CAPTION_SIZE = { width: CAPTION_WIDTH, height: CAPTION_HEIGHT }; + +const CAPTION_STYLE: AnnotationTextStyle = { + color: "#ffffff", + backgroundColor: "rgba(255, 255, 255, 0)", + fontSize: 24, + fontFamily: "Inter", + fontWeight: "normal", + fontStyle: "normal", + textDecoration: "none", + textAlign: "center", +}; + +export function captionSegmentsToAnnotationRegions( + segments: CaptionSegment[], + startNumericId: number, + startZIndex: number, + layout?: CaptionSegmentLayoutOptions, +): AnnotationRegion[] { + // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a + // normalized key and would merge spans while keeping only the first token's text. + const minW = layout?.minWordsPerCaption ?? 2; + const maxW = layout?.maxWordsPerCaption ?? 7; + const granularity = layout?.timestampGranularity ?? "word"; + + const grouped = + granularity === "phrase" + ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW) + : groupTimedCaptionWordsIntoLines(segments, minW, maxW); + + const finalized = finalizeCaptionSegmentsForPlayback(dedupeAdjacentCaptionRepeats(grouped)); + + let nid = startNumericId; + let z = startZIndex; + return finalized.map((seg) => { + const startMs = Math.round(seg.startSec * 1000); + const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1); + return { + id: `annotation-${nid++}`, + startMs, + endMs, + type: "text", + content: seg.text, + annotationSource: "auto-caption", + position: { ...CAPTION_POSITION }, + size: { ...CAPTION_SIZE }, + style: { ...CAPTION_STYLE }, + zIndex: z++, + }; + }); +} diff --git a/src/cli/vendor/captionRegions.ts b/src/cli/vendor/captionRegions.ts deleted file mode 100644 index f8ce2726d2..0000000000 --- a/src/cli/vendor/captionRegions.ts +++ /dev/null @@ -1,606 +0,0 @@ -// Vendored for the CLI: captionSegmentsToAnnotationRegions was removed from @/lib/captioning/annotationsFromCaptions in the 1.8 line (the v4 editor renders captions natively). The CLI still writes caption *annotations* into .openscreen projects, so the converter lives on here verbatim. - -import type { AnnotationRegion, AnnotationTextStyle } from "@/components/video-editor/types"; - -import type { CaptionSegment } from "@/lib/captioning/transcribe"; - -/** Wide lower-third bar; `position.x` is top-left as % of container, so center with (100 - width) / 2. */ -const CAPTION_WIDTH = 92; -const CAPTION_HEIGHT = 12; -const CAPTION_BOTTOM_MARGIN = 2; - -const CAPTION_POSITION = { - x: (100 - CAPTION_WIDTH) / 2, - y: 100 - CAPTION_HEIGHT - CAPTION_BOTTOM_MARGIN, -}; - -const CAPTION_SIZE = { width: CAPTION_WIDTH, height: CAPTION_HEIGHT }; - -const CAPTION_STYLE: AnnotationTextStyle = { - color: "#ffffff", - backgroundColor: "rgba(255, 255, 255, 0)", - fontSize: 24, - fontFamily: "Inter", - fontWeight: "normal", - fontStyle: "normal", - textDecoration: "none", - textAlign: "center", -}; - -/** Nudge caption starts earlier (seconds); Whisper onsets run slightly late. Do not offset ends too, that pulls lines off-screen early. */ -const AUTO_CAPTION_START_BIAS_SEC = 0; - -/** Extra hold after Whisper's segment end (seconds); model end times run early vs trailing vowels. Separate from the start bias. */ -const AUTO_CAPTION_END_HOLD_SEC = 0; - -/** Inside one Whisper phrase, sub-lines can be shorter (do not steal time from neighbors). */ -const WORD_SPLIT_MIN_SPAN_SEC = 0.02; - -/** Brief linger after the last word in a line (seconds); trimmed if it would overlap the next line. */ -const CAPTION_LINE_END_TAIL_SEC = 0; - -/** A real silence between word-level timestamps should start a new caption run. */ -const WORD_RUN_BREAK_GAP_SEC = 0.24; - -/** Min time between consecutive caption regions (seconds); keeps a visible gap so blocks don't read as one clip. Small so short pauses survive. */ -const MIN_CAPTION_TIMELINE_GAP_SEC = 0; - -/** Same text again with almost no gap or overlap; common Whisper/chunk artifact. */ -const DEDUPE_SAME_TEXT_MAX_GAP_SEC = 0.55; - -export const SAME_CONTENT_ECHO_MAX_GAP_SEC = 1.15; - -function normalizeCaptionKey(text: string): string { - return text - .trim() - .replace(/\s+/g, " ") - .replace(/[\u2018\u2019]/g, "'") - .replace(/[\u201C\u201D]/g, '"') - .toLowerCase() - .replace(/[.!?,;:]+$/g, ""); -} - -/** Legacy echo-collapse helper kept for reference while phrase timing uses raw model spans. */ -export function collapseSameContentEchoes(segments: CaptionSegment[]): CaptionSegment[] { - const sorted = [...segments] - .filter((s) => s.text.trim()) - .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); - const out: CaptionSegment[] = []; - const lastIndexByKey = new Map<string, number>(); - - for (const seg of sorted) { - const key = normalizeCaptionKey(seg.text); - const hit = lastIndexByKey.get(key); - if (hit !== undefined) { - const prev = out[hit]!; - if (seg.startSec < prev.endSec + SAME_CONTENT_ECHO_MAX_GAP_SEC) { - prev.startSec = Math.min(prev.startSec, seg.startSec); - prev.endSec = Math.max(prev.endSec, seg.endSec); - continue; - } - } - out.push({ - startSec: seg.startSec, - endSec: seg.endSec, - text: seg.text.trim(), - }); - lastIndexByKey.set(key, out.length - 1); - } - return out; -} - -/** - * Collapse adjacent duplicate lines (overlapping or tiny gap). Does not merge the same phrase - * repeated later in the video when separated by real silence. - */ -function dedupeAdjacentCaptionRepeats(segments: CaptionSegment[]): CaptionSegment[] { - const sorted = [...segments] - .filter((s) => s.text.trim()) - .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); - const out: CaptionSegment[] = []; - for (const seg of sorted) { - const t = seg.text.trim(); - const prev = out[out.length - 1]; - if (prev && normalizeCaptionKey(prev.text) === normalizeCaptionKey(t)) { - const overlap = prev.endSec - seg.startSec; - const gap = seg.startSec - prev.endSec; - if (overlap > 0.015 || gap < DEDUPE_SAME_TEXT_MAX_GAP_SEC) { - prev.startSec = Math.min(prev.startSec, seg.startSec); - prev.endSec = Math.max(prev.endSec, seg.endSec); - continue; - } - } - out.push({ startSec: seg.startSec, endSec: seg.endSec, text: t }); - } - return out; -} - -/** Trim only real overlaps. Avoid synthetic lead/lag so caption timing matches model output. */ -function finalizeCaptionSegmentsForPlayback(segments: CaptionSegment[]): CaptionSegment[] { - const OVERLAP_TRIM_SEC = 0.002; - - const sortedRaw = [...segments] - .filter((s) => s.text.trim()) - .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); - - const a = sortedRaw.map((seg) => { - let s = seg.startSec + AUTO_CAPTION_START_BIAS_SEC; - let e = seg.endSec + AUTO_CAPTION_END_HOLD_SEC; - s = Math.max(0, s); - if (e <= s) e = s + 0.02; - return { startSec: s, endSec: e, text: seg.text.trim() }; - }); - - for (let i = 1; i < a.length; i++) { - if (a[i].startSec < a[i - 1].endSec - OVERLAP_TRIM_SEC) { - a[i - 1].endSec = Math.max(a[i - 1].startSec + 1e-4, a[i].startSec); - } - } - - return a; -} - -/** Default min gap between auto-caption blocks on the timeline (ms); matches `MIN_CAPTION_TIMELINE_GAP_SEC`. */ -export const DEFAULT_AUTO_CAPTION_MIN_GAP_MS = Math.round(MIN_CAPTION_TIMELINE_GAP_SEC * 1000); - -/** - * Enforce a min gap between consecutive `auto-caption` regions (by start time). Shortens the previous - * region's end when possible, else shifts the following region later so blocks can't sit completely flush. - */ -export function reconcileAutoCaptionTimelineGaps( - regions: AnnotationRegion[], - minGapMs: number = DEFAULT_AUTO_CAPTION_MIN_GAP_MS, -): AnnotationRegion[] { - const gap = Math.max(0, Math.round(minGapMs)); - if (regions.length === 0 || gap === 0) return regions; - - const autoCandidates = regions.filter((r) => r.annotationSource === "auto-caption"); - if (autoCandidates.length <= 1) return regions; - - const sorted = [...autoCandidates].sort((a, b) => a.startMs - b.startMs || a.endMs - b.endMs); - const fixed: AnnotationRegion[] = []; - let prev = { ...sorted[0]! }; - fixed.push(prev); - - for (let i = 1; i < sorted.length; i++) { - let cur = { ...sorted[i]! }; - const minStart = prev.endMs + gap; - - if (cur.startMs < minStart) { - const newPrevEnd = cur.startMs - gap; - if (newPrevEnd >= prev.startMs + 1) { - prev = { ...prev, endMs: newPrevEnd }; - fixed[fixed.length - 1] = prev; - } else { - const dur = Math.max(1, cur.endMs - cur.startMs); - cur = { ...cur, startMs: minStart, endMs: minStart + dur }; - } - } - - fixed.push(cur); - prev = cur; - } - - const fixedById = new Map(fixed.map((r) => [r.id, r])); - return regions.map((r) => fixedById.get(r.id) ?? r); -} - -/** Join phrases that are close in time so the editor does not create dozens of separate overlays. */ -export function mergeAdjacentCaptionSegments( - segments: CaptionSegment[], - options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, -): CaptionSegment[] { - const maxGapSec = options?.maxGapSec ?? 1.35; - const maxChars = options?.maxChars ?? 320; - const maxBlockDurationSec = options?.maxBlockDurationSec ?? 12; - - const sorted = [...segments].sort((a, b) => a.startSec - b.startSec); - const out: CaptionSegment[] = []; - - for (const seg of sorted) { - const text = seg.text.trim(); - if (!text) continue; - - const prev = out[out.length - 1]; - if (!prev) { - out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); - continue; - } - - const gap = seg.startSec - prev.endSec; - const mergedText = `${prev.text} ${text}`.trim(); - const mergedEnd = Math.max(prev.endSec, seg.endSec); - const wouldSpan = mergedEnd - prev.startSec; - if (gap <= maxGapSec && mergedText.length <= maxChars && wouldSpan <= maxBlockDurationSec) { - prev.endSec = mergedEnd; - prev.text = mergedText; - } else { - out.push({ startSec: seg.startSec, endSec: seg.endSec, text }); - } - } - - return out; -} - -function partitionPhraseCaptionSegments( - segments: CaptionSegment[], - options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, -): CaptionSegment[][] { - const maxGapSec = options?.maxGapSec ?? 0; - const maxChars = options?.maxChars ?? Number.POSITIVE_INFINITY; - const maxBlockDurationSec = options?.maxBlockDurationSec ?? Number.POSITIVE_INFINITY; - - const sorted = [...segments] - .filter((s) => s.text.trim()) - .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); - if (sorted.length === 0) return []; - - const groups: CaptionSegment[][] = []; - let current: CaptionSegment[] = []; - - for (const seg of sorted) { - const text = seg.text.trim(); - if (!text) continue; - - if (current.length === 0) { - current.push({ ...seg, text }); - continue; - } - - const prev = current[current.length - 1]!; - const groupStart = current[0]!.startSec; - const gap = seg.startSec - prev.endSec; - const currentChars = current.reduce((sum, item) => sum + item.text.length, 0); - const wouldChars = currentChars + 1 + text.length; - const wouldSpan = Math.max(prev.endSec, seg.endSec) - groupStart; - - if (gap <= maxGapSec && wouldChars <= maxChars && wouldSpan <= maxBlockDurationSec) { - current.push({ ...seg, text }); - continue; - } - - groups.push(current); - current = [{ ...seg, text }]; - } - - if (current.length > 0) { - groups.push(current); - } - - return groups; -} - -export interface CaptionSegmentLayoutOptions { - /** Lower bound on words per on-screen caption (default 2). */ - minWordsPerCaption?: number; - /** Upper bound on words per on-screen caption (default 7). */ - maxWordsPerCaption?: number; - /** - * `word`: each `CaptionSegment` is a single token with Whisper word timestamps (default). - * `phrase`: merged phrase spans; use proportional line splitting inside each span. - */ - timestampGranularity?: "word" | "phrase"; -} - -function computeCaptionLineIndexRanges( - wordCount: number, - minWords: number, - maxWords: number, -): Array<{ from: number; to: number }> { - const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); - const maxW = Math.max(minW, Math.floor(maxWords)); - const sliceRanges: Array<{ from: number; to: number }> = []; - let i = 0; - while (i < wordCount) { - const remaining = wordCount - i; - if (remaining <= maxW) { - if (sliceRanges.length > 0 && remaining < minW) { - sliceRanges[sliceRanges.length - 1]!.to = wordCount; - } else { - sliceRanges.push({ from: i, to: wordCount }); - } - break; - } - - let take = maxW; - const after = remaining - take; - if (after > 0 && after < minW) { - take = remaining - minW; - if (take < minW) { - sliceRanges.push({ from: i, to: wordCount }); - break; - } - if (take > maxW) { - take = maxW; - } - } - sliceRanges.push({ from: i, to: i + take }); - i += take; - } - return sliceRanges; -} - -/** - * Groups per-word segments into on-screen lines using each token's Whisper timestamps - * (no proportional stretching across a long phrase span). - */ -export function groupTimedCaptionWordsIntoLines( - segments: CaptionSegment[], - minWords: number, - maxWords: number, -): CaptionSegment[] { - const words = [...segments] - .filter((s) => s.text.trim()) - .sort((a, b) => a.startSec - b.startSec || a.endSec - b.endSec); - if (words.length === 0) return []; - - const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); - const maxW = Math.max(minW, Math.floor(maxWords)); - const out: CaptionSegment[] = []; - - let runStart = 0; - const flushRun = (runEndExclusive: number) => { - const run = words.slice(runStart, runEndExclusive); - if (run.length === 0) return; - const ranges = computeCaptionLineIndexRanges(run.length, minW, maxW); - for (const { from, to } of ranges) { - const slice = run.slice(from, to); - const s = slice[0]!.startSec; - const rawEnd = slice[slice.length - 1]!.endSec; - const e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, rawEnd + CAPTION_LINE_END_TAIL_SEC); - out.push({ - startSec: s, - endSec: e, - text: slice.map((w) => w.text.trim()).join(" "), - }); - } - }; - - for (let i = 1; i < words.length; i++) { - const prev = words[i - 1]!; - const cur = words[i]!; - const gap = cur.startSec - prev.endSec; - if (gap >= WORD_RUN_BREAK_GAP_SEC) { - flushRun(i); - runStart = i; - } - } - flushRun(words.length); - - for (let i = 0; i < out.length - 1; i++) { - if (out[i]!.endSec > out[i + 1]!.startSec + 1e-3) { - out[i]!.endSec = Math.max( - out[i]!.startSec + WORD_SPLIT_MIN_SPAN_SEC, - out[i + 1]!.startSec - 1e-4, - ); - } - } - return out; -} - -/** - * Splits each merged transcription span into shorter captions with about - * `minWords`-`maxWords` words. Times are interpolated by character weight inside the span. - */ -export function splitMergedCaptionsByWordBounds( - merged: CaptionSegment[], - minWords: number, - maxWords: number, -): CaptionSegment[] { - const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); - const maxW = Math.max(minW, Math.floor(maxWords)); - const out: CaptionSegment[] = []; - - for (const seg of merged) { - const words = seg.text.trim().split(/\s+/).filter(Boolean); - if (words.length === 0) continue; - - if (words.length <= maxW) { - out.push({ - startSec: seg.startSec, - endSec: seg.endSec, - text: words.join(" "), - }); - continue; - } - - out.push(...splitOneSegmentByWordBounds(seg.startSec, seg.endSec, words, minW, maxW)); - } - - return out; -} - -function wrapCaptionTextByWordBounds(text: string, minWords: number, maxWords: number): string { - const words = text.trim().split(/\s+/).filter(Boolean); - if (words.length === 0) return ""; - const minW = Math.max(1, Math.min(Math.floor(minWords), Math.floor(maxWords))); - const maxW = Math.max(minW, Math.floor(maxWords)); - const ranges = computeCaptionLineIndexRanges(words.length, minW, maxW); - return ranges.map(({ from, to }) => words.slice(from, to).join(" ")).join("\n"); -} - -function expandPhraseSegmentToPseudoWords(segment: CaptionSegment): CaptionSegment[] { - const words = segment.text.trim().split(/\s+/).filter(Boolean); - if (words.length === 0) return []; - if (words.length === 1) { - return [ - { - startSec: segment.startSec, - endSec: segment.endSec, - text: words[0]!, - }, - ]; - } - - return splitOneSegmentByWordBounds(segment.startSec, segment.endSec, words, 1, 1); -} - -export function groupPhraseCaptionSegmentsIntoLines( - segments: CaptionSegment[], - minWords: number, - maxWords: number, - options?: { maxGapSec?: number; maxChars?: number; maxBlockDurationSec?: number }, -): CaptionSegment[] { - const groups = partitionPhraseCaptionSegments(segments, options); - const out: CaptionSegment[] = []; - - for (const group of groups) { - if (group.length === 1) { - const only = group[0]!; - const wrapped = wrapCaptionTextByWordBounds(only.text, minWords, maxWords).trim(); - if (!wrapped) continue; - const lineTexts = wrapped - .split("\n") - .map((t) => t.trim()) - .filter(Boolean); - const n = lineTexts.length; - const rawDur = only.endSec - only.startSec; - if (n > 1 && rawDur < n * WORD_SPLIT_MIN_SPAN_SEC) { - out.push({ - startSec: only.startSec, - endSec: only.endSec, - text: lineTexts.join(" "), - }); - continue; - } - const dur = Math.max(rawDur, WORD_SPLIT_MIN_SPAN_SEC * n); - if (n <= 1) { - out.push({ - startSec: only.startSec, - endSec: only.endSec, - text: lineTexts[0] ?? wrapped, - }); - continue; - } - for (let i = 0; i < n; i++) { - const startSec = only.startSec + (dur * i) / n; - const boundary = only.startSec + (dur * (i + 1)) / n; - const endSec = - i === n - 1 ? only.endSec : Math.max(startSec + WORD_SPLIT_MIN_SPAN_SEC, boundary); - out.push({ - startSec, - endSec, - text: lineTexts[i]!, - }); - } - continue; - } - - const pseudoWords = group.flatMap(expandPhraseSegmentToPseudoWords); - out.push(...groupTimedCaptionWordsIntoLines(pseudoWords, minWords, maxWords)); - } - - return out; -} - -function splitOneSegmentByWordBounds( - startSec: number, - endSec: number, - words: string[], - minWords: number, - maxWords: number, -): CaptionSegment[] { - const sliceRanges = computeCaptionLineIndexRanges(words.length, minWords, maxWords); - - const dur = Math.max(endSec - startSec, 0.05); - const weights = words.map((w) => Math.max(1, w.length)); - const totalW = weights.reduce((a, b) => a + b, 0); - - const weightSum = (from: number, to: number) => { - let s = 0; - for (let k = from; k < to; k++) s += weights[k] ?? 0; - return s; - }; - - const result: CaptionSegment[] = []; - let prevEnd = startSec; - for (const { from, to } of sliceRanges) { - const wb = weightSum(0, from); - const ws = weightSum(from, to); - let s = startSec + (wb / totalW) * dur; - let e = startSec + ((wb + ws) / totalW) * dur; - s = Math.max(s, prevEnd); - e = Math.max(s + WORD_SPLIT_MIN_SPAN_SEC, e); - e = Math.min(e, endSec); - if (e <= s) { - e = Math.min(endSec, s + WORD_SPLIT_MIN_SPAN_SEC); - } - prevEnd = e; - result.push({ - startSec: s, - endSec: e, - text: words.slice(from, to).join(" "), - }); - } - if (result.length > 0) { - result[result.length - 1].endSec = endSec; - for (let i = 0; i < result.length - 1; i++) { - if (result[i].endSec > result[i + 1].startSec + 0.002) { - result[i].endSec = Math.max(result[i].startSec + 1e-4, result[i + 1].startSec); - } - } - } - return result; -} - -export function captionSegmentsToAnnotationRegions( - segments: CaptionSegment[], - startNumericId: number, - startZIndex: number, - layout?: CaptionSegmentLayoutOptions, -): { regions: AnnotationRegion[]; nextNumericId: number; nextZIndex: number } { - // Don't echo-collapse raw word tokens before grouping: repeated words ("I … I") share a - // normalized key and would merge spans while keeping only the first token's text. - const minW = layout?.minWordsPerCaption ?? 2; - const maxW = layout?.maxWordsPerCaption ?? 7; - const granularity = layout?.timestampGranularity ?? "word"; - - const grouped = - granularity === "phrase" - ? groupPhraseCaptionSegmentsIntoLines(segments, minW, maxW) - : groupTimedCaptionWordsIntoLines(segments, minW, maxW); - - const dedupedOut = dedupeAdjacentCaptionRepeats(grouped); - const finalized = finalizeCaptionSegmentsForPlayback(dedupedOut); - - let nid = startNumericId; - let z = startZIndex; - const regions: AnnotationRegion[] = []; - - for (const seg of finalized) { - const startMs = Math.round(seg.startSec * 1000); - const endMs = Math.max(Math.round(seg.endSec * 1000), startMs + 1); - regions.push({ - id: `annotation-${nid++}`, - startMs, - endMs, - type: "text", - content: seg.text, - annotationSource: "auto-caption", - position: { ...CAPTION_POSITION }, - size: { ...CAPTION_SIZE }, - style: { ...CAPTION_STYLE }, - zIndex: z++, - }); - } - - return { - regions: reconcileAutoCaptionTimelineGaps(regions), - nextNumericId: nid, - nextZIndex: z, - }; -} - -export function maxAnnotationNumericId(regions: AnnotationRegion[]): number { - let max = 0; - for (const r of regions) { - const m = /^annotation-(\d+)$/.exec(r.id); - if (m) max = Math.max(max, Number.parseInt(m[1], 10)); - } - return max; -} - -export function maxAnnotationZIndex(regions: AnnotationRegion[]): number { - if (regions.length === 0) return 0; - return Math.max(...regions.map((r) => r.zIndex)); -} diff --git a/src/cli/vendor/zoomHelpers.ts b/src/cli/vendor/zoomHelpers.ts index 151b080df9..e2fc32a4ac 100644 --- a/src/cli/vendor/zoomHelpers.ts +++ b/src/cli/vendor/zoomHelpers.ts @@ -1,9 +1,6 @@ // Vendored for the CLI: clampFocusToDepth was deleted from -// @/components/video-editor/types in the 1.8 line with no successor, and -// hasNativeCursorRecordingData left with @/lib/cursor/nativeCursor. Both are -// tiny pure predicates the CLI export runner still needs. - -import type { CursorRecordingData } from "@/native/contracts"; +// @/components/video-editor/types in the 1.8 line with no successor. It is a +// tiny pure predicate the CLI export runner still needs. export interface ZoomFocusPoint { cx: number; @@ -18,13 +15,3 @@ function clamp(value: number, min: number, max: number): number { export function clampZoomFocus(focus: ZoomFocusPoint): ZoomFocusPoint { return { cx: clamp(focus.cx, 0, 1), cy: clamp(focus.cy, 0, 1) }; } - -export function hasNativeCursorRecordingData( - recordingData: CursorRecordingData | null | undefined, -): recordingData is CursorRecordingData { - return Boolean( - recordingData && - recordingData.samples.length > 0 && - (recordingData.assets.length > 0 || recordingData.provider === "none"), - ); -} From 885082eb17c158314c0c9829381e386f680ff755 Mon Sep 17 00:00:00 2001 From: Etienne Lescot <etiennelescot@gmail.com> Date: Mon, 3 Aug 2026 09:43:47 +0200 Subject: [PATCH 10/10] test(cli): cover pack/info, and drop the no-op --preview-size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pack` and `info` only touch the project file and its media, but they sat in cliMain.ts behind its `electron` import, so nothing could test them — including the basename-collision fix from 17fde1af. Moved to electron/cli/projectCommands.ts with the writer injected, and covered: colliding screen/webcam basenames, the cursor sidecar, the stale-path sibling fallback, missing media, and info's exit codes. Verified by ablation — reverting the de-dup turns the first test red. An e2e would have been the obvious home for this, but ci.yml runs no Playwright job, so it would be a test nobody runs; vitest is what the Test job executes. --preview-size is gone: it was a documented no-op on the native pipeline, kept for compatibility by a CLI that has never shipped. --- docs/cli.md | 5 - electron/cli/args.test.ts | 4 - electron/cli/args.ts | 17 --- electron/cli/cliMain.ts | 169 +------------------------- electron/cli/projectCommands.test.ts | 139 +++++++++++++++++++++ electron/cli/projectCommands.ts | 174 +++++++++++++++++++++++++++ src/cli/CliExportRunner.tsx | 7 -- src/lib/cliContracts.ts | 2 - 8 files changed, 318 insertions(+), 199 deletions(-) create mode 100644 electron/cli/projectCommands.test.ts create mode 100644 electron/cli/projectCommands.ts diff --git a/docs/cli.md b/docs/cli.md index 0d0f8a598d..9e965bcc18 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -129,7 +129,6 @@ openscreen export demo.openscreen --json | while read line; do ...; done | `--format <mp4\|gif>` | Override the project's stored format | | `--quality <medium\|good\|source>` | MP4 quality | | `--gif-fps <15\|20\|25\|30>`, `--gif-size <medium\|large\|original>` | GIF settings | -| `--preview-size <WxH>` | Reference preview box (default `1280x720`), see below | | `--auto-zoom` | Add automatic zooms from cursor telemetry before rendering — the same dwell-detection engine as the editor's magic wand. Existing zoom regions are kept; suggestions never overlap them | | `--audio <file>` | Mix a voiceover file into the MP4 (mp3/wav/m4a — anything Chromium can decode; AIFF is not supported) | | `--audio-mode <mix\|replace>` | Layer the voiceover over the recording's audio (default `mix`) or replace it | @@ -148,10 +147,6 @@ when it lives in the app's recordings directory or **next to the project file**. Keep `.openscreen` files beside their media (or record via the CLI, which uses the recordings directory). -**`--preview-size`** is accepted for compatibility but is a no-op on the -native pipeline: annotation geometry is percentage-based in the scene -description, so exports no longer depend on any preview box. - **No cancel**: the native compositor has no abort mechanism — killing the CLI mid-export stops output but the render worker runs until process exit. diff --git a/electron/cli/args.test.ts b/electron/cli/args.test.ts index e33acb6724..7f0b18e092 100644 --- a/electron/cli/args.test.ts +++ b/electron/cli/args.test.ts @@ -45,8 +45,6 @@ describe("parseCliArgs", () => { "out.gif", "--gif-fps", "20", - "--preview-size", - "1600x900", "--json", ]); expect(cmd).toMatchObject({ @@ -54,8 +52,6 @@ describe("parseCliArgs", () => { outPath: inCwd("out.gif"), format: "gif", gifFrameRate: 20, - previewWidth: 1600, - previewHeight: 900, json: true, }); }); diff --git a/electron/cli/args.ts b/electron/cli/args.ts index 56fea1ee5f..0dad8d07c4 100644 --- a/electron/cli/args.ts +++ b/electron/cli/args.ts @@ -61,7 +61,6 @@ Export options: --gif-fps <15|20|25|30> GIF frame rate (default: from project) --gif-size <medium|large|original> GIF size preset (default: from project) - --preview-size <WxH> Reference preview box for annotation scaling (default 1280x720) --auto-zoom Add automatic zooms from cursor telemetry (editor's magic wand) --audio <file> Mix a voiceover audio file into the MP4 (mp3/wav/m4a) --audio-mode <mix|replace> @@ -150,8 +149,6 @@ function parseExport(args: string[], cwd: string): CliCommand { quality: null, gifFrameRate: null, gifSizePreset: null, - previewWidth: null, - previewHeight: null, autoZoom: false, audioPath: null, audioMode: "mix", @@ -205,20 +202,6 @@ function parseExport(args: string[], cwd: string): CliCommand { i = next; break; } - case "--preview-size": { - const [value, next] = takeValue(args, i, arg); - const match = /^(\d+)x(\d+)$/.exec(value); - if (!match) throw new Error(`--preview-size must look like 1280x720, got "${value}"`); - const previewWidth = Number(match[1]); - const previewHeight = Number(match[2]); - if (previewWidth <= 0 || previewHeight <= 0) { - throw new Error(`--preview-size dimensions must be positive, got "${value}"`); - } - request.previewWidth = previewWidth; - request.previewHeight = previewHeight; - i = next; - break; - } case "--auto-zoom": request.autoZoom = true; break; diff --git a/electron/cli/cliMain.ts b/electron/cli/cliMain.ts index 611c3cae2b..b5f02afdfa 100644 --- a/electron/cli/cliMain.ts +++ b/electron/cli/cliMain.ts @@ -17,6 +17,7 @@ import { getSelectedDesktopSource, registerIpcHandlers } from "../ipc/handlers"; import { registerSttIpc } from "../stt"; import { ASSET_BASE_URL_ARG } from "../windows"; import { CLI_USAGE, type CliCommand } from "./args"; +import { runInfoCommand, runPackCommand } from "./projectCommands"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const VITE_DEV_SERVER_URL = process.env["VITE_DEV_SERVER_URL"]; @@ -209,169 +210,8 @@ function setupRecordStopSignals(stop: (reason: string) => void): void { } } -interface PackedProjectData { - version?: number; - media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; - videoPath?: string; - editor?: Record<string, unknown>; -} - -/** Copies a project and everything it references into one portable folder. */ -async function runPackCommand(projectPath: string, outDir: string, json: boolean): Promise<number> { - const emit = (message: string) => { - if (!json) safeWrite(process.stdout, `${message}\n`); - }; - - const raw = await fs.readFile(projectPath, "utf8"); - const data = JSON.parse(raw) as PackedProjectData; - const media = data.media ?? (data.videoPath ? { screenVideoPath: data.videoPath } : undefined); - const screenVideoPath = media?.screenVideoPath; - if (!screenVideoPath) { - throw new Error("Project file does not reference a screen video"); - } - - const projectDir = path.dirname(path.resolve(projectPath)); - const resolveSource = async (mediaPath: string): Promise<string> => { - const exists = await fs - .stat(mediaPath) - .then((stats) => stats.isFile()) - .catch(() => false); - if (exists) return mediaPath; - const sibling = path.join(projectDir, path.basename(mediaPath)); - const siblingExists = await fs - .stat(sibling) - .then((stats) => stats.isFile()) - .catch(() => false); - if (siblingExists) return sibling; - throw new Error(`Referenced media not found: ${mediaPath}`); - }; - - await fs.mkdir(outDir, { recursive: true }); - - const copied: string[] = []; - const copyIn = async (sourcePath: string): Promise<string> => { - const ext = path.extname(sourcePath); - const stem = path.basename(sourcePath, ext); - let destination = path.join(outDir, stem + ext); - // Screen and webcam can share a basename across directories; don't overwrite. - for (let n = 1; copied.includes(destination); n++) { - destination = path.join(outDir, `${stem}-${n}${ext}`); - } - if (path.resolve(sourcePath) !== path.resolve(destination)) { - await fs.copyFile(sourcePath, destination); - } - copied.push(destination); - return destination; - }; - - const screenSource = await resolveSource(screenVideoPath); - const newScreenPath = await copyIn(screenSource); - - let newWebcamPath: string | undefined; - if (media.webcamVideoPath) { - newWebcamPath = await copyIn(await resolveSource(media.webcamVideoPath)); - } - - // Cursor telemetry sidecar sits at "<video path>.cursor.json". - const cursorSidecar = `${screenSource}.cursor.json`; - const hasCursorData = await fs - .stat(cursorSidecar) - .then((stats) => stats.isFile()) - .catch(() => false); - if (hasCursorData) { - await copyIn(cursorSidecar); - } - - const packedProject: PackedProjectData = { - ...data, - media: { - ...media, - screenVideoPath: newScreenPath, - ...(newWebcamPath ? { webcamVideoPath: newWebcamPath } : {}), - }, - }; - delete packedProject.videoPath; - const packedProjectPath = path.join(outDir, path.basename(projectPath)); - await fs.writeFile(packedProjectPath, JSON.stringify(packedProject, null, 2), "utf8"); - - if (json) { - safeWrite( - process.stdout, - `${JSON.stringify({ - event: "done", - success: true, - projectPath: packedProjectPath, - files: [packedProjectPath, ...copied], - cursorData: hasCursorData, - })}\n`, - ); - } else { - emit(`Packed project → ${packedProjectPath}`); - for (const file of copied) { - emit(` + ${path.basename(file)}`); - } - if (!hasCursorData) { - emit(" (no cursor telemetry sidecar found)"); - } - emit( - "The folder is self-contained: if the stored paths go stale after moving it, the loader falls back to files next to the project.", - ); - } - return 0; -} - -async function runInfoCommand(projectPath: string, json: boolean): Promise<number> { - const raw = await fs.readFile(projectPath, "utf8"); - const data = JSON.parse(raw) as { - version?: number; - media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; - videoPath?: string; - editor?: Record<string, unknown>; - }; - const editor = data.editor ?? {}; - const count = (key: string) => - Array.isArray(editor[key]) ? (editor[key] as unknown[]).length : 0; - const screenVideoPath = data.media?.screenVideoPath ?? data.videoPath ?? null; - const mediaExists = screenVideoPath - ? await fs - .access(screenVideoPath) - .then(() => true) - .catch(() => false) - : false; - - const summary = { - projectPath, - version: data.version ?? null, - screenVideoPath, - screenVideoExists: mediaExists, - webcamVideoPath: data.media?.webcamVideoPath ?? null, - cursorCaptureMode: data.media?.cursorCaptureMode ?? null, - exportFormat: (editor.exportFormat as string) ?? null, - exportQuality: (editor.exportQuality as string) ?? null, - aspectRatio: (editor.aspectRatio as string) ?? null, - zoomRegions: count("zoomRegions"), - trimRegions: count("trimRegions"), - speedRegions: count("speedRegions"), - annotationRegions: count("annotationRegions"), - }; - - if (json) { - safeWrite(process.stdout, `${JSON.stringify(summary)}\n`); - } else { - safeWrite( - process.stdout, - [ - `Project: ${summary.projectPath} (version ${summary.version ?? "?"})`, - `Video: ${summary.screenVideoPath ?? "(none)"}${mediaExists ? "" : " [MISSING]"}`, - `Webcam: ${summary.webcamVideoPath ?? "(none)"}`, - `Cursor: ${summary.cursorCaptureMode ?? "(unknown)"}`, - `Export: ${summary.exportFormat ?? "?"} / ${summary.exportQuality ?? "?"} / ${summary.aspectRatio ?? "?"}`, - `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations`, - ].join("\n") + "\n", - ); - } - return summary.screenVideoPath && !mediaExists ? 1 : 0; -} +/** Both file-only commands write through the same EPIPE-safe stdout writer. */ +const writeStdout = (text: string) => safeWrite(process.stdout, text); export function runCli(command: CliCommand): void { if (command.kind === "help") { @@ -449,7 +289,7 @@ export function runCli(command: CliCommand): void { .whenReady() .then(async () => { if (command.kind === "info") { - const code = await runInfoCommand(command.projectPath, command.json === true); + const code = await runInfoCommand(command.projectPath, command.json === true, writeStdout); app.exit(code); return; } @@ -459,6 +299,7 @@ export function runCli(command: CliCommand): void { command.projectPath, command.outDir, command.json === true, + writeStdout, ); app.exit(code); return; diff --git a/electron/cli/projectCommands.test.ts b/electron/cli/projectCommands.test.ts new file mode 100644 index 0000000000..0748ad44db --- /dev/null +++ b/electron/cli/projectCommands.test.ts @@ -0,0 +1,139 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { type PackedProjectData, runInfoCommand, runPackCommand } from "./projectCommands"; + +let root = ""; + +/** Absolute path inside the throwaway root, parents created. */ +async function make(relative: string, contents = "video-bytes"): Promise<string> { + const target = path.join(root, relative); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.writeFile(target, contents, "utf8"); + return target; +} + +async function writeProject(relative: string, data: PackedProjectData): Promise<string> { + return make(relative, JSON.stringify(data)); +} + +const readProject = async (file: string): Promise<PackedProjectData> => + JSON.parse(await fs.readFile(file, "utf8")); + +/** Collects CLI output instead of writing to stdout. */ +function recorder() { + const chunks: string[] = []; + const write = (text: string) => { + chunks.push(text); + }; + return { write, text: () => chunks.join("") }; +} + +beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "os-cli-pack-")); +}); + +afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); +}); + +describe("runPackCommand", () => { + it("keeps screen and webcam apart when they share a basename", async () => { + const screen = await make("rec/a/clip.mp4", "screen"); + const webcam = await make("rec/b/clip.mp4", "webcam"); + const project = await writeProject("demo.openscreen", { + version: 1, + media: { screenVideoPath: screen, webcamVideoPath: webcam }, + }); + const outDir = path.join(root, "packed"); + + const out = recorder(); + expect(await runPackCommand(project, outDir, false, out.write)).toBe(0); + + const packed = await readProject(path.join(outDir, "demo.openscreen")); + expect(packed.media?.screenVideoPath).not.toBe(packed.media?.webcamVideoPath); + await expect(fs.readFile(packed.media?.screenVideoPath ?? "", "utf8")).resolves.toBe("screen"); + await expect(fs.readFile(packed.media?.webcamVideoPath ?? "", "utf8")).resolves.toBe("webcam"); + }); + + it("copies the cursor sidecar and drops the legacy videoPath", async () => { + const screen = await make("rec/clip.mp4", "screen"); + await make("rec/clip.mp4.cursor.json", "[]"); + const project = await writeProject("demo.openscreen", { version: 1, videoPath: screen }); + const outDir = path.join(root, "packed"); + + const out = recorder(); + expect(await runPackCommand(project, outDir, true, out.write)).toBe(0); + + const packed = await readProject(path.join(outDir, "demo.openscreen")); + expect(packed.videoPath).toBeUndefined(); + expect(packed.media?.screenVideoPath).toBe(path.join(outDir, "clip.mp4")); + await expect(fs.readFile(path.join(outDir, "clip.mp4.cursor.json"), "utf8")).resolves.toBe( + "[]", + ); + expect(JSON.parse(out.text())).toMatchObject({ event: "done", cursorData: true }); + }); + + it("falls back to media sitting next to the project when the stored path is stale", async () => { + await make("moved/clip.mp4", "screen"); + const project = await writeProject("moved/demo.openscreen", { + version: 1, + media: { screenVideoPath: path.join(root, "gone", "clip.mp4") }, + }); + const outDir = path.join(root, "packed"); + + const out = recorder(); + expect(await runPackCommand(project, outDir, false, out.write)).toBe(0); + await expect(fs.readFile(path.join(outDir, "clip.mp4"), "utf8")).resolves.toBe("screen"); + }); + + it("fails when the referenced media is nowhere to be found", async () => { + const project = await writeProject("demo.openscreen", { + version: 1, + media: { screenVideoPath: path.join(root, "gone", "clip.mp4") }, + }); + + const out = recorder(); + await expect( + runPackCommand(project, path.join(root, "packed"), false, out.write), + ).rejects.toThrow(/Referenced media not found/); + }); +}); + +describe("runInfoCommand", () => { + it("exits 1 when the project's video is missing, 0 when it is there", async () => { + const missing = await writeProject("missing.openscreen", { + version: 1, + media: { screenVideoPath: path.join(root, "gone", "clip.mp4") }, + }); + const present = await writeProject("present.openscreen", { + version: 1, + media: { screenVideoPath: await make("rec/clip.mp4") }, + }); + + const out = recorder(); + expect(await runInfoCommand(missing, false, out.write)).toBe(1); + expect(out.text()).toContain("[MISSING]"); + expect(await runInfoCommand(present, false, out.write)).toBe(0); + }); + + it("counts timeline regions in --json mode", async () => { + const project = await writeProject("demo.openscreen", { + version: 1, + media: { screenVideoPath: await make("rec/clip.mp4") }, + editor: { zoomRegions: [{}, {}], trimRegions: [{}], exportFormat: "mp4" }, + }); + + const out = recorder(); + expect(await runInfoCommand(project, true, out.write)).toBe(0); + expect(JSON.parse(out.text())).toMatchObject({ + zoomRegions: 2, + trimRegions: 1, + speedRegions: 0, + annotationRegions: 0, + exportFormat: "mp4", + screenVideoExists: true, + }); + }); +}); diff --git a/electron/cli/projectCommands.ts b/electron/cli/projectCommands.ts new file mode 100644 index 0000000000..30e211e8c4 --- /dev/null +++ b/electron/cli/projectCommands.ts @@ -0,0 +1,174 @@ +// The two CLI commands that only touch the project file and its media: +// `openscreen pack` and `openscreen info`. They live outside cliMain.ts so they +// carry no `electron` import and stay unit-testable — the caller passes the +// writer, so nothing here knows about process.stdout either. + +import fs from "node:fs/promises"; +import path from "node:path"; + +/** Writes one already-newline-terminated chunk of CLI output. */ +export type CliWriter = (text: string) => void; + +export interface PackedProjectData { + version?: number; + media?: { screenVideoPath?: string; webcamVideoPath?: string; cursorCaptureMode?: string }; + videoPath?: string; + editor?: Record<string, unknown>; +} + +const isFile = (candidate: string): Promise<boolean> => + fs + .stat(candidate) + .then((stats) => stats.isFile()) + .catch(() => false); + +/** Copies a project and everything it references into one portable folder. */ +export async function runPackCommand( + projectPath: string, + outDir: string, + json: boolean, + out: CliWriter, +): Promise<number> { + const emit = (message: string) => { + if (!json) out(`${message}\n`); + }; + + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as PackedProjectData; + const media = data.media ?? (data.videoPath ? { screenVideoPath: data.videoPath } : undefined); + const screenVideoPath = media?.screenVideoPath; + if (!screenVideoPath) { + throw new Error("Project file does not reference a screen video"); + } + + const projectDir = path.dirname(path.resolve(projectPath)); + const resolveSource = async (mediaPath: string): Promise<string> => { + if (await isFile(mediaPath)) return mediaPath; + // Moved project: the stored absolute path is stale but the media travelled + // with the .openscreen file. Same rule as the loader's sibling fallback. + const sibling = path.join(projectDir, path.basename(mediaPath)); + if (await isFile(sibling)) return sibling; + throw new Error(`Referenced media not found: ${mediaPath}`); + }; + + await fs.mkdir(outDir, { recursive: true }); + + const copied: string[] = []; + const copyIn = async (sourcePath: string): Promise<string> => { + const ext = path.extname(sourcePath); + const stem = path.basename(sourcePath, ext); + let destination = path.join(outDir, stem + ext); + // Screen and webcam can share a basename across directories; don't overwrite. + for (let n = 1; copied.includes(destination); n++) { + destination = path.join(outDir, `${stem}-${n}${ext}`); + } + if (path.resolve(sourcePath) !== path.resolve(destination)) { + await fs.copyFile(sourcePath, destination); + } + copied.push(destination); + return destination; + }; + + const screenSource = await resolveSource(screenVideoPath); + const newScreenPath = await copyIn(screenSource); + + let newWebcamPath: string | undefined; + if (media.webcamVideoPath) { + newWebcamPath = await copyIn(await resolveSource(media.webcamVideoPath)); + } + + // Cursor telemetry sidecar sits at "<video path>.cursor.json". + const cursorSidecar = `${screenSource}.cursor.json`; + const hasCursorData = await isFile(cursorSidecar); + if (hasCursorData) { + await copyIn(cursorSidecar); + } + + const packedProject: PackedProjectData = { + ...data, + media: { + ...media, + screenVideoPath: newScreenPath, + ...(newWebcamPath ? { webcamVideoPath: newWebcamPath } : {}), + }, + }; + delete packedProject.videoPath; + const packedProjectPath = path.join(outDir, path.basename(projectPath)); + await fs.writeFile(packedProjectPath, JSON.stringify(packedProject, null, 2), "utf8"); + + if (json) { + out( + `${JSON.stringify({ + event: "done", + success: true, + projectPath: packedProjectPath, + files: [packedProjectPath, ...copied], + cursorData: hasCursorData, + })}\n`, + ); + } else { + emit(`Packed project → ${packedProjectPath}`); + for (const file of copied) { + emit(` + ${path.basename(file)}`); + } + if (!hasCursorData) { + emit(" (no cursor telemetry sidecar found)"); + } + emit( + "The folder is self-contained: if the stored paths go stale after moving it, the loader falls back to files next to the project.", + ); + } + return 0; +} + +/** Prints what a project references and whether its media is still reachable. */ +export async function runInfoCommand( + projectPath: string, + json: boolean, + out: CliWriter, +): Promise<number> { + const raw = await fs.readFile(projectPath, "utf8"); + const data = JSON.parse(raw) as PackedProjectData; + const editor = data.editor ?? {}; + const count = (key: string) => + Array.isArray(editor[key]) ? (editor[key] as unknown[]).length : 0; + const screenVideoPath = data.media?.screenVideoPath ?? data.videoPath ?? null; + const mediaExists = screenVideoPath + ? await fs + .access(screenVideoPath) + .then(() => true) + .catch(() => false) + : false; + + const summary = { + projectPath, + version: data.version ?? null, + screenVideoPath, + screenVideoExists: mediaExists, + webcamVideoPath: data.media?.webcamVideoPath ?? null, + cursorCaptureMode: data.media?.cursorCaptureMode ?? null, + exportFormat: (editor.exportFormat as string) ?? null, + exportQuality: (editor.exportQuality as string) ?? null, + aspectRatio: (editor.aspectRatio as string) ?? null, + zoomRegions: count("zoomRegions"), + trimRegions: count("trimRegions"), + speedRegions: count("speedRegions"), + annotationRegions: count("annotationRegions"), + }; + + if (json) { + out(`${JSON.stringify(summary)}\n`); + } else { + out( + [ + `Project: ${summary.projectPath} (version ${summary.version ?? "?"})`, + `Video: ${summary.screenVideoPath ?? "(none)"}${mediaExists ? "" : " [MISSING]"}`, + `Webcam: ${summary.webcamVideoPath ?? "(none)"}`, + `Cursor: ${summary.cursorCaptureMode ?? "(unknown)"}`, + `Export: ${summary.exportFormat ?? "?"} / ${summary.exportQuality ?? "?"} / ${summary.aspectRatio ?? "?"}`, + `Timeline: ${summary.zoomRegions} zooms, ${summary.trimRegions} trims, ${summary.speedRegions} speed regions, ${summary.annotationRegions} annotations`, + ].join("\n") + "\n", + ); + } + return summary.screenVideoPath && !mediaExists ? 1 : 0; +} diff --git a/src/cli/CliExportRunner.tsx b/src/cli/CliExportRunner.tsx index 2b42da58ee..f9c57c6824 100644 --- a/src/cli/CliExportRunner.tsx +++ b/src/cli/CliExportRunner.tsx @@ -180,13 +180,6 @@ async function runExport(request: CliExportRequest): Promise<CliDoneResult> { const gifSizePreset = request.gifSizePreset ?? editor.gifSizePreset; const outPath = request.outPath ?? replaceExtension(request.projectPath, format === "gif" ? ".gif" : ".mp4"); - if (request.previewWidth !== null || request.previewHeight !== null) { - window.electronAPI.cliLog( - "info", - "--preview-size is a no-op on the native pipeline (annotation geometry is percentage-based) and kept only for CLI compatibility", - ); - } - // Cursor telemetry: only needed to compute --auto-zoom suggestions. The // native compositor discovers the `<video>.cursor.json` sidecar itself. let cursorTelemetry: CursorTelemetryPoint[] = []; diff --git a/src/lib/cliContracts.ts b/src/lib/cliContracts.ts index 111f2ed2a5..186613a859 100644 --- a/src/lib/cliContracts.ts +++ b/src/lib/cliContracts.ts @@ -23,8 +23,6 @@ export interface CliExportRequest { * same way the editor's on-screen preview does. The composition is fitted * into this box, mirroring the editor layout. Defaults to 1280x720. */ - previewWidth: number | null; - previewHeight: number | null; /** * Add automatic zoom regions derived from cursor-dwell telemetry (same * suggestion engine as the editor's magic wand) before rendering. Existing