diff --git a/AGENTS.md b/AGENTS.md index 1f1a63d40..8864fdf57 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ OpenScreen is a free, open-source screen recorder and video editor (Electron + R - Install deps: `npm install` (Node 22.22.1, npm 10.9.4 — see `package.json#engines`) - Start dev: `npm run dev` (Vite dev server; Electron window opens via `vite-plugin-electron`) - Build: `npm run build` (TypeScript check + Vite build + electron-builder) -- Typecheck: `npx tsc --noEmit` (CI runs this; no standalone script) +- Typecheck: `npx tsc --noEmit` — app code only. CI also runs `npx tsc -p tsconfig.test.json --noEmit` in a separate job ("Typecheck (tests)"), so **run both**: test files are invisible to the root config, and a type error in a `*.test.ts` fails CI while the root check stays green. - Test (unit): `npm run test` (Vitest, jsdom env) - Test (browser): `npm run test:browser` (Vitest + Playwright, requires `npm run test:browser:install` first) - Test (e2e): `npm run test:e2e` (Playwright) diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index 351b756d0..df94fc102 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -94,6 +94,26 @@ describe("SttManager", () => { expect(fakeWhisperServer.stop).toHaveBeenCalledOnce(); }); + it("retries setup after a failed one instead of caching the rejection", async () => { + // First run downloads a 253 MB model. Caching a rejected `prepare()` meant + // one dropped connection failed every later transcription in the session — + // including the retry the editor offers — until the app was restarted. + const { ensureModels } = await import("./modelManager"); + const mocked = vi.mocked(ensureModels); + mocked.mockClear(); + mocked.mockRejectedValueOnce(new Error("Failed to download: network unreachable")); + const mgr = new SttManager(); + + await expect(mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" })).rejects.toThrow( + "network unreachable", + ); + // The network came back: the next attempt must actually attempt. + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + + expect(mocked).toHaveBeenCalledTimes(2); + expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); + }); + it("setStatusSink replaces the previous sink (last call wins)", () => { const mgr = new SttManager(); const a = vi.fn(); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 4f3e992aa..641a3abe6 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -59,7 +59,18 @@ export class SttManager { if (options.statusSink) this.statusSink = options.statusSink; if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir; if (!this.initPromise) { - this.initPromise = this.prepare(); + // A REJECTED init must not be cached. `prepare()` downloads a 253 MB + // model on first run, and caching its rejection meant one dropped + // connection poisoned the whole app session: every later transcription + // — including the retry the UI offers, and every remaining asset in the + // auto-transcription queue — awaited the same stale rejection and failed + // in milliseconds, with no way back short of quitting the app. + // Reconnecting the network changed nothing. Clearing the slot on failure + // makes the next attempt a real attempt. + this.initPromise = this.prepare().catch((error) => { + this.initPromise = null; + throw error; + }); } return this.initPromise; } diff --git a/src/components/ai-edition/CaptionsPane.gating.test.tsx b/src/components/ai-edition/CaptionsPane.gating.test.tsx new file mode 100644 index 000000000..b1e87c92b --- /dev/null +++ b/src/components/ai-edition/CaptionsPane.gating.test.tsx @@ -0,0 +1,129 @@ +// Captions are a view of the transcript, so the pane's "Transcribe video" +// button is a retry, not a first step — the background pass has already tried. +// On a media with no audio track that retry can only fail again, so the button +// has to be dead and the pane has to say what is wrong instead of inviting a +// pointless click. + +import "@testing-library/jest-dom"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutDocument } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { useTranscriptionStore } from "@/lib/ai-edition/store/transcriptionStore"; +import { CaptionsPane } from "./CaptionsPane"; + +vi.mock("@/native", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +function documentWith(asset: AxcutAsset): AxcutDocument { + return { + schemaVersion: 7, + project: { + id: "proj_1", + title: "Test", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + primaryAssetId: asset.id, + }, + assets: [asset], + transcript: null, + transcripts: [], + timeline: { + clips: [ + { + id: "clip_1", + assetId: asset.id, + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + legacyEditor: null, + } as unknown as AxcutDocument; +} + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, +}; + +function load(document: AxcutDocument) { + useProjectStore.setState({ + projectId: document.project.id, + document, + status: "ready", + error: null, + dirty: false, + }); +} + +beforeEach(() => { + useTranscriptionStore.getState().reset(); + useProjectStore.getState().clear(); +}); + +afterEach(() => { + cleanup(); +}); + +describe("captions pane gating", () => { + it("offers the retry while the media might still yield a transcript", () => { + load(documentWith(ASSET)); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribe video" })).toBeEnabled(); + }); + + it("shows the queued background run instead of an idle button", () => { + load(documentWith(ASSET)); + useTranscriptionStore.setState({ + projectId: "proj_1", + jobs: { asset_1: { status: "running", language: "auto", manual: false } }, + }); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribing…" })).toBeDisabled(); + }); + + it("kills the retry on a media with no audio track and explains it", () => { + load( + documentWith({ + ...ASSET, + transcriptionFailure: { kind: "no-audio", message: "No audio track found in this video." }, + }), + ); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribe video" })).toBeDisabled(); + expect( + screen.getByText("This media has no audio track — there is nothing to transcribe."), + ).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index 91661bdc4..c1f6d732d 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -15,6 +15,10 @@ import { useScopedT } from "@/contexts/I18nContext"; import type { CaptionTextAlign, CaptionVerticalPosition } from "@/lib/ai-edition/captions"; import { untranslatedUnits } from "@/lib/ai-edition/captions"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { + useTimelineTranscriptGate, + useTranscriptionStore, +} from "@/lib/ai-edition/store/transcriptionStore"; import { useCaptions } from "@/lib/ai-edition/store/useCaptions"; import { nativeBridgeClient } from "@/native"; import { ColorField } from "./ColorField"; @@ -62,15 +66,9 @@ const TRANSLATION_LANGUAGES: ReadonlyArray<{ code: string; label: string }> = [ { code: "zh", label: "中文" }, ]; -interface CaptionsPaneProps { - /** Runs the local Whisper pipeline for the primary asset — owned by the - * shell, which already has the toast + per-asset status plumbing. */ - onTranscribe: () => void; - isTranscribing: boolean; -} - -export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps) { +export function CaptionsPane() { const t = useScopedT("settings"); + const te = useScopedT("editor"); const { settings, translations, @@ -85,6 +83,20 @@ export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps } = useCaptions(); const document = useProjectStore((s) => s.document); const saveDocument = useProjectStore((s) => s.saveDocument); + // Captions are a view of the transcript, and the transcript arrives on its + // own (transcriptionStore's background pass). The pane reads that state + // straight from the store rather than being handed a busy flag: it is the + // same answer everywhere, and "Transcribe" here is only ever a retry. + // + // Resolved over the timeline's assets, not the primary one: `hasTranscript` + // below is already timeline-scoped (useCaptions), and mixing the two scopes + // is what let a silent primary asset dead-end this button for a project whose + // actual footage had speech. + const gate = useTimelineTranscriptGate(); + const requestTimelineTranscripts = useTranscriptionStore((s) => s.requestTimelineTranscripts); + const isTranscribing = gate.state === "pending"; + const silentMedia = gate.state === "blocked" && gate.reason === "no-audio"; + const engineError = gate.state === "blocked" && gate.reason === "failed" ? gate.message : null; const [target, setTarget] = useState(TRANSLATION_LANGUAGES[1].code); const [translating, setTranslating] = useState(false); @@ -191,13 +203,26 @@ export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps }} >

- {t("captions.noTranscript")} + {silentMedia ? te("mediaStage.noAudioTrackHint") : t("captions.noTranscript")}

+ {engineError ? ( +

+ {engineError} +

+ ) : null} @@ -627,7 +652,7 @@ export function TranscriptPane({ key={section.clip.id} index={idx} section={section} - busy={busy} + busy={busyAssetIds.includes(section.clip.assetId)} cueWordId={cueWordId} onSeek={onSeek} onAddTrimRange={onAddTrimRange} @@ -933,6 +958,23 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} + {/* A block whose transcript is being regenerated is read-only — say it, + rather than letting the word stream look live and drop the edits. */} + {busy ? ( + + + {ts("transcript.transcribing")} + + ) : null} {words.length === 0 ? (

- {ts("transcript.noClipTranscript")} + {busy ? ts("transcript.transcribing") : ts("transcript.noClipTranscript")}

) : (
({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, +}; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, +]; + +function renderPane( + overrides: { + isTranscribing?: boolean; + blocked?: { reason: TranscriptGateReason; message?: string }; + } = {}, +) { + return render( + + + , + ); +} + +afterEach(() => { + cleanup(); +}); + +describe("transcript pane gating", () => { + it("offers the button while nothing has been attempted", () => { + renderPane(); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeEnabled(); + }); + + it("shows the background run in progress instead of an idle button", () => { + renderPane({ isTranscribing: true }); + const button = screen.getByRole("button", { name: "Transcribing…" }); + expect(button).toBeDisabled(); + }); + + it("disables the button when the timeline's media have no audio track, and says why", () => { + renderPane({ blocked: { reason: "no-audio" } }); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeDisabled(); + expect(screen.getByText("This media has no audio track")).toBeInTheDocument(); + }); + + it("keeps the retry available after a transient failure, and surfaces the engine message", () => { + renderPane({ blocked: { reason: "failed", message: "whisper-server exited" } }); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeEnabled(); + expect(screen.getByText("whisper-server exited")).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx index af75267ad..c55ae8233 100644 --- a/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx +++ b/src/components/ai-edition/TranscriptPane.keyboardCut.test.tsx @@ -67,7 +67,11 @@ const W2_TRIMMED: AxcutTrimRange = { reason: "", }; -function renderPane(trimRanges: AxcutTrimRange[], onAddTrimRange = vi.fn()) { +function renderPane( + trimRanges: AxcutTrimRange[], + onAddTrimRange = vi.fn(), + busyAssetIds: string[] = [], +) { const view = render( { expect(cutRange(onAddTrimRange)).toEqual([2, 3]); // "trois" }); + it("keeps cutting while ANOTHER asset is being transcribed", () => { + // The background pass runs on its own now, so a run on some other media must + // not quietly turn this block into an editor that ignores Backspace — the + // read-only state is scoped to the asset whose transcript is being rewritten. + const { editor, onAddTrimRange } = renderPane([], vi.fn(), ["asset_other"]); + caretBeforeWordAt(editor, 3); + fireEvent.keyDown(editor, { key: "Backspace" }); + expect(cutRange(onAddTrimRange)).toEqual([2, 3]); + }); + + it("stops cutting, visibly, while THIS asset is being transcribed", () => { + // Its transcript is about to be replaced, so the block is read-only — and it + // says so, instead of swallowing the keystroke in silence. + const { editor, onAddTrimRange, getByText } = renderPane([], vi.fn(), ["asset_1"]); + caretBeforeWordAt(editor, 3); + fireEvent.keyDown(editor, { key: "Backspace" }); + expect(cutRange(onAddTrimRange)).toBeNull(); + expect(editor).toHaveAttribute("aria-busy", "true"); + expect(getByText("Transcribing…")).toBeInTheDocument(); + }); + it("Backspace skips over an already-trimmed word instead of doing nothing", () => { // Hold Backspace and you land here: "deux" is already struck through, so the word // immediately before the caret has nothing left to cut. The keystroke used to @@ -184,7 +209,7 @@ describe("keyboard cut with the caret between words", () => { transcripts={[TRANSCRIPT]} assets={[ASSET]} trimRanges={trims} - busy={false} + busyAssetIds={[]} onSeek={vi.fn()} onAddTrimRange={(_target, startSec, endSec) => setTrims((prev) => [ diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index 36004c5c3..9ce354646 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -73,7 +73,7 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { transcripts={[TRANSCRIPT]} assets={[ASSET]} trimRanges={[]} - busy={false} + busyAssetIds={[]} onSeek={onSeek} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} diff --git a/src/components/ai-edition/TranscriptionStatus.tsx b/src/components/ai-edition/TranscriptionStatus.tsx new file mode 100644 index 000000000..5e17bd276 --- /dev/null +++ b/src/components/ai-edition/TranscriptionStatus.tsx @@ -0,0 +1,81 @@ +// One place that turns an `AssetTranscriptionView` into words and a colour. +// +// The media list (left panel), the media stage and the source-transcript modal +// all report the same six states; before auto-transcription each of them +// spelled its own dot colours and labels out inline, and they had already +// drifted (the left panel knew about "pending", the stage only ever showed a +// spinner). Keeping the vocabulary here means a new state shows up everywhere +// at once. + +import { Loader2 } from "lucide-react"; +import { useScopedT } from "@/contexts/I18nContext"; +import type { AssetTranscriptionView } from "@/lib/ai-edition/transcription/status"; + +/** Human-readable state of one asset's transcript, in the user's language. */ +export function useTranscriptionLabel(): (view: AssetTranscriptionView) => string { + const t = useScopedT("editor"); + return (view) => { + switch (view.status) { + case "ready": + return t("mediaStage.transcriptReady"); + case "queued": + return t("mediaStage.pendingTranscription"); + case "running": + return t("mediaStage.transcribing"); + case "empty": + return t("mediaStage.noSpeechDetected"); + case "failed": + return view.failure?.kind === "error" + ? t("mediaStage.transcriptionFailed") + : t("mediaStage.noAudioTrack"); + default: + return t("mediaStage.noTranscript"); + } + }; +} + +const DOT_COLOR: Record = { + ready: { fill: "var(--success)", halo: "0 0 0 3px var(--success-soft)" }, + queued: { fill: "#f59e0b", halo: "0 0 0 3px rgba(245, 158, 11, 0.2)" }, + running: { fill: "var(--accent)", halo: "0 0 0 3px rgba(16, 185, 129, 0.2)" }, + // A silent media is not a bug — it just has nothing to say. Amber, not red. + empty: { fill: "#f59e0b", halo: "0 0 0 3px rgba(245, 158, 11, 0.2)" }, + failed: { fill: "var(--danger)", halo: "0 0 0 3px rgba(239, 68, 68, 0.2)" }, + idle: { fill: "var(--dim)", halo: "none" }, +}; + +/** Compact status marker: a spinner while a run is in flight, a dot otherwise. */ +export function TranscriptionStatusDot({ + view, + size = 8, +}: { + view: AssetTranscriptionView; + size?: number; +}) { + const label = useTranscriptionLabel()(view); + if (view.status === "running" || view.status === "queued") { + return ( + + ); + } + const { fill, halo } = DOT_COLOR[view.status]; + return ( + + ); +} diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index e3698c75a..79c89f7e0 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -873,9 +873,14 @@ cursor: pointer; text-align: left; } -.recMenuRow:hover { +.recMenuRow:hover:not(:disabled) { background: var(--surface-2); } +/* A gated row (e.g. Smart cuts before its transcript exists) still shows its + reason line, so it must not also light up as if it were clickable. */ +.recMenuRow:disabled { + cursor: not-allowed; +} .recMenuRow.active { background: var(--accent-soft); color: var(--accent); diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index 306d3574a..6f53c4dad 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -78,10 +78,6 @@ interface FloatingInspectorProps { * through a facet body. */ clips: AxcutClip[]; onEditClip: (clip: AxcutClip) => void; - /** Runs the transcription the caption layer reads from. Captions are derived - * from the transcript, so this is the only caption action the shell owns. */ - onTranscribe: () => void; - isTranscribing: boolean; transcriptProps: TranscriptProps; /** Drives the selected-element settings pane (zoom/speed/annotation/trim) — * takes over the inspector, forcing it open, whenever a timeline region is @@ -97,8 +93,6 @@ export function FloatingInspector({ onToggleOpen, clips, onEditClip, - onTranscribe, - isTranscribing, transcriptProps, tl, }: FloatingInspectorProps) { @@ -114,13 +108,7 @@ export function FloatingInspector({ {selection ? ( tl.clearSelection()} /> ) : ( - + )}
) : null} @@ -1029,14 +1017,10 @@ const secondaryBtnStyle: React.CSSProperties = { function FacetBody({ facet, - onTranscribe, - isTranscribing, onCollapse, transcriptProps, }: { facet: Facet; - onTranscribe: () => void; - isTranscribing: boolean; onCollapse: () => void; transcriptProps: TranscriptProps; }) { @@ -1073,10 +1057,7 @@ function FacetBody({ if (facet === "layout") return wrap(collapse, ); if (facet === "cursor") return wrap(collapse, ); if (facet === "transcript") return wrap(collapse, ); - return wrap( - collapse, - , - ); + return wrap(collapse, ); } function wrap(collapse: React.ReactNode, body: React.ReactNode) { diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 540bef3c8..8c4bca65c 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -4,8 +4,17 @@ import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAsset } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { + useAssetTranscriptions, + useTranscriptionStore, +} from "@/lib/ai-edition/store/transcriptionStore"; import { formatSeconds } from "@/lib/ai-edition/timeline/format"; +import type { + AssetTranscriptionStatus, + AssetTranscriptionView, +} from "@/lib/ai-edition/transcription/status"; import { formatBytes } from "@/utils/formatBytes"; +import { TranscriptionStatusDot, useTranscriptionLabel } from "../TranscriptionStatus"; import styles from "./EditorShellV4.module.css"; const ASSET_MIME = "application/x-axcut-asset"; @@ -20,17 +29,17 @@ function basename(path: string): string { return path.split(/[\\/]/).pop() ?? path; } -export function MediaStage({ - assetStatuses, - onRegenerateAsset, -}: { - assetStatuses?: Record; - onRegenerateAsset?: (assetId: string, language: string) => Promise; -}) { +export function MediaStage() { const t = useScopedT("editor"); const projectId = useProjectStore((s) => s.projectId); const document = useProjectStore((s) => s.document); const addAsset = useProjectStore((s) => s.addAsset); + // Transcripts are produced in the background as soon as a media lands here + // (see transcriptionStore) — this stage only reports where each one is at, + // and lets the user force a re-run in another language. + const transcriptions = useAssetTranscriptions(); + const requestTranscription = useTranscriptionStore((s) => s.request); + const transcriptionLabel = useTranscriptionLabel(); const [query, setQuery] = useState(""); const [busy, setBusy] = useState(false); const [selectedId, setSelectedId] = useState(null); @@ -50,6 +59,11 @@ export function MediaStage({ const transcript = selected ? (document?.transcripts?.find((t) => t.assetId === selected.id) ?? null) : null; + const selectedTranscription: AssetTranscriptionView = selected + ? (transcriptions[selected.id] ?? { assetId: selected.id, status: "idle" }) + : { assetId: "", status: "idle" }; + const selectedBusy = + selectedTranscription.status === "running" || selectedTranscription.status === "queued"; const handleImport = async () => { if (!projectId) { @@ -95,7 +109,10 @@ export function MediaStage({ style={{ gridTemplateColumns: detailOpen ? "repeat(2,1fr)" : "repeat(3,1fr)" }} > {filtered.map((asset, i) => { - const status = assetStatuses?.[asset.id] ?? "idle"; + const transcription = transcriptions[asset.id] ?? { + assetId: asset.id, + status: "idle" as AssetTranscriptionStatus, + }; return ( ); @@ -236,24 +247,42 @@ export function MediaStage({ gap: 6, padding: "5px 10px 5px 8px", borderRadius: 9999, - background: transcript ? "var(--success-soft)" : "var(--accent-soft)", - color: transcript ? "var(--success)" : "var(--accent)", + background: + selectedTranscription.status === "failed" + ? "var(--danger-soft)" + : selectedTranscription.status === "ready" + ? "var(--success-soft)" + : "var(--accent-soft)", + color: + selectedTranscription.status === "failed" + ? "var(--danger)" + : selectedTranscription.status === "ready" + ? "var(--success)" + : "var(--accent)", fontSize: 11.5, fontWeight: 600, }} > - - {transcript ? t("mediaStage.transcriptReady") : t("mediaStage.notGeneratedYet")} + + {transcriptionLabel(selectedTranscription)} + {selectedTranscription.failure ? ( +

+ {selectedTranscription.failure.kind === "error" + ? selectedTranscription.failure.message + : t("mediaStage.noAudioTrackHint")} +

+ ) : null} +
{ - if (onRegenerateAsset) void onRegenerateAsset(selected.id, lang); - }} + disabled={selectedBusy} + onClick={() => void requestTranscription(selected.id, lang)} style={{ width: 36, height: 36, @@ -306,10 +333,11 @@ export function MediaStage({ color: "var(--fg-2)", background: "var(--surface-2)", border: "1px solid var(--border)", - cursor: "pointer", + cursor: selectedBusy ? "not-allowed" : "pointer", + opacity: selectedBusy ? 0.6 : 1, }} > - +
@@ -332,7 +360,13 @@ export function MediaStage({ .map((seg) => (seg as { text?: string }).text ?? "") .join(" ") || t("mediaStage.transcriptEmpty") ) : ( - {t("mediaStage.notGeneratedHint")} + + {selectedBusy + ? t("mediaStage.transcribingEllipsis") + : selectedTranscription.status === "failed" + ? t("mediaStage.generationFailedHint") + : t("mediaStage.notGeneratedHint")} + )} diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 268aca40f..ca173b155 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -34,6 +34,7 @@ import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; import { setUiProbeScrubbing } from "@/lib/ai-edition/perf/uiFrameProbe"; import type { AxcutClip } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { useTimelineTranscriptGate } from "@/lib/ai-edition/store/transcriptionStore"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; @@ -337,6 +338,24 @@ export function V4Timeline({ const [aspectMenuOpen, setAspectMenuOpen] = useState(false); const [autoEnhanceOpen, setAutoEnhanceOpen] = useState(false); const [autoBusy, setAutoBusy] = useState(false); + // The AI cut pass reads the transcript, and the transcript is produced in the + // background (see transcriptionStore). Until it is there, the entry says why + // rather than handing the agent a prompt it cannot honour — the failure mode + // that made this button the wrong first click for a new user. + const transcriptGate = useTimelineTranscriptGate(); + const smartCutsBlocked = transcriptGate.state !== "ready"; + const smartCutsHint = + transcriptGate.state === "pending" + ? t("toolbar.smartCutsWaiting") + : transcriptGate.state === "ready" + ? t("toolbar.smartZoomsAndCutsHint") + : transcriptGate.reason === "no-audio" + ? t("toolbar.smartCutsNoAudio") + : transcriptGate.reason === "no-speech" + ? t("toolbar.smartCutsNoSpeech") + : transcriptGate.reason === "failed" + ? t("toolbar.smartCutsFailed") + : t("toolbar.smartCutsNeedsTranscript"); const clips = tl.clips; const total = useMemo( @@ -1167,13 +1186,22 @@ export function V4Timeline({ - diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 8a450b47a..255bad4c1 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "فشل الإنشاء — اختر لغة وأعد الإنشاء.", "noPreviewAvailable": "لا تتوفر معاينة", "restart": "إعادة التشغيل", - "detectedLanguage": "اللغة المكتشفة: {{language}}" + "detectedLanguage": "اللغة المكتشفة: {{language}}", + "noAudioTrack": "لا يوجد مسار صوتي", + "noAudioTrackHint": "لا يحتوي هذا الملف على مسار صوتي — لا يوجد ما يمكن نسخه.", + "noSpeechDetected": "لم يتم اكتشاف كلام" }, "exportDialog": { "title": "تصدير", diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index d2ab9c645..72f3883be 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -255,7 +255,8 @@ "silence": "[صمت {{duration}} ث]", "restoreSilence": "استعادة الصمت ({{duration}} ث)", "trimSilence": "قص الصمت ({{duration}} ث)", - "restoreWord": "استعادة \"{{word}}\"" + "restoreWord": "استعادة \"{{word}}\"", + "noAudio": "لا يحتوي هذا الملف على مسار صوتي" }, "captions": { "show": "إظهار الترجمة", diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json index dc2065459..a47328e86 100644 --- a/src/i18n/locales/ar/timeline.json +++ b/src/i18n/locales/ar/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "تمت إضافة {{count}} تكبير تلقائي", "addedAutoZoomPlural": "تمت إضافة {{count}} تكبيرات تلقائية", "autoZoomFailed": "فشل التكبير التلقائي", - "aiEnhanceRequested": "طُلب من وكيل الذكاء الاصطناعي قص الأوقات الميتة" + "aiEnhanceRequested": "طُلب من وكيل الذكاء الاصطناعي قص الأوقات الميتة", + "smartCutsWaiting": "جارٍ النسخ… سيكون جاهزًا بعد قليل", + "smartCutsNeedsTranscript": "يتطلب نصًا منسوخًا", + "smartCutsNoAudio": "لا يحتوي هذا الملف على صوت", + "smartCutsNoSpeech": "لم يتم اكتشاف كلام", + "smartCutsFailed": "فشل النسخ — أعد المحاولة من قسم الوسائط" } } diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 34c5e6438..2e270ffb3 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Generation failed — pick a language and regenerate.", "noPreviewAvailable": "No preview available", "restart": "Restart", - "detectedLanguage": "Detected language: {{language}}" + "detectedLanguage": "Detected language: {{language}}", + "noAudioTrack": "No audio track", + "noAudioTrackHint": "This media has no audio track — there is nothing to transcribe.", + "noSpeechDetected": "No speech detected" }, "exportDialog": { "title": "Export", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 4b8358ea1..d6fa45887 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -255,7 +255,8 @@ "silence": "[silence {{duration}}s]", "restoreSilence": "Restore silence ({{duration}}s)", "trimSilence": "Trim silence ({{duration}}s)", - "restoreWord": "Restore \"{{word}}\"" + "restoreWord": "Restore \"{{word}}\"", + "noAudio": "This media has no audio track" }, "captions": { "show": "Show captions", diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index 224422f42..d1940b2ea 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Added {{count}} automatic zoom", "addedAutoZoomPlural": "Added {{count}} automatic zooms", "autoZoomFailed": "Auto-zoom failed", - "aiEnhanceRequested": "Asked the AI agent to cut the dead time" + "aiEnhanceRequested": "Asked the AI agent to cut the dead time", + "smartCutsWaiting": "Transcribing… ready in a moment", + "smartCutsNeedsTranscript": "Needs a transcript", + "smartCutsNoAudio": "This media has no audio", + "smartCutsNoSpeech": "No speech detected", + "smartCutsFailed": "Transcription failed — retry it from Media" } } diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index b0f07e8ea..8f5ba5bec 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Error de generación — elige un idioma y vuelve a generar.", "noPreviewAvailable": "Vista previa no disponible", "restart": "Reiniciar", - "detectedLanguage": "Idioma detectado: {{language}}" + "detectedLanguage": "Idioma detectado: {{language}}", + "noAudioTrack": "Sin pista de audio", + "noAudioTrackHint": "Este medio no tiene pista de audio: no hay nada que transcribir.", + "noSpeechDetected": "No se detectó voz" }, "exportDialog": { "title": "Exportar", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index d7cb0cb9a..71ba0b9d5 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -255,7 +255,8 @@ "silence": "[silencio {{duration}} s]", "restoreSilence": "Restaurar silencio ({{duration}} s)", "trimSilence": "Recortar silencio ({{duration}} s)", - "restoreWord": "Restaurar «{{word}}»" + "restoreWord": "Restaurar «{{word}}»", + "noAudio": "Este medio no tiene pista de audio" }, "captions": { "show": "Mostrar subtítulos", diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json index 1a23e1f82..4226ead06 100644 --- a/src/i18n/locales/es/timeline.json +++ b/src/i18n/locales/es/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Se añadió {{count}} zoom automático", "addedAutoZoomPlural": "Se añadieron {{count}} zooms automáticos", "autoZoomFailed": "Error en el zoom automático", - "aiEnhanceRequested": "Se pidió al agente de IA que corte los tiempos muertos" + "aiEnhanceRequested": "Se pidió al agente de IA que corte los tiempos muertos", + "smartCutsWaiting": "Transcribiendo… disponible en un momento", + "smartCutsNeedsTranscript": "Requiere una transcripción", + "smartCutsNoAudio": "Este medio no tiene audio", + "smartCutsNoSpeech": "No se detectó voz", + "smartCutsFailed": "La transcripción falló: reinténtala desde Medios" } } diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 3b03b9871..5cda995b1 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Échec de la génération — choisissez une langue et régénérez.", "noPreviewAvailable": "Aucun aperçu disponible", "restart": "Redémarrer", - "detectedLanguage": "Langue détectée : {{language}}" + "detectedLanguage": "Langue détectée : {{language}}", + "noAudioTrack": "Aucune piste audio", + "noAudioTrackHint": "Ce média n'a pas de piste audio — il n'y a rien à transcrire.", + "noSpeechDetected": "Aucune parole détectée" }, "exportDialog": { "title": "Exporter", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index afba3a562..e94b4f579 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -255,7 +255,8 @@ "silence": "[silence {{duration}} s]", "restoreSilence": "Restaurer le silence ({{duration}} s)", "trimSilence": "Couper le silence ({{duration}} s)", - "restoreWord": "Restaurer « {{word}} »" + "restoreWord": "Restaurer « {{word}} »", + "noAudio": "Ce média n'a pas de piste audio" }, "captions": { "show": "Afficher les sous-titres", diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json index ffabdf22e..210fbfe4c 100644 --- a/src/i18n/locales/fr/timeline.json +++ b/src/i18n/locales/fr/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} zoom automatique ajouté", "addedAutoZoomPlural": "{{count}} zooms automatiques ajoutés", "autoZoomFailed": "Échec du zoom automatique", - "aiEnhanceRequested": "Demandé à l'agent IA de couper les temps morts" + "aiEnhanceRequested": "Demandé à l'agent IA de couper les temps morts", + "smartCutsWaiting": "Transcription en cours… disponible dans un instant", + "smartCutsNeedsTranscript": "Nécessite une transcription", + "smartCutsNoAudio": "Ce média n'a pas d'audio", + "smartCutsNoSpeech": "Aucune parole détectée", + "smartCutsFailed": "Échec de la transcription — relancez-la depuis Médias" } } diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 1b2bbbf0c..ecc998b7d 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Generazione non riuscita — scegli una lingua e rigenera.", "noPreviewAvailable": "Anteprima non disponibile", "restart": "Riavvia", - "detectedLanguage": "Lingua rilevata: {{language}}" + "detectedLanguage": "Lingua rilevata: {{language}}", + "noAudioTrack": "Nessuna traccia audio", + "noAudioTrackHint": "Questo contenuto non ha una traccia audio: non c'è nulla da trascrivere.", + "noSpeechDetected": "Nessun parlato rilevato" }, "exportDialog": { "title": "Esporta", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index eeaa72d60..464adba34 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -255,7 +255,8 @@ "silence": "[silenzio {{duration}} s]", "restoreSilence": "Ripristina silenzio ({{duration}} s)", "trimSilence": "Taglia silenzio ({{duration}} s)", - "restoreWord": "Ripristina «{{word}}»" + "restoreWord": "Ripristina «{{word}}»", + "noAudio": "Questo contenuto non ha una traccia audio" }, "captions": { "show": "Mostra sottotitoli", diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json index 1a02f7d9c..ed9f2f9f8 100644 --- a/src/i18n/locales/it/timeline.json +++ b/src/i18n/locales/it/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Aggiunto {{count}} zoom automatico", "addedAutoZoomPlural": "Aggiunti {{count}} zoom automatici", "autoZoomFailed": "Zoom automatico non riuscito", - "aiEnhanceRequested": "Chiesto all'agente IA di tagliare i tempi morti" + "aiEnhanceRequested": "Chiesto all'agente IA di tagliare i tempi morti", + "smartCutsWaiting": "Trascrizione in corso… disponibile a breve", + "smartCutsNeedsTranscript": "Richiede una trascrizione", + "smartCutsNoAudio": "Questo contenuto non ha audio", + "smartCutsNoSpeech": "Nessun parlato rilevato", + "smartCutsFailed": "Trascrizione non riuscita: riprova da Contenuti multimediali" } } diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 04970a21f..615cf15a5 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "生成に失敗しました — 言語を選んで再生成してください。", "noPreviewAvailable": "プレビューがありません", "restart": "再生位置を先頭に戻す", - "detectedLanguage": "検出された言語: {{language}}" + "detectedLanguage": "検出された言語: {{language}}", + "noAudioTrack": "音声トラックがありません", + "noAudioTrackHint": "このメディアには音声トラックがないため、文字起こしできません。", + "noSpeechDetected": "音声が検出されませんでした" }, "exportDialog": { "title": "エクスポート", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index 0dbc20e65..386d51092 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -255,7 +255,8 @@ "silence": "[無音 {{duration}} 秒]", "restoreSilence": "無音を元に戻す（{{duration}} 秒）", "trimSilence": "無音をトリム（{{duration}} 秒）", - "restoreWord": "「{{word}}」を元に戻す" + "restoreWord": "「{{word}}」を元に戻す", + "noAudio": "このメディアには音声トラックがありません" }, "captions": { "show": "字幕を表示", diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json index 2ab41033c..2f12a3b9b 100644 --- a/src/i18n/locales/ja-JP/timeline.json +++ b/src/i18n/locales/ja-JP/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "自動ズームを {{count}} 件追加しました", "addedAutoZoomPlural": "自動ズームを {{count}} 件追加しました", "autoZoomFailed": "自動ズームに失敗しました", - "aiEnhanceRequested": "AIエージェントに無音部分のカットを依頼しました" + "aiEnhanceRequested": "AIエージェントに無音部分のカットを依頼しました", + "smartCutsWaiting": "文字起こし中… まもなく使えます", + "smartCutsNeedsTranscript": "文字起こしが必要です", + "smartCutsNoAudio": "このメディアには音声がありません", + "smartCutsNoSpeech": "音声が検出されませんでした", + "smartCutsFailed": "文字起こしに失敗しました — メディアから再試行してください" } } diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index e7b986017..190b9af0b 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "생성 실패 — 언어를 선택하고 다시 생성하세요.", "noPreviewAvailable": "미리보기를 사용할 수 없습니다", "restart": "다시 시작", - "detectedLanguage": "감지된 언어: {{language}}" + "detectedLanguage": "감지된 언어: {{language}}", + "noAudioTrack": "오디오 트랙 없음", + "noAudioTrackHint": "이 미디어에는 오디오 트랙이 없어 받아쓸 내용이 없습니다.", + "noSpeechDetected": "음성이 감지되지 않음" }, "exportDialog": { "title": "내보내기", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index 4f3de7c7a..24405508b 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -255,7 +255,8 @@ "silence": "[무음 {{duration}}초]", "restoreSilence": "무음 복원 ({{duration}}초)", "trimSilence": "무음 자르기 ({{duration}}초)", - "restoreWord": "\"{{word}}\" 복원" + "restoreWord": "\"{{word}}\" 복원", + "noAudio": "이 미디어에는 오디오 트랙이 없습니다" }, "captions": { "show": "자막 표시", diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json index 12f0abadc..d05035641 100644 --- a/src/i18n/locales/ko-KR/timeline.json +++ b/src/i18n/locales/ko-KR/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "자동 줌 {{count}}개가 추가되었습니다", "addedAutoZoomPlural": "자동 줌 {{count}}개가 추가되었습니다", "autoZoomFailed": "자동 줌 실패", - "aiEnhanceRequested": "AI 에이전트에 빈 구간 컷을 요청했습니다" + "aiEnhanceRequested": "AI 에이전트에 빈 구간 컷을 요청했습니다", + "smartCutsWaiting": "받아쓰는 중… 곧 사용할 수 있습니다", + "smartCutsNeedsTranscript": "받아쓰기가 필요합니다", + "smartCutsNoAudio": "이 미디어에는 오디오가 없습니다", + "smartCutsNoSpeech": "음성이 감지되지 않음", + "smartCutsFailed": "받아쓰기에 실패했습니다 — 미디어에서 다시 시도하세요" } } diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 5833af8aa..b7d90914d 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Falha na geração — escolha um idioma e gere novamente.", "noPreviewAvailable": "Pré-visualização indisponível", "restart": "Reiniciar", - "detectedLanguage": "Idioma detectado: {{language}}" + "detectedLanguage": "Idioma detectado: {{language}}", + "noAudioTrack": "Sem faixa de áudio", + "noAudioTrackHint": "Esta mídia não tem faixa de áudio — não há nada para transcrever.", + "noSpeechDetected": "Nenhuma fala detectada" }, "exportDialog": { "title": "Exportar", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index b0f492a2e..c3eebc6da 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -255,7 +255,8 @@ "silence": "[silêncio {{duration}} s]", "restoreSilence": "Restaurar silêncio ({{duration}} s)", "trimSilence": "Cortar silêncio ({{duration}} s)", - "restoreWord": "Restaurar \"{{word}}\"" + "restoreWord": "Restaurar \"{{word}}\"", + "noAudio": "Esta mídia não tem faixa de áudio" }, "captions": { "show": "Mostrar legendas", diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json index 76f4cd6db..4a70046d5 100644 --- a/src/i18n/locales/pt-BR/timeline.json +++ b/src/i18n/locales/pt-BR/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} zoom automático adicionado", "addedAutoZoomPlural": "{{count}} zooms automáticos adicionados", "autoZoomFailed": "Falha no zoom automático", - "aiEnhanceRequested": "Pedido ao agente de IA para cortar os tempos mortos" + "aiEnhanceRequested": "Pedido ao agente de IA para cortar os tempos mortos", + "smartCutsWaiting": "Transcrevendo… disponível em instantes", + "smartCutsNeedsTranscript": "Requer uma transcrição", + "smartCutsNoAudio": "Esta mídia não tem áudio", + "smartCutsNoSpeech": "Nenhuma fala detectada", + "smartCutsFailed": "Falha na transcrição — tente de novo em Mídia" } } diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 6ebd18453..b50d5a986 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Ошибка создания — выберите язык и создайте заново.", "noPreviewAvailable": "Предпросмотр недоступен", "restart": "Перезапустить", - "detectedLanguage": "Обнаруженный язык: {{language}}" + "detectedLanguage": "Обнаруженный язык: {{language}}", + "noAudioTrack": "Нет аудиодорожки", + "noAudioTrackHint": "В этом медиафайле нет аудиодорожки — расшифровывать нечего.", + "noSpeechDetected": "Речь не обнаружена" }, "exportDialog": { "title": "Экспорт", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 8ff6591e7..ad63e276d 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -255,7 +255,8 @@ "silence": "[тишина {{duration}} с]", "restoreSilence": "Вернуть тишину ({{duration}} с)", "trimSilence": "Вырезать тишину ({{duration}} с)", - "restoreWord": "Вернуть «{{word}}»" + "restoreWord": "Вернуть «{{word}}»", + "noAudio": "В этом медиафайле нет аудиодорожки" }, "captions": { "show": "Показывать субтитры", diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json index 621c5c059..ebd77431a 100644 --- a/src/i18n/locales/ru/timeline.json +++ b/src/i18n/locales/ru/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Добавлен {{count}} автоматический зум", "addedAutoZoomPlural": "Добавлено {{count}} автоматических зумов", "autoZoomFailed": "Не удалось выполнить автозум", - "aiEnhanceRequested": "Агенту ИИ поручено вырезать паузы" + "aiEnhanceRequested": "Агенту ИИ поручено вырезать паузы", + "smartCutsWaiting": "Идёт расшифровка… скоро будет готово", + "smartCutsNeedsTranscript": "Нужна расшифровка", + "smartCutsNoAudio": "В этом медиафайле нет звука", + "smartCutsNoSpeech": "Речь не обнаружена", + "smartCutsFailed": "Не удалось расшифровать — повторите из раздела «Медиа»" } } diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 067f0f069..0c1d77344 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Oluşturma başarısız oldu — bir dil seçip yeniden oluşturun.", "noPreviewAvailable": "Önizleme kullanılamıyor", "restart": "Yeniden başlat", - "detectedLanguage": "Algılanan dil: {{language}}" + "detectedLanguage": "Algılanan dil: {{language}}", + "noAudioTrack": "Ses parçası yok", + "noAudioTrackHint": "Bu medyada ses parçası yok — metne dökülecek bir şey bulunmuyor.", + "noSpeechDetected": "Konuşma algılanmadı" }, "exportDialog": { "title": "Dışa Aktar", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 8193da831..d397e65a3 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -255,7 +255,8 @@ "silence": "[sessizlik {{duration}} sn]", "restoreSilence": "Sessizliği geri al ({{duration}} sn)", "trimSilence": "Sessizliği kırp ({{duration}} sn)", - "restoreWord": "\"{{word}}\" kelimesini geri al" + "restoreWord": "\"{{word}}\" kelimesini geri al", + "noAudio": "Bu medyada ses parçası yok" }, "captions": { "show": "Altyazıları göster", diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json index a4be44a5b..d8f4431c5 100644 --- a/src/i18n/locales/tr/timeline.json +++ b/src/i18n/locales/tr/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} otomatik yakınlaştırma eklendi", "addedAutoZoomPlural": "{{count}} otomatik yakınlaştırma eklendi", "autoZoomFailed": "Otomatik yakınlaştırma başarısız oldu", - "aiEnhanceRequested": "Yapay zeka aracısından ölü zamanları kırpması istendi" + "aiEnhanceRequested": "Yapay zeka aracısından ölü zamanları kırpması istendi", + "smartCutsWaiting": "Metne dökülüyor… birazdan hazır", + "smartCutsNeedsTranscript": "Bir döküm gerekiyor", + "smartCutsNoAudio": "Bu medyada ses yok", + "smartCutsNoSpeech": "Konuşma algılanmadı", + "smartCutsFailed": "Metne dökme başarısız — Medya bölümünden yeniden deneyin" } } diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index e5e649486..0ad0a044c 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Tạo thất bại — chọn ngôn ngữ và tạo lại.", "noPreviewAvailable": "Không có bản xem trước", "restart": "Bắt đầu lại", - "detectedLanguage": "Ngôn ngữ phát hiện: {{language}}" + "detectedLanguage": "Ngôn ngữ phát hiện: {{language}}", + "noAudioTrack": "Không có bản âm thanh", + "noAudioTrackHint": "Media này không có bản âm thanh — không có gì để phiên âm.", + "noSpeechDetected": "Không phát hiện giọng nói" }, "exportDialog": { "title": "Xuất", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index 2d1d201a8..fcaa03388 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -255,7 +255,8 @@ "silence": "[khoảng lặng {{duration}} giây]", "restoreSilence": "Khôi phục khoảng lặng ({{duration}} giây)", "trimSilence": "Cắt khoảng lặng ({{duration}} giây)", - "restoreWord": "Khôi phục \"{{word}}\"" + "restoreWord": "Khôi phục \"{{word}}\"", + "noAudio": "Media này không có bản âm thanh" }, "captions": { "show": "Hiện phụ đề", diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json index 8ecbb7d4f..4b9218ba7 100644 --- a/src/i18n/locales/vi/timeline.json +++ b/src/i18n/locales/vi/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Đã thêm {{count}} thu phóng tự động", "addedAutoZoomPlural": "Đã thêm {{count}} thu phóng tự động", "autoZoomFailed": "Thu phóng tự động thất bại", - "aiEnhanceRequested": "Đã yêu cầu tác nhân AI cắt thời gian chết" + "aiEnhanceRequested": "Đã yêu cầu tác nhân AI cắt thời gian chết", + "smartCutsWaiting": "Đang phiên âm… sẵn sàng trong giây lát", + "smartCutsNeedsTranscript": "Cần có bản phiên âm", + "smartCutsNoAudio": "Media này không có âm thanh", + "smartCutsNoSpeech": "Không phát hiện giọng nói", + "smartCutsFailed": "Phiên âm thất bại — thử lại trong Media" } } diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 1500049a1..35610a0b6 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "生成失败 — 选择语言并重新生成。", "noPreviewAvailable": "无法预览", "restart": "重新开始", - "detectedLanguage": "检测到的语言：{{language}}" + "detectedLanguage": "检测到的语言：{{language}}", + "noAudioTrack": "无音频轨道", + "noAudioTrackHint": "此媒体没有音频轨道，没有可转录的内容。", + "noSpeechDetected": "未检测到语音" }, "exportDialog": { "title": "导出", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index ca6c78910..013b5e6d1 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -255,7 +255,8 @@ "silence": "[静音 {{duration}} 秒]", "restoreSilence": "恢复静音（{{duration}} 秒）", "trimSilence": "修剪静音（{{duration}} 秒）", - "restoreWord": "恢复“{{word}}”" + "restoreWord": "恢复“{{word}}”", + "noAudio": "此媒体没有音频轨道" }, "captions": { "show": "显示字幕", diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json index f73508cf3..f1065037c 100644 --- a/src/i18n/locales/zh-CN/timeline.json +++ b/src/i18n/locales/zh-CN/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "已添加 {{count}} 个自动缩放", "addedAutoZoomPlural": "已添加 {{count}} 个自动缩放", "autoZoomFailed": "自动缩放失败", - "aiEnhanceRequested": "已请求 AI 代理剪除空白片段" + "aiEnhanceRequested": "已请求 AI 代理剪除空白片段", + "smartCutsWaiting": "正在转录…稍后可用", + "smartCutsNeedsTranscript": "需要转录文本", + "smartCutsNoAudio": "此媒体没有音频", + "smartCutsNoSpeech": "未检测到语音", + "smartCutsFailed": "转录失败 — 请在“媒体”中重试" } } diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index b0b36e5a7..c3e7e55e4 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "產生失敗 — 選擇語言並重新產生。", "noPreviewAvailable": "無法預覽", "restart": "重新開始", - "detectedLanguage": "偵測到的語言：{{language}}" + "detectedLanguage": "偵測到的語言：{{language}}", + "noAudioTrack": "無音訊軌道", + "noAudioTrackHint": "此媒體沒有音訊軌道，沒有可轉錄的內容。", + "noSpeechDetected": "未偵測到語音" }, "exportDialog": { "title": "匯出", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 8fe3c9ffa..82b1d76c9 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -256,7 +256,8 @@ "silence": "[靜音 {{duration}} 秒]", "restoreSilence": "還原靜音（{{duration}} 秒）", "trimSilence": "修剪靜音（{{duration}} 秒）", - "restoreWord": "還原「{{word}}」" + "restoreWord": "還原「{{word}}」", + "noAudio": "此媒體沒有音訊軌道" }, "captions": { "show": "顯示字幕", diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json index cadb63dff..94fce3f4a 100644 --- a/src/i18n/locales/zh-TW/timeline.json +++ b/src/i18n/locales/zh-TW/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "已新增 {{count}} 個自動縮放", "addedAutoZoomPlural": "已新增 {{count}} 個自動縮放", "autoZoomFailed": "自動縮放失敗", - "aiEnhanceRequested": "已請求 AI 代理剪除空白片段" + "aiEnhanceRequested": "已請求 AI 代理剪除空白片段", + "smartCutsWaiting": "正在轉錄…稍後可用", + "smartCutsNeedsTranscript": "需要轉錄文字", + "smartCutsNoAudio": "此媒體沒有音訊", + "smartCutsNoSpeech": "未偵測到語音", + "smartCutsFailed": "轉錄失敗 — 請在「媒體」中重試" } } diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index ecc8e0daa..17c2305a6 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -124,6 +124,19 @@ export const cameraTrackSchema = z .nullable() .default(null); +// Why a media can never be transcribed. Only the DETERMINISTIC verdicts live +// here: a container with no audio track (a screen recording captured with no +// mic and no system audio — the common case) fails identically on every +// attempt, and re-deciding that costs a full audio extraction on each project +// open. Transient failures (engine down, decode hiccup) are deliberately NOT +// persistable and stay in the transcription store for the session, so the next +// load retries them. See `src/lib/ai-edition/transcription/status.ts`. +export const assetTranscriptionFailureSchema = z.object({ + kind: z.enum(["no-audio", "unsupported-audio"]), + message: z.string().default(""), + at: isoDateSchema.optional(), +}); + export const assetSchema = z.object({ id: z.string().min(1), kind: z.literal("video"), @@ -136,6 +149,9 @@ export const assetSchema = z.object({ sizeBytes: z.number().int().nonnegative().optional(), video: assetVideoSchema.optional(), audio: assetAudioSchema.optional(), + // Absent on every document written before auto-transcription; additive, so + // no schema-version bump (an older build simply drops the key on save). + transcriptionFailure: assetTranscriptionFailureSchema.nullish(), cameraTrack: cameraTrackSchema, }); @@ -802,6 +818,7 @@ export type AxcutWord = z.infer; export type AxcutTranscriptSegment = z.infer; export type AxcutTranscript = z.infer; export type AxcutAsset = z.infer; +export type AxcutAssetTranscriptionFailure = z.infer; export type AxcutClip = z.infer; export type AxcutClipCropRegion = z.infer; export type AxcutGap = z.infer; diff --git a/src/lib/ai-edition/store/projectStore.ts b/src/lib/ai-edition/store/projectStore.ts index fec88776f..539449dab 100644 --- a/src/lib/ai-edition/store/projectStore.ts +++ b/src/lib/ai-edition/store/projectStore.ts @@ -5,12 +5,7 @@ import { replaceTimeline as replaceTimelineOp, restoreFullTimeline as restoreFullTimelineOp, } from "../document/timeline"; -import { - type AxcutAsset, - type AxcutDocument, - type AxcutTranscript, - documentSchema, -} from "../schema"; +import { type AxcutAsset, type AxcutDocument, documentSchema } from "../schema"; // ponytail: thin Zustand wrapper over the native-bridge client. Keeps the // current project + revision counter in renderer memory; mutations round-trip @@ -45,7 +40,6 @@ export interface ProjectState { setDocument: (document: AxcutDocument) => void; replaceTimeline: (intervals: Interval[], reason: string) => Promise; restoreFullTimeline: () => Promise; - setTranscript: (transcript: AxcutTranscript) => Promise; setSourceDuration: (sec: number) => void; setCurrentTime: (sec: number) => void; setPlaying: (playing: boolean) => void; @@ -261,21 +255,6 @@ export const useProjectStore = create((set, get) => ({ await get().saveDocument(next); }, - async setTranscript(transcript) { - const doc = get().document; - if (!doc) throw new Error("No project loaded"); - const transcripts = [ - ...doc.transcripts.filter((t) => t.assetId !== transcript.assetId), - transcript, - ]; - const next: AxcutDocument = { - ...doc, - transcript: doc.project.primaryAssetId === transcript.assetId ? transcript : doc.transcript, - transcripts, - }; - await get().saveDocument(next); - }, - setSourceDuration(sec) { set({ sourceDurationSec: sec }); }, diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts new file mode 100644 index 000000000..d8208ddef --- /dev/null +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -0,0 +1,430 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { useProjectStore } from "./projectStore"; +import { useTranscriptionStore, whenTranscriptionIdle } from "./transcriptionStore"; + +const bridgeMocks = vi.hoisted(() => ({ + save: vi.fn(), +})); + +const transcribeMocks = vi.hoisted(() => ({ + transcribeAsset: vi.fn(), +})); + +const toastMocks = vi.hoisted(() => ({ + success: vi.fn(), + error: vi.fn(), +})); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { aiEdition: { save: bridgeMocks.save } }, +})); + +vi.mock("../document/transcribe", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, transcribeAsset: transcribeMocks.transcribeAsset }; +}); + +vi.mock("sonner", () => ({ toast: { success: toastMocks.success, error: toastMocks.error } })); + +function asset(id: string, extra: Record = {}) { + return { + id, + kind: "video" as const, + label: `${id}.mp4`, + originalPath: `/tmp/${id}.mp4`, + cameraTrack: null, + ...extra, + }; +} + +function makeDoc(assetIds: string[], projectId = "proj_1"): AxcutDocument { + return { + schemaVersion: 7, + project: { + id: projectId, + title: "Test", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + primaryAssetId: assetIds[0], + }, + assets: assetIds.map((id) => asset(id)), + transcript: null, + transcripts: [], + timeline: { + clips: [], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + legacyEditor: null, + } as unknown as AxcutDocument; +} + +function transcriptFor(assetId: string): AxcutTranscript { + return { + assetId, + language: "en", + segments: [ + { id: "seg_1", kind: "speech", startSec: 0, endSec: 1, text: "hello", wordIds: ["word_1"] }, + ], + words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "hello" }], + }; +} + +/** A promise plus a 0-arg release, so a mocked run can be held open mid-flight. */ +function deferred(): { promise: Promise; release: () => void } { + let release: () => void = () => { + // Replaced synchronously by the executor below, before this can be called. + }; + const promise = new Promise((resolve) => { + release = () => resolve(); + }); + return { promise, release: () => release() }; +} + +/** Loads a document into the project store the way `loadProject` would. */ +function loadDocument(document: AxcutDocument) { + useProjectStore.setState({ + projectId: document.project.id, + document, + status: "ready", + error: null, + dirty: false, + }); +} + +describe("useTranscriptionStore", () => { + beforeEach(() => { + useTranscriptionStore.getState().reset(); + useProjectStore.getState().clear(); + bridgeMocks.save.mockReset(); + // The bridge echoes back whatever it was handed, like a successful save. + bridgeMocks.save.mockImplementation(async (document: AxcutDocument) => ({ + success: true, + document, + })); + transcribeMocks.transcribeAsset.mockReset(); + toastMocks.success.mockReset(); + toastMocks.error.mockReset(); + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + (window as any).electronAPI = { stt: { transcribe: vi.fn() } }; + }); + + afterEach(() => { + vi.clearAllMocks(); + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + }); + + it("transcribes every asset that has no transcript, one at a time", async () => { + let inFlight = 0; + let maxInFlight = 0; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await Promise.resolve(); + inFlight -= 1; + return transcriptFor(assetId); + }, + ); + loadDocument(makeDoc(["asset_1", "asset_2"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(maxInFlight).toBe(1); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(2); + expect( + useProjectStore + .getState() + .document?.transcripts.map((t) => t.assetId) + .sort(), + ).toEqual(["asset_1", "asset_2"]); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + // The background pass stays quiet on success. + expect(toastMocks.success).not.toHaveBeenCalled(); + }); + + it("does not re-enqueue an asset whose transcript it just wrote", async () => { + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + const { sync } = useTranscriptionStore.getState(); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + // Every document change re-runs sync in the shell — this is the loop guard. + sync(useProjectStore.getState().document); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("remembers a no-audio verdict on the asset and never retries it by itself", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue( + new Error("No audio track found in this video."), + ); + loadDocument(makeDoc(["asset_1"])); + + const { sync } = useTranscriptionStore.getState(); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + const job = useTranscriptionStore.getState().jobs.asset_1; + expect(job?.status).toBe("failed"); + expect(job?.failure?.kind).toBe("no-audio"); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure?.kind).toBe( + "no-audio", + ); + // Silence is an expected outcome, not an incident. + expect(toastMocks.error).not.toHaveBeenCalled(); + + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + }); + + it("skips an asset that already carries a persisted failure on a fresh load", async () => { + const doc = makeDoc(["asset_1"]); + loadDocument({ + ...doc, + assets: [ + asset("asset_1", { transcriptionFailure: { kind: "no-audio", message: "silent" } }), + ] as AxcutDocument["assets"], + }); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("keeps a transient failure in memory only, and toasts it", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(useTranscriptionStore.getState().jobs.asset_1?.failure?.kind).toBe("error"); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure).toBeUndefined(); + expect(toastMocks.error).toHaveBeenCalledTimes(1); + }); + + it("stops the queue on an engine failure instead of failing each asset in turn", async () => { + // The model download died / whisper-server didn't come up: that verdict is + // about the engine, so the remaining assets inherit it rather than each + // spending a full retry budget and stacking an identical toast. + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1", "asset_2", "asset_3"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + const jobs = useTranscriptionStore.getState().jobs; + expect(Object.values(jobs).map((j) => j.status)).toEqual(["failed", "failed", "failed"]); + expect(jobs.asset_3?.failure?.message).toBe("whisper-server exited"); + expect(toastMocks.error).toHaveBeenCalledTimes(1); + }); + + it("request() re-runs a failed asset and clears the remembered verdict", async () => { + transcribeMocks.transcribeAsset.mockRejectedValueOnce( + new Error("No audio track found in this video."), + ); + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure?.kind).toBe( + "no-audio", + ); + + await useTranscriptionStore.getState().request("asset_1", "fr"); + + expect(transcribeMocks.transcribeAsset).toHaveBeenLastCalledWith( + expect.anything(), + "asset_1", + expect.objectContaining({ language: "fr" }), + ); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure).toBeNull(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + // A run the user asked for reports back. + expect(toastMocks.success).toHaveBeenCalledTimes(1); + }); + + it("drops the queue when another project is loaded", async () => { + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + const other = makeDoc(["asset_9"], "proj_2"); + loadDocument(other); + useTranscriptionStore.getState().sync(other); + expect(useTranscriptionStore.getState().projectId).toBe("proj_2"); + await whenTranscriptionIdle(); + + expect(useProjectStore.getState().document?.transcripts.map((t) => t.assetId)).toEqual([ + "asset_9", + ]); + }); + + it("forgets a job when its asset leaves the document", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1", "asset_2"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(Object.keys(useTranscriptionStore.getState().jobs)).toEqual(["asset_1", "asset_2"]); + + const doc = useProjectStore.getState().document as AxcutDocument; + const pruned = { ...doc, assets: doc.assets.filter((a) => a.id === "asset_1") }; + loadDocument(pruned); + useTranscriptionStore.getState().sync(pruned); + + expect(Object.keys(useTranscriptionStore.getState().jobs)).toEqual(["asset_1"]); + }); + + it("runs no background pass without a local STT engine", async () => { + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("lets a manual request supersede the background run of the same asset", async () => { + // The background pass is mid-run on asset_1 (auto language) when the user + // asks for French from the media card. The outgoing run must neither win + // the race nor delete the request that replaced it. + const languages: string[] = []; + const firstRun = deferred(); + transcribeMocks.transcribeAsset.mockImplementation( + async ( + _doc: AxcutDocument, + assetId: string, + options: { language?: string; signal?: AbortSignal }, + ) => { + languages.push(options.language ?? "auto"); + if (languages.length === 1) { + await firstRun.promise; + throw new DOMException("Aborted", "AbortError"); + } + return { ...transcriptFor(assetId), language: options.language ?? "auto" }; + }, + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await Promise.resolve(); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); + + const requested = useTranscriptionStore.getState().request("asset_1", "fr"); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("queued"); + firstRun.release(); + await requested; + await whenTranscriptionIdle(); + + expect(languages).toEqual(["auto", "fr"]); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + expect(useProjectStore.getState().document?.transcripts[0].language).toBe("fr"); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("requestTimelineTranscripts covers the timeline's media, skipping the silent ones", async () => { + // The pane button used to target `primaryAssetId` only — which in a + // recording project is the (often silent) screen capture, leaving the + // talking clip next to it untranscribable from there. + // The background pass is off here so the assertions see only what the + // button itself asked for. + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + const doc = makeDoc(["silent", "voice", "offTimeline"]); + loadDocument({ + ...doc, + assets: [ + asset("silent", { transcriptionFailure: { kind: "no-audio", message: "silent" } }), + asset("voice"), + asset("offTimeline"), + ], + timeline: { + ...doc.timeline, + clips: ["silent", "voice"].map((assetId, i) => ({ + id: `clip_${i}`, + assetId, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + wordRefs: [], + origin: "user", + reason: "", + })), + }, + } as unknown as AxcutDocument); + + await useTranscriptionStore.getState().requestTimelineTranscripts(); + + expect(transcribeMocks.transcribeAsset.mock.calls.map((c) => c[1])).toEqual(["voice"]); + expect(useProjectStore.getState().document?.transcripts.map((t) => t.assetId)).toEqual([ + "voice", + ]); + }); + + it("waits for the background run instead of transcribing the same asset twice", async () => { + const firstRun = deferred(); + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => { + await firstRun.promise; + return transcriptFor(assetId); + }, + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await Promise.resolve(); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); + + const requested = useTranscriptionStore.getState().requestTimelineTranscripts(); + firstRun.release(); + await requested; + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + }); + + it("still honours a manual request without the auto pass", async () => { + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + await useTranscriptionStore.getState().request("asset_1"); + + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + }); +}); diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts new file mode 100644 index 000000000..9d765afc3 --- /dev/null +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -0,0 +1,519 @@ +// Single source of truth for "where is each asset's transcript?". +// +// Transcription is local (whisper.cpp, no network), so there is no reason to +// make the user go and ask for it: every asset that lands in the document — +// imported from the Media tab, or auto-added from a screen recording — is +// queued here and transcribed in the background. The transcript itself still +// lives on the document (`document.transcripts[]`); this store only owns the +// JOB: queued / running / failed, plus the phase for the spinner. Nothing +// derives "is there a transcript?" from here — that answer comes from the +// document, and the two are folded together by `deriveAssetStatus`. +// +// Loop safety, which is the whole difficulty of an auto pass whose result +// mutates the document it reacts to: +// +// - `sync` only ever enqueues an asset that has NO transcript, NO job entry +// (queued / running / failed alike) and NO persisted failure. A finished +// run leaves a transcript on the document, a failed one leaves a `failed` +// entry here, so neither can be picked up twice. +// - the job entry is deleted only AFTER the save has resolved, i.e. after +// the document already carries the transcript. +// - the pump is a single sequential loop (whisper-server is one process and +// audio extraction is memory-hungry), guarded by a module-level promise, +// and it drops any job a run left behind rather than spinning on it. + +import { useEffect, useMemo } from "react"; +import { toast } from "sonner"; +import { create } from "zustand"; +import { DEFAULT_LOCALE, LOCALE_STORAGE_KEY, type Locale } from "@/i18n/config"; +import { getAvailableLocales, translate } from "@/i18n/loader"; +import { transcribeAsset, withTranscript } from "../document/transcribe"; +import type { AxcutDocument } from "../schema"; +import { + type AssetTranscriptionView, + classifyTranscriptionError, + deriveAssetStatus, + findAssetTranscript, + isAbortError, + isPermanentFailure, + resolveTranscriptGate, + type TranscriptGate, + type TranscriptionFailure, + type TranscriptionPhase, + transcriptRelevantAssetIds, +} from "../transcription/status"; +import { useProjectStore } from "./projectStore"; + +export interface TranscriptionJob { + status: "queued" | "running" | "failed"; + /** Set when a run picks the job up. Identifies THIS attempt, so a run that + * finishes after the user asked for another one cannot clear its successor. */ + runId?: number; + phase?: TranscriptionPhase; + /** `"auto"` unless the user forced a language from the media card. */ + language: string; + failure?: TranscriptionFailure; + /** User-triggered runs get a toast on success; the background pass stays quiet. */ + manual: boolean; +} + +interface TranscriptionState { + /** Project the jobs belong to — switching projects drops them all. */ + projectId: string | null; + jobs: Record; + + /** Reconcile the queue with a document. Idempotent; safe to call on every document change. */ + sync: (document: AxcutDocument | null) => void; + /** Transcribe (or re-transcribe) one asset now. Resolves once the run settles. */ + request: (assetId: string, language?: string) => Promise; + /** + * What the panes' "Transcribe now" button asks for: every asset the timeline + * plays that still has no transcript. NOT just the primary asset — a project + * whose first (primary) media is a silent screen capture would otherwise + * leave that button unable to transcribe the talking clip next to it. + */ + requestTimelineTranscripts: () => Promise; + reset: () => void; +} + +/** The local engine is only reachable through the preload bridge. */ +function hasLocalSttEngine(): boolean { + if (typeof window === "undefined") return false; + return typeof window.electronAPI?.stt?.transcribe === "function"; +} + +/** + * Toasts fired outside React still have to speak the user's language. Same + * source as `I18nProvider` (stored preference, else the default), validated so + * a stale value can't push `translate` onto a locale it doesn't have. + */ +function toastText(key: string, vars?: Record): string { + let locale: Locale = DEFAULT_LOCALE; + try { + const stored = localStorage.getItem(LOCALE_STORAGE_KEY); + if (stored && getAvailableLocales().includes(stored as Locale)) locale = stored as Locale; + } catch { + // localStorage may be unavailable — the default locale is a fine answer. + } + return translate(locale, "editor", key, vars); +} + +export const useTranscriptionStore = create((set, get) => ({ + projectId: null, + jobs: {}, + + sync(document) { + if (!document) { + if (get().projectId !== null || Object.keys(get().jobs).length > 0) get().reset(); + return; + } + if (document.project.id !== get().projectId) { + get().reset(); + set({ projectId: document.project.id }); + } + + const assetIds = new Set(document.assets.map((a) => a.id)); + const jobs = get().jobs; + let next: Record | null = null; + const patch = () => { + if (!next) next = { ...jobs }; + return next; + }; + + // An asset the user removed takes its job with it (the run itself is + // dropped by `runJob`, which re-reads the document before starting). + for (const assetId of Object.keys(jobs)) { + if (!assetIds.has(assetId)) delete patch()[assetId]; + } + + if (hasLocalSttEngine()) { + for (const asset of document.assets) { + if (jobs[asset.id]) continue; + if (findAssetTranscript(document, asset.id)) continue; + if (asset.transcriptionFailure) continue; + patch()[asset.id] = { status: "queued", language: "auto", manual: false }; + } + } + + if (next) { + set({ jobs: next }); + void pump(); + } + }, + + request(assetId, language = "auto") { + // A manual run can be the first thing that happens in a project (the + // auto pass is off without a local engine), so adopt the loaded project + // before queueing — `runJob` refuses to write into a document the queue + // doesn't belong to. + const document = useProjectStore.getState().document; + if (document && document.project.id !== get().projectId) get().sync(document); + // Asking again for an asset that is mid-run (regenerate in another + // language while the background pass is on it) supersedes that run + // instead of queueing behind it and losing the language. + if (activeRun?.assetId === assetId) abortActiveRun(); + const settled = waitForSettle(assetId); + set((state) => ({ + jobs: { ...state.jobs, [assetId]: { status: "queued", language, manual: true } }, + })); + void pump(); + return settled; + }, + + requestTimelineTranscripts() { + const document = useProjectStore.getState().document; + if (!document) return Promise.resolve(); + get().sync(document); + const targets = transcriptRelevantAssetIds(document).filter((assetId) => { + if (findAssetTranscript(document, assetId)) return false; + // A media with no audio track can only fail again — asking for it here + // would buy the user a run and an error toast for nothing. The per-asset + // regenerate in the media stage stays available for the stubborn case. + return !document.assets.find((a) => a.id === assetId)?.transcriptionFailure; + }); + if (targets.length === 0) return Promise.resolve(); + return Promise.all( + targets.map((assetId) => { + // Already queued or running: the background pass owns that run, so + // wait for it instead of superseding it with an identical one. + const job = get().jobs[assetId]; + if (job && job.status !== "failed") return waitForSettle(assetId); + return get().request(assetId); + }), + ).then(() => undefined); + }, + + reset() { + abortActiveRun(); + const pending = Object.keys(get().jobs); + set({ projectId: null, jobs: {} }); + for (const assetId of pending) flushSettleWaiters(assetId); + }, +})); + +// ─── The pump ────────────────────────────────────────────────────── +// Module state, not store state: none of it is rendered, and keeping it out of +// the store means a re-render can never observe a half-started run. + +let pumping: Promise | null = null; +let activeRun: { assetId: string; controller: AbortController } | null = null; +const settleWaiters = new Map void>>(); + +function waitForSettle(assetId: string): Promise { + return new Promise((resolve) => { + const waiters = settleWaiters.get(assetId); + if (waiters) waiters.push(resolve); + else settleWaiters.set(assetId, [resolve]); + }); +} + +function flushSettleWaiters(assetId: string): void { + const waiters = settleWaiters.get(assetId); + if (!waiters) return; + settleWaiters.delete(assetId); + for (const resolve of waiters) resolve(); +} + +function abortActiveRun(): void { + activeRun?.controller.abort(); + activeRun = null; +} + +/** + * Hand a queued job to a run: stamps it with the run's id, which every later + * write checks. A `request` made mid-run replaces the entry with a fresh + * (unstamped) one, and that stamp is what stops the outgoing run from + * reporting its own status — or its deletion — over its successor. + */ +function claimJob(projectId: string, assetId: string, runId: number): boolean { + let claimed = false; + useTranscriptionStore.setState((state) => { + if (state.projectId !== projectId) return state; + const job = state.jobs[assetId]; + if (!job || job.status !== "queued") return state; + claimed = true; + return { + jobs: { + ...state.jobs, + [assetId]: { + ...job, + runId, + status: "running", + phase: "extracting-audio", + failure: undefined, + }, + }, + }; + }); + return claimed; +} + +/** Patch the job a run owns. No-op once that run has been superseded. */ +function patchJob(assetId: string, runId: number, patch: Partial): void { + useTranscriptionStore.setState((state) => { + const job = state.jobs[assetId]; + if (!job || job.runId !== runId) return state; + return { jobs: { ...state.jobs, [assetId]: { ...job, ...patch } } }; + }); +} + +/** + * Give every still-queued job the verdict that just came back from the engine. + * Waiters are flushed so a `requestTimelineTranscripts()` awaiting the batch + * settles instead of hanging on runs that will never happen. + */ +function failRemainingQueue(projectId: string, failure: TranscriptionFailure): void { + const queued = Object.entries(useTranscriptionStore.getState().jobs) + .filter(([, job]) => job.status === "queued") + .map(([assetId]) => assetId); + if (queued.length === 0) return; + useTranscriptionStore.setState((state) => { + if (state.projectId !== projectId) return state; + const jobs = { ...state.jobs }; + for (const assetId of queued) { + const job = jobs[assetId]; + if (job?.status !== "queued") continue; + jobs[assetId] = { ...job, status: "failed", phase: undefined, failure }; + } + return { jobs }; + }); + for (const assetId of queued) flushSettleWaiters(assetId); +} + +/** True while `runId` is still the attempt the store is tracking for this asset. */ +function isCurrentRun(assetId: string, runId: number): boolean { + return useTranscriptionStore.getState().jobs[assetId]?.runId === runId; +} + +/** + * Remove a job once it has settled. With a `runId`, only the entry that run + * owns. Waiters are flushed whenever the entry is gone: a caller superseded by + * a newer request is waiting on that newer run, which flushes them in turn. + */ +function dropJob(assetId: string, runId?: number): void { + useTranscriptionStore.setState((state) => { + const job = state.jobs[assetId]; + if (!job) return state; + if (runId !== undefined && job.runId !== runId) return state; + const jobs = { ...state.jobs }; + delete jobs[assetId]; + return { jobs }; + }); + if (useTranscriptionStore.getState().jobs[assetId] === undefined) flushSettleWaiters(assetId); +} + +/** + * Remember a deterministic failure on the asset so the next project open shows + * "no audio" straight away instead of re-extracting the audio to rediscover it. + * Best-effort: a save that loses a race with a user edit is not worth a toast. + */ +async function persistPermanentFailure( + projectId: string, + assetId: string, + failure: TranscriptionFailure, +): Promise { + const kind = failure.kind; + if (!isPermanentFailure(kind)) return; + const project = useProjectStore.getState(); + const doc = project.document; + if (!doc || doc.project.id !== projectId) return; + if (!doc.assets.some((a) => a.id === assetId)) return; + try { + await project.saveDocument({ + ...doc, + assets: doc.assets.map((a) => + a.id === assetId + ? { + ...a, + transcriptionFailure: { + kind, + message: failure.message, + at: new Date().toISOString(), + }, + } + : a, + ), + }); + } catch (error) { + console.warn("[transcription] could not persist the failure on the asset:", error); + } +} + +let runSeq = 0; + +async function runJob(assetId: string, job: TranscriptionJob): Promise { + const projectId = useTranscriptionStore.getState().projectId; + const doc = useProjectStore.getState().document; + if ( + !projectId || + !doc || + doc.project.id !== projectId || + !doc.assets.some((a) => a.id === assetId) + ) { + // Nothing this run could legally write to. Drop it rather than leave it + // queued — `drain` would otherwise pick the same job forever. + dropJob(assetId); + return; + } + + const runId = ++runSeq; + if (!claimJob(projectId, assetId, runId)) return; + const controller = new AbortController(); + activeRun = { assetId, controller }; + + try { + const transcript = await transcribeAsset(doc, assetId, { + language: job.language, + signal: controller.signal, + onStatus: (phase) => patchJob(assetId, runId, { phase: phase as TranscriptionPhase }), + }); + if (controller.signal.aborted) { + dropJob(assetId, runId); + return; + } + // The user may have switched projects while whisper was working — writing + // the transcript now would attach it to the document that is loaded today. + const current = useProjectStore.getState().document; + if (!current || current.project.id !== projectId) { + dropJob(assetId, runId); + return; + } + // One save: the transcript, and (on a successful retry) the removal of + // the verdict remembered on the asset. + await useProjectStore.getState().saveDocument( + withTranscript( + { + ...current, + assets: current.assets.map((a) => + a.id === assetId && a.transcriptionFailure ? { ...a, transcriptionFailure: null } : a, + ), + }, + transcript, + ), + ); + dropJob(assetId, runId); + if (job.manual) toast.success(toastText("mediaStage.transcriptReady")); + } catch (error) { + if (isAbortError(error) || controller.signal.aborted) { + dropJob(assetId, runId); + return; + } + if (!isCurrentRun(assetId, runId)) return; // superseded by a newer request + const failure = classifyTranscriptionError(error); + patchJob(assetId, runId, { status: "failed", phase: undefined, failure }); + flushSettleWaiters(assetId); + await persistPermanentFailure(projectId, assetId, failure); + // A transient failure is about the ENGINE, not about this media: the model + // download died, whisper-server didn't come up. Marching the rest of the + // queue into the same wall would spend a full retry budget per asset and + // stack one identical toast per asset. Fail them with the same verdict + // instead — the gate then reads "failed" (not "queued forever"), and one + // manual retry re-runs them all once the engine is back. + if (failure.kind === "error") failRemainingQueue(projectId, failure); + // A silent recording is an expected outcome, not an incident: the media + // card and every gated button already say so. Only surface the noisy + // (retryable) failures, plus anything the user asked for by hand. + if (failure.kind === "error" || job.manual) { + toast.error(toastText("mediaStage.transcriptionFailed"), { description: failure.message }); + } + } finally { + if (activeRun?.controller === controller) activeRun = null; + } +} + +function nextQueuedJob(): [string, TranscriptionJob] | null { + const { jobs } = useTranscriptionStore.getState(); + for (const [assetId, job] of Object.entries(jobs)) { + if (job.status === "queued") return [assetId, job]; + } + return null; +} + +async function drain(): Promise { + for (;;) { + const next = nextQueuedJob(); + if (!next) return; + const [assetId, job] = next; + await runJob(assetId, job); + // Belt and braces: a run that neither settled nor failed its job would + // make this loop spin. Drop it and move on. + if (useTranscriptionStore.getState().jobs[assetId] === job) { + console.warn("[transcription] job left queued after a run, dropping:", assetId); + dropJob(assetId); + } + } +} + +function pump(): Promise { + if (pumping) return pumping; + pumping = drain().finally(() => { + pumping = null; + }); + return pumping; +} + +/** Test/diagnostic helper: resolves once the queue has drained. */ +export function whenTranscriptionIdle(): Promise { + return pumping ?? Promise.resolve(); +} + +// ─── React bindings ──────────────────────────────────────────────── + +/** + * Mount once (the editor shell does). Keeps the queue reconciled with whatever + * document is loaded — a new project, an imported asset, a removed one. + */ +export function useAutoTranscription(): void { + const document = useProjectStore((s) => s.document); + const sync = useTranscriptionStore((s) => s.sync); + useEffect(() => { + sync(document); + }, [document, sync]); +} + +/** + * Per-asset transcription state, keyed by asset id — one subscription for a + * whole media list instead of a hook per row (which a `.map()` can't have). + * + * There is deliberately no single-asset variant: every consumer either lists + * media (this) or asks about a transcript-dependent action, and the answer for + * an action is `useTimelineTranscriptGate` — resolved over the assets the + * timeline plays, never over one asset picked as representative. + */ +export function useAssetTranscriptions(): Record { + const document = useProjectStore((s) => s.document); + const jobs = useTranscriptionStore((s) => s.jobs); + return useMemo(() => { + const views: Record = {}; + for (const asset of document?.assets ?? []) { + views[asset.id] = deriveAssetStatus({ + assetId: asset.id, + job: jobs[asset.id], + transcript: findAssetTranscript(document, asset.id), + persistedFailure: asset.transcriptionFailure, + }); + } + return views; + }, [document, jobs]); +} + +/** + * Gate for the transcript-dependent timeline actions (Smart cuts): resolved + * over the assets the timeline actually plays. + */ +export function useTimelineTranscriptGate(): TranscriptGate { + const document = useProjectStore((s) => s.document); + const jobs = useTranscriptionStore((s) => s.jobs); + return useMemo(() => { + const views = transcriptRelevantAssetIds(document).map((assetId) => + deriveAssetStatus({ + assetId, + job: jobs[assetId], + transcript: findAssetTranscript(document, assetId), + persistedFailure: + document?.assets.find((a) => a.id === assetId)?.transcriptionFailure ?? null, + }), + ); + return resolveTranscriptGate(views); + }, [document, jobs]); +} diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts new file mode 100644 index 000000000..adc21cd35 --- /dev/null +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it } from "vitest"; +import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { + type AssetTranscriptionView, + classifyTranscriptionError, + deriveAssetStatus, + isPermanentFailure, + resolveTranscriptGate, + transcriptHasSpeech, + transcriptRelevantAssetIds, +} from "./status"; + +function transcript(assetId: string, words: string[]): AxcutTranscript { + return { + assetId, + language: "en", + segments: words.map((text, i) => ({ + id: `seg_${i}`, + kind: "speech" as const, + startSec: i, + endSec: i + 1, + text, + wordIds: [`word_${i}`], + })), + words: words.map((text, i) => ({ + id: `word_${i}`, + segmentId: `seg_${i}`, + startSec: i, + endSec: i + 1, + text, + })), + }; +} + +function view( + assetId: string, + status: AssetTranscriptionView["status"], + failureKind?: "no-audio" | "unsupported-audio" | "error", +): AssetTranscriptionView { + return { + assetId, + status, + failure: failureKind ? { kind: failureKind, message: `${failureKind} boom` } : undefined, + }; +} + +describe("classifyTranscriptionError", () => { + it("recognises a container with no audio track", () => { + const failure = classifyTranscriptionError(new Error("No audio track found in this video.")); + expect(failure.kind).toBe("no-audio"); + expect(isPermanentFailure(failure.kind)).toBe(true); + }); + + it("treats a decode that yielded nothing as no-audio too", () => { + expect( + classifyTranscriptionError(new Error("Decoded zero audio frames from this video.")).kind, + ).toBe("no-audio"); + }); + + it("recognises an audio codec the caption path cannot read", () => { + const failure = classifyTranscriptionError( + new Error("Audio codec not supported for captions: ac-3"), + ); + expect(failure.kind).toBe("unsupported-audio"); + expect(isPermanentFailure(failure.kind)).toBe(true); + }); + + it("treats anything else as a transient error worth retrying", () => { + const failure = classifyTranscriptionError(new Error("whisper-server exited")); + expect(failure.kind).toBe("error"); + expect(failure.message).toBe("whisper-server exited"); + expect(isPermanentFailure(failure.kind)).toBe(false); + }); +}); + +describe("transcriptHasSpeech", () => { + it("is false for a transcript whisper returned empty", () => { + expect(transcriptHasSpeech(transcript("asset_1", []))).toBe(false); + }); + + it("is true as soon as one word came back", () => { + expect(transcriptHasSpeech(transcript("asset_1", ["hello"]))).toBe(true); + }); +}); + +describe("deriveAssetStatus", () => { + it("reports a live job over an existing transcript (regenerate)", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "running", phase: "transcribing" }, + transcript: transcript("asset_1", ["hello"]), + }); + expect(derived).toEqual({ assetId: "asset_1", status: "running", phase: "transcribing" }); + }); + + it("reports ready from the document, with no job at all", () => { + expect( + deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", ["hello"]) }) + .status, + ).toBe("ready"); + }); + + it("distinguishes an empty transcript from a ready one", () => { + expect( + deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", []) }).status, + ).toBe("empty"); + }); + + it("keeps a stored transcript ready when a regenerate over it failed", () => { + // The failed retry left the previous transcript untouched on the document: + // reading the asset as "failed" would disable Smart cuts over a transcript + // that is right there and usable. + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "failed", failure: { kind: "error", message: "whisper-server exited" } }, + transcript: transcript("asset_1", ["hello"]), + }); + expect(derived.status).toBe("ready"); + // …and the failure still travels, so a tooltip can explain the red flash. + expect(derived.failure?.message).toBe("whisper-server exited"); + }); + + it("reports failed only when nothing was ever produced", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "failed", failure: { kind: "error", message: "boom" } }, + }); + expect(derived.status).toBe("failed"); + }); + + it("falls back to the failure remembered on the asset across reloads", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + persistedFailure: { kind: "no-audio", message: "No audio track found in this video." }, + }); + expect(derived.status).toBe("failed"); + expect(derived.failure?.kind).toBe("no-audio"); + }); + + it("is idle when nothing has been attempted", () => { + expect(deriveAssetStatus({ assetId: "asset_1" }).status).toBe("idle"); + }); +}); + +describe("resolveTranscriptGate", () => { + it("blocks with no-media when the project is empty", () => { + expect(resolveTranscriptGate([])).toEqual({ + state: "blocked", + reason: "no-media", + pendingCount: 0, + }); + }); + + it("opens once an asset has speech", () => { + expect(resolveTranscriptGate([view("a", "ready")]).state).toBe("ready"); + }); + + it("waits while any asset is still in flight, even next to a ready one", () => { + const gate = resolveTranscriptGate([view("a", "ready"), view("b", "running")]); + expect(gate.state).toBe("pending"); + expect(gate.pendingCount).toBe(1); + }); + + it("counts queued assets as pending", () => { + expect(resolveTranscriptGate([view("a", "queued"), view("b", "queued")]).pendingCount).toBe(2); + }); + + it("blocks on no-audio when every media is silent", () => { + const gate = resolveTranscriptGate([ + view("a", "failed", "no-audio"), + view("b", "failed", "unsupported-audio"), + ]); + expect(gate.state).toBe("blocked"); + expect(gate.reason).toBe("no-audio"); + }); + + it("blocks on failed (retryable) as soon as one failure is not about silence", () => { + const gate = resolveTranscriptGate([ + view("a", "failed", "no-audio"), + view("b", "failed", "error"), + ]); + expect(gate.reason).toBe("failed"); + expect(gate.message).toContain("boom"); + }); + + it("stays ready when a failed retry sits on top of a usable transcript", () => { + const gate = resolveTranscriptGate([ + { + assetId: "a", + status: "ready", + failure: { kind: "error", message: "whisper-server exited" }, + }, + ]); + expect(gate.state).toBe("ready"); + }); + + it("blocks on no-speech when the transcripts came back empty", () => { + expect(resolveTranscriptGate([view("a", "empty")]).reason).toBe("no-speech"); + }); + + it("blocks on not-started when nothing ran (no local engine)", () => { + expect(resolveTranscriptGate([view("a", "idle")]).reason).toBe("not-started"); + }); +}); + +describe("transcriptRelevantAssetIds", () => { + const base = { + schemaVersion: 7 as const, + project: { + id: "proj_1", + title: "T", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + }, + transcript: null, + transcripts: [], + annotations: [], + zoomRanges: [], + legacyEditor: null, + }; + + function doc(assetIds: string[], clipAssetIds: string[]): AxcutDocument { + return { + ...base, + assets: assetIds.map((id) => ({ + id, + kind: "video" as const, + label: id, + originalPath: `/tmp/${id}.mp4`, + cameraTrack: null, + })), + timeline: { + clips: clipAssetIds.map((assetId, i) => ({ + id: `clip_${i}`, + assetId, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + wordRefs: [], + origin: "system" as const, + reason: "", + })), + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + } as AxcutDocument; + } + + it("only counts the assets the timeline plays", () => { + expect(transcriptRelevantAssetIds(doc(["a", "b"], ["a", "a"]))).toEqual(["a"]); + }); + + it("falls back to the whole media bin while the timeline is empty", () => { + expect(transcriptRelevantAssetIds(doc(["a", "b"], []))).toEqual(["a", "b"]); + }); + + it("ignores clips pointing at a removed asset", () => { + expect(transcriptRelevantAssetIds(doc(["a"], ["ghost"]))).toEqual(["a"]); + }); + + it("has nothing to say about a missing document", () => { + expect(transcriptRelevantAssetIds(null)).toEqual([]); + }); +}); diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts new file mode 100644 index 000000000..fced8390a --- /dev/null +++ b/src/lib/ai-edition/transcription/status.ts @@ -0,0 +1,219 @@ +// Pure status logic for the transcription pipeline: what state one asset's +// transcript is in, and whether a transcript-dependent action (Smart cuts, +// captions…) may run right now. +// +// Kept free of React and of both stores so the rules can be unit-tested on +// plain data — `store/transcriptionStore.ts` owns the queue and the side +// effects, this module owns the vocabulary. + +import type { AxcutDocument, AxcutTranscript } from "../schema"; + +/** Why a transcription run could not produce anything. */ +export type TranscriptionFailureKind = "no-audio" | "unsupported-audio" | "error"; + +export interface TranscriptionFailure { + kind: TranscriptionFailureKind; + /** Raw engine/exception message — surfaced as a tooltip / toast description. */ + message: string; +} + +/** Which half of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ +export type TranscriptionPhase = "extracting-audio" | "transcribing"; + +/** + * A media that has no audio track (or one Whisper cannot read) will fail the + * same way on every attempt, so that verdict is worth remembering: it is + * persisted on the asset and stops the auto pass from re-extracting the audio + * of a silent screen recording on every project open. Everything else + * ("error") is treated as transient and retried on the next load. + */ +export function isPermanentFailure(kind: TranscriptionFailureKind): kind is PersistableFailureKind { + return kind !== "error"; +} + +/** The failure kinds `assetSchema.transcriptionFailure` accepts. */ +export type PersistableFailureKind = Exclude; + +/** + * Map an exception out of `transcribeAsset` onto a failure the UI can explain. + * The two deterministic cases come from `extractMono16kWebDemuxer` — it is the + * only layer that knows whether the container actually holds audio. + */ +export function classifyTranscriptionError(error: unknown): TranscriptionFailure { + const message = error instanceof Error ? error.message : String(error); + if (/no audio track/i.test(message) || /zero audio frames/i.test(message)) { + return { kind: "no-audio", message }; + } + if (/audio codec not supported/i.test(message)) { + return { kind: "unsupported-audio", message }; + } + return { kind: "error", message }; +} + +export function isAbortError(error: unknown): boolean { + return ( + (error instanceof DOMException && error.name === "AbortError") || + (error instanceof Error && error.name === "AbortError") + ); +} + +export type AssetTranscriptionStatus = + /** Nothing attempted yet (no local engine, or the auto pass hasn't reached it). */ + | "idle" + | "queued" + | "running" + /** A transcript exists and holds at least one word. */ + | "ready" + /** A transcript exists but Whisper heard no speech — nothing for the agent to cut on. */ + | "empty" + | "failed"; + +export interface AssetTranscriptionView { + assetId: string; + status: AssetTranscriptionStatus; + phase?: TranscriptionPhase; + failure?: TranscriptionFailure; +} + +/** In-flight (or last-failed) state of one asset's job. Mirrors the store entry. */ +export interface TranscriptionJobLike { + status: "queued" | "running" | "failed"; + phase?: TranscriptionPhase; + failure?: TranscriptionFailure; +} + +export function findAssetTranscript( + document: AxcutDocument | null, + assetId: string, +): AxcutTranscript | null { + if (!document) return null; + return ( + document.transcripts.find((t) => t.assetId === assetId) ?? + (document.transcript?.assetId === assetId ? document.transcript : null) + ); +} + +/** A transcript with no word is "empty", not "ready": captions and AI cuts have nothing to work with. */ +export function transcriptHasSpeech(transcript: AxcutTranscript | null): boolean { + if (!transcript) return false; + return transcript.words.length > 0 || transcript.segments.some((s) => s.text.trim().length > 0); +} + +/** + * Fold the live job (if any), the persisted failure (if any) and the stored + * transcript into the single status the UI renders. Precedence, in order: + * + * 1. A run in flight — a regenerate over an existing transcript must read as + * "running", not "ready". + * 2. A stored transcript — it OUTRANKS a failed job on purpose. A regenerate + * that fails (whisper restart, decode hiccup) leaves the previous + * transcript untouched on the document, and it is still perfectly usable: + * the pane renders it, captions read it, the agent can cut on it. Reading + * that asset as "failed" would have disabled Smart cuts for the rest of the + * session over a transcript that is right there. The failure still travels + * on the view (tooltips surface it), it just doesn't veto the content. + * 3. Only then a failure — nothing was ever produced for this asset. + */ +export function deriveAssetStatus(input: { + assetId: string; + job?: TranscriptionJobLike; + transcript?: AxcutTranscript | null; + persistedFailure?: TranscriptionFailure | null; +}): AssetTranscriptionView { + const { assetId, job, transcript, persistedFailure } = input; + if (job && job.status !== "failed") { + return { assetId, status: job.status, phase: job.phase }; + } + if (transcript) { + return { + assetId, + status: transcriptHasSpeech(transcript) ? "ready" : "empty", + failure: job?.failure, + }; + } + if (job?.status === "failed") { + return { assetId, status: "failed", failure: job.failure }; + } + if (persistedFailure) { + return { assetId, status: "failed", failure: persistedFailure }; + } + return { assetId, status: "idle" }; +} + +export type TranscriptGateState = "ready" | "pending" | "blocked"; + +export type TranscriptGateReason = + /** The project holds no media at all. */ + | "no-media" + /** Every media is silent (no audio track / unreadable audio). */ + | "no-audio" + /** At least one run failed for a reason worth retrying. */ + | "failed" + /** Transcripts exist but hold no speech. */ + | "no-speech" + /** Nothing has been transcribed yet and nothing is running (no local engine). */ + | "not-started"; + +export interface TranscriptGate { + state: TranscriptGateState; + /** Null when `state === "ready"`. */ + reason: TranscriptGateReason | null; + /** Engine message behind a `failed` reason, for the tooltip/description. */ + message?: string; + /** How many assets are still queued or running — drives the "2 remaining" hint. */ + pendingCount: number; +} + +/** + * Decide whether a transcript-dependent action may run over a set of assets. + * + * Pending beats ready on purpose: with one media transcribed and another still + * running, letting the agent loose now would have it plan cuts against half the + * timeline and then watch the document change underneath it. + */ +export function resolveTranscriptGate(views: AssetTranscriptionView[]): TranscriptGate { + if (views.length === 0) { + return { state: "blocked", reason: "no-media", pendingCount: 0 }; + } + const pendingCount = views.filter((v) => v.status === "queued" || v.status === "running").length; + if (pendingCount > 0) { + return { state: "pending", reason: null, pendingCount }; + } + if (views.some((v) => v.status === "ready")) { + return { state: "ready", reason: null, pendingCount: 0 }; + } + const failures = views.filter((v) => v.status === "failed"); + if (failures.length > 0) { + const everyFailureIsSilence = failures.every( + (v) => v.failure?.kind === "no-audio" || v.failure?.kind === "unsupported-audio", + ); + return { + state: "blocked", + reason: everyFailureIsSilence ? "no-audio" : "failed", + message: failures.find((v) => v.failure?.message)?.failure?.message, + pendingCount: 0, + }; + } + if (views.some((v) => v.status === "empty")) { + return { state: "blocked", reason: "no-speech", pendingCount: 0 }; + } + return { state: "blocked", reason: "not-started", pendingCount: 0 }; +} + +/** + * Assets a transcript-dependent timeline action actually depends on: the ones + * the timeline plays. An asset sitting in the media bin but not on the timeline + * must not keep the Smart-cuts entry disabled — and, symmetrically, must not + * make it look ready when the clip on screen has no transcript. Falls back to + * the whole bin while the timeline is still empty. + */ +export function transcriptRelevantAssetIds(document: AxcutDocument | null): string[] { + if (!document) return []; + const onTimeline: string[] = []; + for (const clip of document.timeline.clips) { + if (!onTimeline.includes(clip.assetId)) onTimeline.push(clip.assetId); + } + const known = new Set(document.assets.map((a) => a.id)); + const filtered = onTimeline.filter((id) => known.has(id)); + return filtered.length > 0 ? filtered : document.assets.map((a) => a.id); +} diff --git a/src/lib/captioning/extractMono16k.ts b/src/lib/captioning/extractMono16k.ts index bf2c7320f..1116bb9a2 100644 --- a/src/lib/captioning/extractMono16k.ts +++ b/src/lib/captioning/extractMono16k.ts @@ -57,13 +57,23 @@ async function loadSourceVideoFile(videoUrl: string, signal?: AbortSignal): Prom function mixToMono(audioBuffer: AudioBuffer): Float32Array { const { length, numberOfChannels } = audioBuffer; + if (numberOfChannels === 0) return new Float32Array(length); + // `getChannelData` is a WebIDL call, so calling it INSIDE the sample loop cost + // one call per sample per channel — ~57 M of them for a ten-minute stereo + // recording, seconds of blocked main thread. That was survivable while + // transcription only ran when the user asked for it; it now runs by itself when + // a project opens, and a frozen window (spinners included) is exactly what the + // automatic pass must not look like. Hoisting the channel arrays out of the loop + // leaves plain typed-array indexing. + const channels: Float32Array[] = []; + for (let c = 0; c < numberOfChannels; c++) channels.push(audioBuffer.getChannelData(c)); + // Mono source: the mixdown is a copy. `slice` keeps the caller's contract of + // owning its buffer (the AudioBuffer's own array is reused by the context). + if (numberOfChannels === 1) return channels[0].slice(); const out = new Float32Array(length); - if (numberOfChannels === 0) return out; for (let i = 0; i < length; i++) { let sum = 0; - for (let c = 0; c < numberOfChannels; c++) { - sum += audioBuffer.getChannelData(c)[i]; - } + for (let c = 0; c < numberOfChannels; c++) sum += channels[c][i]; out[i] = sum / numberOfChannels; } return out; diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index c9694ec79..cf60e7c6b 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -13,7 +13,9 @@ the transcript). ```mermaid flowchart LR - A["Recorded audio"] -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] + Z["Asset added
(import or recording)"] -- "sync()" --> Y["transcriptionStore queue
(src/lib/ai-edition/store/transcriptionStore.ts)"] + Y -- "one job at a time" --> A["Recorded audio"] + A -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] B -- "IPC: Float32Array + language" --> C["SttManager
(electron/stt/index.ts)"] C -- "POST /inference (WAV)" --> D["whisper-stt-server
(electron/native/whisper-stt/)"] D -- "whisper_full() + DTW" --> E["SttTranscribeResponse
(segments + wordSegments)"] @@ -33,6 +35,103 @@ result is mapped back onto the `AxcutTranscript` shape the rest of the editor reads. From the moment that transcript is persisted, captions and the words they show cannot drift — captions are a derived view, not a parallel store. +## Auto-transcription + +Recognition is local and free, so the editor does not wait to be asked: every +asset in the document gets a transcript on its own. The queue lives in +[`src/lib/ai-edition/store/transcriptionStore.ts`](../../src/lib/ai-edition/store/transcriptionStore.ts), +mounted once by the shell through `useAutoTranscription()`, and it is the only +thing that calls `transcribeAsset`. + +Why it exists at all: "Smart cuts with AI" (and captions, and the transcript +pane) need a transcript, but nothing produced one until the user found the +Media tab or the transcript pane and pressed a button. The obvious first click +was therefore also the one that could not work. Now the button is either ready, +or it says what it is waiting for. + +What the store owns and what it does not: + +- **The document owns the transcript.** `document.transcripts[]` is still the + source of truth for "is there one"; the store only owns the JOB (queued / + running / failed + the phase for the spinner). `deriveAssetStatus` + ([`transcription/status.ts`](../../src/lib/ai-edition/transcription/status.ts)) + folds the two into the single status the UI renders, and + `resolveTranscriptGate` folds a set of those into the ready / pending / + blocked verdict that enables or disables a transcript-dependent action. + A stored transcript **outranks a failed job**: a regenerate that dies on a + whisper restart leaves the previous transcript in place and usable, and + reading that asset as "failed" would have disabled Smart cuts for the rest of + the session over a transcript sitting right there. +- **Gates are resolved over the assets the TIMELINE plays** + (`transcriptRelevantAssetIds`), never over `project.primaryAssetId`. In a + recording project the primary asset is the screen capture — frequently the + silent one — so a primary-scoped answer had the transcript and captions panes + announce "this media has no audio track" for a project whose actual footage + was mid-transcription. `requestTimelineTranscripts` is the matching action for + the panes' one button: every timeline asset that still lacks a transcript, + skipping the ones already known to be silent and waiting on (rather than + duplicating) a run the background pass already has in flight. +- **One run at a time.** whisper-server is a single process and the audio + extraction path holds decoded frames in renderer memory, so the pump is a + sequential loop, not a fan-out. +- **The auto pass never loops.** `sync` enqueues an asset only when it has no + transcript, no job entry (queued / running / failed alike) and no persisted + failure, and a job is deleted only after the save that carries its transcript + has resolved. Since `sync` runs on every document change — including the one + the run itself produces — that guard is what keeps it from re-triggering. +- **Silence is remembered, glitches are not.** A media with no audio track (a + screen recording captured with no mic and no system audio) fails the same way + every time, so the verdict is written to `asset.transcriptionFailure` and the + auto pass skips it on the next load instead of re-extracting its audio to + rediscover it. Everything else stays in memory for the session and is retried + on the next load. A successful manual retry clears the stored verdict in the + same save that writes the transcript. +- **No local engine, no background pass.** Without `window.electronAPI.stt` + (browser preview, e2e shim) nothing is queued; a manual request still runs. + +What each state means in the UI: a spinner and "Transcribing…" while queued or +running, the media-card dot green (ready) / amber (silent or no speech) / red +(failed) via +[`TranscriptionStatus.tsx`](../../src/components/ai-edition/TranscriptionStatus.tsx), +and — the point of the whole thing — a disabled "Smart cuts" entry whose +subtitle is the reason rather than "With AI". + +### One phase, including the first-run model download + +On a fresh install the GGML model (~253 MB) is not on disk. It is fetched by +`SttManager.prepare()` **inside** the `stt:transcribe` IPC call — i.e. inside a +run this store has already marked `running` — so the user sees one single busy +phase that simply takes longer the first time. That is deliberate: no separate +"downloading" step, no progress bar to stare at, and nothing that looks +clickable in the meantime (`phase: "model"` is emitted by the main process but +deliberately not forwarded to the renderer by `transcribeAsset`). Nothing in the +renderer imposes a timeout that a slow download could trip: the preload does a +bare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, and +whisper-server's 30 s readiness budget only starts once the download resolved. + +Three edges make that promise hold, and each is load-bearing: + +- **A failed setup is not cached.** `SttManager.init` used to memoise the + rejected `prepare()` promise, so one dropped connection during the first + download failed every later transcription in the session — including the retry + the editor offers — until the app was restarted. It now clears the slot on + failure ([`electron/stt/index.ts`](../../electron/stt/index.ts)). +- **A transient failure stops the queue** instead of walking the remaining + assets into the same wall: they inherit the verdict (one toast, one retry + affordance) rather than each spending a full retry budget. +- **Read-only is scoped per asset.** The transcript pane's blocks go read-only + only for the asset whose transcript is being rewritten, and say so (spinner + + "Transcribing…" + dimmed stream). A timeline-wide flag made every other + clip's word stream swallow Backspace and hover-bin clicks in silence for the + whole background pass. + +Still synchronous, and still on the main thread: `mixToMono` +([`src/lib/captioning/extractMono16k.ts`](../../src/lib/captioning/extractMono16k.ts)) +now hoists its channel arrays out of the sample loop (it was making one WebIDL +call per sample per channel), but a long recording will still block the renderer +for a moment during "extracting-audio". Moving the mixdown to a worker is the +next step if it shows up in practice. + ## The STT engine The recogniser is **whisper.cpp**, embedded as a static library inside one