From 99920051ce172572fd0f0b32a6fc8190316094ac Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Sat, 1 Aug 2026 14:31:39 +0200 Subject: [PATCH 1/4] feat(editor): transcribe every media automatically and gate the AI actions on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clicking "Smart cuts with AI" was the natural first move in the editor and the one that could not work: nothing produced a transcript until the user found the Media tab or the transcript pane and asked for one. Recognition is local and free, so the editor now produces transcripts by itself, and every affordance that needs one says what it is waiting for instead of doing nothing. - `store/transcriptionStore.ts` owns the queue: one asset at a time (whisper-server is a single process and audio extraction holds decoded frames in renderer memory), auto-enqueued from `useAutoTranscription()` on any document change. The transcript itself still lives on the document — the store only owns the job. - The auto pass cannot loop: an asset is enqueued only when it has no transcript, no job entry (queued / running / failed alike) and no persisted failure, and a job is dropped only after the save carrying its transcript resolved. Runs are stamped with a run id so a manual regenerate can supersede one mid-flight without the outgoing run clearing its successor. - `transcription/status.ts` holds the vocabulary: `deriveAssetStatus` folds job + document into one status (a stored transcript outranks a failed retry — the previous transcript is still usable), `resolveTranscriptGate` folds a set of those into ready / pending / blocked, resolved over the assets the TIMELINE plays rather than over `primaryAssetId` (which in a recording project is the screen capture, routinely the silent one). - Silent media are remembered: a container with no audio track fails the same way every time, so the verdict lands on `asset.transcriptionFailure` (additive schema field, no version bump) and the auto pass stops re-extracting its audio on every project open. Transient failures stay in memory and retry next load. - No local STT engine (browser preview, e2e shim) means no background pass; a manual request still runs. - Removes `projectStore.setTranscript` (it duplicated `withTranscript`) and the `assetStatuses` / `onRegenerateAsset` / `onTranscribe` / `isTranscribing` prop chains — the panes read the store. 25 tests cover the pure logic, the queue (loop guard, supersede, project switch, no engine, silent-verdict persistence) and the rendered gating. --- .../ai-edition/CaptionsPane.gating.test.tsx | 129 +++++ src/components/ai-edition/CaptionsPane.tsx | 47 +- src/components/ai-edition/LeftPanel.tsx | 110 ++-- src/components/ai-edition/Modals.tsx | 5 +- src/components/ai-edition/NewEditorShell.tsx | 146 ++---- src/components/ai-edition/RightPanes.tsx | 25 +- .../ai-edition/TranscriptPane.gating.test.tsx | 94 ++++ .../ai-edition/TranscriptionStatus.tsx | 81 +++ .../ai-edition/v4/EditorShellV4.module.css | 7 +- .../ai-edition/v4/FloatingInspector.tsx | 23 +- src/components/ai-edition/v4/MediaStage.tsx | 100 ++-- src/components/ai-edition/v4/V4Timeline.tsx | 38 +- src/i18n/locales/ar/editor.json | 5 +- src/i18n/locales/ar/settings.json | 3 +- src/i18n/locales/ar/timeline.json | 7 +- src/i18n/locales/en/editor.json | 5 +- src/i18n/locales/en/settings.json | 3 +- src/i18n/locales/en/timeline.json | 7 +- src/i18n/locales/es/editor.json | 5 +- src/i18n/locales/es/settings.json | 3 +- src/i18n/locales/es/timeline.json | 7 +- src/i18n/locales/fr/editor.json | 5 +- src/i18n/locales/fr/settings.json | 3 +- src/i18n/locales/fr/timeline.json | 7 +- src/i18n/locales/it/editor.json | 5 +- src/i18n/locales/it/settings.json | 3 +- src/i18n/locales/it/timeline.json | 7 +- src/i18n/locales/ja-JP/editor.json | 5 +- src/i18n/locales/ja-JP/settings.json | 3 +- src/i18n/locales/ja-JP/timeline.json | 7 +- src/i18n/locales/ko-KR/editor.json | 5 +- src/i18n/locales/ko-KR/settings.json | 3 +- src/i18n/locales/ko-KR/timeline.json | 7 +- src/i18n/locales/pt-BR/editor.json | 5 +- src/i18n/locales/pt-BR/settings.json | 3 +- src/i18n/locales/pt-BR/timeline.json | 7 +- src/i18n/locales/ru/editor.json | 5 +- src/i18n/locales/ru/settings.json | 3 +- src/i18n/locales/ru/timeline.json | 7 +- src/i18n/locales/tr/editor.json | 5 +- src/i18n/locales/tr/settings.json | 3 +- src/i18n/locales/tr/timeline.json | 7 +- src/i18n/locales/vi/editor.json | 5 +- src/i18n/locales/vi/settings.json | 3 +- src/i18n/locales/vi/timeline.json | 7 +- src/i18n/locales/zh-CN/editor.json | 5 +- src/i18n/locales/zh-CN/settings.json | 3 +- src/i18n/locales/zh-CN/timeline.json | 7 +- src/i18n/locales/zh-TW/editor.json | 5 +- src/i18n/locales/zh-TW/settings.json | 3 +- src/i18n/locales/zh-TW/timeline.json | 7 +- src/lib/ai-edition/schema/index.ts | 17 + src/lib/ai-edition/store/projectStore.ts | 23 +- .../store/transcriptionStore.test.ts | 408 +++++++++++++++ .../ai-edition/store/transcriptionStore.ts | 489 ++++++++++++++++++ .../ai-edition/transcription/status.test.ts | 268 ++++++++++ src/lib/ai-edition/transcription/status.ts | 219 ++++++++ .../transcription-and-captions.md | 65 ++- 58 files changed, 2166 insertions(+), 323 deletions(-) create mode 100644 src/components/ai-edition/CaptionsPane.gating.test.tsx create mode 100644 src/components/ai-edition/TranscriptPane.gating.test.tsx create mode 100644 src/components/ai-edition/TranscriptionStatus.tsx create mode 100644 src/lib/ai-edition/store/transcriptionStore.test.ts create mode 100644 src/lib/ai-edition/store/transcriptionStore.ts create mode 100644 src/lib/ai-edition/transcription/status.test.ts create mode 100644 src/lib/ai-edition/transcription/status.ts diff --git a/src/components/ai-edition/CaptionsPane.gating.test.tsx b/src/components/ai-edition/CaptionsPane.gating.test.tsx new file mode 100644 index 0000000000..b1e87c92bb --- /dev/null +++ b/src/components/ai-edition/CaptionsPane.gating.test.tsx @@ -0,0 +1,129 @@ +// Captions are a view of the transcript, so the pane's "Transcribe video" +// button is a retry, not a first step — the background pass has already tried. +// On a media with no audio track that retry can only fail again, so the button +// has to be dead and the pane has to say what is wrong instead of inviting a +// pointless click. + +import "@testing-library/jest-dom"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutDocument } from "@/lib/ai-edition/schema"; +import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { useTranscriptionStore } from "@/lib/ai-edition/store/transcriptionStore"; +import { CaptionsPane } from "./CaptionsPane"; + +vi.mock("@/native", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +function documentWith(asset: AxcutAsset): AxcutDocument { + return { + schemaVersion: 7, + project: { + id: "proj_1", + title: "Test", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + primaryAssetId: asset.id, + }, + assets: [asset], + transcript: null, + transcripts: [], + timeline: { + clips: [ + { + id: "clip_1", + assetId: asset.id, + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, + ], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + legacyEditor: null, + } as unknown as AxcutDocument; +} + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, +}; + +function load(document: AxcutDocument) { + useProjectStore.setState({ + projectId: document.project.id, + document, + status: "ready", + error: null, + dirty: false, + }); +} + +beforeEach(() => { + useTranscriptionStore.getState().reset(); + useProjectStore.getState().clear(); +}); + +afterEach(() => { + cleanup(); +}); + +describe("captions pane gating", () => { + it("offers the retry while the media might still yield a transcript", () => { + load(documentWith(ASSET)); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribe video" })).toBeEnabled(); + }); + + it("shows the queued background run instead of an idle button", () => { + load(documentWith(ASSET)); + useTranscriptionStore.setState({ + projectId: "proj_1", + jobs: { asset_1: { status: "running", language: "auto", manual: false } }, + }); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribing…" })).toBeDisabled(); + }); + + it("kills the retry on a media with no audio track and explains it", () => { + load( + documentWith({ + ...ASSET, + transcriptionFailure: { kind: "no-audio", message: "No audio track found in this video." }, + }), + ); + render( + + + , + ); + expect(screen.getByRole("button", { name: "Transcribe video" })).toBeDisabled(); + expect( + screen.getByText("This media has no audio track — there is nothing to transcribe."), + ).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/CaptionsPane.tsx b/src/components/ai-edition/CaptionsPane.tsx index 91661bdc4c..c1f6d732dd 100644 --- a/src/components/ai-edition/CaptionsPane.tsx +++ b/src/components/ai-edition/CaptionsPane.tsx @@ -15,6 +15,10 @@ import { useScopedT } from "@/contexts/I18nContext"; import type { CaptionTextAlign, CaptionVerticalPosition } from "@/lib/ai-edition/captions"; import { untranslatedUnits } from "@/lib/ai-edition/captions"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { + useTimelineTranscriptGate, + useTranscriptionStore, +} from "@/lib/ai-edition/store/transcriptionStore"; import { useCaptions } from "@/lib/ai-edition/store/useCaptions"; import { nativeBridgeClient } from "@/native"; import { ColorField } from "./ColorField"; @@ -62,15 +66,9 @@ const TRANSLATION_LANGUAGES: ReadonlyArray<{ code: string; label: string }> = [ { code: "zh", label: "中文" }, ]; -interface CaptionsPaneProps { - /** Runs the local Whisper pipeline for the primary asset — owned by the - * shell, which already has the toast + per-asset status plumbing. */ - onTranscribe: () => void; - isTranscribing: boolean; -} - -export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps) { +export function CaptionsPane() { const t = useScopedT("settings"); + const te = useScopedT("editor"); const { settings, translations, @@ -85,6 +83,20 @@ export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps } = useCaptions(); const document = useProjectStore((s) => s.document); const saveDocument = useProjectStore((s) => s.saveDocument); + // Captions are a view of the transcript, and the transcript arrives on its + // own (transcriptionStore's background pass). The pane reads that state + // straight from the store rather than being handed a busy flag: it is the + // same answer everywhere, and "Transcribe" here is only ever a retry. + // + // Resolved over the timeline's assets, not the primary one: `hasTranscript` + // below is already timeline-scoped (useCaptions), and mixing the two scopes + // is what let a silent primary asset dead-end this button for a project whose + // actual footage had speech. + const gate = useTimelineTranscriptGate(); + const requestTimelineTranscripts = useTranscriptionStore((s) => s.requestTimelineTranscripts); + const isTranscribing = gate.state === "pending"; + const silentMedia = gate.state === "blocked" && gate.reason === "no-audio"; + const engineError = gate.state === "blocked" && gate.reason === "failed" ? gate.message : null; const [target, setTarget] = useState(TRANSLATION_LANGUAGES[1].code); const [translating, setTranslating] = useState(false); @@ -191,13 +203,26 @@ export function CaptionsPane({ onTranscribe, isTranscribing }: CaptionsPaneProps }} >

- {t("captions.noTranscript")} + {silentMedia ? te("mediaStage.noAudioTrackHint") : t("captions.noTranscript")}

+ {engineError ? ( +

+ {engineError} +

+ ) : null} diff --git a/src/components/ai-edition/TranscriptPane.gating.test.tsx b/src/components/ai-edition/TranscriptPane.gating.test.tsx new file mode 100644 index 0000000000..ba319a1f5b --- /dev/null +++ b/src/components/ai-edition/TranscriptPane.gating.test.tsx @@ -0,0 +1,94 @@ +// The transcript pane's empty state is what a user meets before any transcript +// exists — and, since transcription now runs by itself in the background, it is +// also where they wait for one. These assertions pin the three answers it has +// to give: "it's coming", "here's the button", and "this media has no audio, so +// there is nothing to press". + +import "@testing-library/jest-dom"; +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { I18nProvider } from "@/contexts/I18nContext"; +import type { AxcutAsset, AxcutClip } from "@/lib/ai-edition/schema"; +import type { TranscriptGateReason } from "@/lib/ai-edition/transcription/status"; +import { TranscriptPane } from "./RightPanes"; + +vi.mock("@/native/client", () => ({ nativeBridgeClient: { aiEdition: {} } })); +vi.mock("sonner", () => ({ toast: { error: vi.fn() } })); + +const ASSET: AxcutAsset = { + id: "asset_1", + kind: "video", + label: "recording.mp4", + originalPath: "/rec.mp4", + durationSec: 12, + cameraTrack: null, +}; + +const CLIPS: AxcutClip[] = [ + { + id: "clip_1", + assetId: "asset_1", + sourceStartSec: 0, + sourceEndSec: 12, + timelineStartSec: 0, + timelineEndSec: 12, + wordRefs: [], + origin: "user", + reason: "", + }, +]; + +function renderPane( + overrides: { + isTranscribing?: boolean; + blocked?: { reason: TranscriptGateReason; message?: string }; + } = {}, +) { + return render( + + + , + ); +} + +afterEach(() => { + cleanup(); +}); + +describe("transcript pane gating", () => { + it("offers the button while nothing has been attempted", () => { + renderPane(); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeEnabled(); + }); + + it("shows the background run in progress instead of an idle button", () => { + renderPane({ isTranscribing: true }); + const button = screen.getByRole("button", { name: "Transcribing…" }); + expect(button).toBeDisabled(); + }); + + it("disables the button when the timeline's media have no audio track, and says why", () => { + renderPane({ blocked: { reason: "no-audio" } }); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeDisabled(); + expect(screen.getByText("This media has no audio track")).toBeInTheDocument(); + }); + + it("keeps the retry available after a transient failure, and surfaces the engine message", () => { + renderPane({ blocked: { reason: "failed", message: "whisper-server exited" } }); + expect(screen.getByRole("button", { name: "Transcribe now" })).toBeEnabled(); + expect(screen.getByText("whisper-server exited")).toBeInTheDocument(); + }); +}); diff --git a/src/components/ai-edition/TranscriptionStatus.tsx b/src/components/ai-edition/TranscriptionStatus.tsx new file mode 100644 index 0000000000..5e17bd276f --- /dev/null +++ b/src/components/ai-edition/TranscriptionStatus.tsx @@ -0,0 +1,81 @@ +// One place that turns an `AssetTranscriptionView` into words and a colour. +// +// The media list (left panel), the media stage and the source-transcript modal +// all report the same six states; before auto-transcription each of them +// spelled its own dot colours and labels out inline, and they had already +// drifted (the left panel knew about "pending", the stage only ever showed a +// spinner). Keeping the vocabulary here means a new state shows up everywhere +// at once. + +import { Loader2 } from "lucide-react"; +import { useScopedT } from "@/contexts/I18nContext"; +import type { AssetTranscriptionView } from "@/lib/ai-edition/transcription/status"; + +/** Human-readable state of one asset's transcript, in the user's language. */ +export function useTranscriptionLabel(): (view: AssetTranscriptionView) => string { + const t = useScopedT("editor"); + return (view) => { + switch (view.status) { + case "ready": + return t("mediaStage.transcriptReady"); + case "queued": + return t("mediaStage.pendingTranscription"); + case "running": + return t("mediaStage.transcribing"); + case "empty": + return t("mediaStage.noSpeechDetected"); + case "failed": + return view.failure?.kind === "error" + ? t("mediaStage.transcriptionFailed") + : t("mediaStage.noAudioTrack"); + default: + return t("mediaStage.noTranscript"); + } + }; +} + +const DOT_COLOR: Record = { + ready: { fill: "var(--success)", halo: "0 0 0 3px var(--success-soft)" }, + queued: { fill: "#f59e0b", halo: "0 0 0 3px rgba(245, 158, 11, 0.2)" }, + running: { fill: "var(--accent)", halo: "0 0 0 3px rgba(16, 185, 129, 0.2)" }, + // A silent media is not a bug — it just has nothing to say. Amber, not red. + empty: { fill: "#f59e0b", halo: "0 0 0 3px rgba(245, 158, 11, 0.2)" }, + failed: { fill: "var(--danger)", halo: "0 0 0 3px rgba(239, 68, 68, 0.2)" }, + idle: { fill: "var(--dim)", halo: "none" }, +}; + +/** Compact status marker: a spinner while a run is in flight, a dot otherwise. */ +export function TranscriptionStatusDot({ + view, + size = 8, +}: { + view: AssetTranscriptionView; + size?: number; +}) { + const label = useTranscriptionLabel()(view); + if (view.status === "running" || view.status === "queued") { + return ( + + ); + } + const { fill, halo } = DOT_COLOR[view.status]; + return ( + + ); +} diff --git a/src/components/ai-edition/v4/EditorShellV4.module.css b/src/components/ai-edition/v4/EditorShellV4.module.css index e3698c75a3..79c89f7e08 100644 --- a/src/components/ai-edition/v4/EditorShellV4.module.css +++ b/src/components/ai-edition/v4/EditorShellV4.module.css @@ -873,9 +873,14 @@ cursor: pointer; text-align: left; } -.recMenuRow:hover { +.recMenuRow:hover:not(:disabled) { background: var(--surface-2); } +/* A gated row (e.g. Smart cuts before its transcript exists) still shows its + reason line, so it must not also light up as if it were clickable. */ +.recMenuRow:disabled { + cursor: not-allowed; +} .recMenuRow.active { background: var(--accent-soft); color: var(--accent); diff --git a/src/components/ai-edition/v4/FloatingInspector.tsx b/src/components/ai-edition/v4/FloatingInspector.tsx index 306d3574a1..6f53c4dadf 100644 --- a/src/components/ai-edition/v4/FloatingInspector.tsx +++ b/src/components/ai-edition/v4/FloatingInspector.tsx @@ -78,10 +78,6 @@ interface FloatingInspectorProps { * through a facet body. */ clips: AxcutClip[]; onEditClip: (clip: AxcutClip) => void; - /** Runs the transcription the caption layer reads from. Captions are derived - * from the transcript, so this is the only caption action the shell owns. */ - onTranscribe: () => void; - isTranscribing: boolean; transcriptProps: TranscriptProps; /** Drives the selected-element settings pane (zoom/speed/annotation/trim) — * takes over the inspector, forcing it open, whenever a timeline region is @@ -97,8 +93,6 @@ export function FloatingInspector({ onToggleOpen, clips, onEditClip, - onTranscribe, - isTranscribing, transcriptProps, tl, }: FloatingInspectorProps) { @@ -114,13 +108,7 @@ export function FloatingInspector({ {selection ? ( tl.clearSelection()} /> ) : ( - + )} ) : null} @@ -1029,14 +1017,10 @@ const secondaryBtnStyle: React.CSSProperties = { function FacetBody({ facet, - onTranscribe, - isTranscribing, onCollapse, transcriptProps, }: { facet: Facet; - onTranscribe: () => void; - isTranscribing: boolean; onCollapse: () => void; transcriptProps: TranscriptProps; }) { @@ -1073,10 +1057,7 @@ function FacetBody({ if (facet === "layout") return wrap(collapse, ); if (facet === "cursor") return wrap(collapse, ); if (facet === "transcript") return wrap(collapse, ); - return wrap( - collapse, - , - ); + return wrap(collapse, ); } function wrap(collapse: React.ReactNode, body: React.ReactNode) { diff --git a/src/components/ai-edition/v4/MediaStage.tsx b/src/components/ai-edition/v4/MediaStage.tsx index 540bef3c83..8c4bca65cd 100644 --- a/src/components/ai-edition/v4/MediaStage.tsx +++ b/src/components/ai-edition/v4/MediaStage.tsx @@ -4,8 +4,17 @@ import { toast } from "sonner"; import { useScopedT } from "@/contexts/I18nContext"; import type { AxcutAsset } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { + useAssetTranscriptions, + useTranscriptionStore, +} from "@/lib/ai-edition/store/transcriptionStore"; import { formatSeconds } from "@/lib/ai-edition/timeline/format"; +import type { + AssetTranscriptionStatus, + AssetTranscriptionView, +} from "@/lib/ai-edition/transcription/status"; import { formatBytes } from "@/utils/formatBytes"; +import { TranscriptionStatusDot, useTranscriptionLabel } from "../TranscriptionStatus"; import styles from "./EditorShellV4.module.css"; const ASSET_MIME = "application/x-axcut-asset"; @@ -20,17 +29,17 @@ function basename(path: string): string { return path.split(/[\\/]/).pop() ?? path; } -export function MediaStage({ - assetStatuses, - onRegenerateAsset, -}: { - assetStatuses?: Record; - onRegenerateAsset?: (assetId: string, language: string) => Promise; -}) { +export function MediaStage() { const t = useScopedT("editor"); const projectId = useProjectStore((s) => s.projectId); const document = useProjectStore((s) => s.document); const addAsset = useProjectStore((s) => s.addAsset); + // Transcripts are produced in the background as soon as a media lands here + // (see transcriptionStore) — this stage only reports where each one is at, + // and lets the user force a re-run in another language. + const transcriptions = useAssetTranscriptions(); + const requestTranscription = useTranscriptionStore((s) => s.request); + const transcriptionLabel = useTranscriptionLabel(); const [query, setQuery] = useState(""); const [busy, setBusy] = useState(false); const [selectedId, setSelectedId] = useState(null); @@ -50,6 +59,11 @@ export function MediaStage({ const transcript = selected ? (document?.transcripts?.find((t) => t.assetId === selected.id) ?? null) : null; + const selectedTranscription: AssetTranscriptionView = selected + ? (transcriptions[selected.id] ?? { assetId: selected.id, status: "idle" }) + : { assetId: "", status: "idle" }; + const selectedBusy = + selectedTranscription.status === "running" || selectedTranscription.status === "queued"; const handleImport = async () => { if (!projectId) { @@ -95,7 +109,10 @@ export function MediaStage({ style={{ gridTemplateColumns: detailOpen ? "repeat(2,1fr)" : "repeat(3,1fr)" }} > {filtered.map((asset, i) => { - const status = assetStatuses?.[asset.id] ?? "idle"; + const transcription = transcriptions[asset.id] ?? { + assetId: asset.id, + status: "idle" as AssetTranscriptionStatus, + }; return ( ); @@ -236,24 +247,42 @@ export function MediaStage({ gap: 6, padding: "5px 10px 5px 8px", borderRadius: 9999, - background: transcript ? "var(--success-soft)" : "var(--accent-soft)", - color: transcript ? "var(--success)" : "var(--accent)", + background: + selectedTranscription.status === "failed" + ? "var(--danger-soft)" + : selectedTranscription.status === "ready" + ? "var(--success-soft)" + : "var(--accent-soft)", + color: + selectedTranscription.status === "failed" + ? "var(--danger)" + : selectedTranscription.status === "ready" + ? "var(--success)" + : "var(--accent)", fontSize: 11.5, fontWeight: 600, }} > - - {transcript ? t("mediaStage.transcriptReady") : t("mediaStage.notGeneratedYet")} + + {transcriptionLabel(selectedTranscription)} + {selectedTranscription.failure ? ( +

+ {selectedTranscription.failure.kind === "error" + ? selectedTranscription.failure.message + : t("mediaStage.noAudioTrackHint")} +

+ ) : null} +
{ - if (onRegenerateAsset) void onRegenerateAsset(selected.id, lang); - }} + disabled={selectedBusy} + onClick={() => void requestTranscription(selected.id, lang)} style={{ width: 36, height: 36, @@ -306,10 +333,11 @@ export function MediaStage({ color: "var(--fg-2)", background: "var(--surface-2)", border: "1px solid var(--border)", - cursor: "pointer", + cursor: selectedBusy ? "not-allowed" : "pointer", + opacity: selectedBusy ? 0.6 : 1, }} > - +
@@ -332,7 +360,13 @@ export function MediaStage({ .map((seg) => (seg as { text?: string }).text ?? "") .join(" ") || t("mediaStage.transcriptEmpty") ) : ( - {t("mediaStage.notGeneratedHint")} + + {selectedBusy + ? t("mediaStage.transcribingEllipsis") + : selectedTranscription.status === "failed" + ? t("mediaStage.generationFailedHint") + : t("mediaStage.notGeneratedHint")} + )} diff --git a/src/components/ai-edition/v4/V4Timeline.tsx b/src/components/ai-edition/v4/V4Timeline.tsx index 9aa811d6d2..e73b1587bc 100644 --- a/src/components/ai-edition/v4/V4Timeline.tsx +++ b/src/components/ai-edition/v4/V4Timeline.tsx @@ -34,6 +34,7 @@ import { collectNativeFormats } from "@/lib/ai-edition/document/outputFormat"; import { setUiProbeScrubbing } from "@/lib/ai-edition/perf/uiFrameProbe"; import type { AxcutClip } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; +import { useTimelineTranscriptGate } from "@/lib/ai-edition/store/transcriptionStore"; import { useChatPromptBus } from "@/lib/ai-edition/store/useChatPromptBus"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import type { useTimeline } from "@/lib/ai-edition/store/useTimeline"; @@ -334,6 +335,24 @@ export function V4Timeline({ const [aspectMenuOpen, setAspectMenuOpen] = useState(false); const [autoEnhanceOpen, setAutoEnhanceOpen] = useState(false); const [autoBusy, setAutoBusy] = useState(false); + // The AI cut pass reads the transcript, and the transcript is produced in the + // background (see transcriptionStore). Until it is there, the entry says why + // rather than handing the agent a prompt it cannot honour — the failure mode + // that made this button the wrong first click for a new user. + const transcriptGate = useTimelineTranscriptGate(); + const smartCutsBlocked = transcriptGate.state !== "ready"; + const smartCutsHint = + transcriptGate.state === "pending" + ? t("toolbar.smartCutsWaiting") + : transcriptGate.state === "ready" + ? t("toolbar.smartZoomsAndCutsHint") + : transcriptGate.reason === "no-audio" + ? t("toolbar.smartCutsNoAudio") + : transcriptGate.reason === "no-speech" + ? t("toolbar.smartCutsNoSpeech") + : transcriptGate.reason === "failed" + ? t("toolbar.smartCutsFailed") + : t("toolbar.smartCutsNeedsTranscript"); const clips = tl.clips; const total = useMemo( @@ -1144,13 +1163,22 @@ export function V4Timeline({ - diff --git a/src/i18n/locales/ar/editor.json b/src/i18n/locales/ar/editor.json index 8a450b47a7..255bad4c15 100644 --- a/src/i18n/locales/ar/editor.json +++ b/src/i18n/locales/ar/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "فشل الإنشاء — اختر لغة وأعد الإنشاء.", "noPreviewAvailable": "لا تتوفر معاينة", "restart": "إعادة التشغيل", - "detectedLanguage": "اللغة المكتشفة: {{language}}" + "detectedLanguage": "اللغة المكتشفة: {{language}}", + "noAudioTrack": "لا يوجد مسار صوتي", + "noAudioTrackHint": "لا يحتوي هذا الملف على مسار صوتي — لا يوجد ما يمكن نسخه.", + "noSpeechDetected": "لم يتم اكتشاف كلام" }, "exportDialog": { "title": "تصدير", diff --git a/src/i18n/locales/ar/settings.json b/src/i18n/locales/ar/settings.json index d2ab9c645b..72f3883be6 100644 --- a/src/i18n/locales/ar/settings.json +++ b/src/i18n/locales/ar/settings.json @@ -255,7 +255,8 @@ "silence": "[صمت {{duration}} ث]", "restoreSilence": "استعادة الصمت ({{duration}} ث)", "trimSilence": "قص الصمت ({{duration}} ث)", - "restoreWord": "استعادة \"{{word}}\"" + "restoreWord": "استعادة \"{{word}}\"", + "noAudio": "لا يحتوي هذا الملف على مسار صوتي" }, "captions": { "show": "إظهار الترجمة", diff --git a/src/i18n/locales/ar/timeline.json b/src/i18n/locales/ar/timeline.json index dc20654596..a47328e863 100644 --- a/src/i18n/locales/ar/timeline.json +++ b/src/i18n/locales/ar/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "تمت إضافة {{count}} تكبير تلقائي", "addedAutoZoomPlural": "تمت إضافة {{count}} تكبيرات تلقائية", "autoZoomFailed": "فشل التكبير التلقائي", - "aiEnhanceRequested": "طُلب من وكيل الذكاء الاصطناعي قص الأوقات الميتة" + "aiEnhanceRequested": "طُلب من وكيل الذكاء الاصطناعي قص الأوقات الميتة", + "smartCutsWaiting": "جارٍ النسخ… سيكون جاهزًا بعد قليل", + "smartCutsNeedsTranscript": "يتطلب نصًا منسوخًا", + "smartCutsNoAudio": "لا يحتوي هذا الملف على صوت", + "smartCutsNoSpeech": "لم يتم اكتشاف كلام", + "smartCutsFailed": "فشل النسخ — أعد المحاولة من قسم الوسائط" } } diff --git a/src/i18n/locales/en/editor.json b/src/i18n/locales/en/editor.json index 34c5e64380..2e270ffb33 100644 --- a/src/i18n/locales/en/editor.json +++ b/src/i18n/locales/en/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Generation failed — pick a language and regenerate.", "noPreviewAvailable": "No preview available", "restart": "Restart", - "detectedLanguage": "Detected language: {{language}}" + "detectedLanguage": "Detected language: {{language}}", + "noAudioTrack": "No audio track", + "noAudioTrackHint": "This media has no audio track — there is nothing to transcribe.", + "noSpeechDetected": "No speech detected" }, "exportDialog": { "title": "Export", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 4b8358ea18..d6fa45887d 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -255,7 +255,8 @@ "silence": "[silence {{duration}}s]", "restoreSilence": "Restore silence ({{duration}}s)", "trimSilence": "Trim silence ({{duration}}s)", - "restoreWord": "Restore \"{{word}}\"" + "restoreWord": "Restore \"{{word}}\"", + "noAudio": "This media has no audio track" }, "captions": { "show": "Show captions", diff --git a/src/i18n/locales/en/timeline.json b/src/i18n/locales/en/timeline.json index 224422f422..d1940b2eaa 100644 --- a/src/i18n/locales/en/timeline.json +++ b/src/i18n/locales/en/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Added {{count}} automatic zoom", "addedAutoZoomPlural": "Added {{count}} automatic zooms", "autoZoomFailed": "Auto-zoom failed", - "aiEnhanceRequested": "Asked the AI agent to cut the dead time" + "aiEnhanceRequested": "Asked the AI agent to cut the dead time", + "smartCutsWaiting": "Transcribing… ready in a moment", + "smartCutsNeedsTranscript": "Needs a transcript", + "smartCutsNoAudio": "This media has no audio", + "smartCutsNoSpeech": "No speech detected", + "smartCutsFailed": "Transcription failed — retry it from Media" } } diff --git a/src/i18n/locales/es/editor.json b/src/i18n/locales/es/editor.json index b0f07e8ea4..8f5ba5becd 100644 --- a/src/i18n/locales/es/editor.json +++ b/src/i18n/locales/es/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Error de generación — elige un idioma y vuelve a generar.", "noPreviewAvailable": "Vista previa no disponible", "restart": "Reiniciar", - "detectedLanguage": "Idioma detectado: {{language}}" + "detectedLanguage": "Idioma detectado: {{language}}", + "noAudioTrack": "Sin pista de audio", + "noAudioTrackHint": "Este medio no tiene pista de audio: no hay nada que transcribir.", + "noSpeechDetected": "No se detectó voz" }, "exportDialog": { "title": "Exportar", diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index d7cb0cb9ae..71ba0b9d5c 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -255,7 +255,8 @@ "silence": "[silencio {{duration}} s]", "restoreSilence": "Restaurar silencio ({{duration}} s)", "trimSilence": "Recortar silencio ({{duration}} s)", - "restoreWord": "Restaurar «{{word}}»" + "restoreWord": "Restaurar «{{word}}»", + "noAudio": "Este medio no tiene pista de audio" }, "captions": { "show": "Mostrar subtítulos", diff --git a/src/i18n/locales/es/timeline.json b/src/i18n/locales/es/timeline.json index 1a23e1f827..4226ead069 100644 --- a/src/i18n/locales/es/timeline.json +++ b/src/i18n/locales/es/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Se añadió {{count}} zoom automático", "addedAutoZoomPlural": "Se añadieron {{count}} zooms automáticos", "autoZoomFailed": "Error en el zoom automático", - "aiEnhanceRequested": "Se pidió al agente de IA que corte los tiempos muertos" + "aiEnhanceRequested": "Se pidió al agente de IA que corte los tiempos muertos", + "smartCutsWaiting": "Transcribiendo… disponible en un momento", + "smartCutsNeedsTranscript": "Requiere una transcripción", + "smartCutsNoAudio": "Este medio no tiene audio", + "smartCutsNoSpeech": "No se detectó voz", + "smartCutsFailed": "La transcripción falló: reinténtala desde Medios" } } diff --git a/src/i18n/locales/fr/editor.json b/src/i18n/locales/fr/editor.json index 3b03b98718..5cda995b1d 100644 --- a/src/i18n/locales/fr/editor.json +++ b/src/i18n/locales/fr/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Échec de la génération — choisissez une langue et régénérez.", "noPreviewAvailable": "Aucun aperçu disponible", "restart": "Redémarrer", - "detectedLanguage": "Langue détectée : {{language}}" + "detectedLanguage": "Langue détectée : {{language}}", + "noAudioTrack": "Aucune piste audio", + "noAudioTrackHint": "Ce média n'a pas de piste audio — il n'y a rien à transcrire.", + "noSpeechDetected": "Aucune parole détectée" }, "exportDialog": { "title": "Exporter", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index afba3a5621..e94b4f579e 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -255,7 +255,8 @@ "silence": "[silence {{duration}} s]", "restoreSilence": "Restaurer le silence ({{duration}} s)", "trimSilence": "Couper le silence ({{duration}} s)", - "restoreWord": "Restaurer « {{word}} »" + "restoreWord": "Restaurer « {{word}} »", + "noAudio": "Ce média n'a pas de piste audio" }, "captions": { "show": "Afficher les sous-titres", diff --git a/src/i18n/locales/fr/timeline.json b/src/i18n/locales/fr/timeline.json index ffabdf22e8..210fbfe4ce 100644 --- a/src/i18n/locales/fr/timeline.json +++ b/src/i18n/locales/fr/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} zoom automatique ajouté", "addedAutoZoomPlural": "{{count}} zooms automatiques ajoutés", "autoZoomFailed": "Échec du zoom automatique", - "aiEnhanceRequested": "Demandé à l'agent IA de couper les temps morts" + "aiEnhanceRequested": "Demandé à l'agent IA de couper les temps morts", + "smartCutsWaiting": "Transcription en cours… disponible dans un instant", + "smartCutsNeedsTranscript": "Nécessite une transcription", + "smartCutsNoAudio": "Ce média n'a pas d'audio", + "smartCutsNoSpeech": "Aucune parole détectée", + "smartCutsFailed": "Échec de la transcription — relancez-la depuis Médias" } } diff --git a/src/i18n/locales/it/editor.json b/src/i18n/locales/it/editor.json index 1b2bbbf0cc..ecc998b7d4 100644 --- a/src/i18n/locales/it/editor.json +++ b/src/i18n/locales/it/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Generazione non riuscita — scegli una lingua e rigenera.", "noPreviewAvailable": "Anteprima non disponibile", "restart": "Riavvia", - "detectedLanguage": "Lingua rilevata: {{language}}" + "detectedLanguage": "Lingua rilevata: {{language}}", + "noAudioTrack": "Nessuna traccia audio", + "noAudioTrackHint": "Questo contenuto non ha una traccia audio: non c'è nulla da trascrivere.", + "noSpeechDetected": "Nessun parlato rilevato" }, "exportDialog": { "title": "Esporta", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index eeaa72d603..464adba34e 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -255,7 +255,8 @@ "silence": "[silenzio {{duration}} s]", "restoreSilence": "Ripristina silenzio ({{duration}} s)", "trimSilence": "Taglia silenzio ({{duration}} s)", - "restoreWord": "Ripristina «{{word}}»" + "restoreWord": "Ripristina «{{word}}»", + "noAudio": "Questo contenuto non ha una traccia audio" }, "captions": { "show": "Mostra sottotitoli", diff --git a/src/i18n/locales/it/timeline.json b/src/i18n/locales/it/timeline.json index 1a02f7d9c4..ed9f2f9f8e 100644 --- a/src/i18n/locales/it/timeline.json +++ b/src/i18n/locales/it/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Aggiunto {{count}} zoom automatico", "addedAutoZoomPlural": "Aggiunti {{count}} zoom automatici", "autoZoomFailed": "Zoom automatico non riuscito", - "aiEnhanceRequested": "Chiesto all'agente IA di tagliare i tempi morti" + "aiEnhanceRequested": "Chiesto all'agente IA di tagliare i tempi morti", + "smartCutsWaiting": "Trascrizione in corso… disponibile a breve", + "smartCutsNeedsTranscript": "Richiede una trascrizione", + "smartCutsNoAudio": "Questo contenuto non ha audio", + "smartCutsNoSpeech": "Nessun parlato rilevato", + "smartCutsFailed": "Trascrizione non riuscita: riprova da Contenuti multimediali" } } diff --git a/src/i18n/locales/ja-JP/editor.json b/src/i18n/locales/ja-JP/editor.json index 04970a21fe..615cf15a5b 100644 --- a/src/i18n/locales/ja-JP/editor.json +++ b/src/i18n/locales/ja-JP/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "生成に失敗しました — 言語を選んで再生成してください。", "noPreviewAvailable": "プレビューがありません", "restart": "再生位置を先頭に戻す", - "detectedLanguage": "検出された言語: {{language}}" + "detectedLanguage": "検出された言語: {{language}}", + "noAudioTrack": "音声トラックがありません", + "noAudioTrackHint": "このメディアには音声トラックがないため、文字起こしできません。", + "noSpeechDetected": "音声が検出されませんでした" }, "exportDialog": { "title": "エクスポート", diff --git a/src/i18n/locales/ja-JP/settings.json b/src/i18n/locales/ja-JP/settings.json index 0dbc20e65a..386d51092b 100644 --- a/src/i18n/locales/ja-JP/settings.json +++ b/src/i18n/locales/ja-JP/settings.json @@ -255,7 +255,8 @@ "silence": "[無音 {{duration}} 秒]", "restoreSilence": "無音を元に戻す（{{duration}} 秒）", "trimSilence": "無音をトリム（{{duration}} 秒）", - "restoreWord": "「{{word}}」を元に戻す" + "restoreWord": "「{{word}}」を元に戻す", + "noAudio": "このメディアには音声トラックがありません" }, "captions": { "show": "字幕を表示", diff --git a/src/i18n/locales/ja-JP/timeline.json b/src/i18n/locales/ja-JP/timeline.json index 2ab41033ce..2f12a3b9b6 100644 --- a/src/i18n/locales/ja-JP/timeline.json +++ b/src/i18n/locales/ja-JP/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "自動ズームを {{count}} 件追加しました", "addedAutoZoomPlural": "自動ズームを {{count}} 件追加しました", "autoZoomFailed": "自動ズームに失敗しました", - "aiEnhanceRequested": "AIエージェントに無音部分のカットを依頼しました" + "aiEnhanceRequested": "AIエージェントに無音部分のカットを依頼しました", + "smartCutsWaiting": "文字起こし中… まもなく使えます", + "smartCutsNeedsTranscript": "文字起こしが必要です", + "smartCutsNoAudio": "このメディアには音声がありません", + "smartCutsNoSpeech": "音声が検出されませんでした", + "smartCutsFailed": "文字起こしに失敗しました — メディアから再試行してください" } } diff --git a/src/i18n/locales/ko-KR/editor.json b/src/i18n/locales/ko-KR/editor.json index e7b9860171..190b9af0b7 100644 --- a/src/i18n/locales/ko-KR/editor.json +++ b/src/i18n/locales/ko-KR/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "생성 실패 — 언어를 선택하고 다시 생성하세요.", "noPreviewAvailable": "미리보기를 사용할 수 없습니다", "restart": "다시 시작", - "detectedLanguage": "감지된 언어: {{language}}" + "detectedLanguage": "감지된 언어: {{language}}", + "noAudioTrack": "오디오 트랙 없음", + "noAudioTrackHint": "이 미디어에는 오디오 트랙이 없어 받아쓸 내용이 없습니다.", + "noSpeechDetected": "음성이 감지되지 않음" }, "exportDialog": { "title": "내보내기", diff --git a/src/i18n/locales/ko-KR/settings.json b/src/i18n/locales/ko-KR/settings.json index 4f3de7c7a2..24405508b9 100644 --- a/src/i18n/locales/ko-KR/settings.json +++ b/src/i18n/locales/ko-KR/settings.json @@ -255,7 +255,8 @@ "silence": "[무음 {{duration}}초]", "restoreSilence": "무음 복원 ({{duration}}초)", "trimSilence": "무음 자르기 ({{duration}}초)", - "restoreWord": "\"{{word}}\" 복원" + "restoreWord": "\"{{word}}\" 복원", + "noAudio": "이 미디어에는 오디오 트랙이 없습니다" }, "captions": { "show": "자막 표시", diff --git a/src/i18n/locales/ko-KR/timeline.json b/src/i18n/locales/ko-KR/timeline.json index 12f0abadcc..d050356411 100644 --- a/src/i18n/locales/ko-KR/timeline.json +++ b/src/i18n/locales/ko-KR/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "자동 줌 {{count}}개가 추가되었습니다", "addedAutoZoomPlural": "자동 줌 {{count}}개가 추가되었습니다", "autoZoomFailed": "자동 줌 실패", - "aiEnhanceRequested": "AI 에이전트에 빈 구간 컷을 요청했습니다" + "aiEnhanceRequested": "AI 에이전트에 빈 구간 컷을 요청했습니다", + "smartCutsWaiting": "받아쓰는 중… 곧 사용할 수 있습니다", + "smartCutsNeedsTranscript": "받아쓰기가 필요합니다", + "smartCutsNoAudio": "이 미디어에는 오디오가 없습니다", + "smartCutsNoSpeech": "음성이 감지되지 않음", + "smartCutsFailed": "받아쓰기에 실패했습니다 — 미디어에서 다시 시도하세요" } } diff --git a/src/i18n/locales/pt-BR/editor.json b/src/i18n/locales/pt-BR/editor.json index 5833af8aa6..b7d90914d6 100644 --- a/src/i18n/locales/pt-BR/editor.json +++ b/src/i18n/locales/pt-BR/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Falha na geração — escolha um idioma e gere novamente.", "noPreviewAvailable": "Pré-visualização indisponível", "restart": "Reiniciar", - "detectedLanguage": "Idioma detectado: {{language}}" + "detectedLanguage": "Idioma detectado: {{language}}", + "noAudioTrack": "Sem faixa de áudio", + "noAudioTrackHint": "Esta mídia não tem faixa de áudio — não há nada para transcrever.", + "noSpeechDetected": "Nenhuma fala detectada" }, "exportDialog": { "title": "Exportar", diff --git a/src/i18n/locales/pt-BR/settings.json b/src/i18n/locales/pt-BR/settings.json index b0f492a2e3..c3eebc6dae 100644 --- a/src/i18n/locales/pt-BR/settings.json +++ b/src/i18n/locales/pt-BR/settings.json @@ -255,7 +255,8 @@ "silence": "[silêncio {{duration}} s]", "restoreSilence": "Restaurar silêncio ({{duration}} s)", "trimSilence": "Cortar silêncio ({{duration}} s)", - "restoreWord": "Restaurar \"{{word}}\"" + "restoreWord": "Restaurar \"{{word}}\"", + "noAudio": "Esta mídia não tem faixa de áudio" }, "captions": { "show": "Mostrar legendas", diff --git a/src/i18n/locales/pt-BR/timeline.json b/src/i18n/locales/pt-BR/timeline.json index 76f4cd6db1..4a70046d55 100644 --- a/src/i18n/locales/pt-BR/timeline.json +++ b/src/i18n/locales/pt-BR/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} zoom automático adicionado", "addedAutoZoomPlural": "{{count}} zooms automáticos adicionados", "autoZoomFailed": "Falha no zoom automático", - "aiEnhanceRequested": "Pedido ao agente de IA para cortar os tempos mortos" + "aiEnhanceRequested": "Pedido ao agente de IA para cortar os tempos mortos", + "smartCutsWaiting": "Transcrevendo… disponível em instantes", + "smartCutsNeedsTranscript": "Requer uma transcrição", + "smartCutsNoAudio": "Esta mídia não tem áudio", + "smartCutsNoSpeech": "Nenhuma fala detectada", + "smartCutsFailed": "Falha na transcrição — tente de novo em Mídia" } } diff --git a/src/i18n/locales/ru/editor.json b/src/i18n/locales/ru/editor.json index 6ebd184537..b50d5a986a 100644 --- a/src/i18n/locales/ru/editor.json +++ b/src/i18n/locales/ru/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Ошибка создания — выберите язык и создайте заново.", "noPreviewAvailable": "Предпросмотр недоступен", "restart": "Перезапустить", - "detectedLanguage": "Обнаруженный язык: {{language}}" + "detectedLanguage": "Обнаруженный язык: {{language}}", + "noAudioTrack": "Нет аудиодорожки", + "noAudioTrackHint": "В этом медиафайле нет аудиодорожки — расшифровывать нечего.", + "noSpeechDetected": "Речь не обнаружена" }, "exportDialog": { "title": "Экспорт", diff --git a/src/i18n/locales/ru/settings.json b/src/i18n/locales/ru/settings.json index 8ff6591e7e..ad63e276d3 100644 --- a/src/i18n/locales/ru/settings.json +++ b/src/i18n/locales/ru/settings.json @@ -255,7 +255,8 @@ "silence": "[тишина {{duration}} с]", "restoreSilence": "Вернуть тишину ({{duration}} с)", "trimSilence": "Вырезать тишину ({{duration}} с)", - "restoreWord": "Вернуть «{{word}}»" + "restoreWord": "Вернуть «{{word}}»", + "noAudio": "В этом медиафайле нет аудиодорожки" }, "captions": { "show": "Показывать субтитры", diff --git a/src/i18n/locales/ru/timeline.json b/src/i18n/locales/ru/timeline.json index 621c5c059e..ebd77431a5 100644 --- a/src/i18n/locales/ru/timeline.json +++ b/src/i18n/locales/ru/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Добавлен {{count}} автоматический зум", "addedAutoZoomPlural": "Добавлено {{count}} автоматических зумов", "autoZoomFailed": "Не удалось выполнить автозум", - "aiEnhanceRequested": "Агенту ИИ поручено вырезать паузы" + "aiEnhanceRequested": "Агенту ИИ поручено вырезать паузы", + "smartCutsWaiting": "Идёт расшифровка… скоро будет готово", + "smartCutsNeedsTranscript": "Нужна расшифровка", + "smartCutsNoAudio": "В этом медиафайле нет звука", + "smartCutsNoSpeech": "Речь не обнаружена", + "smartCutsFailed": "Не удалось расшифровать — повторите из раздела «Медиа»" } } diff --git a/src/i18n/locales/tr/editor.json b/src/i18n/locales/tr/editor.json index 067f0f0697..0c1d77344f 100644 --- a/src/i18n/locales/tr/editor.json +++ b/src/i18n/locales/tr/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Oluşturma başarısız oldu — bir dil seçip yeniden oluşturun.", "noPreviewAvailable": "Önizleme kullanılamıyor", "restart": "Yeniden başlat", - "detectedLanguage": "Algılanan dil: {{language}}" + "detectedLanguage": "Algılanan dil: {{language}}", + "noAudioTrack": "Ses parçası yok", + "noAudioTrackHint": "Bu medyada ses parçası yok — metne dökülecek bir şey bulunmuyor.", + "noSpeechDetected": "Konuşma algılanmadı" }, "exportDialog": { "title": "Dışa Aktar", diff --git a/src/i18n/locales/tr/settings.json b/src/i18n/locales/tr/settings.json index 8193da8311..d397e65a3b 100644 --- a/src/i18n/locales/tr/settings.json +++ b/src/i18n/locales/tr/settings.json @@ -255,7 +255,8 @@ "silence": "[sessizlik {{duration}} sn]", "restoreSilence": "Sessizliği geri al ({{duration}} sn)", "trimSilence": "Sessizliği kırp ({{duration}} sn)", - "restoreWord": "\"{{word}}\" kelimesini geri al" + "restoreWord": "\"{{word}}\" kelimesini geri al", + "noAudio": "Bu medyada ses parçası yok" }, "captions": { "show": "Altyazıları göster", diff --git a/src/i18n/locales/tr/timeline.json b/src/i18n/locales/tr/timeline.json index a4be44a5ba..d8f4431c55 100644 --- a/src/i18n/locales/tr/timeline.json +++ b/src/i18n/locales/tr/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "{{count}} otomatik yakınlaştırma eklendi", "addedAutoZoomPlural": "{{count}} otomatik yakınlaştırma eklendi", "autoZoomFailed": "Otomatik yakınlaştırma başarısız oldu", - "aiEnhanceRequested": "Yapay zeka aracısından ölü zamanları kırpması istendi" + "aiEnhanceRequested": "Yapay zeka aracısından ölü zamanları kırpması istendi", + "smartCutsWaiting": "Metne dökülüyor… birazdan hazır", + "smartCutsNeedsTranscript": "Bir döküm gerekiyor", + "smartCutsNoAudio": "Bu medyada ses yok", + "smartCutsNoSpeech": "Konuşma algılanmadı", + "smartCutsFailed": "Metne dökme başarısız — Medya bölümünden yeniden deneyin" } } diff --git a/src/i18n/locales/vi/editor.json b/src/i18n/locales/vi/editor.json index e5e6494861..0ad0a044c5 100644 --- a/src/i18n/locales/vi/editor.json +++ b/src/i18n/locales/vi/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "Tạo thất bại — chọn ngôn ngữ và tạo lại.", "noPreviewAvailable": "Không có bản xem trước", "restart": "Bắt đầu lại", - "detectedLanguage": "Ngôn ngữ phát hiện: {{language}}" + "detectedLanguage": "Ngôn ngữ phát hiện: {{language}}", + "noAudioTrack": "Không có bản âm thanh", + "noAudioTrackHint": "Media này không có bản âm thanh — không có gì để phiên âm.", + "noSpeechDetected": "Không phát hiện giọng nói" }, "exportDialog": { "title": "Xuất", diff --git a/src/i18n/locales/vi/settings.json b/src/i18n/locales/vi/settings.json index 2d1d201a8b..fcaa033882 100644 --- a/src/i18n/locales/vi/settings.json +++ b/src/i18n/locales/vi/settings.json @@ -255,7 +255,8 @@ "silence": "[khoảng lặng {{duration}} giây]", "restoreSilence": "Khôi phục khoảng lặng ({{duration}} giây)", "trimSilence": "Cắt khoảng lặng ({{duration}} giây)", - "restoreWord": "Khôi phục \"{{word}}\"" + "restoreWord": "Khôi phục \"{{word}}\"", + "noAudio": "Media này không có bản âm thanh" }, "captions": { "show": "Hiện phụ đề", diff --git a/src/i18n/locales/vi/timeline.json b/src/i18n/locales/vi/timeline.json index 8ecbb7d4fa..4b9218ba7d 100644 --- a/src/i18n/locales/vi/timeline.json +++ b/src/i18n/locales/vi/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "Đã thêm {{count}} thu phóng tự động", "addedAutoZoomPlural": "Đã thêm {{count}} thu phóng tự động", "autoZoomFailed": "Thu phóng tự động thất bại", - "aiEnhanceRequested": "Đã yêu cầu tác nhân AI cắt thời gian chết" + "aiEnhanceRequested": "Đã yêu cầu tác nhân AI cắt thời gian chết", + "smartCutsWaiting": "Đang phiên âm… sẵn sàng trong giây lát", + "smartCutsNeedsTranscript": "Cần có bản phiên âm", + "smartCutsNoAudio": "Media này không có âm thanh", + "smartCutsNoSpeech": "Không phát hiện giọng nói", + "smartCutsFailed": "Phiên âm thất bại — thử lại trong Media" } } diff --git a/src/i18n/locales/zh-CN/editor.json b/src/i18n/locales/zh-CN/editor.json index 1500049a15..35610a0b64 100644 --- a/src/i18n/locales/zh-CN/editor.json +++ b/src/i18n/locales/zh-CN/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "生成失败 — 选择语言并重新生成。", "noPreviewAvailable": "无法预览", "restart": "重新开始", - "detectedLanguage": "检测到的语言：{{language}}" + "detectedLanguage": "检测到的语言：{{language}}", + "noAudioTrack": "无音频轨道", + "noAudioTrackHint": "此媒体没有音频轨道，没有可转录的内容。", + "noSpeechDetected": "未检测到语音" }, "exportDialog": { "title": "导出", diff --git a/src/i18n/locales/zh-CN/settings.json b/src/i18n/locales/zh-CN/settings.json index ca6c78910c..013b5e6d1e 100644 --- a/src/i18n/locales/zh-CN/settings.json +++ b/src/i18n/locales/zh-CN/settings.json @@ -255,7 +255,8 @@ "silence": "[静音 {{duration}} 秒]", "restoreSilence": "恢复静音（{{duration}} 秒）", "trimSilence": "修剪静音（{{duration}} 秒）", - "restoreWord": "恢复“{{word}}”" + "restoreWord": "恢复“{{word}}”", + "noAudio": "此媒体没有音频轨道" }, "captions": { "show": "显示字幕", diff --git a/src/i18n/locales/zh-CN/timeline.json b/src/i18n/locales/zh-CN/timeline.json index f73508cf3c..f1065037cd 100644 --- a/src/i18n/locales/zh-CN/timeline.json +++ b/src/i18n/locales/zh-CN/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "已添加 {{count}} 个自动缩放", "addedAutoZoomPlural": "已添加 {{count}} 个自动缩放", "autoZoomFailed": "自动缩放失败", - "aiEnhanceRequested": "已请求 AI 代理剪除空白片段" + "aiEnhanceRequested": "已请求 AI 代理剪除空白片段", + "smartCutsWaiting": "正在转录…稍后可用", + "smartCutsNeedsTranscript": "需要转录文本", + "smartCutsNoAudio": "此媒体没有音频", + "smartCutsNoSpeech": "未检测到语音", + "smartCutsFailed": "转录失败 — 请在“媒体”中重试" } } diff --git a/src/i18n/locales/zh-TW/editor.json b/src/i18n/locales/zh-TW/editor.json index b0b36e5a79..c3e7e55e46 100644 --- a/src/i18n/locales/zh-TW/editor.json +++ b/src/i18n/locales/zh-TW/editor.json @@ -150,7 +150,10 @@ "generationFailedHint": "產生失敗 — 選擇語言並重新產生。", "noPreviewAvailable": "無法預覽", "restart": "重新開始", - "detectedLanguage": "偵測到的語言：{{language}}" + "detectedLanguage": "偵測到的語言：{{language}}", + "noAudioTrack": "無音訊軌道", + "noAudioTrackHint": "此媒體沒有音訊軌道，沒有可轉錄的內容。", + "noSpeechDetected": "未偵測到語音" }, "exportDialog": { "title": "匯出", diff --git a/src/i18n/locales/zh-TW/settings.json b/src/i18n/locales/zh-TW/settings.json index 8fe3c9ffa1..82b1d76c97 100644 --- a/src/i18n/locales/zh-TW/settings.json +++ b/src/i18n/locales/zh-TW/settings.json @@ -256,7 +256,8 @@ "silence": "[靜音 {{duration}} 秒]", "restoreSilence": "還原靜音（{{duration}} 秒）", "trimSilence": "修剪靜音（{{duration}} 秒）", - "restoreWord": "還原「{{word}}」" + "restoreWord": "還原「{{word}}」", + "noAudio": "此媒體沒有音訊軌道" }, "captions": { "show": "顯示字幕", diff --git a/src/i18n/locales/zh-TW/timeline.json b/src/i18n/locales/zh-TW/timeline.json index cadb63dff3..94fce3f4a4 100644 --- a/src/i18n/locales/zh-TW/timeline.json +++ b/src/i18n/locales/zh-TW/timeline.json @@ -82,6 +82,11 @@ "addedAutoZoom": "已新增 {{count}} 個自動縮放", "addedAutoZoomPlural": "已新增 {{count}} 個自動縮放", "autoZoomFailed": "自動縮放失敗", - "aiEnhanceRequested": "已請求 AI 代理剪除空白片段" + "aiEnhanceRequested": "已請求 AI 代理剪除空白片段", + "smartCutsWaiting": "正在轉錄…稍後可用", + "smartCutsNeedsTranscript": "需要轉錄文字", + "smartCutsNoAudio": "此媒體沒有音訊", + "smartCutsNoSpeech": "未偵測到語音", + "smartCutsFailed": "轉錄失敗 — 請在「媒體」中重試" } } diff --git a/src/lib/ai-edition/schema/index.ts b/src/lib/ai-edition/schema/index.ts index ecc8e0daa3..17c2305a6b 100644 --- a/src/lib/ai-edition/schema/index.ts +++ b/src/lib/ai-edition/schema/index.ts @@ -124,6 +124,19 @@ export const cameraTrackSchema = z .nullable() .default(null); +// Why a media can never be transcribed. Only the DETERMINISTIC verdicts live +// here: a container with no audio track (a screen recording captured with no +// mic and no system audio — the common case) fails identically on every +// attempt, and re-deciding that costs a full audio extraction on each project +// open. Transient failures (engine down, decode hiccup) are deliberately NOT +// persistable and stay in the transcription store for the session, so the next +// load retries them. See `src/lib/ai-edition/transcription/status.ts`. +export const assetTranscriptionFailureSchema = z.object({ + kind: z.enum(["no-audio", "unsupported-audio"]), + message: z.string().default(""), + at: isoDateSchema.optional(), +}); + export const assetSchema = z.object({ id: z.string().min(1), kind: z.literal("video"), @@ -136,6 +149,9 @@ export const assetSchema = z.object({ sizeBytes: z.number().int().nonnegative().optional(), video: assetVideoSchema.optional(), audio: assetAudioSchema.optional(), + // Absent on every document written before auto-transcription; additive, so + // no schema-version bump (an older build simply drops the key on save). + transcriptionFailure: assetTranscriptionFailureSchema.nullish(), cameraTrack: cameraTrackSchema, }); @@ -802,6 +818,7 @@ export type AxcutWord = z.infer; export type AxcutTranscriptSegment = z.infer; export type AxcutTranscript = z.infer; export type AxcutAsset = z.infer; +export type AxcutAssetTranscriptionFailure = z.infer; export type AxcutClip = z.infer; export type AxcutClipCropRegion = z.infer; export type AxcutGap = z.infer; diff --git a/src/lib/ai-edition/store/projectStore.ts b/src/lib/ai-edition/store/projectStore.ts index fec88776f0..539449dab9 100644 --- a/src/lib/ai-edition/store/projectStore.ts +++ b/src/lib/ai-edition/store/projectStore.ts @@ -5,12 +5,7 @@ import { replaceTimeline as replaceTimelineOp, restoreFullTimeline as restoreFullTimelineOp, } from "../document/timeline"; -import { - type AxcutAsset, - type AxcutDocument, - type AxcutTranscript, - documentSchema, -} from "../schema"; +import { type AxcutAsset, type AxcutDocument, documentSchema } from "../schema"; // ponytail: thin Zustand wrapper over the native-bridge client. Keeps the // current project + revision counter in renderer memory; mutations round-trip @@ -45,7 +40,6 @@ export interface ProjectState { setDocument: (document: AxcutDocument) => void; replaceTimeline: (intervals: Interval[], reason: string) => Promise; restoreFullTimeline: () => Promise; - setTranscript: (transcript: AxcutTranscript) => Promise; setSourceDuration: (sec: number) => void; setCurrentTime: (sec: number) => void; setPlaying: (playing: boolean) => void; @@ -261,21 +255,6 @@ export const useProjectStore = create((set, get) => ({ await get().saveDocument(next); }, - async setTranscript(transcript) { - const doc = get().document; - if (!doc) throw new Error("No project loaded"); - const transcripts = [ - ...doc.transcripts.filter((t) => t.assetId !== transcript.assetId), - transcript, - ]; - const next: AxcutDocument = { - ...doc, - transcript: doc.project.primaryAssetId === transcript.assetId ? transcript : doc.transcript, - transcripts, - }; - await get().saveDocument(next); - }, - setSourceDuration(sec) { set({ sourceDurationSec: sec }); }, diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts new file mode 100644 index 0000000000..61bd4b6bd2 --- /dev/null +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -0,0 +1,408 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { useProjectStore } from "./projectStore"; +import { useTranscriptionStore, whenTranscriptionIdle } from "./transcriptionStore"; + +const bridgeMocks = vi.hoisted(() => ({ + save: vi.fn(), +})); + +const transcribeMocks = vi.hoisted(() => ({ + transcribeAsset: vi.fn(), +})); + +const toastMocks = vi.hoisted(() => ({ + success: vi.fn(), + error: vi.fn(), +})); + +vi.mock("@/native/client", () => ({ + nativeBridgeClient: { aiEdition: { save: bridgeMocks.save } }, +})); + +vi.mock("../document/transcribe", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, transcribeAsset: transcribeMocks.transcribeAsset }; +}); + +vi.mock("sonner", () => ({ toast: { success: toastMocks.success, error: toastMocks.error } })); + +function asset(id: string, extra: Record = {}) { + return { + id, + kind: "video" as const, + label: `${id}.mp4`, + originalPath: `/tmp/${id}.mp4`, + cameraTrack: null, + ...extra, + }; +} + +function makeDoc(assetIds: string[], projectId = "proj_1"): AxcutDocument { + return { + schemaVersion: 7, + project: { + id: projectId, + title: "Test", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + primaryAssetId: assetIds[0], + }, + assets: assetIds.map((id) => asset(id)), + transcript: null, + transcripts: [], + timeline: { + clips: [], + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + annotations: [], + zoomRanges: [], + legacyEditor: null, + } as unknown as AxcutDocument; +} + +function transcriptFor(assetId: string): AxcutTranscript { + return { + assetId, + language: "en", + segments: [ + { id: "seg_1", kind: "speech", startSec: 0, endSec: 1, text: "hello", wordIds: ["word_1"] }, + ], + words: [{ id: "word_1", segmentId: "seg_1", startSec: 0, endSec: 1, text: "hello" }], + }; +} + +/** Loads a document into the project store the way `loadProject` would. */ +function loadDocument(document: AxcutDocument) { + useProjectStore.setState({ + projectId: document.project.id, + document, + status: "ready", + error: null, + dirty: false, + }); +} + +describe("useTranscriptionStore", () => { + beforeEach(() => { + useTranscriptionStore.getState().reset(); + useProjectStore.getState().clear(); + bridgeMocks.save.mockReset(); + // The bridge echoes back whatever it was handed, like a successful save. + bridgeMocks.save.mockImplementation(async (document: AxcutDocument) => ({ + success: true, + document, + })); + transcribeMocks.transcribeAsset.mockReset(); + toastMocks.success.mockReset(); + toastMocks.error.mockReset(); + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + (window as any).electronAPI = { stt: { transcribe: vi.fn() } }; + }); + + afterEach(() => { + vi.clearAllMocks(); + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + }); + + it("transcribes every asset that has no transcript, one at a time", async () => { + let inFlight = 0; + let maxInFlight = 0; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + await Promise.resolve(); + inFlight -= 1; + return transcriptFor(assetId); + }, + ); + loadDocument(makeDoc(["asset_1", "asset_2"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(maxInFlight).toBe(1); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(2); + expect( + useProjectStore + .getState() + .document?.transcripts.map((t) => t.assetId) + .sort(), + ).toEqual(["asset_1", "asset_2"]); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + // The background pass stays quiet on success. + expect(toastMocks.success).not.toHaveBeenCalled(); + }); + + it("does not re-enqueue an asset whose transcript it just wrote", async () => { + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + const { sync } = useTranscriptionStore.getState(); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + // Every document change re-runs sync in the shell — this is the loop guard. + sync(useProjectStore.getState().document); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("remembers a no-audio verdict on the asset and never retries it by itself", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue( + new Error("No audio track found in this video."), + ); + loadDocument(makeDoc(["asset_1"])); + + const { sync } = useTranscriptionStore.getState(); + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + const job = useTranscriptionStore.getState().jobs.asset_1; + expect(job?.status).toBe("failed"); + expect(job?.failure?.kind).toBe("no-audio"); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure?.kind).toBe( + "no-audio", + ); + // Silence is an expected outcome, not an incident. + expect(toastMocks.error).not.toHaveBeenCalled(); + + sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + }); + + it("skips an asset that already carries a persisted failure on a fresh load", async () => { + const doc = makeDoc(["asset_1"]); + loadDocument({ + ...doc, + assets: [ + asset("asset_1", { transcriptionFailure: { kind: "no-audio", message: "silent" } }), + ] as AxcutDocument["assets"], + }); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("keeps a transient failure in memory only, and toasts it", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(useTranscriptionStore.getState().jobs.asset_1?.failure?.kind).toBe("error"); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure).toBeUndefined(); + expect(toastMocks.error).toHaveBeenCalledTimes(1); + }); + + it("request() re-runs a failed asset and clears the remembered verdict", async () => { + transcribeMocks.transcribeAsset.mockRejectedValueOnce( + new Error("No audio track found in this video."), + ); + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure?.kind).toBe( + "no-audio", + ); + + await useTranscriptionStore.getState().request("asset_1", "fr"); + + expect(transcribeMocks.transcribeAsset).toHaveBeenLastCalledWith( + expect.anything(), + "asset_1", + expect.objectContaining({ language: "fr" }), + ); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + expect(useProjectStore.getState().document?.assets[0].transcriptionFailure).toBeNull(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + // A run the user asked for reports back. + expect(toastMocks.success).toHaveBeenCalledTimes(1); + }); + + it("drops the queue when another project is loaded", async () => { + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + const other = makeDoc(["asset_9"], "proj_2"); + loadDocument(other); + useTranscriptionStore.getState().sync(other); + expect(useTranscriptionStore.getState().projectId).toBe("proj_2"); + await whenTranscriptionIdle(); + + expect(useProjectStore.getState().document?.transcripts.map((t) => t.assetId)).toEqual([ + "asset_9", + ]); + }); + + it("forgets a job when its asset leaves the document", async () => { + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1", "asset_2"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + expect(Object.keys(useTranscriptionStore.getState().jobs)).toEqual(["asset_1", "asset_2"]); + + const doc = useProjectStore.getState().document as AxcutDocument; + const pruned = { ...doc, assets: doc.assets.filter((a) => a.id === "asset_1") }; + loadDocument(pruned); + useTranscriptionStore.getState().sync(pruned); + + expect(Object.keys(useTranscriptionStore.getState().jobs)).toEqual(["asset_1"]); + }); + + it("runs no background pass without a local STT engine", async () => { + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + loadDocument(makeDoc(["asset_1"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).not.toHaveBeenCalled(); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("lets a manual request supersede the background run of the same asset", async () => { + // The background pass is mid-run on asset_1 (auto language) when the user + // asks for French from the media card. The outgoing run must neither win + // the race nor delete the request that replaced it. + const languages: string[] = []; + let releaseFirst: (() => void) | null = null; + transcribeMocks.transcribeAsset.mockImplementation( + async ( + _doc: AxcutDocument, + assetId: string, + options: { language?: string; signal?: AbortSignal }, + ) => { + languages.push(options.language ?? "auto"); + if (languages.length === 1) { + await new Promise((resolve) => { + releaseFirst = resolve; + }); + throw new DOMException("Aborted", "AbortError"); + } + return { ...transcriptFor(assetId), language: options.language ?? "auto" }; + }, + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await Promise.resolve(); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); + + const requested = useTranscriptionStore.getState().request("asset_1", "fr"); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("queued"); + releaseFirst?.(); + await requested; + await whenTranscriptionIdle(); + + expect(languages).toEqual(["auto", "fr"]); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + expect(useProjectStore.getState().document?.transcripts[0].language).toBe("fr"); + expect(useTranscriptionStore.getState().jobs).toEqual({}); + }); + + it("requestTimelineTranscripts covers the timeline's media, skipping the silent ones", async () => { + // The pane button used to target `primaryAssetId` only — which in a + // recording project is the (often silent) screen capture, leaving the + // talking clip next to it untranscribable from there. + // The background pass is off here so the assertions see only what the + // button itself asked for. + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + const doc = makeDoc(["silent", "voice", "offTimeline"]); + loadDocument({ + ...doc, + assets: [ + asset("silent", { transcriptionFailure: { kind: "no-audio", message: "silent" } }), + asset("voice"), + asset("offTimeline"), + ], + timeline: { + ...doc.timeline, + clips: ["silent", "voice"].map((assetId, i) => ({ + id: `clip_${i}`, + assetId, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + wordRefs: [], + origin: "user", + reason: "", + })), + }, + } as unknown as AxcutDocument); + + await useTranscriptionStore.getState().requestTimelineTranscripts(); + + expect(transcribeMocks.transcribeAsset.mock.calls.map((c) => c[1])).toEqual(["voice"]); + expect(useProjectStore.getState().document?.transcripts.map((t) => t.assetId)).toEqual([ + "voice", + ]); + }); + + it("waits for the background run instead of transcribing the same asset twice", async () => { + let releaseFirst: (() => void) | null = null; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => { + if (!releaseFirst) { + await new Promise((resolve) => { + releaseFirst = resolve; + }); + } + return transcriptFor(assetId); + }, + ); + loadDocument(makeDoc(["asset_1"])); + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await Promise.resolve(); + expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); + + const requested = useTranscriptionStore.getState().requestTimelineTranscripts(); + releaseFirst?.(); + await requested; + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + }); + + it("still honours a manual request without the auto pass", async () => { + // biome-ignore lint/suspicious/noExplicitAny: test-only stub of the preload bridge + delete (window as any).electronAPI; + transcribeMocks.transcribeAsset.mockImplementation( + async (_doc: AxcutDocument, assetId: string) => transcriptFor(assetId), + ); + loadDocument(makeDoc(["asset_1"])); + + await useTranscriptionStore.getState().request("asset_1"); + + expect(useProjectStore.getState().document?.transcripts).toHaveLength(1); + }); +}); diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts new file mode 100644 index 0000000000..c09dba4b8e --- /dev/null +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -0,0 +1,489 @@ +// Single source of truth for "where is each asset's transcript?". +// +// Transcription is local (whisper.cpp, no network), so there is no reason to +// make the user go and ask for it: every asset that lands in the document — +// imported from the Media tab, or auto-added from a screen recording — is +// queued here and transcribed in the background. The transcript itself still +// lives on the document (`document.transcripts[]`); this store only owns the +// JOB: queued / running / failed, plus the phase for the spinner. Nothing +// derives "is there a transcript?" from here — that answer comes from the +// document, and the two are folded together by `deriveAssetStatus`. +// +// Loop safety, which is the whole difficulty of an auto pass whose result +// mutates the document it reacts to: +// +// - `sync` only ever enqueues an asset that has NO transcript, NO job entry +// (queued / running / failed alike) and NO persisted failure. A finished +// run leaves a transcript on the document, a failed one leaves a `failed` +// entry here, so neither can be picked up twice. +// - the job entry is deleted only AFTER the save has resolved, i.e. after +// the document already carries the transcript. +// - the pump is a single sequential loop (whisper-server is one process and +// audio extraction is memory-hungry), guarded by a module-level promise, +// and it drops any job a run left behind rather than spinning on it. + +import { useEffect, useMemo } from "react"; +import { toast } from "sonner"; +import { create } from "zustand"; +import { DEFAULT_LOCALE, LOCALE_STORAGE_KEY, type Locale } from "@/i18n/config"; +import { getAvailableLocales, translate } from "@/i18n/loader"; +import { transcribeAsset, withTranscript } from "../document/transcribe"; +import type { AxcutDocument } from "../schema"; +import { + type AssetTranscriptionView, + classifyTranscriptionError, + deriveAssetStatus, + findAssetTranscript, + isAbortError, + isPermanentFailure, + resolveTranscriptGate, + type TranscriptGate, + type TranscriptionFailure, + type TranscriptionPhase, + transcriptRelevantAssetIds, +} from "../transcription/status"; +import { useProjectStore } from "./projectStore"; + +export interface TranscriptionJob { + status: "queued" | "running" | "failed"; + /** Set when a run picks the job up. Identifies THIS attempt, so a run that + * finishes after the user asked for another one cannot clear its successor. */ + runId?: number; + phase?: TranscriptionPhase; + /** `"auto"` unless the user forced a language from the media card. */ + language: string; + failure?: TranscriptionFailure; + /** User-triggered runs get a toast on success; the background pass stays quiet. */ + manual: boolean; +} + +interface TranscriptionState { + /** Project the jobs belong to — switching projects drops them all. */ + projectId: string | null; + jobs: Record; + + /** Reconcile the queue with a document. Idempotent; safe to call on every document change. */ + sync: (document: AxcutDocument | null) => void; + /** Transcribe (or re-transcribe) one asset now. Resolves once the run settles. */ + request: (assetId: string, language?: string) => Promise; + /** + * What the panes' "Transcribe now" button asks for: every asset the timeline + * plays that still has no transcript. NOT just the primary asset — a project + * whose first (primary) media is a silent screen capture would otherwise + * leave that button unable to transcribe the talking clip next to it. + */ + requestTimelineTranscripts: () => Promise; + reset: () => void; +} + +/** The local engine is only reachable through the preload bridge. */ +function hasLocalSttEngine(): boolean { + if (typeof window === "undefined") return false; + return typeof window.electronAPI?.stt?.transcribe === "function"; +} + +/** + * Toasts fired outside React still have to speak the user's language. Same + * source as `I18nProvider` (stored preference, else the default), validated so + * a stale value can't push `translate` onto a locale it doesn't have. + */ +function toastText(key: string, vars?: Record): string { + let locale: Locale = DEFAULT_LOCALE; + try { + const stored = localStorage.getItem(LOCALE_STORAGE_KEY); + if (stored && getAvailableLocales().includes(stored as Locale)) locale = stored as Locale; + } catch { + // localStorage may be unavailable — the default locale is a fine answer. + } + return translate(locale, "editor", key, vars); +} + +export const useTranscriptionStore = create((set, get) => ({ + projectId: null, + jobs: {}, + + sync(document) { + if (!document) { + if (get().projectId !== null || Object.keys(get().jobs).length > 0) get().reset(); + return; + } + if (document.project.id !== get().projectId) { + get().reset(); + set({ projectId: document.project.id }); + } + + const assetIds = new Set(document.assets.map((a) => a.id)); + const jobs = get().jobs; + let next: Record | null = null; + const patch = () => { + if (!next) next = { ...jobs }; + return next; + }; + + // An asset the user removed takes its job with it (the run itself is + // dropped by `runJob`, which re-reads the document before starting). + for (const assetId of Object.keys(jobs)) { + if (!assetIds.has(assetId)) delete patch()[assetId]; + } + + if (hasLocalSttEngine()) { + for (const asset of document.assets) { + if (jobs[asset.id]) continue; + if (findAssetTranscript(document, asset.id)) continue; + if (asset.transcriptionFailure) continue; + patch()[asset.id] = { status: "queued", language: "auto", manual: false }; + } + } + + if (next) { + set({ jobs: next }); + void pump(); + } + }, + + request(assetId, language = "auto") { + // A manual run can be the first thing that happens in a project (the + // auto pass is off without a local engine), so adopt the loaded project + // before queueing — `runJob` refuses to write into a document the queue + // doesn't belong to. + const document = useProjectStore.getState().document; + if (document && document.project.id !== get().projectId) get().sync(document); + // Asking again for an asset that is mid-run (regenerate in another + // language while the background pass is on it) supersedes that run + // instead of queueing behind it and losing the language. + if (activeRun?.assetId === assetId) abortActiveRun(); + const settled = waitForSettle(assetId); + set((state) => ({ + jobs: { ...state.jobs, [assetId]: { status: "queued", language, manual: true } }, + })); + void pump(); + return settled; + }, + + requestTimelineTranscripts() { + const document = useProjectStore.getState().document; + if (!document) return Promise.resolve(); + get().sync(document); + const targets = transcriptRelevantAssetIds(document).filter((assetId) => { + if (findAssetTranscript(document, assetId)) return false; + // A media with no audio track can only fail again — asking for it here + // would buy the user a run and an error toast for nothing. The per-asset + // regenerate in the media stage stays available for the stubborn case. + return !document.assets.find((a) => a.id === assetId)?.transcriptionFailure; + }); + if (targets.length === 0) return Promise.resolve(); + return Promise.all( + targets.map((assetId) => { + // Already queued or running: the background pass owns that run, so + // wait for it instead of superseding it with an identical one. + const job = get().jobs[assetId]; + if (job && job.status !== "failed") return waitForSettle(assetId); + return get().request(assetId); + }), + ).then(() => undefined); + }, + + reset() { + abortActiveRun(); + const pending = Object.keys(get().jobs); + set({ projectId: null, jobs: {} }); + for (const assetId of pending) flushSettleWaiters(assetId); + }, +})); + +// ─── The pump ────────────────────────────────────────────────────── +// Module state, not store state: none of it is rendered, and keeping it out of +// the store means a re-render can never observe a half-started run. + +let pumping: Promise | null = null; +let activeRun: { assetId: string; controller: AbortController } | null = null; +const settleWaiters = new Map void>>(); + +function waitForSettle(assetId: string): Promise { + return new Promise((resolve) => { + const waiters = settleWaiters.get(assetId); + if (waiters) waiters.push(resolve); + else settleWaiters.set(assetId, [resolve]); + }); +} + +function flushSettleWaiters(assetId: string): void { + const waiters = settleWaiters.get(assetId); + if (!waiters) return; + settleWaiters.delete(assetId); + for (const resolve of waiters) resolve(); +} + +function abortActiveRun(): void { + activeRun?.controller.abort(); + activeRun = null; +} + +/** + * Hand a queued job to a run: stamps it with the run's id, which every later + * write checks. A `request` made mid-run replaces the entry with a fresh + * (unstamped) one, and that stamp is what stops the outgoing run from + * reporting its own status — or its deletion — over its successor. + */ +function claimJob(projectId: string, assetId: string, runId: number): boolean { + let claimed = false; + useTranscriptionStore.setState((state) => { + if (state.projectId !== projectId) return state; + const job = state.jobs[assetId]; + if (!job || job.status !== "queued") return state; + claimed = true; + return { + jobs: { + ...state.jobs, + [assetId]: { + ...job, + runId, + status: "running", + phase: "extracting-audio", + failure: undefined, + }, + }, + }; + }); + return claimed; +} + +/** Patch the job a run owns. No-op once that run has been superseded. */ +function patchJob(assetId: string, runId: number, patch: Partial): void { + useTranscriptionStore.setState((state) => { + const job = state.jobs[assetId]; + if (!job || job.runId !== runId) return state; + return { jobs: { ...state.jobs, [assetId]: { ...job, ...patch } } }; + }); +} + +/** True while `runId` is still the attempt the store is tracking for this asset. */ +function isCurrentRun(assetId: string, runId: number): boolean { + return useTranscriptionStore.getState().jobs[assetId]?.runId === runId; +} + +/** + * Remove a job once it has settled. With a `runId`, only the entry that run + * owns. Waiters are flushed whenever the entry is gone: a caller superseded by + * a newer request is waiting on that newer run, which flushes them in turn. + */ +function dropJob(assetId: string, runId?: number): void { + useTranscriptionStore.setState((state) => { + const job = state.jobs[assetId]; + if (!job) return state; + if (runId !== undefined && job.runId !== runId) return state; + const jobs = { ...state.jobs }; + delete jobs[assetId]; + return { jobs }; + }); + if (useTranscriptionStore.getState().jobs[assetId] === undefined) flushSettleWaiters(assetId); +} + +/** + * Remember a deterministic failure on the asset so the next project open shows + * "no audio" straight away instead of re-extracting the audio to rediscover it. + * Best-effort: a save that loses a race with a user edit is not worth a toast. + */ +async function persistPermanentFailure( + projectId: string, + assetId: string, + failure: TranscriptionFailure, +): Promise { + const kind = failure.kind; + if (!isPermanentFailure(kind)) return; + const project = useProjectStore.getState(); + const doc = project.document; + if (!doc || doc.project.id !== projectId) return; + if (!doc.assets.some((a) => a.id === assetId)) return; + try { + await project.saveDocument({ + ...doc, + assets: doc.assets.map((a) => + a.id === assetId + ? { + ...a, + transcriptionFailure: { + kind, + message: failure.message, + at: new Date().toISOString(), + }, + } + : a, + ), + }); + } catch (error) { + console.warn("[transcription] could not persist the failure on the asset:", error); + } +} + +let runSeq = 0; + +async function runJob(assetId: string, job: TranscriptionJob): Promise { + const projectId = useTranscriptionStore.getState().projectId; + const doc = useProjectStore.getState().document; + if ( + !projectId || + !doc || + doc.project.id !== projectId || + !doc.assets.some((a) => a.id === assetId) + ) { + // Nothing this run could legally write to. Drop it rather than leave it + // queued — `drain` would otherwise pick the same job forever. + dropJob(assetId); + return; + } + + const runId = ++runSeq; + if (!claimJob(projectId, assetId, runId)) return; + const controller = new AbortController(); + activeRun = { assetId, controller }; + + try { + const transcript = await transcribeAsset(doc, assetId, { + language: job.language, + signal: controller.signal, + onStatus: (phase) => patchJob(assetId, runId, { phase: phase as TranscriptionPhase }), + }); + if (controller.signal.aborted) { + dropJob(assetId, runId); + return; + } + // The user may have switched projects while whisper was working — writing + // the transcript now would attach it to the document that is loaded today. + const current = useProjectStore.getState().document; + if (!current || current.project.id !== projectId) { + dropJob(assetId, runId); + return; + } + // One save: the transcript, and (on a successful retry) the removal of + // the verdict remembered on the asset. + await useProjectStore.getState().saveDocument( + withTranscript( + { + ...current, + assets: current.assets.map((a) => + a.id === assetId && a.transcriptionFailure ? { ...a, transcriptionFailure: null } : a, + ), + }, + transcript, + ), + ); + dropJob(assetId, runId); + if (job.manual) toast.success(toastText("mediaStage.transcriptReady")); + } catch (error) { + if (isAbortError(error) || controller.signal.aborted) { + dropJob(assetId, runId); + return; + } + if (!isCurrentRun(assetId, runId)) return; // superseded by a newer request + const failure = classifyTranscriptionError(error); + patchJob(assetId, runId, { status: "failed", phase: undefined, failure }); + flushSettleWaiters(assetId); + await persistPermanentFailure(projectId, assetId, failure); + // A silent recording is an expected outcome, not an incident: the media + // card and every gated button already say so. Only surface the noisy + // (retryable) failures, plus anything the user asked for by hand. + if (failure.kind === "error" || job.manual) { + toast.error(toastText("mediaStage.transcriptionFailed"), { description: failure.message }); + } + } finally { + if (activeRun?.controller === controller) activeRun = null; + } +} + +function nextQueuedJob(): [string, TranscriptionJob] | null { + const { jobs } = useTranscriptionStore.getState(); + for (const [assetId, job] of Object.entries(jobs)) { + if (job.status === "queued") return [assetId, job]; + } + return null; +} + +async function drain(): Promise { + for (;;) { + const next = nextQueuedJob(); + if (!next) return; + const [assetId, job] = next; + await runJob(assetId, job); + // Belt and braces: a run that neither settled nor failed its job would + // make this loop spin. Drop it and move on. + if (useTranscriptionStore.getState().jobs[assetId] === job) { + console.warn("[transcription] job left queued after a run, dropping:", assetId); + dropJob(assetId); + } + } +} + +function pump(): Promise { + if (pumping) return pumping; + pumping = drain().finally(() => { + pumping = null; + }); + return pumping; +} + +/** Test/diagnostic helper: resolves once the queue has drained. */ +export function whenTranscriptionIdle(): Promise { + return pumping ?? Promise.resolve(); +} + +// ─── React bindings ──────────────────────────────────────────────── + +/** + * Mount once (the editor shell does). Keeps the queue reconciled with whatever + * document is loaded — a new project, an imported asset, a removed one. + */ +export function useAutoTranscription(): void { + const document = useProjectStore((s) => s.document); + const sync = useTranscriptionStore((s) => s.sync); + useEffect(() => { + sync(document); + }, [document, sync]); +} + +/** + * Per-asset transcription state, keyed by asset id — one subscription for a + * whole media list instead of a hook per row (which a `.map()` can't have). + * + * There is deliberately no single-asset variant: every consumer either lists + * media (this) or asks about a transcript-dependent action, and the answer for + * an action is `useTimelineTranscriptGate` — resolved over the assets the + * timeline plays, never over one asset picked as representative. + */ +export function useAssetTranscriptions(): Record { + const document = useProjectStore((s) => s.document); + const jobs = useTranscriptionStore((s) => s.jobs); + return useMemo(() => { + const views: Record = {}; + for (const asset of document?.assets ?? []) { + views[asset.id] = deriveAssetStatus({ + assetId: asset.id, + job: jobs[asset.id], + transcript: findAssetTranscript(document, asset.id), + persistedFailure: asset.transcriptionFailure, + }); + } + return views; + }, [document, jobs]); +} + +/** + * Gate for the transcript-dependent timeline actions (Smart cuts): resolved + * over the assets the timeline actually plays. + */ +export function useTimelineTranscriptGate(): TranscriptGate { + const document = useProjectStore((s) => s.document); + const jobs = useTranscriptionStore((s) => s.jobs); + return useMemo(() => { + const views = transcriptRelevantAssetIds(document).map((assetId) => + deriveAssetStatus({ + assetId, + job: jobs[assetId], + transcript: findAssetTranscript(document, assetId), + persistedFailure: + document?.assets.find((a) => a.id === assetId)?.transcriptionFailure ?? null, + }), + ); + return resolveTranscriptGate(views); + }, [document, jobs]); +} diff --git a/src/lib/ai-edition/transcription/status.test.ts b/src/lib/ai-edition/transcription/status.test.ts new file mode 100644 index 0000000000..adc21cd354 --- /dev/null +++ b/src/lib/ai-edition/transcription/status.test.ts @@ -0,0 +1,268 @@ +import { describe, expect, it } from "vitest"; +import type { AxcutDocument, AxcutTranscript } from "../schema"; +import { + type AssetTranscriptionView, + classifyTranscriptionError, + deriveAssetStatus, + isPermanentFailure, + resolveTranscriptGate, + transcriptHasSpeech, + transcriptRelevantAssetIds, +} from "./status"; + +function transcript(assetId: string, words: string[]): AxcutTranscript { + return { + assetId, + language: "en", + segments: words.map((text, i) => ({ + id: `seg_${i}`, + kind: "speech" as const, + startSec: i, + endSec: i + 1, + text, + wordIds: [`word_${i}`], + })), + words: words.map((text, i) => ({ + id: `word_${i}`, + segmentId: `seg_${i}`, + startSec: i, + endSec: i + 1, + text, + })), + }; +} + +function view( + assetId: string, + status: AssetTranscriptionView["status"], + failureKind?: "no-audio" | "unsupported-audio" | "error", +): AssetTranscriptionView { + return { + assetId, + status, + failure: failureKind ? { kind: failureKind, message: `${failureKind} boom` } : undefined, + }; +} + +describe("classifyTranscriptionError", () => { + it("recognises a container with no audio track", () => { + const failure = classifyTranscriptionError(new Error("No audio track found in this video.")); + expect(failure.kind).toBe("no-audio"); + expect(isPermanentFailure(failure.kind)).toBe(true); + }); + + it("treats a decode that yielded nothing as no-audio too", () => { + expect( + classifyTranscriptionError(new Error("Decoded zero audio frames from this video.")).kind, + ).toBe("no-audio"); + }); + + it("recognises an audio codec the caption path cannot read", () => { + const failure = classifyTranscriptionError( + new Error("Audio codec not supported for captions: ac-3"), + ); + expect(failure.kind).toBe("unsupported-audio"); + expect(isPermanentFailure(failure.kind)).toBe(true); + }); + + it("treats anything else as a transient error worth retrying", () => { + const failure = classifyTranscriptionError(new Error("whisper-server exited")); + expect(failure.kind).toBe("error"); + expect(failure.message).toBe("whisper-server exited"); + expect(isPermanentFailure(failure.kind)).toBe(false); + }); +}); + +describe("transcriptHasSpeech", () => { + it("is false for a transcript whisper returned empty", () => { + expect(transcriptHasSpeech(transcript("asset_1", []))).toBe(false); + }); + + it("is true as soon as one word came back", () => { + expect(transcriptHasSpeech(transcript("asset_1", ["hello"]))).toBe(true); + }); +}); + +describe("deriveAssetStatus", () => { + it("reports a live job over an existing transcript (regenerate)", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "running", phase: "transcribing" }, + transcript: transcript("asset_1", ["hello"]), + }); + expect(derived).toEqual({ assetId: "asset_1", status: "running", phase: "transcribing" }); + }); + + it("reports ready from the document, with no job at all", () => { + expect( + deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", ["hello"]) }) + .status, + ).toBe("ready"); + }); + + it("distinguishes an empty transcript from a ready one", () => { + expect( + deriveAssetStatus({ assetId: "asset_1", transcript: transcript("asset_1", []) }).status, + ).toBe("empty"); + }); + + it("keeps a stored transcript ready when a regenerate over it failed", () => { + // The failed retry left the previous transcript untouched on the document: + // reading the asset as "failed" would disable Smart cuts over a transcript + // that is right there and usable. + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "failed", failure: { kind: "error", message: "whisper-server exited" } }, + transcript: transcript("asset_1", ["hello"]), + }); + expect(derived.status).toBe("ready"); + // …and the failure still travels, so a tooltip can explain the red flash. + expect(derived.failure?.message).toBe("whisper-server exited"); + }); + + it("reports failed only when nothing was ever produced", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + job: { status: "failed", failure: { kind: "error", message: "boom" } }, + }); + expect(derived.status).toBe("failed"); + }); + + it("falls back to the failure remembered on the asset across reloads", () => { + const derived = deriveAssetStatus({ + assetId: "asset_1", + persistedFailure: { kind: "no-audio", message: "No audio track found in this video." }, + }); + expect(derived.status).toBe("failed"); + expect(derived.failure?.kind).toBe("no-audio"); + }); + + it("is idle when nothing has been attempted", () => { + expect(deriveAssetStatus({ assetId: "asset_1" }).status).toBe("idle"); + }); +}); + +describe("resolveTranscriptGate", () => { + it("blocks with no-media when the project is empty", () => { + expect(resolveTranscriptGate([])).toEqual({ + state: "blocked", + reason: "no-media", + pendingCount: 0, + }); + }); + + it("opens once an asset has speech", () => { + expect(resolveTranscriptGate([view("a", "ready")]).state).toBe("ready"); + }); + + it("waits while any asset is still in flight, even next to a ready one", () => { + const gate = resolveTranscriptGate([view("a", "ready"), view("b", "running")]); + expect(gate.state).toBe("pending"); + expect(gate.pendingCount).toBe(1); + }); + + it("counts queued assets as pending", () => { + expect(resolveTranscriptGate([view("a", "queued"), view("b", "queued")]).pendingCount).toBe(2); + }); + + it("blocks on no-audio when every media is silent", () => { + const gate = resolveTranscriptGate([ + view("a", "failed", "no-audio"), + view("b", "failed", "unsupported-audio"), + ]); + expect(gate.state).toBe("blocked"); + expect(gate.reason).toBe("no-audio"); + }); + + it("blocks on failed (retryable) as soon as one failure is not about silence", () => { + const gate = resolveTranscriptGate([ + view("a", "failed", "no-audio"), + view("b", "failed", "error"), + ]); + expect(gate.reason).toBe("failed"); + expect(gate.message).toContain("boom"); + }); + + it("stays ready when a failed retry sits on top of a usable transcript", () => { + const gate = resolveTranscriptGate([ + { + assetId: "a", + status: "ready", + failure: { kind: "error", message: "whisper-server exited" }, + }, + ]); + expect(gate.state).toBe("ready"); + }); + + it("blocks on no-speech when the transcripts came back empty", () => { + expect(resolveTranscriptGate([view("a", "empty")]).reason).toBe("no-speech"); + }); + + it("blocks on not-started when nothing ran (no local engine)", () => { + expect(resolveTranscriptGate([view("a", "idle")]).reason).toBe("not-started"); + }); +}); + +describe("transcriptRelevantAssetIds", () => { + const base = { + schemaVersion: 7 as const, + project: { + id: "proj_1", + title: "T", + createdAt: "2026-06-25T10:00:00.000Z", + updatedAt: "2026-06-25T10:00:00.000Z", + }, + transcript: null, + transcripts: [], + annotations: [], + zoomRanges: [], + legacyEditor: null, + }; + + function doc(assetIds: string[], clipAssetIds: string[]): AxcutDocument { + return { + ...base, + assets: assetIds.map((id) => ({ + id, + kind: "video" as const, + label: id, + originalPath: `/tmp/${id}.mp4`, + cameraTrack: null, + })), + timeline: { + clips: clipAssetIds.map((assetId, i) => ({ + id: `clip_${i}`, + assetId, + sourceStartSec: 0, + sourceEndSec: 10, + timelineStartSec: i * 10, + timelineEndSec: i * 10 + 10, + wordRefs: [], + origin: "system" as const, + reason: "", + })), + gaps: [], + trimRanges: [], + muteRanges: [], + speedRanges: [], + captionRanges: [], + }, + } as AxcutDocument; + } + + it("only counts the assets the timeline plays", () => { + expect(transcriptRelevantAssetIds(doc(["a", "b"], ["a", "a"]))).toEqual(["a"]); + }); + + it("falls back to the whole media bin while the timeline is empty", () => { + expect(transcriptRelevantAssetIds(doc(["a", "b"], []))).toEqual(["a", "b"]); + }); + + it("ignores clips pointing at a removed asset", () => { + expect(transcriptRelevantAssetIds(doc(["a"], ["ghost"]))).toEqual(["a"]); + }); + + it("has nothing to say about a missing document", () => { + expect(transcriptRelevantAssetIds(null)).toEqual([]); + }); +}); diff --git a/src/lib/ai-edition/transcription/status.ts b/src/lib/ai-edition/transcription/status.ts new file mode 100644 index 0000000000..fced8390a4 --- /dev/null +++ b/src/lib/ai-edition/transcription/status.ts @@ -0,0 +1,219 @@ +// Pure status logic for the transcription pipeline: what state one asset's +// transcript is in, and whether a transcript-dependent action (Smart cuts, +// captions…) may run right now. +// +// Kept free of React and of both stores so the rules can be unit-tested on +// plain data — `store/transcriptionStore.ts` owns the queue and the side +// effects, this module owns the vocabulary. + +import type { AxcutDocument, AxcutTranscript } from "../schema"; + +/** Why a transcription run could not produce anything. */ +export type TranscriptionFailureKind = "no-audio" | "unsupported-audio" | "error"; + +export interface TranscriptionFailure { + kind: TranscriptionFailureKind; + /** Raw engine/exception message — surfaced as a tooltip / toast description. */ + message: string; +} + +/** Which half of the pipeline a running job is in (mirrors `TranscribeAssetOptions.onStatus`). */ +export type TranscriptionPhase = "extracting-audio" | "transcribing"; + +/** + * A media that has no audio track (or one Whisper cannot read) will fail the + * same way on every attempt, so that verdict is worth remembering: it is + * persisted on the asset and stops the auto pass from re-extracting the audio + * of a silent screen recording on every project open. Everything else + * ("error") is treated as transient and retried on the next load. + */ +export function isPermanentFailure(kind: TranscriptionFailureKind): kind is PersistableFailureKind { + return kind !== "error"; +} + +/** The failure kinds `assetSchema.transcriptionFailure` accepts. */ +export type PersistableFailureKind = Exclude; + +/** + * Map an exception out of `transcribeAsset` onto a failure the UI can explain. + * The two deterministic cases come from `extractMono16kWebDemuxer` — it is the + * only layer that knows whether the container actually holds audio. + */ +export function classifyTranscriptionError(error: unknown): TranscriptionFailure { + const message = error instanceof Error ? error.message : String(error); + if (/no audio track/i.test(message) || /zero audio frames/i.test(message)) { + return { kind: "no-audio", message }; + } + if (/audio codec not supported/i.test(message)) { + return { kind: "unsupported-audio", message }; + } + return { kind: "error", message }; +} + +export function isAbortError(error: unknown): boolean { + return ( + (error instanceof DOMException && error.name === "AbortError") || + (error instanceof Error && error.name === "AbortError") + ); +} + +export type AssetTranscriptionStatus = + /** Nothing attempted yet (no local engine, or the auto pass hasn't reached it). */ + | "idle" + | "queued" + | "running" + /** A transcript exists and holds at least one word. */ + | "ready" + /** A transcript exists but Whisper heard no speech — nothing for the agent to cut on. */ + | "empty" + | "failed"; + +export interface AssetTranscriptionView { + assetId: string; + status: AssetTranscriptionStatus; + phase?: TranscriptionPhase; + failure?: TranscriptionFailure; +} + +/** In-flight (or last-failed) state of one asset's job. Mirrors the store entry. */ +export interface TranscriptionJobLike { + status: "queued" | "running" | "failed"; + phase?: TranscriptionPhase; + failure?: TranscriptionFailure; +} + +export function findAssetTranscript( + document: AxcutDocument | null, + assetId: string, +): AxcutTranscript | null { + if (!document) return null; + return ( + document.transcripts.find((t) => t.assetId === assetId) ?? + (document.transcript?.assetId === assetId ? document.transcript : null) + ); +} + +/** A transcript with no word is "empty", not "ready": captions and AI cuts have nothing to work with. */ +export function transcriptHasSpeech(transcript: AxcutTranscript | null): boolean { + if (!transcript) return false; + return transcript.words.length > 0 || transcript.segments.some((s) => s.text.trim().length > 0); +} + +/** + * Fold the live job (if any), the persisted failure (if any) and the stored + * transcript into the single status the UI renders. Precedence, in order: + * + * 1. A run in flight — a regenerate over an existing transcript must read as + * "running", not "ready". + * 2. A stored transcript — it OUTRANKS a failed job on purpose. A regenerate + * that fails (whisper restart, decode hiccup) leaves the previous + * transcript untouched on the document, and it is still perfectly usable: + * the pane renders it, captions read it, the agent can cut on it. Reading + * that asset as "failed" would have disabled Smart cuts for the rest of the + * session over a transcript that is right there. The failure still travels + * on the view (tooltips surface it), it just doesn't veto the content. + * 3. Only then a failure — nothing was ever produced for this asset. + */ +export function deriveAssetStatus(input: { + assetId: string; + job?: TranscriptionJobLike; + transcript?: AxcutTranscript | null; + persistedFailure?: TranscriptionFailure | null; +}): AssetTranscriptionView { + const { assetId, job, transcript, persistedFailure } = input; + if (job && job.status !== "failed") { + return { assetId, status: job.status, phase: job.phase }; + } + if (transcript) { + return { + assetId, + status: transcriptHasSpeech(transcript) ? "ready" : "empty", + failure: job?.failure, + }; + } + if (job?.status === "failed") { + return { assetId, status: "failed", failure: job.failure }; + } + if (persistedFailure) { + return { assetId, status: "failed", failure: persistedFailure }; + } + return { assetId, status: "idle" }; +} + +export type TranscriptGateState = "ready" | "pending" | "blocked"; + +export type TranscriptGateReason = + /** The project holds no media at all. */ + | "no-media" + /** Every media is silent (no audio track / unreadable audio). */ + | "no-audio" + /** At least one run failed for a reason worth retrying. */ + | "failed" + /** Transcripts exist but hold no speech. */ + | "no-speech" + /** Nothing has been transcribed yet and nothing is running (no local engine). */ + | "not-started"; + +export interface TranscriptGate { + state: TranscriptGateState; + /** Null when `state === "ready"`. */ + reason: TranscriptGateReason | null; + /** Engine message behind a `failed` reason, for the tooltip/description. */ + message?: string; + /** How many assets are still queued or running — drives the "2 remaining" hint. */ + pendingCount: number; +} + +/** + * Decide whether a transcript-dependent action may run over a set of assets. + * + * Pending beats ready on purpose: with one media transcribed and another still + * running, letting the agent loose now would have it plan cuts against half the + * timeline and then watch the document change underneath it. + */ +export function resolveTranscriptGate(views: AssetTranscriptionView[]): TranscriptGate { + if (views.length === 0) { + return { state: "blocked", reason: "no-media", pendingCount: 0 }; + } + const pendingCount = views.filter((v) => v.status === "queued" || v.status === "running").length; + if (pendingCount > 0) { + return { state: "pending", reason: null, pendingCount }; + } + if (views.some((v) => v.status === "ready")) { + return { state: "ready", reason: null, pendingCount: 0 }; + } + const failures = views.filter((v) => v.status === "failed"); + if (failures.length > 0) { + const everyFailureIsSilence = failures.every( + (v) => v.failure?.kind === "no-audio" || v.failure?.kind === "unsupported-audio", + ); + return { + state: "blocked", + reason: everyFailureIsSilence ? "no-audio" : "failed", + message: failures.find((v) => v.failure?.message)?.failure?.message, + pendingCount: 0, + }; + } + if (views.some((v) => v.status === "empty")) { + return { state: "blocked", reason: "no-speech", pendingCount: 0 }; + } + return { state: "blocked", reason: "not-started", pendingCount: 0 }; +} + +/** + * Assets a transcript-dependent timeline action actually depends on: the ones + * the timeline plays. An asset sitting in the media bin but not on the timeline + * must not keep the Smart-cuts entry disabled — and, symmetrically, must not + * make it look ready when the clip on screen has no transcript. Falls back to + * the whole bin while the timeline is still empty. + */ +export function transcriptRelevantAssetIds(document: AxcutDocument | null): string[] { + if (!document) return []; + const onTimeline: string[] = []; + for (const clip of document.timeline.clips) { + if (!onTimeline.includes(clip.assetId)) onTimeline.push(clip.assetId); + } + const known = new Set(document.assets.map((a) => a.id)); + const filtered = onTimeline.filter((id) => known.has(id)); + return filtered.length > 0 ? filtered : document.assets.map((a) => a.id); +} diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index c9694ec791..a2f1e385dd 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -13,7 +13,9 @@ the transcript). ```mermaid flowchart LR - A["Recorded audio"] -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] + Z["Asset added
(import or recording)"] -- "sync()" --> Y["transcriptionStore queue
(src/lib/ai-edition/store/transcriptionStore.ts)"] + Y -- "one job at a time" --> A["Recorded audio"] + A -- "extract mono 16 kHz" --> B["transcribeAsset
(src/lib/ai-edition/document/transcribe.ts)"] B -- "IPC: Float32Array + language" --> C["SttManager
(electron/stt/index.ts)"] C -- "POST /inference (WAV)" --> D["whisper-stt-server
(electron/native/whisper-stt/)"] D -- "whisper_full() + DTW" --> E["SttTranscribeResponse
(segments + wordSegments)"] @@ -33,6 +35,67 @@ result is mapped back onto the `AxcutTranscript` shape the rest of the editor reads. From the moment that transcript is persisted, captions and the words they show cannot drift — captions are a derived view, not a parallel store. +## Auto-transcription + +Recognition is local and free, so the editor does not wait to be asked: every +asset in the document gets a transcript on its own. The queue lives in +[`src/lib/ai-edition/store/transcriptionStore.ts`](../../src/lib/ai-edition/store/transcriptionStore.ts), +mounted once by the shell through `useAutoTranscription()`, and it is the only +thing that calls `transcribeAsset`. + +Why it exists at all: "Smart cuts with AI" (and captions, and the transcript +pane) need a transcript, but nothing produced one until the user found the +Media tab or the transcript pane and pressed a button. The obvious first click +was therefore also the one that could not work. Now the button is either ready, +or it says what it is waiting for. + +What the store owns and what it does not: + +- **The document owns the transcript.** `document.transcripts[]` is still the + source of truth for "is there one"; the store only owns the JOB (queued / + running / failed + the phase for the spinner). `deriveAssetStatus` + ([`transcription/status.ts`](../../src/lib/ai-edition/transcription/status.ts)) + folds the two into the single status the UI renders, and + `resolveTranscriptGate` folds a set of those into the ready / pending / + blocked verdict that enables or disables a transcript-dependent action. + A stored transcript **outranks a failed job**: a regenerate that dies on a + whisper restart leaves the previous transcript in place and usable, and + reading that asset as "failed" would have disabled Smart cuts for the rest of + the session over a transcript sitting right there. +- **Gates are resolved over the assets the TIMELINE plays** + (`transcriptRelevantAssetIds`), never over `project.primaryAssetId`. In a + recording project the primary asset is the screen capture — frequently the + silent one — so a primary-scoped answer had the transcript and captions panes + announce "this media has no audio track" for a project whose actual footage + was mid-transcription. `requestTimelineTranscripts` is the matching action for + the panes' one button: every timeline asset that still lacks a transcript, + skipping the ones already known to be silent and waiting on (rather than + duplicating) a run the background pass already has in flight. +- **One run at a time.** whisper-server is a single process and the audio + extraction path holds decoded frames in renderer memory, so the pump is a + sequential loop, not a fan-out. +- **The auto pass never loops.** `sync` enqueues an asset only when it has no + transcript, no job entry (queued / running / failed alike) and no persisted + failure, and a job is deleted only after the save that carries its transcript + has resolved. Since `sync` runs on every document change — including the one + the run itself produces — that guard is what keeps it from re-triggering. +- **Silence is remembered, glitches are not.** A media with no audio track (a + screen recording captured with no mic and no system audio) fails the same way + every time, so the verdict is written to `asset.transcriptionFailure` and the + auto pass skips it on the next load instead of re-extracting its audio to + rediscover it. Everything else stays in memory for the session and is retried + on the next load. A successful manual retry clears the stored verdict in the + same save that writes the transcript. +- **No local engine, no background pass.** Without `window.electronAPI.stt` + (browser preview, e2e shim) nothing is queued; a manual request still runs. + +What each state means in the UI: a spinner and "Transcribing…" while queued or +running, the media-card dot green (ready) / amber (silent or no speech) / red +(failed) via +[`TranscriptionStatus.tsx`](../../src/components/ai-edition/TranscriptionStatus.tsx), +and — the point of the whole thing — a disabled "Smart cuts" entry whose +subtitle is the reason rather than "With AI". + ## The STT engine The recogniser is **whisper.cpp**, embedded as a static library inside one From ad2a6ebd8f80a620206b97e47584abd9d29e2301 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Sat, 1 Aug 2026 15:02:42 +0200 Subject: [PATCH 2/4] fix(editor): hold the "one busy phase" promise through the first-run download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 253 MB model download happens inside the `stt:transcribe` IPC — i.e. inside a run the store has already marked `running` — so it correctly reads as one longer "Transcribing…" phase, with no separate step and nothing falsely clickable. The edges of that first run did not hold up, though: - `SttManager.init` cached a REJECTED `prepare()`, so one dropped connection during the download failed every later transcription in the session: the remaining assets in the queue flipped red in the same frame, and the retry the editor offers was a dead control until the app was restarted — reconnecting changed nothing. The slot is cleared on failure now. - A transient failure now stops the queue: the still-queued assets inherit the verdict instead of each spending a full retry budget and stacking an identical toast. It is the engine that failed, not their media. - The transcript pane's read-only state is scoped PER ASSET again. Widening it to the timeline-wide gate made every other clip's word stream swallow Backspace and hover-bin clicks for the whole background pass, with nothing on screen to say why — the exact "looks live, ignores you" failure mode. A block being rewritten now shows a spinner + "Transcribing…" and dims its stream, and the per-clip empty line no longer tells the user to regenerate an asset that is mid-run. - `mixToMono` hoists its channel arrays out of the sample loop. One WebIDL call per sample per channel (~57 M for a ten-minute stereo recording) froze the window — spinners included — for seconds, which was survivable while the pass was user-triggered and is not now that it is automatic. Found by an adversarial review of the first-run sequence; 4 new tests cover the init retry, the queue stop and the per-asset read-only scoping. --- electron/stt/index.test.ts | 20 ++++++++++ electron/stt/index.ts | 13 ++++++- src/components/ai-edition/NewEditorShell.tsx | 14 ++++++- src/components/ai-edition/RightPanes.tsx | 39 ++++++++++++++++--- .../ai-edition/TranscriptPane.gating.test.tsx | 2 +- .../TranscriptPane.keyboardCut.test.tsx | 31 +++++++++++++-- .../TranscriptPane.sharedMedia.test.tsx | 2 +- .../store/transcriptionStore.test.ts | 17 ++++++++ .../ai-edition/store/transcriptionStore.ts | 30 ++++++++++++++ src/lib/captioning/extractMono16k.ts | 18 +++++++-- .../transcription-and-captions.md | 36 +++++++++++++++++ 11 files changed, 206 insertions(+), 16 deletions(-) diff --git a/electron/stt/index.test.ts b/electron/stt/index.test.ts index 351b756d0c..df94fc102d 100644 --- a/electron/stt/index.test.ts +++ b/electron/stt/index.test.ts @@ -94,6 +94,26 @@ describe("SttManager", () => { expect(fakeWhisperServer.stop).toHaveBeenCalledOnce(); }); + it("retries setup after a failed one instead of caching the rejection", async () => { + // First run downloads a 253 MB model. Caching a rejected `prepare()` meant + // one dropped connection failed every later transcription in the session — + // including the retry the editor offers — until the app was restarted. + const { ensureModels } = await import("./modelManager"); + const mocked = vi.mocked(ensureModels); + mocked.mockClear(); + mocked.mockRejectedValueOnce(new Error("Failed to download: network unreachable")); + const mgr = new SttManager(); + + await expect(mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" })).rejects.toThrow( + "network unreachable", + ); + // The network came back: the next attempt must actually attempt. + await mgr.init({ modelsBaseDir: "/tmp/fake-stt-models" }); + + expect(mocked).toHaveBeenCalledTimes(2); + expect(fakeWhisperServer.start).toHaveBeenCalledOnce(); + }); + it("setStatusSink replaces the previous sink (last call wins)", () => { const mgr = new SttManager(); const a = vi.fn(); diff --git a/electron/stt/index.ts b/electron/stt/index.ts index 4f3e992aa9..641a3abe6e 100644 --- a/electron/stt/index.ts +++ b/electron/stt/index.ts @@ -59,7 +59,18 @@ export class SttManager { if (options.statusSink) this.statusSink = options.statusSink; if (options.modelsBaseDir) this.modelsBaseDir = options.modelsBaseDir; if (!this.initPromise) { - this.initPromise = this.prepare(); + // A REJECTED init must not be cached. `prepare()` downloads a 253 MB + // model on first run, and caching its rejection meant one dropped + // connection poisoned the whole app session: every later transcription + // — including the retry the UI offers, and every remaining asset in the + // auto-transcription queue — awaited the same stale rejection and failed + // in milliseconds, with no way back short of quitting the app. + // Reconnecting the network changed nothing. Clearing the slot on failure + // makes the next attempt a real attempt. + this.initPromise = this.prepare().catch((error) => { + this.initPromise = null; + throw error; + }); } return this.initPromise; } diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index acbcf02857..356e250d7e 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -15,6 +15,7 @@ import { import { type AxcutClip, documentSchema } from "@/lib/ai-edition/schema"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { + useAssetTranscriptions, useAutoTranscription, useTimelineTranscriptGate, useTranscriptionStore, @@ -139,6 +140,17 @@ export function NewEditorShell() { // routinely silent, and keying the transcript pane off it made the pane claim // "no audio track" for a project whose actual footage was mid-transcription. const transcriptGate = useTimelineTranscriptGate(); + // Per asset, for the transcript pane: only the block whose transcript is + // actually being rewritten goes read-only. The gate answers "may a + // transcript-dependent ACTION run?", which is a different question. + const transcriptions = useAssetTranscriptions(); + const busyAssetIds = useMemo( + () => + Object.values(transcriptions) + .filter((v) => v.status === "running" || v.status === "queued") + .map((v) => v.assetId), + [transcriptions], + ); const tl = useTimeline(); useUndoRedoShortcuts(() => { // ponytail: placeholder, wire when undo stack merges with history @@ -1002,7 +1014,7 @@ export function NewEditorShell() { transcripts: document?.transcripts ?? [], assets: document?.assets ?? [], trimRanges: document?.timeline?.trimRanges ?? [], - busy: transcriptGate.state === "pending", + busyAssetIds, onSeek: handleSeek, onAddTrimRange: handleAddTrimRange, onRemoveTrimRange: handleRemoveTrimRange, diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 3de8beea42..0490f31053 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -9,6 +9,7 @@ import { FileText, HelpCircle, Layout as LayoutIcon, + Loader2, MousePointerClick, Palette, Sliders, @@ -524,7 +525,7 @@ export function TranscriptPane({ transcripts, assets, trimRanges, - busy, + busyAssetIds, onSeek, onAddTrimRange, onRemoveTrimRange, @@ -537,7 +538,12 @@ export function TranscriptPane({ transcripts: AxcutTranscript[]; assets: AxcutAsset[]; trimRanges: AxcutTrimRange[]; - busy: boolean; + /** Assets whose transcript is being (re)generated right now — their block is + * read-only while the run is in flight, since it is about to be replaced. + * PER ASSET on purpose: a timeline-wide flag made every other clip's word + * stream silently swallow Backspace and hover-bin clicks for the whole + * background pass, with nothing on screen to say why. */ + busyAssetIds: readonly string[]; onSeek: (sec: number) => void; onAddTrimRange: (target: TrimTarget, startSec: number, endSec: number, reason: string) => void; onRemoveTrimRange: (trimId: string) => void; @@ -646,7 +652,7 @@ export function TranscriptPane({ key={section.clip.id} index={idx} section={section} - busy={busy} + busy={busyAssetIds.includes(section.clip.assetId)} cueWordId={cueWordId} onSeek={onSeek} onAddTrimRange={onAddTrimRange} @@ -952,6 +958,23 @@ const TranscriptClipBlock = memo(function TranscriptClipBlock({ {ts("transcript.clipLabel", { index: index + 1 })} · {sourceRangeLabel} + {/* A block whose transcript is being regenerated is read-only — say it, + rather than letting the word stream look live and drop the edits. */} + {busy ? ( + + + {ts("transcript.transcribing")} + + ) : null} {words.length === 0 ? (

- {ts("transcript.noClipTranscript")} + {busy ? ts("transcript.transcribing") : ts("transcript.noClipTranscript")}

) : (
{ expect(cutRange(onAddTrimRange)).toEqual([2, 3]); // "trois" }); + it("keeps cutting while ANOTHER asset is being transcribed", () => { + // The background pass runs on its own now, so a run on some other media must + // not quietly turn this block into an editor that ignores Backspace — the + // read-only state is scoped to the asset whose transcript is being rewritten. + const { editor, onAddTrimRange } = renderPane([], vi.fn(), ["asset_other"]); + caretBeforeWordAt(editor, 3); + fireEvent.keyDown(editor, { key: "Backspace" }); + expect(cutRange(onAddTrimRange)).toEqual([2, 3]); + }); + + it("stops cutting, visibly, while THIS asset is being transcribed", () => { + // Its transcript is about to be replaced, so the block is read-only — and it + // says so, instead of swallowing the keystroke in silence. + const { editor, onAddTrimRange, getByText } = renderPane([], vi.fn(), ["asset_1"]); + caretBeforeWordAt(editor, 3); + fireEvent.keyDown(editor, { key: "Backspace" }); + expect(cutRange(onAddTrimRange)).toBeNull(); + expect(editor).toHaveAttribute("aria-busy", "true"); + expect(getByText("Transcribing…")).toBeInTheDocument(); + }); + it("Backspace skips over an already-trimmed word instead of doing nothing", () => { // Hold Backspace and you land here: "deux" is already struck through, so the word // immediately before the caret has nothing left to cut. The keystroke used to @@ -184,7 +209,7 @@ describe("keyboard cut with the caret between words", () => { transcripts={[TRANSCRIPT]} assets={[ASSET]} trimRanges={trims} - busy={false} + busyAssetIds={[]} onSeek={vi.fn()} onAddTrimRange={(_target, startSec, endSec) => setTrims((prev) => [ diff --git a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx index 36004c5c38..9ce354646a 100644 --- a/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx +++ b/src/components/ai-edition/TranscriptPane.sharedMedia.test.tsx @@ -73,7 +73,7 @@ function renderPane(onSeek: (sec: number) => void = vi.fn()) { transcripts={[TRANSCRIPT]} assets={[ASSET]} trimRanges={[]} - busy={false} + busyAssetIds={[]} onSeek={onSeek} onAddTrimRange={vi.fn()} onRemoveTrimRange={vi.fn()} diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts index 61bd4b6bd2..25e2b25484 100644 --- a/src/lib/ai-edition/store/transcriptionStore.test.ts +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -211,6 +211,23 @@ describe("useTranscriptionStore", () => { expect(toastMocks.error).toHaveBeenCalledTimes(1); }); + it("stops the queue on an engine failure instead of failing each asset in turn", async () => { + // The model download died / whisper-server didn't come up: that verdict is + // about the engine, so the remaining assets inherit it rather than each + // spending a full retry budget and stacking an identical toast. + transcribeMocks.transcribeAsset.mockRejectedValue(new Error("whisper-server exited")); + loadDocument(makeDoc(["asset_1", "asset_2", "asset_3"])); + + useTranscriptionStore.getState().sync(useProjectStore.getState().document); + await whenTranscriptionIdle(); + + expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); + const jobs = useTranscriptionStore.getState().jobs; + expect(Object.values(jobs).map((j) => j.status)).toEqual(["failed", "failed", "failed"]); + expect(jobs.asset_3?.failure?.message).toBe("whisper-server exited"); + expect(toastMocks.error).toHaveBeenCalledTimes(1); + }); + it("request() re-runs a failed asset and clears the remembered verdict", async () => { transcribeMocks.transcribeAsset.mockRejectedValueOnce( new Error("No audio track found in this video."), diff --git a/src/lib/ai-edition/store/transcriptionStore.ts b/src/lib/ai-edition/store/transcriptionStore.ts index c09dba4b8e..9d765afc30 100644 --- a/src/lib/ai-edition/store/transcriptionStore.ts +++ b/src/lib/ai-edition/store/transcriptionStore.ts @@ -257,6 +257,29 @@ function patchJob(assetId: string, runId: number, patch: Partial job.status === "queued") + .map(([assetId]) => assetId); + if (queued.length === 0) return; + useTranscriptionStore.setState((state) => { + if (state.projectId !== projectId) return state; + const jobs = { ...state.jobs }; + for (const assetId of queued) { + const job = jobs[assetId]; + if (job?.status !== "queued") continue; + jobs[assetId] = { ...job, status: "failed", phase: undefined, failure }; + } + return { jobs }; + }); + for (const assetId of queued) flushSettleWaiters(assetId); +} + /** True while `runId` is still the attempt the store is tracking for this asset. */ function isCurrentRun(assetId: string, runId: number): boolean { return useTranscriptionStore.getState().jobs[assetId]?.runId === runId; @@ -380,6 +403,13 @@ async function runJob(assetId: string, job: TranscriptionJob): Promise { patchJob(assetId, runId, { status: "failed", phase: undefined, failure }); flushSettleWaiters(assetId); await persistPermanentFailure(projectId, assetId, failure); + // A transient failure is about the ENGINE, not about this media: the model + // download died, whisper-server didn't come up. Marching the rest of the + // queue into the same wall would spend a full retry budget per asset and + // stack one identical toast per asset. Fail them with the same verdict + // instead — the gate then reads "failed" (not "queued forever"), and one + // manual retry re-runs them all once the engine is back. + if (failure.kind === "error") failRemainingQueue(projectId, failure); // A silent recording is an expected outcome, not an incident: the media // card and every gated button already say so. Only surface the noisy // (retryable) failures, plus anything the user asked for by hand. diff --git a/src/lib/captioning/extractMono16k.ts b/src/lib/captioning/extractMono16k.ts index bf2c7320fc..1116bb9a2a 100644 --- a/src/lib/captioning/extractMono16k.ts +++ b/src/lib/captioning/extractMono16k.ts @@ -57,13 +57,23 @@ async function loadSourceVideoFile(videoUrl: string, signal?: AbortSignal): Prom function mixToMono(audioBuffer: AudioBuffer): Float32Array { const { length, numberOfChannels } = audioBuffer; + if (numberOfChannels === 0) return new Float32Array(length); + // `getChannelData` is a WebIDL call, so calling it INSIDE the sample loop cost + // one call per sample per channel — ~57 M of them for a ten-minute stereo + // recording, seconds of blocked main thread. That was survivable while + // transcription only ran when the user asked for it; it now runs by itself when + // a project opens, and a frozen window (spinners included) is exactly what the + // automatic pass must not look like. Hoisting the channel arrays out of the loop + // leaves plain typed-array indexing. + const channels: Float32Array[] = []; + for (let c = 0; c < numberOfChannels; c++) channels.push(audioBuffer.getChannelData(c)); + // Mono source: the mixdown is a copy. `slice` keeps the caller's contract of + // owning its buffer (the AudioBuffer's own array is reused by the context). + if (numberOfChannels === 1) return channels[0].slice(); const out = new Float32Array(length); - if (numberOfChannels === 0) return out; for (let i = 0; i < length; i++) { let sum = 0; - for (let c = 0; c < numberOfChannels; c++) { - sum += audioBuffer.getChannelData(c)[i]; - } + for (let c = 0; c < numberOfChannels; c++) sum += channels[c][i]; out[i] = sum / numberOfChannels; } return out; diff --git a/technical-documentation/architecture/transcription-and-captions.md b/technical-documentation/architecture/transcription-and-captions.md index a2f1e385dd..cf60e7c6bd 100644 --- a/technical-documentation/architecture/transcription-and-captions.md +++ b/technical-documentation/architecture/transcription-and-captions.md @@ -96,6 +96,42 @@ running, the media-card dot green (ready) / amber (silent or no speech) / red and — the point of the whole thing — a disabled "Smart cuts" entry whose subtitle is the reason rather than "With AI". +### One phase, including the first-run model download + +On a fresh install the GGML model (~253 MB) is not on disk. It is fetched by +`SttManager.prepare()` **inside** the `stt:transcribe` IPC call — i.e. inside a +run this store has already marked `running` — so the user sees one single busy +phase that simply takes longer the first time. That is deliberate: no separate +"downloading" step, no progress bar to stare at, and nothing that looks +clickable in the meantime (`phase: "model"` is emitted by the main process but +deliberately not forwarded to the renderer by `transcribeAsset`). Nothing in the +renderer imposes a timeout that a slow download could trip: the preload does a +bare `ipcRenderer.invoke`, `fetchWithRetry` has no per-request deadline, and +whisper-server's 30 s readiness budget only starts once the download resolved. + +Three edges make that promise hold, and each is load-bearing: + +- **A failed setup is not cached.** `SttManager.init` used to memoise the + rejected `prepare()` promise, so one dropped connection during the first + download failed every later transcription in the session — including the retry + the editor offers — until the app was restarted. It now clears the slot on + failure ([`electron/stt/index.ts`](../../electron/stt/index.ts)). +- **A transient failure stops the queue** instead of walking the remaining + assets into the same wall: they inherit the verdict (one toast, one retry + affordance) rather than each spending a full retry budget. +- **Read-only is scoped per asset.** The transcript pane's blocks go read-only + only for the asset whose transcript is being rewritten, and say so (spinner + + "Transcribing…" + dimmed stream). A timeline-wide flag made every other + clip's word stream swallow Backspace and hover-bin clicks in silence for the + whole background pass. + +Still synchronous, and still on the main thread: `mixToMono` +([`src/lib/captioning/extractMono16k.ts`](../../src/lib/captioning/extractMono16k.ts)) +now hoists its channel arrays out of the sample loop (it was making one WebIDL +call per sample per channel), but a long recording will still block the renderer +for a moment during "extracting-audio". Moving the mixdown to a worker is the +next step if it shows up in practice. + ## The STT engine The recogniser is **whisper.cpp**, embedded as a static library inside one From f2c89092f4c280bb672e1321ec0eef2794d50921 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Sat, 1 Aug 2026 15:15:39 +0200 Subject: [PATCH 3/4] fix(test): type the deferred release so the tests typecheck `tsc -p tsconfig.test.json` (its own CI job) rejected the two held-open-run tests: assigning the resolver inside a Promise executor leaves the outer `let releaseFirst: (() => void) | null` narrowed to `null`, so `releaseFirst?.()` is a call on `never`. A small `deferred()` helper hands back a stable 0-arg release instead. Also fixes the instruction that let this reach CI: AGENTS.md said `npx tsc --noEmit` was what CI runs, and that config does not see test files at all. --- .claude/launch.json | 3 +-- AGENTS.md | 2 +- .../store/transcriptionStore.test.ts | 27 ++++++++++--------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.claude/launch.json b/.claude/launch.json index 16c0d4afe8..0cbf829326 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -12,8 +12,7 @@ "name": "desktop-dev", "runtimeExecutable": "npm", "runtimeArgs": ["run", "dev", "--", "--port", "5201", "--strictPort"], - "port": 5201, - "env": { "NO_ELECTRON": "1" } + "port": 5201 }, { "name": "v4preview", diff --git a/AGENTS.md b/AGENTS.md index 1f1a63d40e..8864fdf576 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ OpenScreen is a free, open-source screen recorder and video editor (Electron + R - Install deps: `npm install` (Node 22.22.1, npm 10.9.4 — see `package.json#engines`) - Start dev: `npm run dev` (Vite dev server; Electron window opens via `vite-plugin-electron`) - Build: `npm run build` (TypeScript check + Vite build + electron-builder) -- Typecheck: `npx tsc --noEmit` (CI runs this; no standalone script) +- Typecheck: `npx tsc --noEmit` — app code only. CI also runs `npx tsc -p tsconfig.test.json --noEmit` in a separate job ("Typecheck (tests)"), so **run both**: test files are invisible to the root config, and a type error in a `*.test.ts` fails CI while the root check stays green. - Test (unit): `npm run test` (Vitest, jsdom env) - Test (browser): `npm run test:browser` (Vitest + Playwright, requires `npm run test:browser:install` first) - Test (e2e): `npm run test:e2e` (Playwright) diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts index 25e2b25484..47786837ab 100644 --- a/src/lib/ai-edition/store/transcriptionStore.test.ts +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -76,6 +76,15 @@ function transcriptFor(assetId: string): AxcutTranscript { }; } +/** A promise plus a 0-arg release, so a mocked run can be held open mid-flight. */ +function deferred(): { promise: Promise; release: () => void } { + let release: () => void = () => {}; + const promise = new Promise((resolve) => { + release = () => resolve(); + }); + return { promise, release: () => release() }; +} + /** Loads a document into the project store the way `loadProject` would. */ function loadDocument(document: AxcutDocument) { useProjectStore.setState({ @@ -308,7 +317,7 @@ describe("useTranscriptionStore", () => { // asks for French from the media card. The outgoing run must neither win // the race nor delete the request that replaced it. const languages: string[] = []; - let releaseFirst: (() => void) | null = null; + const firstRun = deferred(); transcribeMocks.transcribeAsset.mockImplementation( async ( _doc: AxcutDocument, @@ -317,9 +326,7 @@ describe("useTranscriptionStore", () => { ) => { languages.push(options.language ?? "auto"); if (languages.length === 1) { - await new Promise((resolve) => { - releaseFirst = resolve; - }); + await firstRun.promise; throw new DOMException("Aborted", "AbortError"); } return { ...transcriptFor(assetId), language: options.language ?? "auto" }; @@ -332,7 +339,7 @@ describe("useTranscriptionStore", () => { const requested = useTranscriptionStore.getState().request("asset_1", "fr"); expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("queued"); - releaseFirst?.(); + firstRun.release(); await requested; await whenTranscriptionIdle(); @@ -386,14 +393,10 @@ describe("useTranscriptionStore", () => { }); it("waits for the background run instead of transcribing the same asset twice", async () => { - let releaseFirst: (() => void) | null = null; + const firstRun = deferred(); transcribeMocks.transcribeAsset.mockImplementation( async (_doc: AxcutDocument, assetId: string) => { - if (!releaseFirst) { - await new Promise((resolve) => { - releaseFirst = resolve; - }); - } + await firstRun.promise; return transcriptFor(assetId); }, ); @@ -403,7 +406,7 @@ describe("useTranscriptionStore", () => { expect(useTranscriptionStore.getState().jobs.asset_1?.status).toBe("running"); const requested = useTranscriptionStore.getState().requestTimelineTranscripts(); - releaseFirst?.(); + firstRun.release(); await requested; expect(transcribeMocks.transcribeAsset).toHaveBeenCalledTimes(1); From 3406048c87a3f9079d6d849c202875f9ed21a939 Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Sat, 1 Aug 2026 15:16:42 +0200 Subject: [PATCH 4/4] chore: keep the dev launch config out of this PR, and silence a lint warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `.claude/` is gitignored (while `launch.json` is tracked), so its local desktop-dev tweak slipped in through `git add -u` — reverted here, it stays a local change. Also puts a comment in `deferred()`'s placeholder so it stops tripping noEmptyBlockStatements. --- .claude/launch.json | 3 ++- src/lib/ai-edition/store/transcriptionStore.test.ts | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.claude/launch.json b/.claude/launch.json index 0cbf829326..16c0d4afe8 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -12,7 +12,8 @@ "name": "desktop-dev", "runtimeExecutable": "npm", "runtimeArgs": ["run", "dev", "--", "--port", "5201", "--strictPort"], - "port": 5201 + "port": 5201, + "env": { "NO_ELECTRON": "1" } }, { "name": "v4preview", diff --git a/src/lib/ai-edition/store/transcriptionStore.test.ts b/src/lib/ai-edition/store/transcriptionStore.test.ts index 47786837ab..d8208ddeff 100644 --- a/src/lib/ai-edition/store/transcriptionStore.test.ts +++ b/src/lib/ai-edition/store/transcriptionStore.test.ts @@ -78,7 +78,9 @@ function transcriptFor(assetId: string): AxcutTranscript { /** A promise plus a 0-arg release, so a mocked run can be held open mid-flight. */ function deferred(): { promise: Promise; release: () => void } { - let release: () => void = () => {}; + let release: () => void = () => { + // Replaced synchronously by the executor below, before this can be called. + }; const promise = new Promise((resolve) => { release = () => resolve(); });