diff --git a/README.md b/README.md index aaad823..9b65935 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Most editors keep a project in a format built for their own app, not for you or - **Colour**: brightness, contrast, saturation and temperature on any clip, logo or picture-in-picture, and one click to give every clip the same look, for flat log footage or two cameras that do not match. The preview draws it with the same maths the export renders. - **Transitions**: put a dissolve, a fade through black or white, a wipe, a slide, a blur or a pixelate on any cut, from the mark between two clips on the timeline, or on every cut at once. A transition is centred on its cut and moves no clip, so titles and music stay where you put them, and the preview draws each one frame for frame as it exports. - **Captions**: switch them on for the whole video, pick one of twelve languages and one style. The words come from each clip's transcript, so captions follow every trim, split and reorder. -- **AI, in one conversation**: add your clips and say what you want beside the cut, in your own words: it builds the cut from them ("a 30 second video from my clips") or changes the one you have. It remembers the conversation, so "shorter" or "a square one too" knows what it refers to, and it asks back instead of guessing when a request could mean two things. The model calls a fixed set of checked operations, so the edit is always a valid document. One request, one undo. +- **AI, in one conversation**: add your clips and say what you want beside the cut, in your own words: it builds the cut from them ("a 30 second video from my clips") or changes the one you have. It remembers the conversation, so "shorter" or "a square one too" knows what it refers to, and it asks back instead of guessing when a request could mean two things. A request that would watch a lot of footage says how much first and, on a yes, runs in the background: keep editing or close the tab, and it never lands on top of a change you made meanwhile. The model calls a fixed set of checked operations, so the edit is always a valid document. One request, one undo. - **AI assist**: Auto-cut watches several clips together and assembles the strongest sequence for what the video is for, captions included; Clean up trims one clip down to its good parts. - **Export to MP4** in draft, standard or high quality, on Clawnify's managed edit service. - **Share by link**: one link per project plays an export to anyone you send it to, with no sign-in, and a Download button. The link stays on the export you shared until you move it to a newer one, so a draft never reaches viewers; turning it off ends it for everyone. diff --git a/agent.md b/agent.md index f7ac053..cec054c 100644 --- a/agent.md +++ b/agent.md @@ -224,6 +224,21 @@ more than two clips, it answers with `asked: true` and a question or plan in `said`, and changes nothing. Answer by sending the reply as the next `instruction`. +A request that would watch more than about three minutes of footage is not +answered inline: the reply has `asked: true` and a `job` with `status: +"proposed"`, and `said` says how much it would watch. Run it with +`POST …/ask/jobs/{job}/run`, or send `background: true` with the instruction +to skip the proposal when you already know you want it. Then read +`GET /api/projects/{id}/ask` until the job is no longer `queued` or `running`. +A job runs a few steps at a time; while it runs, `doing` says what it is doing +now and `applied` lists what it has done so far. It ends as: + +- `applied`: the cut was changed (or the answer was a question, in the conversation) +- `stale`: someone changed the cut while it ran, so the result was kept aside, + never written over their edit. Apply it with `…/apply` only if the person wants it +- `failed` (with `error`, e.g. "stopped without finishing" when it has not moved + for 20 minutes) or `cancelled` + Google Drive limits how often a file shared with the link can be downloaded, and on a heavy day it refuses the files of a whole folder for a while. It refuses the whole file but still serves pieces of it, so a refused file of up @@ -301,8 +316,12 @@ in line, and asks Drive again now for the clips waiting on it. `DELETE /api/proj | GET | `/api/projects/{id}/highlights/export` | `format=csv\|xml`: a sheet, or a Premiere Pro / Resolve timeline of the picks | | PUT | `/api/projects/{id}` | Update `{ name?, brief?, edl? }` — the EDL is validated on save | | POST | `/api/projects/{id}/autocut` | `{ asset_ids, prompt? }` — AI assembles the main track from several clips | -| POST | `/api/projects/{id}/instruct` | `{ instruction }` — change the existing cut in words → `{ edl, said, applied, asked }`; `asked: true` means `said` is a question back and nothing was changed | -| GET | `/api/projects/{id}/ask` | The project's AI conversation, oldest first → `{ messages: [{ role, text, author, applied, asked, failed }] }` | +| POST | `/api/projects/{id}/instruct` | `{ instruction, background? }` — change the existing cut in words → `{ edl, said, applied, asked, job? }`; `asked: true` means `said` is a question back (or, with `job`, a proposal) and nothing was changed. `background: true` → `202` with a queued `job` | +| GET | `/api/projects/{id}/ask` | The project's AI conversation, oldest first, and its recent background jobs → `{ messages: [{ role, text, author, applied, asked, failed, job_id }], jobs }` | +| GET | `/api/projects/{id}/ask/jobs/{job}` | One background job with `base_edl` (the cut it started from), `draft_edl` (its cut so far, while it runs) and `result_edl` | +| POST | `/api/projects/{id}/ask/jobs/{job}/run` | Yes to a proposal: run it in the background → `202` | +| POST | `/api/projects/{id}/ask/jobs/{job}/cancel` | Drop a proposal, stop a queued or running job (its result is not applied), or let go of a kept one | +| POST | `/api/projects/{id}/ask/jobs/{job}/apply` | Apply a result kept aside because the cut changed while it ran | | DELETE | `/api/projects/{id}/ask` | Start the conversation over; the cut is untouched | | DELETE | `/api/projects/{id}` | Delete a project, its export history and its footage from Drive; `202 { remaining }` means call again | | POST | `/api/projects/{id}/export` | Export `{ quality?, format? }` → returns the job, `status: "exporting"` | diff --git a/clawnify.json b/clawnify.json index 162037a..819cdac 100644 --- a/clawnify.json +++ b/clawnify.json @@ -22,7 +22,8 @@ { "path": "/api/drive/imports/step", "methods": ["POST"] }, { "path": "/s/**", "methods": ["GET", "HEAD"] }, { "path": "/s/*/comments", "methods": ["POST"] }, - { "path": "/api/footage/step", "methods": ["POST"] } + { "path": "/api/footage/step", "methods": ["POST"] }, + { "path": "/api/ask/step", "methods": ["POST"] } ], "server_routes": ["/s/*"] }, diff --git a/schema.sql b/schema.sql index ddf4671..23dfa95 100644 --- a/schema.sql +++ b/schema.sql @@ -428,3 +428,41 @@ CREATE TABLE IF NOT EXISTS ask_messages ( ); CREATE INDEX IF NOT EXISTS idx_ask_messages_project ON ask_messages(project_id, created_at); + +-- A request that watches more footage than a quick answer allows runs in the +-- background (src/server/ask-jobs.ts), a few steps per queue delivery. `proposed`: told what it would watch, +-- waiting for a yes. `queued` -> `running` -> `applied` (the cut was changed), +-- `stale` (the cut changed while it ran, so it was kept aside: `result_edl`, +-- applied only on request), `failed` or `cancelled`. `base_edl` is the cut it +-- started from, so it never overwrites an edit made meanwhile. +CREATE TABLE IF NOT EXISTS ask_jobs ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))), + project_id TEXT NOT NULL, + instruction TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('proposed', 'queued', 'running', 'applied', 'stale', 'failed', 'cancelled')), + clips INTEGER NOT NULL DEFAULT 0, + seconds REAL, + base_edl TEXT, + result_edl TEXT, + said TEXT, + applied TEXT, + error TEXT, + author TEXT, + -- The request in progress (src/server/instruct.ts InstructState), saved + -- after every step, and what it is doing now in words. + state TEXT, + doing TEXT, + -- A delivery holds the job while it runs a step; past this time another may. + leased_until TEXT, + -- When the next step was last booked on the queue. + step_booked_at TEXT, + started_at TEXT, + finished_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_ask_jobs_project ON ask_jobs(project_id, created_at); + +-- The background job an AI message belongs to, when it does. +ALTER TABLE ask_messages ADD COLUMN job_id TEXT; diff --git a/src/client/assistant.tsx b/src/client/assistant.tsx index 89381cd..6061f27 100644 --- a/src/client/assistant.tsx +++ b/src/client/assistant.tsx @@ -3,13 +3,16 @@ // Ask what you want in your own words; the editor works out the steps and // applies them as one undo step. It remembers the conversation, so a // follow-up ("shorter", "a square one too") knows what it refers to, and it -// asks back instead of guessing. Auto-cut and AI reframe still run in their +// asks back instead of guessing. A request that would watch a lot of footage +// says so first and, on a yes, runs in the background (src/server/ask-jobs.ts): +// the editor keeps working, the tab can close, and the result never lands on +// top of a change made meanwhile. Auto-cut and AI reframe still run in their // own dialogs, opened from here, until the conversation can run them itself. -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { Crop, Loader2, RotateCcw, Send, Sparkles, X } from "lucide-react"; import { api, type Edl } from "./edit"; -import { btnGhost, btnIcon, btnPrimary } from "./ui"; +import { btnGhost, btnIcon, btnPrimary, btnSecondary } from "./ui"; export interface AskMessage { id: string; @@ -19,8 +22,22 @@ export interface AskMessage { applied: string[]; asked: boolean; failed: boolean; + job_id?: string | null; } +export type AskJobStatus = "proposed" | "queued" | "running" | "applied" | "stale" | "failed" | "cancelled"; + +export interface AskJob { + id: string; + status: AskJobStatus; + applied: string[]; + /** What it is doing now, while it runs. */ + doing?: string | null; + error: string | null; +} + +const ACTIVE: AskJobStatus[] = ["queued", "running"]; + /** Requests that show what it can do, sent as written. */ const SUGGESTIONS = [ "Make a 30 second video from my clips", @@ -29,8 +46,100 @@ const SUGGESTIONS = [ "Add a title for the first 3 seconds", ]; +/** JSON with keys in order: the server stores a cut with its keys in another order. */ +function canonical(v: unknown): string { + if (Array.isArray(v)) return `[${v.map(canonical).join(",")}]`; + if (v && typeof v === "object") { + return `{${Object.keys(v as object) + .sort() + .filter((k) => (v as Record)[k] !== undefined) + .map((k) => `${JSON.stringify(k)}:${canonical((v as Record)[k])}`) + .join(",")}}`; + } + return JSON.stringify(v); +} + +/** + * The project's conversation and its background jobs. Lives in the editor, + * not the panel, so a job that finishes while the panel is closed still lands: + * its result becomes an undo step when the cut on screen is the one it started + * from, and otherwise waits for the person ("Apply anyway"). + */ +export function useAsk(projectId: string, getEdl: () => Edl, onApplied: (edl: Edl) => void) { + const [messages, setMessages] = useState(null); + const [jobs, setJobs] = useState([]); + const [loadErr, setLoadErr] = useState(""); + /** Jobs whose result the editor did not take, because the cut changed first. */ + const [notTaken, setNotTaken] = useState>(new Set()); + const seenActive = useRef(new Set()); + const handled = useRef(new Set()); + const latest = useRef({ getEdl, onApplied }); + latest.current = { getEdl, onApplied }; + + const refresh = useCallback(async () => { + try { + const r = await api.get<{ messages: AskMessage[]; jobs: AskJob[] }>(`/api/projects/${projectId}/ask`); + setMessages(r.messages); + setJobs(r.jobs); + setLoadErr(""); + for (const j of r.jobs) { + if (ACTIVE.includes(j.status)) seenActive.current.add(j.id); + // Finished while this editor was open: take the result into the cut on screen. + if (j.status === "applied" && j.applied.length > 0 && seenActive.current.has(j.id) && !handled.current.has(j.id)) { + handled.current.add(j.id); + const detail = await api.get<{ base_edl: Edl | null; result_edl: Edl | null }>(`/api/projects/${projectId}/ask/jobs/${j.id}`); + if (detail.result_edl && detail.base_edl && canonical(latest.current.getEdl()) === canonical(detail.base_edl)) { + latest.current.onApplied(detail.result_edl); + } else if (detail.result_edl) { + // An edit made here got to the server after the job wrote: it kept + // the person's version. Offer the job's. + setNotTaken((s) => new Set(s).add(j.id)); + } + } + } + } catch (e) { + setLoadErr(String((e as Error).message)); + setMessages((m) => m ?? []); + } + }, [projectId]); + + useEffect(() => { + void refresh(); + }, [refresh]); + + const active = jobs.some((j) => ACTIVE.includes(j.status)); + useEffect(() => { + if (!active) return; + const t = setInterval(() => void refresh(), 4000); + return () => clearInterval(t); + }, [active, refresh]); + + const act = useCallback( + async (jobId: string, action: "run" | "cancel" | "apply") => { + const out = await api.send<{ result_edl?: Edl | null }>("POST", `/api/projects/${projectId}/ask/jobs/${jobId}/${action}`); + if (action === "run") seenActive.current.add(jobId); + if (action === "apply" && out.result_edl) { + handled.current.add(jobId); + latest.current.onApplied(out.result_edl); + setNotTaken((s) => { + const n = new Set(s); + n.delete(jobId); + return n; + }); + } + await refresh(); + }, + [projectId, refresh], + ); + + return { messages, setMessages, jobs, active, notTaken, loadErr, refresh, act }; +} + +export type Ask = ReturnType; + export function AssistantPanel({ projectId, + ask, pane, saving, canReframe, @@ -40,6 +149,7 @@ export function AssistantPanel({ onClose, }: { projectId: string; + ask: Ask; pane: "library" | "canvas" | "inspector"; /** A change is still on its way to the server: the request would read the cut before it. */ saving: boolean; @@ -49,34 +159,28 @@ export function AssistantPanel({ onReframe: () => void; onClose: () => void; }) { - const [messages, setMessages] = useState(null); + const { messages, setMessages, jobs, notTaken, loadErr, refresh, act } = ask; const [draft, setDraft] = useState(""); const [running, setRunning] = useState(false); - const [loadErr, setLoadErr] = useState(""); + const [acting, setActing] = useState(null); + const [actErr, setActErr] = useState(""); const list = useRef(null); const input = useRef(null); useEffect(() => { - let live = true; - api - .get<{ messages: AskMessage[] }>(`/api/projects/${projectId}/ask`) - .then((r) => live && setMessages(r.messages)) - .catch((e) => live && (setLoadErr(String((e as Error).message)), setMessages([]))); input.current?.focus(); - return () => { - live = false; - }; - }, [projectId]); + }, []); useEffect(() => { list.current?.scrollTo({ top: list.current.scrollHeight }); - }, [messages, running]); + }, [messages, running, jobs]); const send = async (text: string) => { const instruction = text.trim(); if (!instruction || running || saving) return; setDraft(""); setRunning(true); + setActErr(""); const mine: AskMessage = { id: `local-${Date.now()}`, role: "user", text: instruction, author: null, applied: [], asked: false, failed: false }; setMessages((m) => [...(m ?? []), mine]); try { @@ -87,28 +191,92 @@ export function AssistantPanel({ ); // Only a request that changed something becomes an undo step. if (out.applied.length > 0) onApplied(out.edl); - setMessages((m) => [ - ...(m ?? []), - { id: `local-${Date.now()}-a`, role: "assistant", text: out.said, author: null, applied: out.applied, asked: out.asked, failed: false }, - ]); - } catch (e) { - setMessages((m) => [ - ...(m ?? []), - { id: `local-${Date.now()}-f`, role: "assistant", text: String((e as Error).message), author: null, applied: [], asked: false, failed: true }, - ]); + } catch { + /* the failure is in the conversation, read back below */ } finally { + await refresh(); setRunning(false); input.current?.focus(); } }; + const doAct = async (jobId: string, action: "run" | "cancel" | "apply") => { + setActing(`${jobId}:${action}`); + setActErr(""); + try { + await act(jobId, action); + } catch (e) { + setActErr(String((e as Error).message)); + } finally { + setActing(null); + } + }; + const startOver = async () => { await api.send("DELETE", `/api/projects/${projectId}/ask`).catch(() => {}); - setMessages([]); + await refresh(); input.current?.focus(); }; const empty = messages !== null && messages.length === 0; + // A job's controls go under the last message about it. + const lastOfJob = new Map(); + for (const m of messages ?? []) if (m.job_id) lastOfJob.set(m.job_id, m.id); + const jobById = new Map(jobs.map((j) => [j.id, j])); + + const controls = (job: AskJob) => { + const busy = (a: string) => acting === `${job.id}:${a}`; + if (job.status === "proposed") { + return ( +
+ + +
+ ); + } + if (ACTIVE.includes(job.status)) { + return ( +
+
+ + + {job.status === "queued" ? "Starting in the background…" : `${job.doing ?? "Working"}…`} + + +
+ {job.applied.length > 0 && ( +
    + {job.applied.map((a, i) => ( +
  • ✓ {a}
  • + ))} +
+ )} +

Keep editing if you like: your changes stay, and this then waits for you to apply it.

+
+ ); + } + if (job.status === "stale" || notTaken.has(job.id)) { + return ( +
+ + {job.status === "stale" && ( + + )} +
+ ); + } + return null; + }; return (