From ef74c08bd1bf2400e868e6cccd884e0bf3079f81 Mon Sep 17 00:00:00 2001 From: Riccardo Pallaoro <95774148+pallaoro@users.noreply.github.com> Date: Tue, 10 Nov 2026 12:11:18 +0000 Subject: [PATCH 1/3] Run long AI requests in the background A request that would watch more than about three minutes of footage no longer runs inline: it changes nothing and says how much it would watch. On a yes it becomes a job the platform queue runs (POST /api/ask/step), from the cut as it is then. The result is written only over the cut it started from; if someone edited meanwhile it is kept aside and offered (Apply anyway). The editor, not the panel, watches its jobs, so a result that lands while the panel is closed still becomes one undo step. Agents can skip the proposal with background: true and read the job from GET /api/projects/{id}/ask. --- README.md | 2 +- agent.md | 20 +++- clawnify.json | 3 +- schema.sql | 30 +++++ src/client/assistant.tsx | 238 +++++++++++++++++++++++++++++++++------ src/client/edit.tsx | 23 ++-- src/server/ask-jobs.ts | 206 +++++++++++++++++++++++++++++++++ src/server/index.ts | 219 ++++++++++++++++++++++++++--------- src/server/instruct.ts | 60 +++++++++- test/e2e/run.mjs | 67 ++++++++++- test/instruct.test.ts | 32 +++++- 11 files changed, 795 insertions(+), 105 deletions(-) create mode 100644 src/server/ask-jobs.ts diff --git a/README.md b/README.md index aaad823..9b65935 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Most editors keep a project in a format built for their own app, not for you or - **Colour**: brightness, contrast, saturation and temperature on any clip, logo or picture-in-picture, and one click to give every clip the same look, for flat log footage or two cameras that do not match. The preview draws it with the same maths the export renders. - **Transitions**: put a dissolve, a fade through black or white, a wipe, a slide, a blur or a pixelate on any cut, from the mark between two clips on the timeline, or on every cut at once. A transition is centred on its cut and moves no clip, so titles and music stay where you put them, and the preview draws each one frame for frame as it exports. - **Captions**: switch them on for the whole video, pick one of twelve languages and one style. The words come from each clip's transcript, so captions follow every trim, split and reorder. -- **AI, in one conversation**: add your clips and say what you want beside the cut, in your own words: it builds the cut from them ("a 30 second video from my clips") or changes the one you have. It remembers the conversation, so "shorter" or "a square one too" knows what it refers to, and it asks back instead of guessing when a request could mean two things. The model calls a fixed set of checked operations, so the edit is always a valid document. One request, one undo. +- **AI, in one conversation**: add your clips and say what you want beside the cut, in your own words: it builds the cut from them ("a 30 second video from my clips") or changes the one you have. It remembers the conversation, so "shorter" or "a square one too" knows what it refers to, and it asks back instead of guessing when a request could mean two things. A request that would watch a lot of footage says how much first and, on a yes, runs in the background: keep editing or close the tab, and it never lands on top of a change you made meanwhile. The model calls a fixed set of checked operations, so the edit is always a valid document. One request, one undo. - **AI assist**: Auto-cut watches several clips together and assembles the strongest sequence for what the video is for, captions included; Clean up trims one clip down to its good parts. - **Export to MP4** in draft, standard or high quality, on Clawnify's managed edit service. - **Share by link**: one link per project plays an export to anyone you send it to, with no sign-in, and a Download button. The link stays on the export you shared until you move it to a newer one, so a draft never reaches viewers; turning it off ends it for everyone. diff --git a/agent.md b/agent.md index f7ac053..2cd37fa 100644 --- a/agent.md +++ b/agent.md @@ -224,6 +224,18 @@ more than two clips, it answers with `asked: true` and a question or plan in `said`, and changes nothing. Answer by sending the reply as the next `instruction`. +A request that would watch more than about three minutes of footage is not +answered inline: the reply has `asked: true` and a `job` with `status: +"proposed"`, and `said` says how much it would watch. Run it with +`POST …/ask/jobs/{job}/run`, or send `background: true` with the instruction +to skip the proposal when you already know you want it. Then read +`GET /api/projects/{id}/ask` until the job is no longer `queued` or `running`: + +- `applied`: the cut was changed (or the answer was a question, in the conversation) +- `stale`: someone changed the cut while it ran, so the result was kept aside, + never written over their edit. Apply it with `…/apply` only if the person wants it +- `failed` (with `error`) or `cancelled` + Google Drive limits how often a file shared with the link can be downloaded, and on a heavy day it refuses the files of a whole folder for a while. It refuses the whole file but still serves pieces of it, so a refused file of up @@ -301,8 +313,12 @@ in line, and asks Drive again now for the clips waiting on it. `DELETE /api/proj | GET | `/api/projects/{id}/highlights/export` | `format=csv\|xml`: a sheet, or a Premiere Pro / Resolve timeline of the picks | | PUT | `/api/projects/{id}` | Update `{ name?, brief?, edl? }` — the EDL is validated on save | | POST | `/api/projects/{id}/autocut` | `{ asset_ids, prompt? }` — AI assembles the main track from several clips | -| POST | `/api/projects/{id}/instruct` | `{ instruction }` — change the existing cut in words → `{ edl, said, applied, asked }`; `asked: true` means `said` is a question back and nothing was changed | -| GET | `/api/projects/{id}/ask` | The project's AI conversation, oldest first → `{ messages: [{ role, text, author, applied, asked, failed }] }` | +| POST | `/api/projects/{id}/instruct` | `{ instruction, background? }` — change the existing cut in words → `{ edl, said, applied, asked, job? }`; `asked: true` means `said` is a question back (or, with `job`, a proposal) and nothing was changed. `background: true` → `202` with a queued `job` | +| GET | `/api/projects/{id}/ask` | The project's AI conversation, oldest first, and its recent background jobs → `{ messages: [{ role, text, author, applied, asked, failed, job_id }], jobs }` | +| GET | `/api/projects/{id}/ask/jobs/{job}` | One background job with `base_edl` (the cut it started from) and `result_edl` | +| POST | `/api/projects/{id}/ask/jobs/{job}/run` | Yes to a proposal: run it in the background → `202` | +| POST | `/api/projects/{id}/ask/jobs/{job}/cancel` | Drop a proposal, stop a queued or running job (its result is not applied), or let go of a kept one | +| POST | `/api/projects/{id}/ask/jobs/{job}/apply` | Apply a result kept aside because the cut changed while it ran | | DELETE | `/api/projects/{id}/ask` | Start the conversation over; the cut is untouched | | DELETE | `/api/projects/{id}` | Delete a project, its export history and its footage from Drive; `202 { remaining }` means call again | | POST | `/api/projects/{id}/export` | Export `{ quality?, format? }` → returns the job, `status: "exporting"` | diff --git a/clawnify.json b/clawnify.json index 162037a..819cdac 100644 --- a/clawnify.json +++ b/clawnify.json @@ -22,7 +22,8 @@ { "path": "/api/drive/imports/step", "methods": ["POST"] }, { "path": "/s/**", "methods": ["GET", "HEAD"] }, { "path": "/s/*/comments", "methods": ["POST"] }, - { "path": "/api/footage/step", "methods": ["POST"] } + { "path": "/api/footage/step", "methods": ["POST"] }, + { "path": "/api/ask/step", "methods": ["POST"] } ], "server_routes": ["/s/*"] }, diff --git a/schema.sql b/schema.sql index ddf4671..9c35806 100644 --- a/schema.sql +++ b/schema.sql @@ -428,3 +428,33 @@ CREATE TABLE IF NOT EXISTS ask_messages ( ); CREATE INDEX IF NOT EXISTS idx_ask_messages_project ON ask_messages(project_id, created_at); + +-- A request that watches more footage than a quick answer allows runs in the +-- background (src/server/ask-jobs.ts). `proposed`: told what it would watch, +-- waiting for a yes. `queued` -> `running` -> `applied` (the cut was changed), +-- `stale` (the cut changed while it ran, so it was kept aside: `result_edl`, +-- applied only on request), `failed` or `cancelled`. `base_edl` is the cut it +-- started from, so it never overwrites an edit made meanwhile. +CREATE TABLE IF NOT EXISTS ask_jobs ( + id TEXT PRIMARY KEY DEFAULT (lower(hex(randomblob(8)))), + project_id TEXT NOT NULL, + instruction TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('proposed', 'queued', 'running', 'applied', 'stale', 'failed', 'cancelled')), + clips INTEGER NOT NULL DEFAULT 0, + seconds REAL, + base_edl TEXT, + result_edl TEXT, + said TEXT, + applied TEXT, + error TEXT, + author TEXT, + started_at TEXT, + finished_at TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_ask_jobs_project ON ask_jobs(project_id, created_at); + +-- The background job an AI message belongs to, when it does. +ALTER TABLE ask_messages ADD COLUMN job_id TEXT; diff --git a/src/client/assistant.tsx b/src/client/assistant.tsx index 89381cd..68c3926 100644 --- a/src/client/assistant.tsx +++ b/src/client/assistant.tsx @@ -3,13 +3,16 @@ // Ask what you want in your own words; the editor works out the steps and // applies them as one undo step. It remembers the conversation, so a // follow-up ("shorter", "a square one too") knows what it refers to, and it -// asks back instead of guessing. Auto-cut and AI reframe still run in their +// asks back instead of guessing. A request that would watch a lot of footage +// says so first and, on a yes, runs in the background (src/server/ask-jobs.ts): +// the editor keeps working, the tab can close, and the result never lands on +// top of a change made meanwhile. Auto-cut and AI reframe still run in their // own dialogs, opened from here, until the conversation can run them itself. -import { useEffect, useRef, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { Crop, Loader2, RotateCcw, Send, Sparkles, X } from "lucide-react"; import { api, type Edl } from "./edit"; -import { btnGhost, btnIcon, btnPrimary } from "./ui"; +import { btnGhost, btnIcon, btnPrimary, btnSecondary } from "./ui"; export interface AskMessage { id: string; @@ -19,8 +22,20 @@ export interface AskMessage { applied: string[]; asked: boolean; failed: boolean; + job_id?: string | null; } +export type AskJobStatus = "proposed" | "queued" | "running" | "applied" | "stale" | "failed" | "cancelled"; + +export interface AskJob { + id: string; + status: AskJobStatus; + applied: string[]; + error: string | null; +} + +const ACTIVE: AskJobStatus[] = ["queued", "running"]; + /** Requests that show what it can do, sent as written. */ const SUGGESTIONS = [ "Make a 30 second video from my clips", @@ -29,8 +44,100 @@ const SUGGESTIONS = [ "Add a title for the first 3 seconds", ]; +/** JSON with keys in order: the server stores a cut with its keys in another order. */ +function canonical(v: unknown): string { + if (Array.isArray(v)) return `[${v.map(canonical).join(",")}]`; + if (v && typeof v === "object") { + return `{${Object.keys(v as object) + .sort() + .filter((k) => (v as Record)[k] !== undefined) + .map((k) => `${JSON.stringify(k)}:${canonical((v as Record)[k])}`) + .join(",")}}`; + } + return JSON.stringify(v); +} + +/** + * The project's conversation and its background jobs. Lives in the editor, + * not the panel, so a job that finishes while the panel is closed still lands: + * its result becomes an undo step when the cut on screen is the one it started + * from, and otherwise waits for the person ("Apply anyway"). + */ +export function useAsk(projectId: string, getEdl: () => Edl, onApplied: (edl: Edl) => void) { + const [messages, setMessages] = useState(null); + const [jobs, setJobs] = useState([]); + const [loadErr, setLoadErr] = useState(""); + /** Jobs whose result the editor did not take, because the cut changed first. */ + const [notTaken, setNotTaken] = useState>(new Set()); + const seenActive = useRef(new Set()); + const handled = useRef(new Set()); + const latest = useRef({ getEdl, onApplied }); + latest.current = { getEdl, onApplied }; + + const refresh = useCallback(async () => { + try { + const r = await api.get<{ messages: AskMessage[]; jobs: AskJob[] }>(`/api/projects/${projectId}/ask`); + setMessages(r.messages); + setJobs(r.jobs); + setLoadErr(""); + for (const j of r.jobs) { + if (ACTIVE.includes(j.status)) seenActive.current.add(j.id); + // Finished while this editor was open: take the result into the cut on screen. + if (j.status === "applied" && j.applied.length > 0 && seenActive.current.has(j.id) && !handled.current.has(j.id)) { + handled.current.add(j.id); + const detail = await api.get<{ base_edl: Edl | null; result_edl: Edl | null }>(`/api/projects/${projectId}/ask/jobs/${j.id}`); + if (detail.result_edl && detail.base_edl && canonical(latest.current.getEdl()) === canonical(detail.base_edl)) { + latest.current.onApplied(detail.result_edl); + } else if (detail.result_edl) { + // An edit made here got to the server after the job wrote: it kept + // the person's version. Offer the job's. + setNotTaken((s) => new Set(s).add(j.id)); + } + } + } + } catch (e) { + setLoadErr(String((e as Error).message)); + setMessages((m) => m ?? []); + } + }, [projectId]); + + useEffect(() => { + void refresh(); + }, [refresh]); + + const active = jobs.some((j) => ACTIVE.includes(j.status)); + useEffect(() => { + if (!active) return; + const t = setInterval(() => void refresh(), 4000); + return () => clearInterval(t); + }, [active, refresh]); + + const act = useCallback( + async (jobId: string, action: "run" | "cancel" | "apply") => { + const out = await api.send<{ result_edl?: Edl | null }>("POST", `/api/projects/${projectId}/ask/jobs/${jobId}/${action}`); + if (action === "run") seenActive.current.add(jobId); + if (action === "apply" && out.result_edl) { + handled.current.add(jobId); + latest.current.onApplied(out.result_edl); + setNotTaken((s) => { + const n = new Set(s); + n.delete(jobId); + return n; + }); + } + await refresh(); + }, + [projectId, refresh], + ); + + return { messages, setMessages, jobs, active, notTaken, loadErr, refresh, act }; +} + +export type Ask = ReturnType; + export function AssistantPanel({ projectId, + ask, pane, saving, canReframe, @@ -40,6 +147,7 @@ export function AssistantPanel({ onClose, }: { projectId: string; + ask: Ask; pane: "library" | "canvas" | "inspector"; /** A change is still on its way to the server: the request would read the cut before it. */ saving: boolean; @@ -49,34 +157,28 @@ export function AssistantPanel({ onReframe: () => void; onClose: () => void; }) { - const [messages, setMessages] = useState(null); + const { messages, setMessages, jobs, notTaken, loadErr, refresh, act } = ask; const [draft, setDraft] = useState(""); const [running, setRunning] = useState(false); - const [loadErr, setLoadErr] = useState(""); + const [acting, setActing] = useState(null); + const [actErr, setActErr] = useState(""); const list = useRef(null); const input = useRef(null); useEffect(() => { - let live = true; - api - .get<{ messages: AskMessage[] }>(`/api/projects/${projectId}/ask`) - .then((r) => live && setMessages(r.messages)) - .catch((e) => live && (setLoadErr(String((e as Error).message)), setMessages([]))); input.current?.focus(); - return () => { - live = false; - }; - }, [projectId]); + }, []); useEffect(() => { list.current?.scrollTo({ top: list.current.scrollHeight }); - }, [messages, running]); + }, [messages, running, jobs]); const send = async (text: string) => { const instruction = text.trim(); if (!instruction || running || saving) return; setDraft(""); setRunning(true); + setActErr(""); const mine: AskMessage = { id: `local-${Date.now()}`, role: "user", text: instruction, author: null, applied: [], asked: false, failed: false }; setMessages((m) => [...(m ?? []), mine]); try { @@ -87,28 +189,83 @@ export function AssistantPanel({ ); // Only a request that changed something becomes an undo step. if (out.applied.length > 0) onApplied(out.edl); - setMessages((m) => [ - ...(m ?? []), - { id: `local-${Date.now()}-a`, role: "assistant", text: out.said, author: null, applied: out.applied, asked: out.asked, failed: false }, - ]); - } catch (e) { - setMessages((m) => [ - ...(m ?? []), - { id: `local-${Date.now()}-f`, role: "assistant", text: String((e as Error).message), author: null, applied: [], asked: false, failed: true }, - ]); + } catch { + /* the failure is in the conversation, read back below */ } finally { + await refresh(); setRunning(false); input.current?.focus(); } }; + const doAct = async (jobId: string, action: "run" | "cancel" | "apply") => { + setActing(`${jobId}:${action}`); + setActErr(""); + try { + await act(jobId, action); + } catch (e) { + setActErr(String((e as Error).message)); + } finally { + setActing(null); + } + }; + const startOver = async () => { await api.send("DELETE", `/api/projects/${projectId}/ask`).catch(() => {}); - setMessages([]); + await refresh(); input.current?.focus(); }; const empty = messages !== null && messages.length === 0; + // A job's controls go under the last message about it. + const lastOfJob = new Map(); + for (const m of messages ?? []) if (m.job_id) lastOfJob.set(m.job_id, m.id); + const jobById = new Map(jobs.map((j) => [j.id, j])); + + const controls = (job: AskJob) => { + const busy = (a: string) => acting === `${job.id}:${a}`; + if (job.status === "proposed") { + return ( +
+ + +
+ ); + } + if (ACTIVE.includes(job.status)) { + return ( +
+
+ + {job.status === "queued" ? "Starting in the background…" : "Working in the background…"} + +
+

Keep editing if you like: your changes stay, and this then waits for you to apply it.

+
+ ); + } + if (job.status === "stale" || notTaken.has(job.id)) { + return ( +
+ + {job.status === "stale" && ( + + )} +
+ ); + } + return null; + }; return (