diff --git a/apps/site/convex/_generated/api.d.ts b/apps/site/convex/_generated/api.d.ts index 3b2cefa..6c3d793 100644 --- a/apps/site/convex/_generated/api.d.ts +++ b/apps/site/convex/_generated/api.d.ts @@ -30,6 +30,7 @@ import type * as sessionEphemera from "../sessionEphemera.js"; import type * as sessionParticipants from "../sessionParticipants.js"; import type * as sessions from "../sessions.js"; import type * as topics from "../topics.js"; +import type * as transcript from "../transcript.js"; import type { ApiFromModules, @@ -60,6 +61,7 @@ declare const fullApi: ApiFromModules<{ sessionParticipants: typeof sessionParticipants; sessions: typeof sessions; topics: typeof topics; + transcript: typeof transcript; }>; /** diff --git a/apps/site/convex/agent.ts b/apps/site/convex/agent.ts index 7215832..286f7e3 100644 --- a/apps/site/convex/agent.ts +++ b/apps/site/convex/agent.ts @@ -3,7 +3,6 @@ import { openai } from "@ai-sdk/openai"; import { AiSdkExecutor, type AiSdkStreamUpdate } from "@projectors/aisdk-executor"; import { - ROOT_GENERATOR_ID, actionResult, createMachine, executeCommand, @@ -27,29 +26,25 @@ import { WRITE_APP_SURFACE_ACTION_NAME, hydrateSiteInstance, panesState, - readCardData, siteCharter, } from "../src/agent/charter"; -import type { MessageActor } from "./messageActor"; import { escapeConvexJson, restoreConvexJson } from "./convexJson"; import { SITE_MODEL_ID, sitePromptExecutorConfig } from "./executorConfig"; -import { isTranscriptVisible } from "../src/agent/transcript-visibility"; import { createRepoBash, type RepoBash } from "./repoBash"; import { MAX_RUNNER_MS, isStaleLeaseError, type InboxItemSettle, + type PendingInboxItem, } from "./runnerShared"; +import { + assistantMessageKey, + readStringField, + shouldPersistAssistantMessage, +} from "./transcript"; const STREAM_WRITE_INTERVAL_MS = 250; -type PendingInboxItem = { - itemId: Id<"agentInbox">; - kind: "message" | "command"; - payload: unknown; - actor: MessageActor; -}; - // The session's single runner. Claims the lease (or exits if a live runner // already holds it), keeps ONE machine in memory for its whole tenure, and // loops: drain inbox items into the machine, run the machine step by step @@ -261,14 +256,13 @@ async function drainAndPersist( ): Promise { let surfaceUpdated = false; let reachedDeadline = false; - let lastAssistant: - | { frame: Frame; frameId: Id<"frames">; message: FrameMessage; messageIndex: number } - | null = null; + let lastAssistant: { frame: Frame; frameId: Id<"frames">; messageIndex: number } | null = null; try { for await (const frame of runMachine(machine)) { const settleItems = settlesByFrameId.get(frame.id); settlesByFrameId.delete(frame.id); + // Frame, transcript rows, and item settles land in this one transaction. const frameId: Id<"frames"> = await ctx.runMutation(internal.sessions.appendRunnerFrame, { sessionId, generation, @@ -277,8 +271,13 @@ async function drainAndPersist( }); for (const message of frame.messages) { - if (message.type === "action" && message.name === "writeAppSurface") { - if (message.kind === "result" && message.success) surfaceUpdated = true; + if ( + message.type === "action" && + message.name === WRITE_APP_SURFACE_ACTION_NAME && + message.kind === "result" && + message.success + ) { + surfaceUpdated = true; } } for (const [messageIndex, message] of frame.messages.entries()) { @@ -288,12 +287,10 @@ async function drainAndPersist( shouldPersistAssistantMessage(frame, message) && text.trim() ) { - lastAssistant = { frame, frameId, message, messageIndex }; + lastAssistant = { frame, frameId, messageIndex }; } } - await persistFrameMessages(ctx, { sessionId, frame, frameId }); - // Stop only after the yielded frame and its transcript writes are // durable. The successor reconstructs any remaining work from the log. if (shouldHandoff()) { @@ -313,24 +310,15 @@ async function drainAndPersist( // A successful writeAppSurface anywhere in the drain stamps its last // assistant message, so the transcript can offer "open the app pane" at the - // end of the response. Idempotent re-add: same key, patched row. + // end of the response. Idempotent re-persist through the same code path as + // the original transcript write. if (surfaceUpdated && lastAssistant) { - const text = typeof lastAssistant.message.text === "string" ? lastAssistant.message.text : ""; - const card = readCardData(lastAssistant.message); - await ctx.runMutation(internal.messages.add, { + await ctx.runMutation(internal.sessions.stampUpdatedSurface, { sessionId, - role: "assistant", - content: text, + generation, + frame: escapeConvexJson(lastAssistant.frame), frameId: lastAssistant.frameId, - ...(card ? { card: { title: card.title, source: card.source } } : {}), - idempotencyKey: frameMessageKey( - "assistant", - lastAssistant.frame, - lastAssistant.message, - lastAssistant.messageIndex, - ), - updatedSurface: true, - streamState: "complete", + messageIndex: lastAssistant.messageIndex, }); } return reachedDeadline; @@ -488,7 +476,7 @@ function createStreamWriter( await ctx.runMutation(internal.messages.appendStreamDelta, { sessionId, generation, - messageKey: assistantStreamKey(messageId), + messageKey: assistantMessageKey(messageId), text: delta, streamSeq: update.streamSeq, }); @@ -512,7 +500,7 @@ function createStreamWriter( await ctx.runMutation(internal.messages.markStreamFailed, { sessionId, generation, - messageKey: assistantStreamKey(messageId), + messageKey: assistantMessageKey(messageId), state, }).catch(() => {}); } @@ -549,108 +537,8 @@ function requireSessionId(input: unknown): Id<"sessions"> { return sessionId as Id<"sessions">; } -async function persistFrameMessages( - ctx: ActionCtx, - { - sessionId, - frame, - frameId, - }: { - sessionId: Id<"sessions">; - frame: Frame; - frameId: Id<"frames">; - }, -): Promise { - for (const [messageIndex, message] of frame.messages.entries()) { - const text = typeof message.text === "string" ? message.text : ""; - if (message.type === "user" && text.trim() && isTranscriptVisible(message)) { - const actor = readMessageActor(message); - const clientMessageId = readStringField(message, "clientMessageId"); - await ctx.runMutation(internal.messages.add, { - sessionId, - role: "user", - content: text, - frameId, - ...(actor ? { actor } : {}), - ...(clientMessageId ? { clientMessageId } : {}), - idempotencyKey: frameMessageKey("user", frame, message, messageIndex), - }); - } - - if ( - message.type === "assistant" && - shouldPersistAssistantMessage(frame, message) && - text.trim() - ) { - // postCard rides an assistant message as a data content part; lift it - // onto the message row so the transcript renders the card (content - // stays the prose equivalent, same doctrine as explainer widgets). - const card = readCardData(message); - await ctx.runMutation(internal.messages.add, { - sessionId, - role: "assistant", - content: text, - frameId, - ...(card ? { card: { title: card.title, source: card.source } } : {}), - idempotencyKey: frameMessageKey("assistant", frame, message, messageIndex), - // The durable write settles any in-flight stream row for this message: - // frame messages never carry streamState, so it must be set here or a - // streamed message stays "streaming" forever. - streamState: "complete", - }); - } - } -} - -function shouldPersistAssistantMessage(frame: Frame, message: FrameMessage): boolean { - if (message.audience === "self") return false; - return frame.generatorId === undefined || frame.generatorId === ROOT_GENERATOR_ID; -} - -function frameMessageKey( - prefix: "user" | "assistant", - frame: Frame, - message: FrameMessage, - messageIndex: number, -): string { - const messageId = readStringField(message, "messageId"); - if (messageId && prefix === "assistant") return assistantStreamKey(messageId); - if (messageId) return `${prefix}:${messageId}`; - return `${prefix}:${frame.id}:${messageIndex}`; -} - -function assistantStreamKey(messageId: string): string { - return `assistant:${messageId}`; -} - -function readStringField(value: unknown, key: string): string | undefined { - if (!value || typeof value !== "object") return undefined; - const field = (value as Record)[key]; - return typeof field === "string" && field.length > 0 ? field : undefined; -} - function readBooleanField(value: unknown, key: string): boolean | undefined { if (!value || typeof value !== "object") return undefined; const field = (value as Record)[key]; return typeof field === "boolean" ? field : undefined; } - - -function readMessageActor(message: FrameMessage): MessageActor | undefined { - const actor = message.actor; - if (!actor || typeof actor !== "object") return undefined; - const { id, kind, label, profileUrl } = actor as Record; - if ( - typeof id !== "string" || - (kind !== "anonymous" && kind !== "github") || - typeof label !== "string" - ) { - return undefined; - } - return { - id, - kind, - label, - ...(typeof profileUrl === "string" ? { profileUrl } : {}), - }; -} diff --git a/apps/site/convex/convexJson.test.ts b/apps/site/convex/convexJson.test.ts new file mode 100644 index 0000000..a8bde7e --- /dev/null +++ b/apps/site/convex/convexJson.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { escapeConvexJson, restoreConvexJson } from "./convexJson"; + +describe("Convex JSON encoding", () => { + it("stores schema-bearing fields as strings and restores them losslessly", () => { + let nestedSchema: Record = { type: "number" }; + for (let depth = 0; depth < 25; depth += 1) { + nestedSchema = { + type: "object", + properties: { value: nestedSchema }, + }; + } + const value = { + action: { + stateSchema: { + type: "object", + properties: { + snake: nestedSchema, + }, + }, + }, + child: { + states: [{ stateKey: "game", schema: { $schema: "https://json-schema.org/draft/2020-12/schema", type: "object" } }], + }, + }; + + const escaped = escapeConvexJson(value) as typeof value; + + expect(typeof escaped.action.stateSchema).toBe("string"); + expect(typeof escaped.child.states[0]?.schema).toBe("string"); + expect(restoreConvexJson(escaped)).toEqual(value); + }); + + it("leaves ordinary objects structured", () => { + const value = { state: { nested: { value: 1 } } }; + + expect(escapeConvexJson(value)).toEqual(value); + }); +}); diff --git a/apps/site/convex/convexJson.ts b/apps/site/convex/convexJson.ts index 56fe26c..45a0d9d 100644 --- a/apps/site/convex/convexJson.ts +++ b/apps/site/convex/convexJson.ts @@ -1,6 +1,13 @@ const ESCAPE_PREFIX = "__projector_convex_escaped_key__"; const DOLLAR_PREFIX = `${ESCAPE_PREFIX}dollar__`; const PREFIX_PREFIX = `${ESCAPE_PREFIX}prefix__`; +const SCHEMA_PREFIX = `${ESCAPE_PREFIX}json_schema__`; + +// Serialized nodes and spawn actions use different field names for the same +// JSON Schema payload. Keep every other object queryable in Convex, but store +// these known schema-bearing values as one string so schema depth does not +// count against Convex's document nesting limit. +const SCHEMA_KEYS = new Set(["schema", "inputSchema", "stateSchema", "params"]); export function escapeConvexJson(value: T): T { if (Array.isArray(value)) { @@ -12,10 +19,12 @@ export function escapeConvexJson(value: T): T { } return Object.fromEntries( - Object.entries(value).map(([key, entryValue]) => [ - escapeConvexKey(key), - escapeConvexJson(entryValue), - ]), + Object.entries(value).map(([key, entryValue]) => { + const encodedValue = isSchemaValue(key, entryValue) + ? `${SCHEMA_PREFIX}${JSON.stringify(entryValue)}` + : escapeConvexJson(entryValue); + return [escapeConvexKey(key), encodedValue]; + }), ) as T; } @@ -29,13 +38,26 @@ export function restoreConvexJson(value: T): T { } return Object.fromEntries( - Object.entries(value).map(([key, entryValue]) => [ - restoreConvexKey(key), - restoreConvexJson(entryValue), - ]), + Object.entries(value).map(([key, entryValue]) => { + const restoredKey = restoreConvexKey(key); + return [ + restoredKey, + isEncodedSchema(restoredKey, entryValue) + ? JSON.parse(entryValue.slice(SCHEMA_PREFIX.length)) + : restoreConvexJson(entryValue), + ]; + }), ) as T; } +function isSchemaValue(key: string, value: unknown): value is Record { + return SCHEMA_KEYS.has(key) && value !== null && typeof value === "object"; +} + +function isEncodedSchema(key: string, value: unknown): value is string { + return SCHEMA_KEYS.has(key) && typeof value === "string" && value.startsWith(SCHEMA_PREFIX); +} + export function stripClientSchemas(value: T): T { if (Array.isArray(value)) { return value.map((item) => stripClientSchemas(item)) as T; diff --git a/apps/site/convex/inbox.ts b/apps/site/convex/inbox.ts index 902a9d0..b5a3293 100644 --- a/apps/site/convex/inbox.ts +++ b/apps/site/convex/inbox.ts @@ -18,9 +18,12 @@ import { import type { Id } from "./_generated/dataModel"; import { ACCESS_ERROR, authorizeSessionWrite } from "./access"; import { anonymousActor } from "./actors"; -import { messageActorValidator, requireClientMessageId, type MessageActor } from "./messageActor"; -import { addMessageInternal } from "./messages"; +import { requireClientMessageId, type MessageActor } from "./messageActor"; +import { addMessageInternal, markSessionStreamsFailed } from "./messages"; import { escapeConvexJson } from "./convexJson"; +import { getLatestSessionFrameDoc, restoreFrame } from "./frameHistory"; +import { appendMachineFrameInternal } from "./sessions"; +import type { Frame } from "@projectors/core"; import { LEASE_TTL_MS, MAX_CONSECUTIVE_RUNNER_FAILURES, @@ -28,9 +31,10 @@ import { assertRunnerLease, getRunnerLease, inboxItemSettleValidator, - renewRunnerLease, + pendingInboxItemValidator, settleInboxItems, } from "./runnerShared"; +import { userMessageKey } from "./transcript"; const MAX_PENDING_ITEMS = 100; @@ -42,7 +46,7 @@ const MAX_PENDING_ITEMS = 100; */ async function ensureRunner(ctx: MutationCtx, sessionId: Id<"sessions">): Promise { const lease = await getRunnerLease(ctx, sessionId); - if (lease && lease.active !== false && lease.expiresAt > Date.now()) return; + if (lease && lease.active && lease.expiresAt > Date.now()) return; // A fresh user enqueue re-opens a circuit that stopped after repeated // infrastructure failures. if (lease && (lease.consecutiveFailures ?? 0) > 0) { @@ -141,7 +145,7 @@ async function enqueueUserMessage( content: trimmed, actor, clientMessageId: normalizedClientMessageId, - idempotencyKey: `user:${messageId}`, + idempotencyKey: userMessageKey(messageId), }); return await enqueueItem(ctx, { @@ -214,7 +218,7 @@ export const claim = internalMutation({ handler: async (ctx, { sessionId }) => { const now = Date.now(); const lease = await getRunnerLease(ctx, sessionId); - if (lease && lease.active !== false && lease.expiresAt > now) return null; + if (lease && lease.active && lease.expiresAt > now) return null; const generation = (lease?.generation ?? 0) + 1; if (lease) { await ctx.db.patch(lease._id, { @@ -245,26 +249,22 @@ export const claim = internalMutation({ export const takePending = internalMutation({ args: { sessionId: v.id("sessions"), generation: v.number() }, - returns: v.array( - v.object({ - itemId: v.id("agentInbox"), - kind: v.union(v.literal("message"), v.literal("command")), - payload: v.any(), - actor: messageActorValidator, - }), - ), + returns: v.array(pendingInboxItemValidator), handler: async (ctx, { sessionId, generation }) => { - await renewRunnerLease(ctx, sessionId, generation); + const lease = await assertRunnerLease(ctx, sessionId, generation); + // This runs at every step boundary; renew only when meaningfully aged so + // the common empty poll stays read-only on the lease row. + const now = Date.now(); + if (lease.expiresAt - now < LEASE_TTL_MS / 2) { + await ctx.db.patch(lease._id, { expiresAt: now + LEASE_TTL_MS, renewedAt: now }); + } const items = await ctx.db .query("agentInbox") .withIndex("by_session_status", (q) => q.eq("sessionId", sessionId).eq("status", "pending"), ) .take(MAX_PENDING_ITEMS); - const actionable = items.filter( - (item): item is typeof item & { kind: "message" | "command" } => item.kind !== "topic", - ); - return actionable.map((item) => ({ + return items.map((item) => ({ itemId: item._id, kind: item.kind, payload: item.payload, @@ -336,6 +336,54 @@ export const retryAfterFailure = internalMutation({ const delay = RUNNER_RETRY_BASE_DELAY_MS * 2 ** (consecutiveFailures - 1); await ctx.scheduler.runAfter(delay, internal.agent.runSession, { sessionId }); } else { + const session = await ctx.db.get(sessionId); + if (!session) throw new Error("Session not found"); + const activation = await latestOutstandingActivation(ctx, session); + const terminalFrameId = activation + ? await appendMachineFrameInternal(ctx, { + sessionId, + session, + frame: { + inert: true, + ...(activation.generatorId ? { generatorId: activation.generatorId } : {}), + activationId: activation.activationId, + metadata: { + runnerFailure: true, + consecutiveFailures, + }, + messages: [ + { + type: "work", + kind: "completion", + activationId: activation.activationId, + ...(activation.generatorId ? { generatorId: activation.generatorId } : {}), + reason: "error", + }, + ], + }, + }) + : undefined; + await markSessionStreamsFailed(ctx, sessionId); + const pendingItems = await ctx.db + .query("agentInbox") + .withIndex("by_session_status", (q) => + q.eq("sessionId", sessionId).eq("status", "pending"), + ) + .take(MAX_PENDING_ITEMS); + await settleInboxItems( + ctx, + sessionId, + pendingItems.map((item) => ({ itemId: item._id, error })), + ); + await addMessageInternal(ctx, { + sessionId, + role: "assistant", + content: + "That turn hit an error before Projector could finish. Anything it completed is still available; try sending again.", + ...(terminalFrameId ? { frameId: terminalFrameId } : {}), + idempotencyKey: `runner-error:${activation?.activationId ?? generation}`, + streamState: "error", + }); console.error("Runner retry budget exhausted", { sessionId, generation, @@ -347,6 +395,26 @@ export const retryAfterFailure = internalMutation({ }, }); +async function latestOutstandingActivation( + ctx: MutationCtx, + session: { _id: Id<"sessions"> }, +): Promise<{ activationId: string; generatorId?: string } | undefined> { + const latestFrameDoc = await getLatestSessionFrameDoc(ctx, session._id); + if (!latestFrameDoc?.activationId) return undefined; + const latestFrame = restoreFrame(latestFrameDoc) as Frame; + const alreadyCompleted = latestFrame.messages.some( + (message) => + message.type === "work" && + message.kind === "completion" && + message.activationId === latestFrameDoc.activationId, + ); + if (alreadyCompleted) return undefined; + return { + activationId: latestFrameDoc.activationId, + ...(latestFrameDoc.generatorId ? { generatorId: latestFrameDoc.generatorId } : {}), + }; +} + // Deadline handoff: the runner is near the action ceiling; pass the session to // a fresh action (which claims the next generation) instead of dying mid-work. export const handoff = internalMutation({ @@ -366,7 +434,7 @@ export const reap = internalMutation({ handler: async (ctx, { sessionId, generation }) => { const lease = await getRunnerLease(ctx, sessionId); // A newer generation owns the session, or the runner exited cleanly. - if (!lease || lease.active === false || lease.generation !== generation) return null; + if (!lease || !lease.active || lease.generation !== generation) return null; const now = Date.now(); if (lease.expiresAt > now) { // Still healthy — chase the renewed expiry. diff --git a/apps/site/convex/messages.ts b/apps/site/convex/messages.ts index 8fcdc04..d52bef2 100644 --- a/apps/site/convex/messages.ts +++ b/apps/site/convex/messages.ts @@ -27,6 +27,7 @@ type DbCtx = MutationCtx | QueryCtx; type MessageDoc = Doc<"messages">; const MAX_SESSION_MESSAGES = 2000; +const MAX_OPEN_STREAMS_TO_FAIL = 20; // At the 250ms writer cadence this covers the full ten-minute action limit. const MAX_STREAM_DELTAS = 4096; // Reap abandoned streaming rows comfortably past the action time ceiling. @@ -87,16 +88,16 @@ export const add = internalMutation({ export const appendStreamDelta = internalMutation({ args: { sessionId: v.id("sessions"), - // Runner lease fence: a zombie runner superseded by a newer generation - // must not keep streaming into the transcript. - generation: v.optional(v.number()), + // Runner lease fence, required: only the live generation may stream into + // the transcript — a zombie runner superseded by a newer claim must not. + generation: v.number(), messageKey: v.string(), text: v.string(), streamSeq: v.number(), }, returns: v.id("messages"), handler: async (ctx, { sessionId, generation, messageKey, text, streamSeq }) => { - if (generation !== undefined) await assertRunnerLease(ctx, sessionId, generation); + await assertRunnerLease(ctx, sessionId, generation); const existingIndex = await ctx.db .query("messageIndex") .withIndex("by_session_idempotency_key", (q) => @@ -157,24 +158,44 @@ export const markStreamFailed = internalMutation({ q.eq("sessionId", sessionId).eq("idempotencyKey", messageKey), ) .first(); - if (!index) return null; - const message = await ctx.db.get(index.messageId); - if (message?.streamState === "streaming") { - const deltas = await ctx.db - .query("messageStreamDeltas") - .withIndex("by_message_and_stream_seq", (q) => q.eq("messageId", message._id)) - .order("asc") - .take(MAX_STREAM_DELTAS); - await ctx.db.patch(message._id, { - content: message.content + deltas.map((delta) => delta.text).join(""), - streamState: state, - }); - await Promise.all(deltas.map((delta) => ctx.db.delete(delta._id))); - } + if (index) await markStreamFailedInternal(ctx, index.messageId, state); return null; }, }); +export async function markSessionStreamsFailed( + ctx: MutationCtx, + sessionId: Id<"sessions">, +): Promise { + const indexes = await ctx.db + .query("messageIndex") + .withIndex("by_session", (q) => q.eq("sessionId", sessionId)) + .order("desc") + .take(MAX_OPEN_STREAMS_TO_FAIL); + for (const index of indexes) { + await markStreamFailedInternal(ctx, index.messageId, "error"); + } +} + +async function markStreamFailedInternal( + ctx: MutationCtx, + messageId: Id<"messages">, + state: "cancelled" | "error", +): Promise { + const message = await ctx.db.get(messageId); + if (message?.streamState !== "streaming") return; + const deltas = await ctx.db + .query("messageStreamDeltas") + .withIndex("by_message_and_stream_seq", (q) => q.eq("messageId", message._id)) + .order("asc") + .take(MAX_STREAM_DELTAS); + await ctx.db.patch(message._id, { + content: message.content + deltas.map((delta) => delta.text).join(""), + streamState: state, + }); + await Promise.all(deltas.map((delta) => ctx.db.delete(delta._id))); +} + // Shared by the agent action (via the mutation) and sendCommand (directly): // idempotent message upsert keyed by messageIndex idempotency key. export async function addMessageInternal( diff --git a/apps/site/convex/repoSnapshot.generated.ts b/apps/site/convex/repoSnapshot.generated.ts index 6275a1b..39f1de9 100644 --- a/apps/site/convex/repoSnapshot.generated.ts +++ b/apps/site/convex/repoSnapshot.generated.ts @@ -1,2 +1,2 @@ // Generated by scripts/generate-repo-snapshot.ts. Do not edit. -export default {"revision":"889dfe739e5bc8498653d64b59c659c754139e53-dirty","fileCount":345,"byteCount":2580830,"skippedLarge":0,"skippedBinary":1,"skippedSymlink":1,"files":{"/.claude/plugins/jj/.claude-plugin/plugin.json":"{\n \"name\": \"jj\",\n \"description\": \"Jujutsu (jj) version control commands - a modern alternative to git\",\n \"version\": \"1.0.0\",\n \"author\": {\n \"name\": \"Zack\"\n }\n}\n","/.claude/plugins/jj/commands/commit.md":"---\nallowed-tools: Bash(jj:*)\ndescription: Create a jj commit with a descriptive message\nargument-hint: [message]\n---\n\nCreate a jj commit:\n\n1. Run `jj diff` to see changes\n2. Run `jj log -r @ -T builtin_log_oneline` for context\n3. Draft a descriptive commit message (or use provided message: $ARGUMENTS)\n4. Run `jj commit -m \"\"`\n","/.claude/plugins/jj/commands/rebase-main.md":"---\nallowed-tools: Bash(jj:*)\ndescription: Rebase current changes onto main\n---\n\nRebase current changes onto main:\n\n1. Run `jj rebase --onto main`\n2. Check for conflicts with `jj status`\n3. Report the result\n","/.claude/plugins/jj/commands/squash.md":"---\nallowed-tools: Bash(jj:*)\ndescription: Squash current revision into parent\n---\n\nSquash current revision into parent:\n\n1. Run `jj squash`\n2. Report the result\n","/.claude/plugins/jj/skills/jj/SKILL.md":"---\nallowed-tools: Bash(jj:*)\ndescription: General jj (Jujutsu) guidance and help\nargument-hint: [question]\n---\n\n# Jujutsu (jj) Quick Reference\n\nUse `jj` instead of `git` for all version control operations.\n\n## Key Concepts\n\n- **No staging area** - Files are automatically tracked. Just edit and commit.\n- **`@`** = working copy (current change)\n- **`@-`** = parent of working copy\n- **Changes auto-amend** - Editing files updates the current change automatically\n- **`jj new`** = Start a new empty change (like starting fresh work)\n- **`jj commit`** = Finalize current change with a message and start a new one\n\n## Essential Commands\n\n| Task | Command |\n| ------------------------ | ------------------------------- |\n| Set commit message | `jj describe -m \"message\"` |\n| Finalize and continue | `jj commit -m \"message\"` |\n| Start new change | `jj new` or `jj new main` |\n| Amend into parent | `jj squash` |\n| Interactive amend | `jj squash -i` |\n| View status | `jj st` |\n| View log | `jj log` |\n| View diff | `jj diff` |\n| Push to remote | `jj git push --bookmark ` |\n| Undo last operation | `jj undo` |\n| Rebase onto destination | `jj rebase -d ` |\n| Abandon change | `jj abandon` |\n\n## Common Revsets\n\n- `@` - Working copy\n- `@-` - Parent of working copy\n- `@--` - Grandparent\n- `main` - The main bookmark\n- `main..@` - Commits between main and working copy\n- `::@` - All ancestors of working copy\n- `@::` - All descendants of working copy\n\n## Git Command Translations\n\n| Git | jj |\n| ---------------------------- | ------------------------------- |\n| `git status` | `jj st` |\n| `git diff HEAD` | `jj diff` |\n| `git commit -a` | `jj commit` |\n| `git commit --amend -a` | `jj squash` |\n| `git log --oneline --graph` | `jj log` |\n| `git stash` | `jj new @-` (old change stays) |\n| `git checkout -b topic main` | `jj new main` |\n| `git rebase B A` | `jj rebase -b A -d B` |\n| `git reset --hard` | `jj abandon` or `jj restore` |\n| `git branch` | `jj bookmark list` |\n| `git branch ` | `jj bookmark create ` |\n\n## Request\n\n$ARGUMENTS\n","/.github/workflows/claude-code-review.yml":"name: Claude Code Review\n\non:\n pull_request:\n types: [opened, synchronize, ready_for_review, reopened]\n # Optional: Only run on specific file changes\n # paths:\n # - \"src/**/*.ts\"\n # - \"src/**/*.tsx\"\n # - \"src/**/*.js\"\n # - \"src/**/*.jsx\"\n\njobs:\n claude-review:\n # Optional: Filter by PR author\n # if: |\n # github.event.pull_request.user.login == 'external-contributor' ||\n # github.event.pull_request.user.login == 'new-developer' ||\n # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR'\n\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: read\n issues: read\n id-token: write\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Run Claude Code Review\n id: claude-review\n uses: anthropics/claude-code-action@v1\n with:\n claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n plugin_marketplaces: 'https://github.com/anthropics/claude-code.git'\n plugins: 'code-review@claude-code-plugins'\n prompt: '/code-review:code-review ${{ github.repository }}/pull/${{ github.event.pull_request.number }}'\n # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md\n # or https://code.claude.com/docs/en/cli-reference for available options\n\n","/.github/workflows/claude.yml":"name: Claude Code\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n issues:\n types: [opened, assigned]\n pull_request_review:\n types: [submitted]\n\njobs:\n claude:\n if: |\n (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||\n (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: read\n issues: read\n id-token: write\n actions: read # Required for Claude to read CI results on PRs\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Run Claude Code\n id: claude\n uses: anthropics/claude-code-action@v1\n with:\n claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}\n\n # This is an optional setting that allows Claude to read CI results on PRs\n additional_permissions: |\n actions: read\n\n # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it.\n # prompt: 'Update the pull request description to include a summary of changes.'\n\n # Optional: Add claude_args to customize behavior and configuration\n # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md\n # or https://code.claude.com/docs/en/cli-reference for available options\n # claude_args: '--allowed-tools Bash(gh pr:*)'\n\n","/.gitignore":"# dependencies (bun install)\nnode_modules\n\n# output\nout\ndist\n*.tgz\n\n# code coverage\ncoverage\n*.lcov\n\n# logs\nlogs\n_.log\nreport.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json\n\n# dotenv environment variable files\n.env\n.env.development.local\n.env.test.local\n.env.production.local\n.env.local\n\n# caches\n.eslintcache\n.cache\n*.tsbuildinfo\n\n# IntelliJ based IDEs\n.idea\n\n# Finder (MacOS) folder config\n.DS_Store\n\n# local scripts\n.local-scripts\n\n/sidecar/\n.vercel\n","/.sidecar":"version = 1\nremote = \"https://github.com/anteprojector/projector-sidecar.git\"\npath = \"sidecar\"\nbranch = \"main\"\ninbox = \"sidecar-inbox/{user}/{random}\"\nredaction = \"secrets\"\n","/.zed/settings.json":"{\n \"file_scan_inclusions\": [\n \".env*\",\n \"sidecar/**\"\n ]\n}\n","/AGENTS.md":"# AGENTS.md\n\nThe goal of this repo is to house the projector agent framework, and related packages and apps. \n\nThis repo is early, evolving quickly, and has no production usage. So lets move fast, break things, and aggresively throw out old code. Never implement backwards compatibility during refactors.\n\nKeep tests sparse, mostly focused on asserting outwardly observable framework behavior.\n","/LICENSE":"MIT License\n\nCopyright (c) 2026 Zachary Story\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n","/README.md":"# projector\n\nProjector is an agent framework for building state-complete agents.\n\nAgents are multiplayer apps: the user and the LLM are the first two actors. Projector gives them one shared model of the world, backed by a durable frame log and rendered through projections.\n\nAll you need is state.\n\n## Shape\n\n- State complete: the agent is described by recoverable application state.\n- Durable frame log: every meaningful transition can be replayed.\n- Projection based: each actor sees the slice of state, tools, and instructions meant for them.\n- Client/server unified: client and server respresentations are typesafe representations of the same machine.\n\nThis repo is pre-release and moving quickly. Proper docs will come later.\n","/TODO.md":"# TODO\n\n## Executor-Owned Streaming Metadata\n\nStatus: design pending.\n\nKeep streaming primarily in executor packages for now. Executors can decide\nwhether a provider supports incremental output, how to enable it, and how to\nsurface partial updates to the host. The core machine should not yet commit to a\ngeneric `runMachine(..., { streamWhenAvailable })` API.\n\nCore still needs a small amount of framework support so executor-owned streaming\ncan finalize durably:\n\n- Add machine-scoped metadata for final actor messages produced by executors.\n- Prefer `outputId` for the stable logical output identifier. It is clearer than\n `messageId` because apps often already have app-level message IDs.\n- Treat `outputId` as projector-owned and activation-scoped, for example derived\n from `activationId` plus an output index.\n- Keep provider stream attempts separate from logical outputs. A `streamId` may\n change across retries, while the final logical `outputId` should remain stable\n for the activation output.\n- Allow final messages to carry stream completion metadata such as state and\n final sequence number.\n- Extend implicit text output handling so an executor that returns final text can\n also return metadata for the final durable message. Without this, core maps\n `value: string` into a plain assistant message and loses the stream\n correlation.\n- Consider adding `ExecutorRunRequest` helpers such as `createOutputId(index)`\n and `createStreamId()` so executor packages do not invent incompatible ID\n schemes.\n\nRationale:\n\n- Partial stream deltas should not become projector frames. Token-level frames\n would backpressure provider streaming on durable writes and would pollute\n model-visible history.\n- The durable source of truth should remain the final projector frame.\n- Apps should own storage-specific idempotency keys. Core should provide stable\n machine metadata that apps can use to derive those keys.\n- `outputId` is intentionally not an app message ID. Hosts may map it to Convex\n rows, SQL rows, telemetry events, or ignore it entirely.\n\nCurrent naming preference:\n\n- `outputId`: preferred. Short, readable, and scoped to executor output.\n- `activationOutputId`: precise but longer.\n- `machineOutputId`: clear but a little awkward.\n- `messageId`: avoid, because it collides conceptually with app-owned messages.\n- `asId`: avoid for now; it is too opaque despite being compact.\n\nOpen design questions:\n\n- Whether message metadata should be top-level, such as `outputId` and `stream`,\n or namespaced, such as `machine.outputId` and `machine.stream`.\n- Whether stream completion metadata belongs only on assistant messages or on all\n actor messages.\n- Whether core should expose reusable stream event types even if event emission\n remains executor-owned.\n","/apps/sandbox-agent/.gitignore":"node_modules/\ndist/\n.env\n*.log\n","/apps/sandbox-agent/README.md":"# sandbox-agent\n\nLiveKit worker for the projector sandbox.\n\nThe web app dispatches this worker when the sandbox voice toggle is enabled. The worker joins the LiveKit room, starts an OpenAI Realtime session, syncs compiled projector instructions/tools through `@projectors/livekit-realtime-executor`, and persists voice transcripts back to the same Convex session used by text chat.\n\n## Setup\n\nCreate `apps/sandbox-agent/.env`:\n\n```bash\nLIVEKIT_API_KEY=your-livekit-api-key\nLIVEKIT_API_SECRET=your-livekit-api-secret\nLIVEKIT_URL=wss://your-project.livekit.cloud\nOPENAI_API_KEY=your-openai-api-key\nCONVEX_URL=https://your-deployment.convex.cloud\nENABLE_REALTIME_MODEL=true\n```\n\nOptional:\n\n```bash\nOPENAI_DISCRETE_MODEL=gpt-5.5\nOPENAI_REALTIME_MODEL=gpt-realtime-2\nOPENAI_REALTIME_VOICE=alloy\nOPENAI_REALTIME_VAD_THRESHOLD=0.65\nOPENAI_REALTIME_VAD_SILENCE_DURATION_MS=800\nOPENAI_REALTIME_VAD_PREFIX_PADDING_MS=300\nOPENAI_REALTIME_INPUT_NOISE_REDUCTION=near_field\nOPENAI_REALTIME_INTERRUPT_RESPONSE=true\nOPENAI_REALTIME_MAX_RESPONSE_OUTPUT_TOKENS=inf\nDEBUG_REALTIME_EVENTS=false\n```\n\nFor noisier rooms, raise `OPENAI_REALTIME_VAD_THRESHOLD` toward `0.8` so audio must be louder before it is treated as speech. Increase `OPENAI_REALTIME_VAD_SILENCE_DURATION_MS` if the agent cuts in too aggressively after brief pauses. Set `OPENAI_REALTIME_INPUT_NOISE_REDUCTION` to `near_field` for a headset/close mic, `far_field` for a room mic, or `off` to disable OpenAI input noise reduction. Set `DEBUG_REALTIME_EVENTS=true` when diagnosing cut-off responses; set `OPENAI_REALTIME_INTERRUPT_RESPONSE=false` to test whether VAD/barge-in is cancelling the assistant turn.\n\nThe agent name is `sandbox-agent`; `apps/sandbox/convex/livekitAgentActions.ts` dispatches that exact name.\n\n## Run\n\nFrom this directory:\n\n```bash\nbun run dev\n```\n\nThen run the sandbox app and enable `voice` in the terminal toolbar. The browser will request a LiveKit token from Convex, join the room, and dispatch this worker.\n\nFor a production-style run:\n\n```bash\nbun run build\nbun run start\n```\n","/apps/sandbox-agent/package.json":"{\n \"name\": \"@projectors/sandbox-agent\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"type\": \"module\",\n \"license\": \"MIT\",\n \"scripts\": {\n \"dev\": \"./scripts/kill-zombies.sh && bun run src/agent.ts dev\",\n \"start\": \"bun run src/agent.ts start\",\n \"build\": \"tsgo\",\n \"download-files\": \"tsx src/agent.ts download-files\",\n \"kill-zombies\": \"./scripts/kill-zombies.sh\"\n },\n \"dependencies\": {\n \"@ai-sdk/openai\": \"catalog:\",\n \"@livekit/agents\": \"catalog:\",\n \"@livekit/agents-plugin-openai\": \"catalog:\",\n \"@livekit/rtc-node\": \"catalog:\",\n \"@projectors/livekit-realtime-executor\": \"workspace:*\",\n \"@projectors/aisdk-executor\": \"workspace:*\",\n \"convex\": \"^1.31.2\",\n \"@projectors/sandbox\": \"workspace:*\",\n \"@projectors/core\": \"workspace:*\",\n \"sharp\": \"0.34.5\",\n \"zod\": \"catalog:\"\n },\n \"devDependencies\": {\n \"@types/node\": \"^22\",\n \"tsx\": \"^4.20.0\",\n \"typescript\": \"^5\"\n }\n}\n","/apps/sandbox-agent/scripts/kill-zombies.sh":"#!/bin/bash\n# Kill any zombie voice agent processes before starting a new one\n\n# Get PIDs of LiveKit agent worker processes (excluding this script and grep)\nZOMBIE_PIDS=$(ps aux | grep \"job_proc_lazy_main.js.*agent.ts\" | grep -v grep | awk '{print $2}')\n\nif [ -n \"$ZOMBIE_PIDS\" ]; then\n echo \"Found zombie agent processes: $ZOMBIE_PIDS\"\n echo \"$ZOMBIE_PIDS\" | xargs kill -9 2>/dev/null\n echo \"Killed zombie processes\"\nelse\n echo \"No zombie agent processes found\"\nfi\n\n# Also try pkill as a fallback\npkill -9 -f \"job_proc_lazy_main.js.*sandbox-agent\" 2>/dev/null\n\necho \"Cleanup complete\"\n","/apps/sandbox-agent/skills/example/SKILL.md":"---\nname: example\ndescription: An example skill sandboxnstrating the SKILL.md format\n---\n\n# Example Skill\n\nThis is an example skill that sandboxnstrates the SKILL.md format. When this skill is loaded, you should follow these instructions to help the user.\n\n## Guidelines\n\n1. Always greet the user warmly\n2. Explain what you can help with\n3. Ask clarifying questions if needed\n\n## Available Actions\n\n- **Explain concepts**: Provide clear explanations of topics\n- **Give examples**: Illustrate points with concrete examples\n- **Answer questions**: Respond to user queries directly\n\n## Best Practices\n\n- Be concise and direct\n- Use simple language\n- Provide actionable guidance\n","/apps/sandbox-agent/src/agent.ts":"import {\n WorkerOptions,\n cli,\n defineAgent,\n type JobContext,\n voice,\n} from \"@livekit/agents\";\nimport { ParticipantKind, RoomEvent, type RemoteParticipant } from \"@livekit/rtc-node\";\nimport { openai as aiSdkOpenAI } from \"@ai-sdk/openai\";\nimport * as openai from \"@livekit/agents-plugin-openai\";\nimport { LiveKitRealtimeExecutor } from \"@projectors/livekit-realtime-executor\";\nimport type { LiveKitAgentLike, LiveKitRoomLike, LiveKitSessionLike } from \"@projectors/livekit-realtime-executor\";\nimport { AiSdkExecutor } from \"@projectors/aisdk-executor\";\nimport { ConvexClient } from \"convex/browser\";\nimport { fileURLToPath } from \"node:url\";\nimport {\n ROOT_GENERATOR_ID,\n createMachine,\n executeCommand,\n runMachine,\n type Executor,\n type Frame,\n type FrameDraft,\n syncMachineRuntime,\n} from \"@projectors/core\";\nimport type { ClientMachineMessage } from \"@projectors/core/client\";\nimport {\n attachmentSummary,\n storedAttachmentsFromContentParts,\n userContentPartsForAttachments,\n type SandboxAttachmentData,\n type StoredSandboxAttachmentData,\n} from \"./attachments.js\";\nimport {\n createSandboxCharter,\n getAgentControlsState,\n hydrateSandboxInstance,\n type CameraSensorImage,\n type SandboxMessage,\n} from \"./projector-sandbox.js\";\nimport { attachCameraSampler, type CameraSamplerHandle } from \"./vision.js\";\n\nconst sandboxApiModule = \"@projectors/sandbox/convex/_generated/api.js\";\nconst { api } = (await import(sandboxApiModule)) as any;\n\nconst AGENT_NAME = \"sandbox-agent\";\nconst ENABLE_REALTIME = process.env.ENABLE_REALTIME_MODEL !== \"false\";\nconst DISCRETE_MODEL = process.env.OPENAI_DISCRETE_MODEL ?? \"gpt-5.5\";\nconst OPENAI_NO_REASONING_OPTIONS = { openai: { reasoningEffort: \"none\" } } as const;\nconst REALTIME_VAD_THRESHOLD = readVadThresholdEnv(\"OPENAI_REALTIME_VAD_THRESHOLD\", 0.65);\nconst REALTIME_VAD_SILENCE_DURATION_MS = readIntegerEnv(\"OPENAI_REALTIME_VAD_SILENCE_DURATION_MS\", 800);\nconst REALTIME_VAD_PREFIX_PADDING_MS = readIntegerEnv(\"OPENAI_REALTIME_VAD_PREFIX_PADDING_MS\", 300);\nconst REALTIME_INTERRUPT_RESPONSE = readBooleanEnv(\"OPENAI_REALTIME_INTERRUPT_RESPONSE\", true);\nconst REALTIME_MAX_RESPONSE_OUTPUT_TOKENS = readRealtimeMaxOutputTokensEnv(\n \"OPENAI_REALTIME_MAX_RESPONSE_OUTPUT_TOKENS\",\n \"inf\",\n);\nconst REALTIME_INPUT_NOISE_REDUCTION = readRealtimeNoiseReductionEnv(\n \"OPENAI_REALTIME_INPUT_NOISE_REDUCTION\",\n \"near_field\",\n);\nconst MESSAGE_TOPIC = \"sandbox.message.v1\";\nconst COMMAND_RPC_METHOD = \"sandbox.command.v1\";\nconst WORKER_LEASE_TTL_MS = 15_000;\nconst WORKER_LEASE_HEARTBEAT_MS = 5_000;\nconst DEBUG_REALTIME_EVENTS = process.env.DEBUG_REALTIME_EVENTS === \"true\";\n\ntype SerializedInstance = Parameters[0];\ntype Id = string & { __tableName: TableName };\n\ntype AgentInit = {\n sessionId: Id<\"sessions\">;\n frameId: Id<\"frames\">;\n instance: SerializedInstance;\n messages: SandboxMessage[];\n} | null;\n\ntype AgentWorkerRoomLeaseSnapshot = {\n roomName: string;\n agentWorkerLeaseToken?: string;\n} | null;\n\ntype RealtimeModelOptions = ConstructorParameters[0] & {\n maxResponseOutputTokens?: number | \"inf\";\n};\n\nclass SandboxVoiceAgent extends voice.Agent {\n constructor() {\n super({\n instructions: \"Initializing projector sandbox voice agent.\",\n });\n }\n}\n\nexport default defineAgent({\n entry: async (ctx: JobContext) => {\n console.log(\"[sandbox-agent] entry starting\");\n assertEnv(\"CONVEX_URL\");\n assertEnv(\"OPENAI_API_KEY\");\n\n await ctx.connect();\n const roomName = ctx.room.name;\n if (!roomName) {\n throw new Error(\"LiveKit room has no name\");\n }\n console.log(`[sandbox-agent] connected to room ${roomName}`);\n\n const convex = new ConvexClient(process.env.CONVEX_URL!);\n const workerId = `sandbox-agent:${process.pid}:${crypto.randomUUID()}`;\n const leaseToken = crypto.randomUUID();\n let leaseActive = false;\n let stoppingForLostLease = false;\n let leaseHeartbeat: ReturnType | undefined;\n let leaseSubscription: { unsubscribe: () => void } | undefined;\n let activeSession: voice.AgentSession | undefined;\n let latestCameraImage: CameraSensorImage | undefined;\n let cameraSampler: CameraSamplerHandle | undefined;\n const cameraSensor = {\n latestImage: () => latestCameraImage,\n };\n const stopCameraSampler = async () => {\n const sampler = cameraSampler;\n cameraSampler = undefined;\n latestCameraImage = undefined;\n if (sampler) {\n await sampler.stop();\n }\n };\n\n const releaseLease = () => {\n if (leaseSubscription) {\n leaseSubscription.unsubscribe();\n leaseSubscription = undefined;\n }\n if (leaseHeartbeat) {\n clearInterval(leaseHeartbeat);\n leaseHeartbeat = undefined;\n }\n if (!leaseActive) return;\n leaseActive = false;\n void convex.mutation(api.livekitAgent.releaseAgentWorkerLease, {\n roomName,\n leaseToken,\n }).catch((error) => {\n console.warn(\"[sandbox-agent] failed to release worker lease\", error);\n });\n };\n\n const stopForLostLease = async () => {\n if (stoppingForLostLease) return;\n stoppingForLostLease = true;\n console.warn(`[sandbox-agent] worker lease lost for room ${roomName}; disconnecting`);\n releaseLease();\n await Promise.allSettled([\n activeSession?.close(),\n stopCameraSampler(),\n ctx.room.disconnect(),\n ]);\n setTimeout(() => process.exit(0), 250).unref();\n };\n\n const renewLease = async (): Promise => {\n if (!leaseActive) return false;\n const renewed = await convex.mutation(api.livekitAgent.renewAgentWorkerLease, {\n roomName,\n leaseToken,\n leaseTtlMs: WORKER_LEASE_TTL_MS,\n });\n if (!renewed) {\n await stopForLostLease();\n return false;\n }\n return true;\n };\n\n const assertLease = async () => {\n if (!leaseActive || !(await renewLease())) {\n throw new Error(\"Agent worker lease is no longer active\");\n }\n };\n\n const claimedLease = await convex.mutation(api.livekitAgent.claimAgentWorkerLease, {\n roomName,\n workerId,\n leaseToken,\n leaseTtlMs: WORKER_LEASE_TTL_MS,\n });\n if (!claimedLease) {\n console.warn(`[sandbox-agent] another worker owns room ${roomName}; exiting`);\n await ctx.room.disconnect();\n return;\n }\n\n leaseActive = true;\n leaseSubscription = convex.onUpdate(\n api.livekitAgent.getAgentWorkerRoom,\n { sessionId: claimedLease.sessionId },\n (room: AgentWorkerRoomLeaseSnapshot) => {\n if (!leaseActive) return;\n if (!room || room.roomName !== roomName || room.agentWorkerLeaseToken !== leaseToken) {\n void stopForLostLease();\n }\n },\n (error) => {\n console.error(\"[sandbox-agent] worker lease subscription failed\", error);\n void stopForLostLease();\n },\n );\n leaseHeartbeat = setInterval(() => {\n void renewLease().catch((error) => {\n console.error(\"[sandbox-agent] failed to renew worker lease\", error);\n void stopForLostLease();\n });\n }, WORKER_LEASE_HEARTBEAT_MS);\n ctx.room.once(RoomEvent.Disconnected, releaseLease);\n console.log(`[sandbox-agent] claimed worker lease ${workerId} for room ${roomName}`);\n\n try {\n const init = (await convex.query(api.livekitAgent.getAgentInit, { roomName })) as AgentInit;\n if (!init) {\n throw new Error(`No sandbox session is associated with LiveKit room \"${roomName}\"`);\n }\n console.log(`[sandbox-agent] loaded session ${init.sessionId} with ${init.messages.length} messages`);\n\n let root = hydrateSandboxInstance(init.instance, init.sessionId);\n let referenceFrameId = init.frameId;\n const contextFrames = (await convex.query(api.sessions.listMachineContextFrames, {\n sessionId: init.sessionId,\n })) as Frame[];\n console.log(`[sandbox-agent] initialized machine with ${contextFrames.length} context frame(s)`);\n\n const isLiveMode = (): boolean => getAgentControlsState(root).liveMode;\n const shouldUseRealtime = (): boolean => ENABLE_REALTIME && isLiveMode();\n const shouldStreamText = (): boolean => getAgentControlsState(root).streamingEnabled;\n\n const addSandboxMessage = (args: {\n role: \"user\" | \"assistant\";\n content: string;\n attachments?: StoredSandboxAttachmentData[];\n frameId?: Id<\"frames\">;\n mode: \"text\" | \"voice\";\n idempotencyKey?: string;\n streamState?: \"streaming\" | \"complete\" | \"error\";\n streamSeq?: number;\n }) => convex.mutation(api.messages.add, {\n sessionId: init.sessionId,\n ...args,\n });\n const persistStreamingMessageUpdate = (\n role: \"user\" | \"assistant\",\n mode: \"text\" | \"voice\",\n update: {\n request?: {\n generatorId?: string;\n output?: { audience?: unknown };\n };\n messageId: string;\n text: string;\n streamState: \"streaming\" | \"complete\" | \"cancelled\" | \"error\";\n streamSeq: number;\n },\n ) => {\n if (!leaseActive) return;\n if (!shouldStreamText()) return;\n // Barge-in partials stay in the frame log as interrupted context, but\n // are not durable transcript messages (the executor owns that split).\n if (update.streamState === \"cancelled\") return;\n if (role === \"assistant\" && !shouldPersistAssistantStreamingUpdate(update)) return;\n return addSandboxMessage({\n role,\n content: update.text,\n mode,\n idempotencyKey: `${role}:${update.messageId}`,\n streamState: update.streamState,\n streamSeq: update.streamSeq,\n });\n };\n const agentDiscreteExecutor = new AiSdkExecutor({\n model: aiSdkOpenAI(DISCRETE_MODEL),\n maxOutputTokens: 4096,\n stream: shouldStreamText,\n providerOptions: OPENAI_NO_REASONING_OPTIONS,\n onStreamUpdate: (update) => persistStreamingMessageUpdate(\"assistant\", \"text\", update),\n });\n const memoryExecutor = new AiSdkExecutor({\n model: aiSdkOpenAI(DISCRETE_MODEL),\n maxOutputTokens: 1024,\n providerOptions: OPENAI_NO_REASONING_OPTIONS,\n });\n const isMemoryRequest = (request: { inference: { tools: Array<{ name: string }> } }) =>\n request.inference.tools.some((tool) => tool.name === \"saveMemories\");\n const discreteExecutor: Executor = {\n run: (request) =>\n isMemoryRequest(request)\n ? memoryExecutor.run(request)\n : agentDiscreteExecutor.run(request),\n realizePrompt: (request) =>\n isMemoryRequest(request)\n ? memoryExecutor.realizePrompt(request)\n : agentDiscreteExecutor.realizePrompt(request),\n };\n const session = createVoiceSession();\n activeSession = session;\n const agent = new SandboxVoiceAgent();\n const liveKitExecutor = new LiveKitRealtimeExecutor({\n debug: DEBUG_REALTIME_EVENTS,\n session: session as unknown as LiveKitSessionLike,\n agent: agent as unknown as LiveKitAgentLike,\n room: ctx.room as unknown as LiveKitRoomLike,\n discreteExecutor,\n realtime: { enabled: () => shouldUseRealtime() },\n input: {\n messageTopic: MESSAGE_TOPIC,\n parseDataMessage: (payload) => parseLiveKitSandboxMessage(payload),\n },\n eventNames: {\n dataReceived: RoomEvent.DataReceived as string,\n },\n onAssistantTranscriptUpdate: (update) =>\n persistStreamingMessageUpdate(\"assistant\", \"voice\", update),\n onUserTranscriptUpdate: (update) =>\n persistStreamingMessageUpdate(\"user\", \"voice\", update),\n });\n ctx.room.once(RoomEvent.Disconnected, () => {\n liveKitExecutor.disconnect();\n void stopCameraSampler().catch((error) => {\n console.warn(\"[sandbox-agent] failed to stop camera sampler\", error);\n });\n });\n const createSandboxMachine = (frames: Frame[]) =>\n createMachine({\n id: init.sessionId,\n instance: root,\n charter: createSandboxCharter({ cameraSensor }),\n executor: liveKitExecutor,\n frames,\n });\n let machine = createSandboxMachine(contextFrames);\n let unsubscribeMachine: (() => void) | undefined;\n\n const persistFrameMessages = async (\n frame: Frame,\n frameId: Id<\"frames\">,\n ) => {\n const mode = frameMessageMode(frame);\n for (const message of frame.messages) {\n const text = typeof message.text === \"string\" ? message.text : \"\";\n if (message.type === \"user\" && text.trim()) {\n const streamState = normalizeStreamState(message.streamState);\n const attachments = storedAttachmentsFromContentParts(message.content);\n await addSandboxMessage({\n role: \"user\",\n content: text,\n ...(attachments.length ? { attachments } : {}),\n frameId,\n mode,\n idempotencyKey: idempotencyKey(\"user\", message),\n ...(streamState ? { streamState } : {}),\n ...(typeof message.streamSeq === \"number\" ? { streamSeq: message.streamSeq } : {}),\n });\n }\n\n if (\n message.type === \"assistant\" &&\n shouldPersistAssistantMessage(frame, message) &&\n (text.trim() || hasMessageId(message))\n ) {\n const streamState = normalizeStreamState(message.streamState);\n await addSandboxMessage({\n role: \"assistant\",\n content: text,\n frameId,\n mode,\n idempotencyKey: idempotencyKey(\"assistant\", message),\n ...(streamState ? { streamState } : {}),\n ...(typeof message.streamSeq === \"number\" ? { streamSeq: message.streamSeq } : {}),\n });\n }\n }\n };\n\n const refreshDurableMachineState = async (reason: string) => {\n const refreshed = (await convex.query(api.livekitAgent.getAgentInit, { roomName })) as AgentInit;\n if (!refreshed) {\n throw new Error(`No sandbox session is associated with LiveKit room \"${roomName}\"`);\n }\n\n root = hydrateSandboxInstance(refreshed.instance, init.sessionId);\n referenceFrameId = refreshed.frameId;\n const refreshedFrames = (await convex.query(api.sessions.listMachineContextFrames, {\n sessionId: init.sessionId,\n })) as Frame[];\n\n unsubscribeMachine?.();\n machine = createSandboxMachine(refreshedFrames);\n unsubscribeMachine = machine.subscribe(scheduleMachineHost);\n await syncMachineRuntime(machine, {\n generatorId: ROOT_GENERATOR_ID,\n visibleFrames: [],\n });\n console.warn(`[sandbox-agent] refreshed durable session state after ${reason}`);\n };\n\n const persistMachineFrame = async (frame: Frame): Promise | undefined> => {\n await assertLease();\n let frameId: Id<\"frames\">;\n try {\n frameId = await convex.mutation(api.sessions.appendMachineFrame, {\n sessionId: init.sessionId,\n referenceFrameId,\n frame,\n });\n } catch (error) {\n await refreshDurableMachineState(\"append failure\");\n return undefined;\n }\n referenceFrameId = frameId;\n await persistFrameMessages(frame, frameId);\n return frameId;\n };\n\n const runMachineHost = async () => {\n for await (const frame of runMachine(machine)) {\n const frameId = await persistMachineFrame(frame);\n if (!frameId) return;\n if (!frame.inert) {\n await syncMachineRuntime(machine, {\n generatorId: ROOT_GENERATOR_ID,\n visibleFrames: [frame],\n });\n applyRoomIoParticipant();\n }\n }\n };\n\n let hostScheduled = false;\n let hostTail: Promise = Promise.resolve();\n const scheduleMachineHost = () => {\n if (hostScheduled) return;\n hostScheduled = true;\n hostTail = hostTail\n .catch(() => undefined)\n .then(async () => {\n hostScheduled = false;\n await runMachineHost();\n })\n .catch((error) => {\n console.error(\"[sandbox-agent] machine host failed\", error);\n });\n };\n unsubscribeMachine = machine.subscribe(scheduleMachineHost);\n\n ctx.room.localParticipant?.registerRpcMethod(COMMAND_RPC_METHOD, async (data) => {\n await assertLease();\n const payload = parseCommandRpcPayload(data.payload);\n if (payload.sessionId !== init.sessionId) {\n return JSON.stringify({\n success: false,\n error: \"Command session does not match LiveKit worker session\",\n callId: payload.message.callId,\n });\n }\n\n const result = await executeCommand(machine, payload.message);\n await hostTail;\n return JSON.stringify(result);\n });\n ctx.room.once(RoomEvent.Disconnected, () => {\n ctx.room.localParticipant?.unregisterRpcMethod(COMMAND_RPC_METHOD);\n });\n\n let selectedVoiceParticipantIdentity = selectVoiceParticipantIdentity(\n ctx.room.remoteParticipants.values(),\n init.sessionId,\n );\n let appliedRoomIoParticipantIdentity: string | null | undefined;\n const applyRoomIoParticipant = () => {\n const identity = selectedVoiceParticipantIdentity;\n const roomIO = (session as unknown as { roomIO?: { setParticipant?: (participantIdentity: string | null) => void } }).roomIO;\n if (!roomIO?.setParticipant) return;\n if (appliedRoomIoParticipantIdentity === identity) return;\n if (identity === null && appliedRoomIoParticipantIdentity === undefined) return;\n appliedRoomIoParticipantIdentity = identity;\n roomIO.setParticipant(identity);\n console.log(`[sandbox-agent] voice participant ${identity ?? \"\"}`);\n };\n const setSelectedVoiceParticipant = (identity: string | null) => {\n selectedVoiceParticipantIdentity = identity;\n applyRoomIoParticipant();\n };\n\n // RoomIO must observe participant events before we call its private setParticipant;\n // otherwise its init task can wait forever and never publish the output audio track.\n const deferRoomIoParticipantSync = () => {\n queueMicrotask(() => {\n setSelectedVoiceParticipant(selectVoiceParticipantIdentity(\n ctx.room.remoteParticipants.values(),\n init.sessionId,\n ));\n });\n };\n\n const handleParticipantConnected = (participant: RemoteParticipant) => {\n if (!isVoiceParticipant(participant, init.sessionId)) return;\n deferRoomIoParticipantSync();\n };\n const handleParticipantDisconnected = (participant: RemoteParticipant) => {\n if (participant.identity !== selectedVoiceParticipantIdentity) return;\n deferRoomIoParticipantSync();\n };\n\n ctx.room.on(RoomEvent.LocalTrackPublished, (publication) => {\n console.log(\n `[sandbox-agent] local track published sid=${String(publication.sid ?? \"\")} source=${String(publication.source ?? \"\")}`,\n );\n });\n ctx.room.on(RoomEvent.LocalTrackSubscribed, (track) => {\n console.log(`[sandbox-agent] local track subscribed sid=${String(track.sid ?? \"\")}`);\n });\n ctx.room.on(RoomEvent.ParticipantConnected, handleParticipantConnected);\n ctx.room.on(RoomEvent.ParticipantDisconnected, handleParticipantDisconnected);\n\n if (!shouldUseRealtime()) {\n await syncMachineRuntime(machine, {\n generatorId: ROOT_GENERATOR_ID,\n visibleFrames: [],\n });\n }\n\n session.on(voice.AgentSessionEventTypes.AgentStateChanged, (event) => {\n console.log(`[sandbox-agent] state ${event.oldState} -> ${event.newState}`);\n });\n session.on(voice.AgentSessionEventTypes.SpeechCreated, (event) => {\n const speechId = readSpeechHandleId(event);\n console.log(`[sandbox-agent] speech created ${speechId ?? \"\"} source=${event.source}`);\n });\n session.on(voice.AgentSessionEventTypes.FunctionToolsExecuted, (event) => {\n console.log(`[sandbox-agent] function tools executed ${summarizeLiveKitFunctionToolsExecuted(event)}`);\n });\n session.on(voice.AgentSessionEventTypes.Error, (event) => {\n console.error(\"[sandbox-agent] LiveKit session error\", event.error);\n });\n\n await session.start({\n agent,\n room: ctx.room,\n inputOptions: {\n closeOnDisconnect: false,\n participantKinds: [ParticipantKind.STANDARD],\n ...(selectedVoiceParticipantIdentity ? { participantIdentity: selectedVoiceParticipantIdentity } : {}),\n },\n });\n applyRoomIoParticipant();\n if (DEBUG_REALTIME_EVENTS) {\n attachRealtimeDebugLogging(agent);\n }\n\n cameraSampler = attachCameraSampler({\n room: ctx.room,\n onImage: (image) => {\n latestCameraImage = image;\n void syncMachineRuntime(machine, {\n generatorId: ROOT_GENERATOR_ID,\n visibleFrames: [],\n }).catch((error) => {\n console.warn(\"[sandbox-agent] failed to sync camera sensor\", error);\n });\n },\n });\n\n await syncMachineRuntime(machine, {\n generatorId: ROOT_GENERATOR_ID,\n visibleFrames: [],\n });\n\n scheduleMachineHost();\n console.log(`[sandbox-agent] voice session started for ${roomName}; liveMode=${isLiveMode()}`);\n console.log(`[sandbox-agent] connected to ${roomName} with ${liveKitExecutor.connection.inference?.tools.length ?? 0} projected tools`);\n } catch (error) {\n await stopCameraSampler().catch((stopError) => {\n console.warn(\"[sandbox-agent] failed to stop camera sampler\", stopError);\n });\n releaseLease();\n throw error;\n }\n },\n});\n\nfunction createVoiceSession(): voice.AgentSession {\n if (!ENABLE_REALTIME) {\n return new voice.AgentSession({\n stt: new openai.STT(),\n llm: new openai.LLM({ model: DISCRETE_MODEL }),\n tts: new openai.TTS({ voice: \"alloy\" }),\n });\n }\n\n const realtimeModelOptions: RealtimeModelOptions = {\n model: process.env.OPENAI_REALTIME_MODEL ?? \"gpt-realtime-2\",\n voice: process.env.OPENAI_REALTIME_VOICE ?? \"alloy\",\n inputAudioNoiseReduction: REALTIME_INPUT_NOISE_REDUCTION\n ? { type: REALTIME_INPUT_NOISE_REDUCTION }\n : undefined,\n turnDetection: {\n type: \"server_vad\",\n threshold: REALTIME_VAD_THRESHOLD,\n prefix_padding_ms: REALTIME_VAD_PREFIX_PADDING_MS,\n silence_duration_ms: REALTIME_VAD_SILENCE_DURATION_MS,\n interrupt_response: REALTIME_INTERRUPT_RESPONSE,\n },\n inputAudioTranscription: {\n model: \"whisper-1\",\n },\n maxResponseOutputTokens: REALTIME_MAX_RESPONSE_OUTPUT_TOKENS,\n };\n\n return new voice.AgentSession({\n llm: new openai.realtime.RealtimeModel(realtimeModelOptions),\n // Realtime normally returns audio; TTS is a fallback for text-only realtime responses.\n tts: new openai.TTS({ voice: \"alloy\" }),\n });\n}\n\nfunction readNumberEnv(name: string, fallback: number): number {\n const value = process.env[name];\n if (value === undefined || value.trim() === \"\") {\n return fallback;\n }\n const parsed = Number(value);\n if (!Number.isFinite(parsed)) {\n throw new Error(`${name} must be a finite number`);\n }\n return parsed;\n}\n\nfunction readVadThresholdEnv(name: string, fallback: number): number {\n const parsed = readNumberEnv(name, fallback);\n if (parsed < 0 || parsed > 1) {\n throw new Error(`${name} must be between 0 and 1`);\n }\n return parsed;\n}\n\nfunction readIntegerEnv(name: string, fallback: number): number {\n const parsed = readNumberEnv(name, fallback);\n if (!Number.isInteger(parsed)) {\n throw new Error(`${name} must be an integer`);\n }\n return parsed;\n}\n\nfunction readBooleanEnv(name: string, fallback: boolean): boolean {\n const value = process.env[name]?.trim().toLowerCase();\n if (!value) return fallback;\n if ([\"1\", \"true\", \"yes\", \"on\"].includes(value)) return true;\n if ([\"0\", \"false\", \"no\", \"off\"].includes(value)) return false;\n throw new Error(`${name} must be true or false`);\n}\n\nfunction readRealtimeMaxOutputTokensEnv(name: string, fallback: number | \"inf\"): number | \"inf\" {\n const value = process.env[name]?.trim().toLowerCase();\n if (!value) return fallback;\n if (value === \"inf\" || value === \"infinity\") return \"inf\";\n const parsed = Number(value);\n if (!Number.isInteger(parsed) || parsed <= 0) {\n throw new Error(`${name} must be a positive integer, inf, or infinity`);\n }\n return parsed;\n}\n\nfunction readRealtimeNoiseReductionEnv(\n name: string,\n fallback: \"near_field\" | \"far_field\" | undefined,\n): \"near_field\" | \"far_field\" | undefined {\n const value = process.env[name]?.trim();\n if (!value) {\n return fallback;\n }\n if (value === \"off\" || value === \"none\" || value === \"false\") {\n return undefined;\n }\n if (value === \"near_field\" || value === \"far_field\") {\n return value;\n }\n throw new Error(`${name} must be near_field, far_field, off, none, or false`);\n}\n\nfunction selectVoiceParticipantIdentity(participants: Iterable, sessionId: string): string | null {\n let selected: { identity: string; joinedAtMs: bigint; priority: number } | undefined;\n for (const participant of participants) {\n const priority = voiceParticipantPriority(participant, sessionId);\n if (priority === 0) continue;\n const joinedAtMs = participantJoinedAtMs(participant);\n if (!selected || priority > selected.priority || (priority === selected.priority && joinedAtMs >= selected.joinedAtMs)) {\n selected = { identity: participant.identity, joinedAtMs, priority };\n }\n }\n return selected?.identity ?? null;\n}\n\nfunction isVoiceParticipant(participant: RemoteParticipant, sessionId: string): boolean {\n return voiceParticipantPriority(participant, sessionId) > 0;\n}\n\nfunction voiceParticipantPriority(participant: RemoteParticipant, sessionId: string): number {\n if (participant.kind !== ParticipantKind.STANDARD) return 0;\n const stableIdentity = `user-${sessionId}`;\n if (participant.identity === stableIdentity) return 3;\n if (participant.identity.startsWith(`${stableIdentity}-`)) return 2;\n return 0;\n}\n\nfunction participantJoinedAtMs(participant: RemoteParticipant): bigint {\n const info = (participant as RemoteParticipant & { info?: { joinedAtMs?: bigint; joinedAt?: bigint } }).info;\n if (info?.joinedAtMs && info.joinedAtMs > 0n) return info.joinedAtMs;\n if (info?.joinedAt && info.joinedAt > 0n) return info.joinedAt * 1000n;\n return 0n;\n}\n\nfunction frameMessageMode(frame: Frame): \"text\" | \"voice\" {\n return frame.provenance?.execution?.mode === \"voice\" ? \"voice\" : \"text\";\n}\n\nfunction shouldPersistAssistantMessage(frame: Frame, message: Frame[\"messages\"][number]): boolean {\n if (message.audience === \"self\") return false;\n return !frame.generatorId || frame.generatorId === ROOT_GENERATOR_ID;\n}\n\nfunction shouldPersistAssistantStreamingUpdate(update: {\n request?: {\n generatorId?: string;\n output?: { audience?: unknown };\n };\n}): boolean {\n if (update.request?.output?.audience === \"self\") return false;\n return !update.request?.generatorId || update.request.generatorId === ROOT_GENERATOR_ID;\n}\n\nfunction idempotencyKey(prefix: string, source: unknown): string {\n if (source && typeof source === \"object\") {\n const record = source as Record;\n const messageId = record.messageId;\n if (typeof messageId === \"string\" && messageId) return `${prefix}:${messageId}`;\n const createdAt = record.createdAt;\n const text = record.text;\n if (createdAt !== undefined && typeof text === \"string\") return `${prefix}:${createdAt}:${text}`;\n }\n return `${prefix}:${crypto.randomUUID()}`;\n}\n\nfunction hasMessageId(source: unknown): boolean {\n return Boolean(source && typeof source === \"object\" && typeof (source as Record).messageId === \"string\");\n}\n\nfunction normalizeStreamState(value: unknown): \"streaming\" | \"complete\" | \"error\" | undefined {\n return value === \"streaming\" || value === \"complete\" || value === \"error\" ? value : undefined;\n}\n\nfunction readSpeechHandleId(event: unknown): string | undefined {\n if (!event || typeof event !== \"object\") return undefined;\n const speechHandle = (event as Record).speechHandle;\n if (!speechHandle || typeof speechHandle !== \"object\") return undefined;\n const id = (speechHandle as Record).id;\n return typeof id === \"string\" ? id : undefined;\n}\n\nfunction attachRealtimeDebugLogging(agent: SandboxVoiceAgent): void {\n const realtimeSession = (\n agent as unknown as {\n _agentActivity?: {\n realtimeLLMSession?: {\n on?: (event: string, listener: (...args: unknown[]) => void) => void;\n };\n };\n }\n )._agentActivity?.realtimeLLMSession;\n if (!realtimeSession?.on) {\n console.log(\"[sandbox-agent] realtime debug logging skipped; no realtime session\");\n return;\n }\n\n realtimeSession.on(\"generation_created\", (event) => {\n const record = asRecord(event);\n console.log(\n `[sandbox-agent] realtime generation_created responseId=${String(record.responseId ?? \"\")} userInitiated=${String(record.userInitiated ?? \"\")}`,\n );\n });\n realtimeSession.on(\"input_speech_started\", () => {\n console.warn(\"[sandbox-agent] realtime input_speech_started\");\n });\n realtimeSession.on(\"input_speech_stopped\", (event) => {\n const record = asRecord(event);\n console.warn(\n `[sandbox-agent] realtime input_speech_stopped userTranscriptionEnabled=${String(record.userTranscriptionEnabled ?? \"\")}`,\n );\n });\n realtimeSession.on(\"input_audio_transcription_completed\", (event) => {\n const record = asRecord(event);\n const transcript = typeof record.transcript === \"string\" ? record.transcript : \"\";\n console.log(\n `[sandbox-agent] realtime input_audio_transcription_completed itemId=${String(record.itemId ?? \"\")} chars=${transcript.length}`,\n );\n });\n realtimeSession.on(\"openai_client_event_queued\", (event) => {\n const summary = summarizeRealtimeClientEvent(event);\n if (summary) console.log(`[sandbox-agent] openai client ${summary}`);\n });\n realtimeSession.on(\"openai_server_event_received\", (event) => {\n const summary = summarizeRealtimeServerEvent(event);\n if (summary) console.log(`[sandbox-agent] openai server ${summary}`);\n });\n realtimeSession.on(\"metrics_collected\", (metrics) => {\n const record = asRecord(metrics);\n const type = typeof record.type === \"string\" ? record.type : metrics?.constructor?.name;\n console.log(`[sandbox-agent] realtime metrics_collected type=${type ?? \"\"}`);\n });\n}\n\nfunction summarizeLiveKitFunctionToolsExecuted(event: unknown): string {\n const record = asRecord(event);\n const calls = Array.isArray(record.functionCalls) ? record.functionCalls : [];\n const outputs = Array.isArray(record.functionCallOutputs) ? record.functionCallOutputs : [];\n const callSummary = calls\n .map((call) => {\n const item = asRecord(call);\n return `${String(item.name ?? \"\")}:${String(item.callId ?? \"\")}`;\n })\n .join(\",\");\n const outputSummary = outputs\n .map((output) => {\n const item = asRecord(output);\n const text = typeof item.output === \"string\" ? item.output : \"\";\n return `${String(item.callId ?? \"\")}:error=${String(item.isError ?? false)}:chars=${text.length}`;\n })\n .join(\",\");\n return `calls=${calls.length}${callSummary ? `[${callSummary}]` : \"\"} outputs=${outputs.length}${outputSummary ? `[${outputSummary}]` : \"\"}`;\n}\n\nfunction asRecord(value: unknown): Record {\n return value && typeof value === \"object\" ? value as Record : {};\n}\n\nfunction summarizeRealtimeClientEvent(event: unknown): string | undefined {\n const record = asRecord(event);\n const type = typeof record.type === \"string\" ? record.type : undefined;\n if (!type) return undefined;\n if (type === \"session.update\") {\n const session = asRecord(record.session);\n const createResponse = readNestedValue(session, [\"audio\", \"input\", \"turn_detection\", \"create_response\"]);\n return `${type} max_output_tokens=${String(session.max_output_tokens ?? session.max_response_output_tokens ?? \"\")} create_response=${createResponse === \"\" ? \"default_true\" : createResponse} interrupt_response=${readNestedValue(session, [\"audio\", \"input\", \"turn_detection\", \"interrupt_response\"])}`;\n }\n if (\n type === \"response.cancel\" ||\n type === \"conversation.item.truncate\" ||\n type === \"response.create\"\n ) {\n return `${type} ${formatRecordSummary(record, [\"item_id\", \"response_id\", \"audio_end_ms\"])}`;\n }\n if (type === \"conversation.item.create\") {\n return `${type} item=${summarizeRealtimeItem(record.item)} previous_item_id=${String(record.previous_item_id ?? \"\")}`;\n }\n if (type === \"conversation.item.delete\") {\n return `${type} item_id=${String(record.item_id ?? \"\")}`;\n }\n return undefined;\n}\n\nfunction summarizeRealtimeServerEvent(event: unknown): string | undefined {\n const record = asRecord(event);\n const type = typeof record.type === \"string\" ? record.type : undefined;\n if (!type) return undefined;\n\n if (type === \"response.done\") {\n const response = asRecord(record.response);\n return `${type} id=${String(response.id ?? \"\")} status=${String(response.status ?? \"\")} status_details=${safeJson(response.status_details)} usage=${safeJson(response.usage)} output=${summarizeRealtimeOutput(response.output)}`;\n }\n\n if (\n type === \"input_audio_buffer.speech_started\" ||\n type === \"input_audio_buffer.speech_stopped\" ||\n type === \"conversation.item.truncated\" ||\n type === \"response.audio_transcript.done\" ||\n type === \"response.output_audio_transcript.done\" ||\n type === \"response.audio.done\" ||\n type === \"response.output_audio.done\" ||\n type === \"error\"\n ) {\n return `${type} ${formatRecordSummary(record, [\"item_id\", \"response_id\", \"audio_end_ms\", \"error\"])}`;\n }\n\n if (type === \"conversation.item.added\" || type === \"conversation.item.created\") {\n return `${type} item=${summarizeRealtimeItem(record.item)}`;\n }\n\n if (type === \"response.output_item.added\" || type === \"response.output_item.done\") {\n return `${type} response_id=${String(record.response_id ?? \"\")} item=${summarizeRealtimeItem(record.item)}`;\n }\n\n if (type === \"response.function_call_arguments.done\") {\n return `${type} response_id=${String(record.response_id ?? \"\")} item_id=${String(record.item_id ?? \"\")} call_id=${String(record.call_id ?? \"\")} name=${String(record.name ?? \"\")} args_chars=${typeof record.arguments === \"string\" ? record.arguments.length : 0}`;\n }\n\n if (\n type === \"response.audio_transcript.delta\" ||\n type === \"response.output_audio_transcript.delta\" ||\n type === \"response.text.delta\" ||\n type === \"response.output_text.delta\"\n ) {\n const delta = typeof record.delta === \"string\" ? record.delta : \"\";\n return `${type} response_id=${String(record.response_id ?? \"\")} item_id=${String(record.item_id ?? \"\")} chars=${delta.length}`;\n }\n\n return undefined;\n}\n\nfunction summarizeRealtimeOutput(output: unknown): string {\n if (!Array.isArray(output)) return \"\";\n return output\n .map((item) => {\n const record = asRecord(item);\n return `${String(record.type ?? \"\")}:${String(record.role ?? record.name ?? \"\")}:${summarizeRealtimeContent(record.content)}`;\n })\n .join(\",\");\n}\n\nfunction summarizeRealtimeItem(item: unknown): string {\n const record = asRecord(item);\n const type = typeof record.type === \"string\" ? record.type : \"\";\n if (type === \"function_call\") {\n return `${type}:${String(record.name ?? \"\")}:call_id=${String(record.call_id ?? \"\")}:args_chars=${typeof record.arguments === \"string\" ? record.arguments.length : 0}`;\n }\n if (type === \"function_call_output\") {\n return `${type}:call_id=${String(record.call_id ?? \"\")}:output_chars=${typeof record.output === \"string\" ? record.output.length : 0}`;\n }\n if (type === \"message\") {\n return `${type}:${String(record.role ?? \"\")}:${summarizeRealtimeContent(record.content)} id=${String(record.id ?? \"\")}`;\n }\n return type;\n}\n\nfunction summarizeRealtimeContent(contentValue: unknown): string {\n const content = Array.isArray(contentValue) ? contentValue : contentValue ? [contentValue] : [];\n let textChars = 0;\n let transcriptChars = 0;\n const types: string[] = [];\n const images: string[] = [];\n const imageUrls: string[] = [];\n for (const part of content) {\n const partRecord = asRecord(part);\n const type = typeof partRecord.type === \"string\" ? partRecord.type : \"\";\n types.push(type);\n if (typeof partRecord.text === \"string\") textChars += partRecord.text.length;\n if (typeof partRecord.transcript === \"string\") transcriptChars += partRecord.transcript.length;\n if (type === \"input_image\") {\n images.push(describeRealtimeImageUrl(partRecord.image_url));\n imageUrls.push(describeRawRealtimeImageUrl(partRecord.image_url));\n }\n }\n return `parts=${types.join(\"|\") || \"\"} text_chars=${textChars} transcript_chars=${transcriptChars} images=${images.length}${images.length ? `[${images.join(\",\")}]` : \"\"}${imageUrls.length ? ` image_urls=${safeJson(imageUrls)}` : \"\"}`;\n}\n\nfunction describeRealtimeImageUrl(value: unknown): string {\n if (typeof value !== \"string\") return \"\";\n if (value.startsWith(\"data:\")) return \"data-url\";\n try {\n const url = new URL(value);\n return `${url.protocol}//${url.host}${url.pathname.slice(0, 80)}`;\n } catch {\n return value.slice(0, 80);\n }\n}\n\nfunction describeRawRealtimeImageUrl(value: unknown): string {\n if (typeof value !== \"string\") return \"\";\n if (value.startsWith(\"data:\")) return `data-url:${value.length}chars`;\n return value;\n}\n\nfunction formatRecordSummary(record: Record, keys: string[]): string {\n return keys\n .filter((key) => record[key] !== undefined)\n .map((key) => `${key}=${safeJson(record[key])}`)\n .join(\" \");\n}\n\nfunction readNestedValue(record: Record, path: string[]): string {\n let value: unknown = record;\n for (const key of path) {\n value = asRecord(value)[key];\n }\n return value === undefined ? \"\" : String(value);\n}\n\nfunction safeJson(value: unknown): string {\n if (value === undefined) return \"\";\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nfunction assertEnv(name: string): void {\n if (!process.env[name]) {\n throw new Error(`${name} is required`);\n }\n}\n\nfunction parseLiveKitSandboxMessage(payload: Uint8Array): string | FrameDraft | undefined {\n try {\n const parsed = JSON.parse(new TextDecoder().decode(payload)) as {\n content?: unknown;\n attachments?: unknown;\n };\n const content = typeof parsed.content === \"string\" ? parsed.content.trim() : \"\";\n const attachments = normalizeLiveKitAttachments(parsed.attachments);\n if (DEBUG_REALTIME_EVENTS) {\n console.log(\n `[sandbox-agent] livekit data message parsed content_chars=${content.length} attachments=${summarizeSandboxAttachments(attachments)}`,\n );\n }\n if (!content && attachments.length === 0) return undefined;\n const text = content || attachmentSummary(attachments);\n return {\n messages: [\n {\n type: \"user\",\n content: userContentPartsForAttachments(content, attachments),\n text,\n audience: \"broadcast\",\n source: { external: true, transport: \"livekit\" },\n },\n ],\n };\n } catch (error) {\n if (DEBUG_REALTIME_EVENTS) {\n console.warn(\"[sandbox-agent] failed to parse livekit data message\", error);\n }\n return undefined;\n }\n}\n\nfunction summarizeSandboxAttachments(attachments: readonly SandboxAttachmentData[]): string {\n if (attachments.length === 0) return \"0\";\n return attachments\n .map((attachment) =>\n `${attachment.kind}:${attachment.name}:type=${attachment.contentType}:url=${attachment.url ? \"yes\" : \"no\"}:dataUrl=${attachment.dataUrl ? `${attachment.dataUrl.length}chars` : \"no\"}`\n )\n .join(\",\");\n}\n\nfunction normalizeLiveKitAttachments(value: unknown): SandboxAttachmentData[] {\n if (!Array.isArray(value)) return [];\n const attachments: SandboxAttachmentData[] = [];\n for (const item of value) {\n if (!item || typeof item !== \"object\") continue;\n const record = item as Record;\n const storageId = typeof record.storageId === \"string\" ? record.storageId : \"\";\n const name = typeof record.name === \"string\" ? record.name : \"\";\n const contentType = typeof record.contentType === \"string\" && record.contentType\n ? record.contentType\n : \"application/octet-stream\";\n const size = typeof record.size === \"number\" ? record.size : 0;\n const kind = record.kind === \"image\" ? \"image\" : \"file\";\n const url = typeof record.url === \"string\" && record.url ? record.url : null;\n const dataUrl = typeof record.dataUrl === \"string\" && record.dataUrl.startsWith(\"data:image/\")\n ? record.dataUrl\n : undefined;\n if (!storageId || !name) continue;\n attachments.push({ storageId, url, name, contentType, size, kind, ...(dataUrl ? { dataUrl } : {}) });\n }\n return attachments;\n}\n\nfunction parseCommandRpcPayload(payload: string): {\n sessionId: string;\n message: ClientMachineMessage;\n} {\n const parsed = JSON.parse(payload) as {\n sessionId?: unknown;\n message?: unknown;\n };\n if (typeof parsed.sessionId !== \"string\") {\n throw new Error(\"Command RPC payload is missing sessionId\");\n }\n if (!parsed.message || typeof parsed.message !== \"object\") {\n throw new Error(\"Command RPC payload is missing message\");\n }\n const message = parsed.message as Partial;\n if (\n message.type !== \"action\" ||\n message.kind !== \"request\" ||\n message.action !== \"command\" ||\n typeof message.name !== \"string\"\n ) {\n throw new Error(\"Command RPC message must be a command\");\n }\n return {\n sessionId: parsed.sessionId,\n message: parsed.message as ClientMachineMessage,\n };\n}\n\nif (import.meta.main) {\n cli.runApp(\n new WorkerOptions({\n agent: fileURLToPath(import.meta.url),\n agentName: AGENT_NAME,\n }),\n );\n}\n","/apps/sandbox-agent/src/attachments.ts":"import {\n dataContent,\n imageContent,\n textContent,\n type ContentPart,\n} from \"@projectors/core\";\n\nexport type SandboxAttachmentKind = \"image\" | \"file\";\n\nexport type SandboxAttachmentData = {\n storageId: string;\n name: string;\n contentType: string;\n size: number;\n kind: SandboxAttachmentKind;\n url: string | null;\n dataUrl?: string;\n};\n\nexport type StoredSandboxAttachmentData = Omit;\n\nexport function attachmentSummary(attachments: readonly Pick[]): string {\n return `Attached: ${attachments.map((attachment) => attachment.name).join(\", \")}`;\n}\n\nexport function userContentPartsForAttachments(\n content: string,\n attachments: readonly SandboxAttachmentData[],\n): ContentPart[] {\n const parts: ContentPart[] = [];\n const trimmed = content.trim();\n if (trimmed) parts.push(textContent(trimmed));\n for (const attachment of attachments) {\n const { dataUrl: _dataUrl, ...attachmentMetadata } = attachment;\n parts.push(dataContent(attachmentMetadata, { label: \"Attachment\" }));\n const imageData = attachment.kind === \"image\" ? attachment.dataUrl ?? attachment.url : null;\n if (imageData) {\n parts.push(imageContent(imageData, {\n mediaType: attachment.contentType,\n label: attachment.name,\n }));\n }\n }\n return parts;\n}\n\nexport function storedAttachmentsFromContentParts(\n parts: readonly ContentPart[] | undefined,\n): StoredSandboxAttachmentData[] {\n if (!parts?.length) return [];\n const attachments: StoredSandboxAttachmentData[] = [];\n for (const part of parts) {\n if (part.type !== \"data\" || part.label !== \"Attachment\") continue;\n const attachment = normalizeAttachmentData(part.data);\n if (!attachment) continue;\n attachments.push({\n storageId: attachment.storageId,\n name: attachment.name,\n contentType: attachment.contentType,\n size: attachment.size,\n kind: attachment.kind,\n });\n }\n return attachments;\n}\n\nexport function normalizeAttachmentData(value: unknown): SandboxAttachmentData | undefined {\n if (!value || typeof value !== \"object\") return undefined;\n const record = value as Record;\n if (\n typeof record.storageId !== \"string\" ||\n typeof record.name !== \"string\" ||\n typeof record.contentType !== \"string\" ||\n typeof record.size !== \"number\"\n ) {\n return undefined;\n }\n return {\n storageId: record.storageId,\n name: record.name,\n contentType: record.contentType,\n size: record.size,\n kind: record.kind === \"image\" ? \"image\" : \"file\",\n url: typeof record.url === \"string\" && record.url ? record.url : null,\n ...(typeof record.dataUrl === \"string\" && record.dataUrl.startsWith(\"data:image/\")\n ? { dataUrl: record.dataUrl }\n : {}),\n };\n}\n","/apps/sandbox-agent/src/convexJson.ts":"const ESCAPE_PREFIX = \"__projector_convex_escaped_key__\";\nconst DOLLAR_PREFIX = `${ESCAPE_PREFIX}dollar__`;\n\nexport function escapeConvexJson(value: T): T {\n if (Array.isArray(value)) {\n return value.map((item) => escapeConvexJson(item)) as T;\n }\n\n if (!value || typeof value !== \"object\") {\n return value;\n }\n\n return Object.fromEntries(\n Object.entries(value).map(([key, entryValue]) => [\n key.startsWith(\"$\") ? `${DOLLAR_PREFIX}${key.slice(1)}` : key,\n escapeConvexJson(entryValue),\n ]),\n ) as T;\n}\n","/apps/sandbox-agent/src/projector-sandbox.ts":"import { z } from \"zod\";\nimport {\n ROOT_GENERATOR_ID,\n actionResult,\n appendState,\n createAction,\n createCharter,\n createComputedPart,\n createHistoryProjectionFunction,\n createLayout,\n createNode,\n createRoot,\n createSlot,\n createSourceInstance,\n createState,\n hydrateInstance,\n inspectCompiledProjectionTree,\n imageContent,\n messages,\n messagesBeforeLastCompletion,\n messagesSinceLastCompletion,\n patchState,\n recencyRegion,\n resolveStates,\n serializeInstance,\n textAssistantMessage,\n textContent,\n type CompiledProjectionTree,\n type Charter,\n type GeneratorId,\n type Instance,\n type SerializedInstance,\n} from \"@projectors/core\";\nimport {\n createMachineClientSnapshot,\n realizeClientInstances,\n type MachineClientSnapshot,\n type MachineSyncState,\n} from \"@projectors/core/client\";\n\nconst sandboxStateSchema = z.object({\n themeHue: z.number(),\n turns: z.number(),\n});\n\nexport type SandboxState = z.infer;\n\nconst agentControlsStateSchema = z.object({\n liveMode: z.boolean(),\n cameraEnabled: z.boolean(),\n memoryEnabled: z.boolean(),\n streamingEnabled: z.boolean(),\n testCounter: z.number(),\n});\n\nexport type AgentControlsState = z.infer;\n\nconst memorySchema = z.object({\n text: z.string().min(1).max(500),\n});\n\nconst memoriesStateSchema = z.array(memorySchema);\n\nexport type Memory = z.infer;\nexport type MemoriesState = z.infer;\n\nconst sandboxParamsSchema = z.object({\n sessionId: z.string(),\n});\n\ntype SandboxParamsSchema = typeof sandboxParamsSchema;\n\nexport type SandboxMessage = {\n role: \"user\" | \"assistant\";\n content: string;\n createdAt: number;\n frameId: string;\n mode?: \"text\" | \"voice\";\n streamState?: \"streaming\" | \"complete\" | \"error\";\n streamSeq?: number;\n};\n\nexport type SandboxClientSnapshot = MachineClientSnapshot & {\n projectionTree: CompiledProjectionTree;\n};\n\nexport type CameraSensorImage = {\n dataUrl: string;\n mimeType: \"image/jpeg\";\n capturedAt: number;\n width: number;\n height: number;\n participantIdentity: string;\n trackSid: string;\n};\n\nexport type CameraSensorDataSource = {\n latestImage(): CameraSensorImage | undefined;\n};\n\nlet cameraSensorDataSource: CameraSensorDataSource | undefined;\n\nconst sandboxState = createState({\n key: \"sandbox\",\n schema: sandboxStateSchema,\n init: {\n themeHue: 126,\n turns: 0,\n } satisfies SandboxState,\n projection: { slot: recencyRegion },\n});\n\nconst agentControlsState = createState({\n key: \"agentControls\",\n schema: agentControlsStateSchema,\n init: {\n liveMode: false,\n cameraEnabled: false,\n memoryEnabled: false,\n streamingEnabled: true,\n testCounter: 0,\n } satisfies AgentControlsState,\n projection: { slot: recencyRegion },\n});\n\nconst memoriesState = createState({\n key: \"memories\",\n schema: memoriesStateSchema,\n init: [] satisfies MemoriesState,\n projection: { slot: recencyRegion },\n});\n\nexport const setVoiceEnabled = createAction({\n state: agentControlsState,\n name: \"setVoiceEnabled\",\n description: \"Toggle voice mode for the sandbox session.\",\n inputSchema: z.object({ enabled: z.boolean() }),\n run: ({ enabled }, ctx) => {\n ctx.updateState?.(patchState({ liveMode: enabled }));\n },\n});\n\nexport const enableLiveMode = createAction({\n state: agentControlsState,\n name: \"enableLiveMode\",\n description: \"Enable live voice mode for the sandbox session.\",\n inputSchema: z.object({}),\n run: (_input, ctx) => {\n const state = agentControlsStateSchema.parse(ctx.state);\n if (!state.liveMode) {\n ctx.updateState?.(patchState({ liveMode: true }));\n }\n return state.liveMode ? \"Live mode is already enabled.\" : \"Live mode enabled.\";\n },\n});\n\nexport const cameraSnapshotPart = createComputedPart({\n name: \"cameraSnapshot\",\n slot: recencyRegion,\n compute: () => {\n const image = cameraSensorDataSource?.latestImage();\n if (!image) {\n return \"Camera sensor is enabled, but no camera snapshot has been sampled yet.\";\n }\n return [\n textContent(renderCameraSensorImage(image)),\n imageContent(image.dataUrl, {\n mediaType: image.mimeType,\n label: \"latest camera sensor snapshot\",\n }),\n ];\n },\n});\n\nexport const cameraSensorNode = createNode({\n key: \"cameraSensor\",\n name: \"CameraSensorNode\",\n instructions:\n \"The user's camera is enabled. Use the latest camera snapshot only when relevant. When asked about the camera, answer directly from the currently available snapshot. Do not say you will check or look later; if no usable snapshot is available, say you cannot see the current camera view.\",\n parts: [cameraSnapshotPart],\n});\n\nexport const setCameraEnabled = createAction({\n state: agentControlsState,\n name: \"setCameraEnabled\",\n description: \"Toggle camera sampling in live mode.\",\n inputSchema: z.object({ enabled: z.boolean() }),\n run: ({ enabled }, ctx) => {\n const state = agentControlsStateSchema.parse(ctx.state);\n if (state.cameraEnabled !== enabled) {\n ctx.updateState?.(patchState({ cameraEnabled: enabled }));\n }\n if (enabled) {\n ctx.instance.cede(cameraSensorNode);\n ctx.instance.spawn(cameraSensorNode);\n } else {\n ctx.instance.cede(cameraSensorNode);\n }\n },\n});\n\nexport const enableCamera = createAction({\n state: agentControlsState,\n name: \"enableCamera\",\n description: \"Enable camera sampling for the sandbox session.\",\n inputSchema: z.object({}),\n run: (_input, ctx) => {\n const state = agentControlsStateSchema.parse(ctx.state);\n if (!state.cameraEnabled) {\n ctx.updateState?.(patchState({ cameraEnabled: true }));\n }\n ctx.instance.cede(cameraSensorNode);\n ctx.instance.spawn(cameraSensorNode);\n return state.cameraEnabled ? \"Camera is already enabled.\" : \"Camera enabled.\";\n },\n});\n\nexport const setMemoryEnabled = createAction({\n state: agentControlsState,\n name: \"setMemoryEnabled\",\n description: \"Toggle durable memory extraction for the sandbox session.\",\n inputSchema: z.object({ enabled: z.boolean() }),\n run: ({ enabled }, ctx) => {\n const state = agentControlsStateSchema.parse(ctx.state);\n if (state.memoryEnabled !== enabled) {\n ctx.updateState?.(patchState({ memoryEnabled: enabled }));\n }\n if (enabled) {\n ctx.instance.cede(memoryMemberNode);\n ctx.instance.spawn(memoryMemberNode);\n } else {\n ctx.instance.cede(memoryMemberNode);\n }\n },\n});\n\nexport const setStreamingEnabled = createAction({\n state: agentControlsState,\n name: \"setStreamingEnabled\",\n description: \"Toggle streaming-style assistant output.\",\n inputSchema: z.object({ enabled: z.boolean() }),\n run: ({ enabled }, ctx) => {\n ctx.updateState?.(patchState({ streamingEnabled: enabled }));\n },\n});\n\nexport const incrementTestCounter = createAction({\n state: agentControlsState,\n name: \"incrementTestCounter\",\n description: \"Increment the agent controls test counter.\",\n inputSchema: z.object({ amount: z.number().default(1) }),\n run: async ({ amount }, ctx) => {\n await delay(3000);\n ctx.updateState?.((current) => {\n const state = agentControlsStateSchema.parse(current);\n return patchState({ testCounter: state.testCounter + amount });\n });\n },\n});\n\nexport const setThemeHue = createAction({\n state: sandboxState,\n name: \"setThemeHue\",\n description: \"Change the terminal accent hue.\",\n inputSchema: z.object({ hue: z.number().min(0).max(360) }),\n run: ({ hue }, ctx) => {\n ctx.updateState?.(patchState({ themeHue: hue }));\n },\n});\n\nexport const pingTool = createAction({\n state: null,\n name: \"ping\",\n description: \"Respond with pong.\",\n inputSchema: z.object({}),\n run: () => pongAssistantMessage(),\n});\n\nexport const pingCommand = createAction({\n state: null,\n name: \"ping\",\n description: \"Respond with pong.\",\n inputSchema: z.object({}),\n run: (): unknown => {\n const message = pongAssistantMessage();\n return actionResult({ value: message, messages: [message] });\n },\n});\n\nexport const echoSessionIdTool = createAction({\n state: null,\n params: sandboxParamsSchema,\n name: \"echoSessionId\",\n description: \"Echo the current sandbox session id.\",\n inputSchema: z.object({}),\n run: (_input, ctx) => sessionIdAssistantMessage(ctx.params.sessionId),\n});\n\nexport const echoSessionIdCommand = createAction({\n state: null,\n params: sandboxParamsSchema,\n name: \"echoSessionId\",\n description: \"Echo the current sandbox session id.\",\n inputSchema: z.object({}),\n run: (_input, ctx): unknown => {\n const message = sessionIdAssistantMessage(ctx.params.sessionId);\n return actionResult({ value: message, messages: [message] });\n },\n});\n\nexport const saveMemories = createAction({\n state: memoriesState,\n name: \"saveMemories\",\n description:\n \"Save durable user memories extracted from recent conversation messages.\",\n inputSchema: z.object({\n memories: z.array(memorySchema).max(10),\n }),\n run: ({ memories }, ctx) => {\n const existing = memoriesStateSchema.parse(ctx.state);\n const additions = newMemories(existing, memories);\n const [firstAddition, ...remainingAdditions] = additions;\n if (firstAddition) {\n ctx.updateState?.(appendState(firstAddition, ...remainingAdditions));\n }\n return additions.length === 0\n ? \"No new memories saved.\"\n : `Saved ${additions.length} memories.`;\n },\n});\n\nexport const memoryHistoryProjection = createHistoryProjectionFunction({\n name: \"memory\",\n method: (ctx) => {\n // History rendering is layout-owned with no per-node override, so this\n // single policy dispatches: only the memory generator (the charter's one\n // parent-completion trigger) sees the extraction prompt; every other\n // generator keeps plain message history.\n const triggers = Array.isArray(ctx.trigger) ? ctx.trigger : [ctx.trigger];\n if (!triggers.some((trigger) => trigger.type === \"parent-completion\")) {\n return messages(ctx);\n }\n const previousMessages = messagesBeforeLastCompletion(ctx);\n const newMessages = messagesSinceLastCompletion(ctx);\n const memories = memoriesStateSchema.safeParse(ctx.states.memories).success\n ? (ctx.states.memories as MemoriesState)\n : [];\n const prompt = [\n \"Below is a conversation log between a user and an agent.\",\n \"Analyze the new messages and call saveMemories with any durable new memories.\",\n \"\",\n \"Rules:\",\n \"- Save only stable user facts, preferences, names, and recurring context likely to help in future unrelated conversations.\",\n \"- When uncertain, save nothing.\",\n \"- Ignore one-off tasks, temporary details, and assistant claims.\",\n \"- Do not duplicate current memories.\",\n \"- If there are no new durable memories, do not call any tool.\",\n \"\",\n \"Current memories:\",\n renderMemories(memories),\n \"\",\n \"Conversation history before the last memory update:\",\n renderActorMessages(previousMessages),\n \"\",\n \"New messages since the last memory update:\",\n renderActorMessages(newMessages),\n ].join(\"\\n\");\n\n return [\n {\n type: \"user\",\n content: [textContent(prompt)],\n text: prompt,\n },\n ];\n },\n});\n\n// The sandbox layout mirrors the implicit default layout; it exists to carry the\n// memory history projection, which is layout-owned in the parts model.\nconst sandboxLayout = createLayout({\n name: \"sandbox\",\n default: true,\n historyProjection: memoryHistoryProjection,\n regions: {\n preamble: [createSlot(\"body\", { default: true })],\n recency: [createSlot(\"context\", { default: true, volatile: true })],\n },\n});\n\nexport const memoryMemberNode = createNode({\n key: \"memory\",\n name: \"MemoryNode\",\n instructions:\n \"Maintain durable user memories. Save only concise, stable, generally useful user facts. Do not create a memory for every message.\",\n states: [memoriesState],\n tools: [saveMemories],\n // boundaryProjection defaults to \"hidden\": the memory generator is a\n // private sub-machine and nothing crosses into the parent document (the\n // replacement for the removed replaceProjection).\n runtime: {\n type: \"generator\",\n trigger: { type: \"parent-completion\" },\n concurrency: \"serial\",\n activationHistory: \"snapshot\",\n outputAudienceDefault: \"self\",\n },\n});\n\nexport const agentControlsMemberNode = createNode({\n key: \"agentControls\",\n name: \"Agent Controls\",\n // The old projection hid this node's dev-facing instructions from the\n // model (\"Expose client commands for voice, camera, streaming, memory, and\n // test controls.\"), so it contributes no prose — just state and actions.\n states: [agentControlsState],\n tools: [enableLiveMode, enableCamera],\n commands: [\n setVoiceEnabled,\n setCameraEnabled,\n setMemoryEnabled,\n setStreamingEnabled,\n incrementTestCounter,\n ],\n});\n\nexport const sandboxBaseNode = createNode({\n key: \"sandboxBase\",\n name: \"Projector Sandbox Agent\",\n params: sandboxParamsSchema,\n instructions:\n \"You are a friendly conversation buddy. Be natural, concise, and curious. Do not volunteer internal state, tools, framework details, or state changes during ordinary conversation. If the user asks about your internals, capabilities, memory, tools, state, or how you work, answer directly and transparently with the information available to you.\",\n states: [sandboxState],\n tools: [pingTool, echoSessionIdTool],\n commands: [pingCommand, echoSessionIdCommand, setThemeHue],\n members: [agentControlsMemberNode],\n});\n\nexport function createSandboxCharter(\n options: { cameraSensor?: CameraSensorDataSource } = {},\n): Charter {\n if (\"cameraSensor\" in options) {\n cameraSensorDataSource = options.cameraSensor;\n }\n return createCharter({\n key: \"projector-sandbox\",\n version: \"1\",\n params: sandboxParamsSchema,\n nodes: [\n sandboxBaseNode,\n memoryMemberNode,\n agentControlsMemberNode,\n cameraSensorNode,\n ],\n tools: [pingTool, echoSessionIdTool, saveMemories, enableLiveMode, enableCamera],\n commands: [\n pingCommand,\n echoSessionIdCommand,\n setVoiceEnabled,\n setCameraEnabled,\n setMemoryEnabled,\n setStreamingEnabled,\n incrementTestCounter,\n setThemeHue,\n ],\n states: [sandboxState, memoriesState, agentControlsState],\n layouts: [sandboxLayout],\n computedParts: [cameraSnapshotPart],\n historyProjections: [memoryHistoryProjection],\n });\n}\n\nexport function createInitialSandboxSourceInstance(): Instance {\n const sandboxBase = createSourceInstance({\n id: `sandbox-${crypto.randomUUID()}`,\n node: sandboxBaseNode,\n });\n resolveStates(sandboxBase);\n return sandboxBase;\n}\n\nexport function createSandboxMachineRoot(\n source: Instance,\n params: z.input,\n): Instance {\n reconcileAgentControlMembers(source);\n const root = createRoot(createSandboxCharter(), [source], params);\n resolveStates(root);\n return root;\n}\n\nexport function createInitialSandboxMachineRoot(\n sessionId = `session-${crypto.randomUUID()}`,\n): Instance {\n return createSandboxMachineRoot(createInitialSandboxSourceInstance(), { sessionId });\n}\n\nexport function createInitialSandboxInstance(\n sessionId = `session-${crypto.randomUUID()}`,\n): Instance {\n return createInitialSandboxMachineRoot(sessionId);\n}\n\nexport function createInitialSerializedInstance(): SerializedInstance {\n return serializeSandboxSourceInstance(createInitialSandboxSourceInstance());\n}\n\nexport function hydrateSandboxSourceInstance(serialized: SerializedInstance): Instance {\n const instance = hydrateInstance(serialized, createSandboxCharter());\n resolveStates(instance);\n return instance;\n}\n\nexport function serializeSandboxSourceInstance(instance: Instance): SerializedInstance {\n resolveStates(instance);\n return serializeInstance(instance, createSandboxCharter());\n}\n\nexport function hydrateSandboxInstance(\n serialized: SerializedInstance,\n sessionId: string,\n): Instance {\n return createSandboxMachineRoot(hydrateSandboxSourceInstance(serialized), {\n sessionId,\n });\n}\n\nexport function serializeSandboxInstance(instance: Instance): SerializedInstance {\n const source = findSandboxSourceInstance(instance);\n if (!source) {\n throw new Error(\"Sandbox instance tree has no source instance\");\n }\n return serializeSandboxSourceInstance(source);\n}\n\nexport function getSandboxRootGenerator(_instance: Instance): GeneratorId {\n return ROOT_GENERATOR_ID;\n}\n\nexport function createSandboxClientSnapshot(\n serialized: SerializedInstance,\n sessionId: string,\n syncState: MachineSyncState = { recentCommandResidue: [] },\n): SandboxClientSnapshot {\n const instance = hydrateSandboxInstance(serialized, sessionId);\n return {\n ...createMachineClientSnapshot(realizeClientInstances(instance as Instance), syncState),\n projectionTree: inspectCompiledProjectionTree(instance, {\n charter: createSandboxCharter(),\n }),\n };\n}\n\nfunction findSandboxSourceInstance(instance: Instance): Instance | undefined {\n if (instance.isSource) {\n return instance;\n }\n for (const child of instance.children ?? []) {\n const source = findSandboxSourceInstance(child);\n if (source) {\n return source;\n }\n }\n return undefined;\n}\n\nfunction reconcileAgentControlMembers(instance: Instance): void {\n const controls = agentControlsStateSchema.safeParse(\n readResolvedState(instance, \"agentControls\"),\n );\n if (!controls.success) return;\n\n reconcileOptionalMember(instance, cameraSensorNode, controls.data.cameraEnabled);\n reconcileOptionalMember(instance, memoryMemberNode, controls.data.memoryEnabled);\n resolveStates(instance);\n}\n\nfunction reconcileOptionalMember(\n instance: Instance,\n node: typeof cameraSensorNode | typeof memoryMemberNode,\n enabled: boolean,\n): void {\n const existingChildren = instance.children ?? [];\n const matchingChild = existingChildren.find((child) => child.node.key === node.key);\n const children = existingChildren.filter((child) => child.node.key !== node.key);\n if (enabled) {\n children.push(matchingChild ?? { id: `${node.key}-${crypto.randomUUID()}`, node });\n }\n instance.children = children.length > 0 ? children : undefined;\n}\n\nexport function getSandboxState(instance: Instance): SandboxState {\n return sandboxStateSchema.parse(readResolvedState(instance, \"sandbox\"));\n}\n\nexport function setSandboxState(instance: Instance, next: SandboxState): void {\n sandboxStateSchema.parse(next);\n writeResolvedState(instance, \"sandbox\", next);\n}\n\nexport function getAgentControlsState(instance: Instance): AgentControlsState {\n return agentControlsStateSchema.parse(\n readResolvedState(instance, \"agentControls\"),\n );\n}\n\nexport function setAgentControlsState(\n instance: Instance,\n next: AgentControlsState,\n): void {\n agentControlsStateSchema.parse(next);\n writeResolvedState(instance, \"agentControls\", next);\n}\n\nexport function getMemoriesState(instance: Instance): MemoriesState {\n return memoriesStateSchema.parse(readResolvedState(instance, \"memories\"));\n}\n\nexport function setMemoriesState(\n instance: Instance,\n next: MemoriesState,\n): void {\n memoriesStateSchema.parse(next);\n writeResolvedState(instance, \"memories\", next);\n}\n\nfunction readResolvedState(instance: Instance, key: string): unknown {\n const state = resolveStates(instance).find(\n (candidate) => candidate.address.stateKey === key,\n );\n if (!state) {\n throw new Error(`Unknown sandbox state \"${key}\"`);\n }\n return state.container.value;\n}\n\nfunction writeResolvedState(\n instance: Instance,\n key: string,\n value: unknown,\n): void {\n const state = resolveStates(instance).find(\n (candidate) => candidate.address.stateKey === key,\n );\n if (!state) {\n throw new Error(`Unknown sandbox state \"${key}\"`);\n }\n state.container.value = value;\n}\n\nfunction pongAssistantMessage() {\n return textAssistantMessage(\"pong\");\n}\n\nfunction sessionIdAssistantMessage(sessionId: string) {\n return textAssistantMessage(sessionId);\n}\n\nfunction newMemories(\n existing: MemoriesState,\n next: MemoriesState,\n): MemoriesState {\n const seen = new Set(\n existing.map((memory) => normalizeMemoryText(memory.text)),\n );\n const additions: MemoriesState = [];\n for (const memory of next) {\n const text = memory.text.trim();\n const normalized = normalizeMemoryText(text);\n if (!text || seen.has(normalized)) {\n continue;\n }\n seen.add(normalized);\n additions.push({ text });\n }\n return additions;\n}\n\nfunction normalizeMemoryText(text: string): string {\n return text\n .trim()\n .replace(/\\s+/g, \" \")\n .replace(/[.!?]+$/g, \"\")\n .toLowerCase();\n}\n\nfunction renderMemories(memories: MemoriesState): string {\n if (memories.length === 0) {\n return \"(none)\";\n }\n return memories\n .map((memory, index) => `${index + 1}. ${memory.text}`)\n .join(\"\\n\");\n}\n\nfunction renderCameraSensorImage(image: CameraSensorImage): string {\n return [\n \"Latest camera sensor snapshot:\",\n `- capturedAt: ${new Date(image.capturedAt).toISOString()}`,\n `- mimeType: ${image.mimeType}`,\n `- dimensions: ${image.width}x${image.height}`,\n `- participant: ${image.participantIdentity}`,\n `- trackSid: ${image.trackSid}`,\n \"- image: attached as native multimodal content\",\n ].join(\"\\n\");\n}\n\nfunction renderActorMessages(\n messages: Array & {\n type: string;\n text?: string;\n name?: string;\n value?: unknown;\n }>,\n): string {\n if (messages.length === 0) {\n return \"(none)\";\n }\n return messages\n .map((message) => {\n if (message.type === \"user\") return `User: ${message.text ?? \"\"}`;\n if (message.type === \"assistant\")\n return `Assistant: ${message.text ?? \"\"}`;\n if (message.type === \"action\")\n return `Action ${message.action ?? \"unknown\"} ${message.kind ?? \"message\"} ${message.name ?? \"unknown\"}: ${stringifyValue(message.value ?? message.error ?? message.input)}`;\n return `${message.type}: ${stringifyValue(message)}`;\n })\n .join(\"\\n\");\n}\n\nfunction stringifyValue(value: unknown): string {\n if (value === undefined) return \"\";\n if (typeof value === \"string\") return value;\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nfunction delay(ms: number): Promise {\n return new Promise((resolve) => setTimeout(resolve, ms));\n}\n","/apps/sandbox-agent/src/vision.ts":"import {\n RoomEvent,\n TrackKind,\n TrackSource,\n VideoBufferType,\n VideoStream,\n type Participant,\n type RemoteParticipant,\n type RemoteTrack,\n type RemoteTrackPublication,\n type Room,\n type TrackPublication,\n type VideoFrame,\n type VideoFrameEvent,\n} from \"@livekit/rtc-node\";\nimport sharp from \"sharp\";\nimport type { CameraSensorImage } from \"./projector-sandbox.js\";\n\nexport type CameraSamplerHandle = {\n stop(): Promise;\n};\n\nexport function attachCameraSampler({\n room,\n onImage,\n config,\n}: {\n room: Room;\n onImage: (image: CameraSensorImage | undefined) => void;\n config?: {\n fps?: number;\n maxDimension?: number;\n jpegQuality?: number;\n };\n}): CameraSamplerHandle {\n const fps = config?.fps ?? 1;\n const maxDimension = config?.maxDimension ?? 1024;\n const jpegQuality = config?.jpegQuality ?? 92;\n\n let activeTrackSid: string | null = null;\n let activeParticipantIdentity: string | null = null;\n let frameReader: ReadableStreamDefaultReader | null = null;\n let latestFrame: VideoFrame | null = null;\n let sampledFirstFrameForTrack = false;\n let samplingTimer: ReturnType | null = null;\n let sampleInProgress = false;\n let stopped = false;\n\n const stopTimer = () => {\n if (!samplingTimer) return;\n clearInterval(samplingTimer);\n samplingTimer = null;\n };\n\n const startTimer = () => {\n stopTimer();\n const intervalMs = Math.max(1, Math.round(1000 / Math.max(0.001, fps)));\n samplingTimer = setInterval(() => {\n void sampleLatestFrame();\n }, intervalMs);\n };\n\n const stopSampling = async () => {\n stopTimer();\n latestFrame = null;\n sampledFirstFrameForTrack = false;\n activeTrackSid = null;\n activeParticipantIdentity = null;\n onImage(undefined);\n\n if (!frameReader) return;\n try {\n await frameReader.cancel();\n } catch {\n // ignore cancellation races\n }\n try {\n frameReader.releaseLock();\n } catch {\n // ignore already-released readers\n }\n frameReader = null;\n };\n\n const sampleLatestFrame = async () => {\n if (stopped || sampleInProgress || !latestFrame || !activeTrackSid || !activeParticipantIdentity) {\n return;\n }\n\n sampleInProgress = true;\n try {\n const image = await encodeFrameAsJpegDataUrl(latestFrame, {\n maxDimension,\n jpegQuality,\n });\n onImage({\n ...image,\n capturedAt: Date.now(),\n participantIdentity: activeParticipantIdentity,\n trackSid: activeTrackSid,\n });\n } catch (error) {\n console.warn(\"[camera-sampler] Failed to sample camera frame\", error);\n } finally {\n sampleInProgress = false;\n }\n };\n\n const startSampling = async (\n track: RemoteTrack,\n publication: RemoteTrackPublication,\n participant: RemoteParticipant,\n ) => {\n if (stopped) return;\n if (!isUserParticipant(participant)) return;\n if (!isCameraVideoPublication(publication)) return;\n if (!publication.sid) return;\n if (activeTrackSid === publication.sid) return;\n\n await stopSampling();\n activeTrackSid = publication.sid;\n activeParticipantIdentity = participant.identity;\n sampledFirstFrameForTrack = false;\n\n try {\n const stream = new VideoStream(track);\n frameReader = stream.getReader();\n void readFrames(publication.sid, frameReader);\n startTimer();\n } catch (error) {\n console.warn(\"[camera-sampler] Failed to start camera sampling\", error);\n await stopSampling();\n }\n };\n\n const readFrames = async (\n trackSid: string,\n reader: ReadableStreamDefaultReader,\n ) => {\n try {\n while (!stopped && frameReader === reader && activeTrackSid === trackSid) {\n const { done, value } = await reader.read();\n if (done) break;\n if (value?.frame) {\n latestFrame = value.frame;\n if (!sampledFirstFrameForTrack) {\n sampledFirstFrameForTrack = true;\n void sampleLatestFrame();\n }\n }\n }\n } catch (error) {\n if (!stopped) {\n console.warn(\"[camera-sampler] Frame reader stopped\", error);\n }\n }\n };\n\n const onTrackSubscribed = (\n track: RemoteTrack,\n publication: RemoteTrackPublication,\n participant: RemoteParticipant,\n ) => {\n if (track.kind !== TrackKind.KIND_VIDEO) return;\n void startSampling(track, publication, participant);\n };\n\n const subscribeToCameraPublication = (\n publication: TrackPublication,\n participant: Participant,\n ) => {\n if (!isUserParticipant(participant)) return;\n if (!isCameraVideoPublication(publication)) return;\n\n const remotePublication = publication as RemoteTrackPublication;\n if (!remotePublication.subscribed) {\n remotePublication.setSubscribed(true);\n }\n\n const track = remotePublication.track as RemoteTrack | undefined;\n if (track) {\n void startSampling(track, remotePublication, participant as RemoteParticipant);\n }\n };\n\n const onTrackPublished = (\n publication: RemoteTrackPublication,\n participant: RemoteParticipant,\n ) => {\n subscribeToCameraPublication(publication, participant);\n };\n\n const onTrackUnsubscribed = (\n _track: RemoteTrack,\n publication: RemoteTrackPublication,\n participant: RemoteParticipant,\n ) => {\n if (\n publication.sid === activeTrackSid &&\n participant.identity === activeParticipantIdentity\n ) {\n void stopSampling();\n }\n };\n\n const onTrackMuted = (publication: TrackPublication, participant: Participant) => {\n if (\n publication.sid === activeTrackSid &&\n participant.identity === activeParticipantIdentity\n ) {\n void stopSampling();\n }\n };\n\n const onTrackUnmuted = (publication: TrackPublication, participant: Participant) => {\n if (!isUserParticipant(participant)) return;\n if (!isCameraVideoPublication(publication)) return;\n if (publication.sid && publication.sid === activeTrackSid) return;\n\n const track = publication.track as RemoteTrack | undefined;\n if (!track) {\n subscribeToCameraPublication(publication, participant);\n return;\n }\n void startSampling(\n track,\n publication as RemoteTrackPublication,\n participant as RemoteParticipant,\n );\n };\n\n room.on(RoomEvent.TrackSubscribed, onTrackSubscribed);\n room.on(RoomEvent.TrackPublished, onTrackPublished);\n room.on(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);\n room.on(RoomEvent.TrackMuted, onTrackMuted);\n room.on(RoomEvent.TrackUnmuted, onTrackUnmuted);\n\n for (const participant of room.remoteParticipants.values()) {\n if (!isUserParticipant(participant)) continue;\n for (const publication of participant.trackPublications.values()) {\n if (!isCameraVideoPublication(publication)) continue;\n subscribeToCameraPublication(publication, participant);\n break;\n }\n }\n\n return {\n stop: async () => {\n if (stopped) return;\n stopped = true;\n room.off(RoomEvent.TrackSubscribed, onTrackSubscribed);\n room.off(RoomEvent.TrackPublished, onTrackPublished);\n room.off(RoomEvent.TrackUnsubscribed, onTrackUnsubscribed);\n room.off(RoomEvent.TrackMuted, onTrackMuted);\n room.off(RoomEvent.TrackUnmuted, onTrackUnmuted);\n await stopSampling();\n },\n };\n}\n\nfunction isUserParticipant(participant: { identity: string }): boolean {\n return participant.identity.startsWith(\"user-\");\n}\n\nfunction isCameraVideoPublication(publication: TrackPublication): boolean {\n return (\n publication.kind === TrackKind.KIND_VIDEO &&\n publication.source === TrackSource.SOURCE_CAMERA\n );\n}\n\nasync function encodeFrameAsJpegDataUrl(\n frame: VideoFrame,\n options: { maxDimension: number; jpegQuality: number },\n): Promise> {\n const rgba = frame.type === VideoBufferType.RGBA\n ? frame\n : frame.convert(VideoBufferType.RGBA);\n const scale = Math.min(\n 1,\n options.maxDimension / Math.max(rgba.width, rgba.height),\n );\n const width = Math.max(1, Math.round(rgba.width * scale));\n const height = Math.max(1, Math.round(rgba.height * scale));\n\n let pipeline = sharp(rgba.data, {\n raw: { width: rgba.width, height: rgba.height, channels: 4 },\n });\n if (scale < 1) {\n pipeline = pipeline.resize(width, height, { fit: \"inside\" });\n }\n\n const jpeg = await pipeline.jpeg({ quality: options.jpegQuality }).toBuffer();\n return {\n dataUrl: `data:image/jpeg;base64,${jpeg.toString(\"base64\")}`,\n mimeType: \"image/jpeg\",\n width,\n height,\n };\n}\n","/apps/sandbox-agent/tsconfig.json":"{\n \"compilerOptions\": {\n \"target\": \"ES2022\",\n \"module\": \"NodeNext\",\n \"moduleResolution\": \"NodeNext\",\n \"allowImportingTsExtensions\": true,\n \"verbatimModuleSyntax\": true,\n \"noEmit\": true,\n \"esModuleInterop\": true,\n \"strict\": true,\n \"skipLibCheck\": true,\n \"outDir\": \"dist\",\n \"rootDir\": \"src\",\n \"declaration\": true\n },\n \"include\": [\"src/**/*\"],\n \"exclude\": [\"node_modules\", \"dist\"]\n}\n","/apps/sandbox/.gitignore":"# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.\n\n# dependencies\n/node_modules\n/.pnp\n.pnp.*\n.yarn/*\n!.yarn/patches\n!.yarn/plugins\n!.yarn/releases\n!.yarn/versions\n\n# testing\n/coverage\n\n# next.js\n/.next/\n/out/\n\n# production\n/build\n\n# misc\n.DS_Store\n*.pem\n\n# debug\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\n.pnpm-debug.log*\n\n# env files (can opt-in for committing if needed)\n.env*\n\n# vercel\n.vercel\n\n# typescript\n*.tsbuildinfo\nnext-env.d.ts\n","/apps/sandbox/README.md":"# sandbox\n\nProjector sandbox app with text chat, projected state/commands, and LiveKit voice mode.\n\n## Setup\n\nInstall workspace dependencies from the repo root:\n\n```bash\nbun install\n```\n\nCreate `apps/sandbox/.env.local` with Convex and LiveKit values:\n\n```bash\nNEXT_PUBLIC_CONVEX_URL=https://your-deployment.convex.cloud\nCONVEX_DEPLOYMENT=your-convex-deployment\nLIVEKIT_API_KEY=your-livekit-api-key\nLIVEKIT_API_SECRET=your-livekit-api-secret\nLIVEKIT_URL=wss://your-project.livekit.cloud\n```\n\nSet the same LiveKit values in the Convex dashboard environment variables because `convex/livekitAgentActions.ts` mints room tokens and dispatches the agent from Convex.\n\n## Run\n\nStart Convex:\n\n```bash\ncd apps/sandbox\nnpx convex dev\n```\n\nStart the Next app in another terminal:\n\n```bash\ncd apps/sandbox\nbun run dev\n```\n\nOpen the printed Next URL, usually `http://localhost:3000`.\n\nVoice mode also requires the worker from `apps/sandbox-agent` to be running. See `apps/sandbox-agent/README.md`.\n","/apps/sandbox/SANDBOX_APP_LEARNINGS.md":"# Sandbox App Learnings\n\nMessy append-only notes extracted from recent sessions around `apps/sandbox`,\n`apps/sandbox-agent`, LiveKit, streaming, voice, and durable projector state.\n\nLast reviewed: 2026-06-13.\n\n## 2026-06-12/13 Session Harvest\n\nPrimary sources were recent Codex session logs under:\n\n- `/Users/zack/.codex/sessions/2026/06/12`\n- `/Users/zack/.codex/sessions/2026/06/13`\n\nThe highest-signal sessions covered:\n\n- Convex `$schema` failures during sandbox session creation.\n- Sandbox DB refactor from turns/steps to frames.\n- AI SDK executor and sandbox-agent integration.\n- LiveKit data-channel text transport.\n- Worker leases and duplicate LiveKit agent handling.\n- `liveMode` stale-state regressions.\n- Streaming slowness from durable writes per delta.\n- Voice restart failures caused by LiveKit `RoomIO` participant binding ordering.\n- Realtime VAD/noise-reduction tuning.\n- Session-scoped projector client state via `ProjectorProvider`.\n\n## Durable Projector State\n\nThe sandbox should treat Convex as the durable source of truth. LiveKit rooms,\nOpenAI realtime sessions, browser connections, and executor sockets are runtime\ntransport details. They can reconnect or restart, but the semantic state should\ncome from the Convex frame/session log.\n\nThe current durable model is frame-based:\n\n- `sessions.contextEpoch`\n- `sessions.familyRootSessionId`\n- `sessions.forkedFromSessionId`\n- `sessions.forkedFromFrameId`\n- `frames`\n- `frameIndex`\n- `projectorInstanceLog`\n- `messages`\n- `messageIndex`\n\nThe old `machineTurns` / `machineSteps` vocabulary was removed from the sandbox.\nMessages now point at frames. Branching is modeled as session cloning, while\nmachine-visible history is bounded by `contextEpoch`.\n\nImportant implementation split:\n\n- `frames` records canonical actor/executor frames with a best-effort `referenceFrameId`.\n- `frameIndex` records which frames are visible in each session and context epoch.\n- `messageIndex` records which messages are visible in each session.\n- `projectorInstanceLog` records durable source instance snapshots after frame-contained instance messages.\n- `sessions.get` reconstructs the current instance by selecting the latest persisted source instance snapshot.\n\nThis matters because not every UI-visible message or stream patch should become\na projector frame. Durable projector frames should represent meaningful machine\nstate transitions, not every token/delta.\n\n## Convex `$schema` Boundary\n\nConvex rejects field names beginning with `$`. This applies to documents,\nmutation/action arguments, and function return values. It is not enough to\nescape inside a mutation handler if the raw value is passed as a mutation arg,\nbecause Convex validates args before the handler runs.\n\nLearned fixes:\n\n- Escape JSON-schema-bearing values before crossing any Convex action-to-mutation or external-client mutation boundary.\n- Restore escaped values for server-side logic before running projector behavior.\n- Strip schema metadata from browser-facing query results when the UI does not need it.\n- Do not return full schema-bearing snapshots from actions.\n\nCurrent helper lives in `apps/sandbox/convex/convexJson.ts`.\n\nValues that commonly need care:\n\n- serialized projector `instance`\n- `clientSnapshot`\n- `displayInstance`\n- frame `metadata`\n- frame `messages`\n- `syncState`\n- projected command/state schemas with `$schema` or `inputSchema`\n\nDurable serialized instances should ideally be refs to registered nodes, for\nexample `node:sandboxRoot`, so they do not carry full JSON Schema by accident.\nClient snapshots can still contain schemas for UI metadata, so the Convex\nboundary still needs escaping/stripping.\n\n## Avoid Stale Whole-Instance Writes\n\nOne failure mode: enabling `liveMode`, then sending a message, reverted\n`liveMode` back to false.\n\nRoot cause: a send path processed against a stale action/client snapshot and\nwrote a whole new projector session state derived from the old instance.\n\nThe fix was to remove broad \"write this whole instance\" paths and route message\nor client-command updates through Convex mutations that read the latest head\nframe at mutation time:\n\n- `api.sessions.applyUserMessage`\n- `api.sessions.applyClientMessage`\n\nThose mutations:\n\n- load the latest session head state from Convex\n- apply the user message or client command\n- append the resulting instance log / frame\n\nRemoved or avoided APIs:\n\n- `updateProjectorSession`\n- `updateInstance`\n- `editCurrentInstance`\n- `finalizeFrame`\n\nRule of thumb: message sends should apply to the latest durable head frame, not\nto whatever snapshot the browser, action, or worker happened to have when the\nrequest started.\n\n## LiveKit Text Transport\n\nLiveKit RPC was not reliable for this use case. It failed with `Method not\nsupported at destination`.\n\nThe working path is LiveKit data-channel messages:\n\n- Browser publishes reliable data on topic `sandbox.message.v1`.\n- Payload is JSON like `{ content, sentAt }`.\n- `apps/sandbox-agent` listens for `RoomEvent.DataReceived`.\n- Agent validates topic/payload and processes text through the same projector\n state and executor flow as other text sends.\n\nBrowser side details:\n\n- Keep a sender function only when the room is connected and the worker is ready.\n- When storing a sender function in React state, wrap the setter:\n `setSendLiveKitMessage(sender ? () => sender : null)`.\n- Passing a function directly to a React setter causes React to call it as an updater. This produced empty `{ sentAt }` packets and hid the real sender.\n\nAgent side details:\n\n- Log entry start, room connection, loaded session, data topic/size, and processing start.\n- Accept topicless packets only if the payload is otherwise valid, but ignore wrong non-empty topics.\n- `LiveKit AgentSession` should use `inputOptions.closeOnDisconnect = false` so a browser reconnect does not close the whole agent session.\n\n## LiveKit Text Is Not Automatically Voice\n\nLiveKit transport and voice/realtime mode are separate toggles.\n\nImportant behavior:\n\n- LiveKit data-channel text can be used while `liveMode` is false.\n- If `liveMode` is false, the agent should not create/use realtime voice behavior for the reply.\n- Text should run through the discrete AI SDK executor.\n- `LiveKitRealtimeExecutor` realtime delegation is gated by both `ENABLE_REALTIME_MODEL` and projected `agentControls.liveMode`.\n\nThe current `handleLiveKitTextMessage` path sends text through:\n\n- `liveKitExecutor.run(...)` when `isLiveMode()` is true\n- `agentDiscreteExecutor.run(...)` when `isLiveMode()` is false\n\nThis lets the Dev tab switch message transport between Convex and LiveKit\nwithout forcing audio/realtime responses.\n\n## Worker Dispatch And Leases\n\nLiveKit can over-dispatch or leave multiple agent participants around. The\nrobust pattern is to let LiveKit be best-effort, but make Convex the authority\nfor which worker may mutate or run inference for a session.\n\nCurrent table: `agentWorkerRooms`.\n\nImportant fields:\n\n- `roomName`\n- `agentDispatchId`\n- `agentDispatchCreatedAt`\n- `agentDispatchLockExpiresAt`\n- `agentReconnectAttempt`\n- `agentNextDispatchAt`\n- `agentWorkerId`\n- `agentWorkerLeaseToken`\n- `agentWorkerLeaseExpiresAt`\n- `agentWorkerHeartbeatAt`\n\nDispatch side:\n\n- `getToken` / `ensureAgentDispatched` use stable room name `sandbox-${sessionId}`.\n- `ensureAgentDispatchedImpl` claims an in-DB dispatch lock before creating or reconciling dispatches.\n- It checks for an existing live worker lease before dispatching.\n- It lists/removes duplicate LiveKit agent participants and duplicate dispatches best-effort.\n- It uses backoff via `agentNextDispatchAt` / `agentReconnectAttempt`.\n\nWorker side:\n\n- On agent entry, claim `api.livekitAgent.claimAgentWorkerLease`.\n- Heartbeat with `renewAgentWorkerLease` every few seconds.\n- Subscribe to the room row and stop if the lease token or room changes.\n- Assert/renew the lease before mutating Convex or running inference.\n- Release the lease on disconnect.\n\nRule: only the lease holder may run the room machine loop or write frames/messages.\n\n## Room Rotation Was A Trap\n\nWe tried rotating LiveKit room names on recovery. It addressed one symptom but\nintroduced reconnect churn:\n\n- `getToken` rotated the room on every connection attempt.\n- The client compared the new room against older Convex status.\n- The client disconnected and retried.\n- That caused connect/rotate/disconnect loops.\n\nFinal direction: keep stable room names (`sandbox-${sessionId}`) and fix the actual\nLiveKit `RoomIO` participant ordering bug. Recovery should be dispatch/lease\nbased, not room-name churn.\n\n## Voice Restart Failure And RoomIO Ordering\n\nHardest voice bug: first worker in a room could speak, but after restarting\n`apps/sandbox-agent`, the new worker heard the user and got realtime transcripts /\nmetrics, yet produced no audible output.\n\nKey diagnostic signal:\n\n- Realtime `generation_created` appeared.\n- User transcription completed.\n- Realtime metrics completed.\n- But no `local track published` / `local track subscribed` for output audio.\n\nLikely root cause found in the session:\n\n- `session.start()` creates LiveKit `RoomIO`.\n- `RoomIO.start()` launches async `RoomIO.init()`.\n- Our app had its own `ParticipantConnected` handler and called private `roomIO.setParticipant(...)`.\n- That could race ahead of RoomIO's own participant handling.\n- Input could work, but RoomIO output initialization could get stranded before publishing `roomio_audio`.\n\nSettled fix:\n\n- Keep using LiveKit RoomIO.\n- Let RoomIO observe participant events first.\n- Defer our participant rebinding with `queueMicrotask`.\n- Initialize `activeVoiceParticipantIdentity` from existing participants before `session.start`.\n- Pass `inputOptions.participantIdentity` when known.\n- Use `participantKinds: [ParticipantKind.STANDARD]`.\n- Keep `closeOnDisconnect: false`.\n\nImportant code comment from `apps/sandbox-agent/src/agent.ts`:\n\n```ts\n// RoomIO must observe participant events before we call its private setParticipant;\n// otherwise its init task can wait forever and never publish the output audio track.\n```\n\nKeep passive diagnostics:\n\n- `LocalTrackPublished`\n- `LocalTrackSubscribed`\n- agent state changes\n- `SpeechCreated`\n- realtime `generation_created`\n- realtime input transcription completion\n- realtime metrics\n- session error events\n\nRemove/avoid automatic watchdog recovery as a primary fix. Watchdogs can create\nreconnect churn and hide the real LiveKit-level failure. The lease/dispatch\nsystem should handle genuinely dead workers.\n\n## Streaming Should Not Persist A Frame Per Delta\n\nStreaming was painfully slow because each model delta was backpressured by\ndurable writes.\n\nBad pattern:\n\n- For each AI SDK `textStream` delta, append a projector frame.\n- Each frame append does Convex work and advances session head.\n- Each delta waits for Convex before pulling the next provider delta.\n- Visible streaming cadence becomes Convex mutation cadence.\n\nWorking pattern:\n\n- Streaming deltas are out-of-band UI/message updates.\n- Intermediate chunks call `onStreamUpdate`.\n- `messages.add` patches a message by stable idempotency key.\n- Only the final complete assistant message enqueues a projector frame.\n\nCurrent guard:\n\n- `messages.add` stores `streamSeq`.\n- Existing message patches are ignored if the incoming `streamSeq` is older\n than the stored one.\n- This prevents late/stale stream patches from overwriting final content.\n\nImportant distinction:\n\n- Convex `messages` can show transient streaming text.\n- Durable projector frames should wait for final assistant content.\n\nThe direct Convex `sessionActions.sendMessage` path was noted as non-streaming\nat the time. The sandbox-agent path is where streaming was fixed first.\n\n## Message Idempotency\n\nFor durable and streaming message writes, use idempotency keys.\n\nUseful keys:\n\n- `assistant:${messageId}`\n- `user:${messageId}`\n- fallback from createdAt/text where available\n- random fallback only when no stable source exists\n\nIdempotency is important because streaming updates patch the same logical\nmessage repeatedly, and repeated LiveKit/browser sends should not create\nduplicate visible rows where an update was intended.\n\n## History Rehydration\n\nOn worker start:\n\n- Load `getAgentInit` by room name.\n- Restore the current projector instance.\n- Load persisted messages.\n- Convert persisted complete messages to actor history.\n\nStreaming intermediate messages should not become model history. In the current\nagent, `messagesToActorHistory` ignores messages with `streamState` that is not\n`complete`.\n\nThis matters after restarts: the new worker should rebuild semantic context from\ndurable completed messages/frames, not from half-streamed UI text.\n\n## Realtime / Voice Configuration\n\nThe sandbox-agent was too twitchy. Typing could trigger speech detection.\n\nCurrent configurable knobs:\n\n```bash\nOPENAI_REALTIME_VAD_THRESHOLD=0.65\nOPENAI_REALTIME_VAD_SILENCE_DURATION_MS=800\nOPENAI_REALTIME_VAD_PREFIX_PADDING_MS=300\nOPENAI_REALTIME_INPUT_NOISE_REDUCTION=near_field\nOPENAI_REALTIME_INTERRUPT_RESPONSE=true\nOPENAI_REALTIME_MAX_RESPONSE_OUTPUT_TOKENS=inf\n```\n\nNotes:\n\n- Higher VAD threshold means audio must be louder to count as speech.\n- Try `0.75` or `0.8` in noisy environments.\n- `near_field` is better for close/headset mics.\n- `far_field` is better for room/laptop mics.\n- `off`, `none`, or `false` disables input noise reduction in current parsing.\n- `OPENAI_REALTIME_INTERRUPT_RESPONSE=false` is useful when diagnosing whether\n accidental VAD/barge-in is cutting off assistant speech.\n- `OPENAI_REALTIME_MAX_RESPONSE_OUTPUT_TOKENS` defaults to `inf`; set it to a\n positive integer only when intentionally capping realtime replies.\n- `DEBUG_REALTIME_EVENTS=true` logs raw OpenAI Realtime response status,\n status details, transcript/audio done events, speech-start events, and\n client-side cancel/truncate events.\n\nCurrent realtime model setup:\n\n- `OPENAI_REALTIME_MODEL` defaults to `gpt-realtime-2`.\n- `OPENAI_REALTIME_VOICE` defaults to `alloy`.\n- `inputAudioTranscription.model` is `whisper-1`.\n\n## Projector Client State In React\n\nModule-level mutable session transport caused `No active session` errors after\nagent reconnect/remount.\n\nBad shape:\n\n```ts\nlet currentSessionId = ...\nlet currentSendClientMessage = ...\nconst effigy = createMachineEffigy(async (message) => {\n if (!currentSessionId || !currentSendClientMessage) throw new Error(\"No active session\");\n ...\n});\n```\n\nWhy it failed:\n\n- old mounts, HMR, route transitions, or cleanup effects could clear shared module globals\n- active UI still had a command path\n- the command path read null global transport\n\nCurrent better shape:\n\n- `ProjectorProvider` owns session-scoped projector client state.\n- It keeps one optimistic effigy per mounted provider tree.\n- It stores `sessionId` and `sendClientMessage` in refs.\n- It clears pending optimistic overlays when the session changes.\n- Consumers use `useProjector()`.\n\nRule: context-scoped client object, not module singleton with global mutable\ntransport.\n\n## Debugging Access\n\nLocal machine had Convex CLI/auth for the sandbox deployment.\n\nUseful debugging commands from `apps/sandbox`:\n\n- `bunx convex data --limit 1`\n- `bunx convex function-spec`\n- Convex deployment seen in session: `https://disciplined-bird-407.convex.cloud`\n\nTreat DB writes/mutations as explicit-only during debugging.\n\n## Practical Checks\n\nCommon verification commands used during these sessions:\n\n```bash\nbun run --filter @projectors/sandbox typecheck\nbun run --filter @projectors/sandbox-agent build\nbun run --filter @projectors/aisdk-executor typecheck\nbun run --filter @projectors/aisdk-executor test\nbun run --filter @projectors/livekit-realtime-executor typecheck\nbun run --filter @projectors/livekit-realtime-executor test\n```\n\nESLint note: at one point `apps/sandbox` had ESLint 9 but no flat\n`eslint.config.*`, so lint did not run cleanly.\n\n## Manual Voice Restart Regression\n\nThe important manual regression for reliable voice:\n\n1. Start sandbox app and `apps/sandbox-agent`.\n2. Create a session.\n3. Enable LiveKit/live mode.\n4. Speak once.\n5. Confirm audible response.\n6. Confirm agent logs `local track published` and `local track subscribed`.\n7. Kill `apps/sandbox-agent`.\n8. Restart `apps/sandbox-agent`.\n9. Without creating a new session, speak again.\n10. Confirm audible response and fresh local track publish/subscribe logs.\n11. Send a LiveKit text message while live mode is on.\n12. Confirm normal response and no reconnect loop.\n\nBad logs to watch for:\n\n- repeated room-name changes for the same session\n- infinite reconnecting loops\n- repeated timeout/unhealthy-worker churn\n- realtime generation with no local output track published in that worker lifecycle\n\n## Current Mental Model\n\nDurability:\n\n- Convex frame/session/message tables are the source of truth.\n- Provider sessions and LiveKit rooms are reconstructable runtime state.\n\nStreaming:\n\n- stream to `messages` for UI\n- persist final assistant result as the projector frame\n- guard updates with `streamSeq`\n\nVoice:\n\n- realtime is active only when projected `liveMode` says so\n- LiveKit transport can carry text without realtime voice\n- `RoomIO` participant ordering matters\n\nWorkers:\n\n- LiveKit dispatch is best-effort\n- Convex lease decides who can run and write\n- stable room names plus lease/dispatch reconciliation beat room rotation\n\nReact client:\n\n- projector command transport must be scoped to the mounted session provider\n- no module-level mutable session globals\n","/apps/sandbox/app/ConvexClientProvider.tsx":"\"use client\";\n\nimport { ConvexProvider, ConvexReactClient } from \"convex/react\";\nimport type { ReactNode } from \"react\";\n\nconst convex = new ConvexReactClient(process.env.NEXT_PUBLIC_CONVEX_URL!);\n\nexport function ConvexClientProvider({ children }: { children: ReactNode }) {\n return {children};\n}\n","/apps/sandbox/app/HomeClient.tsx":"\"use client\";\n\nimport { useCallback, useEffect, useRef, useState } from \"react\";\nimport { useAction, useConvex, useMutation, useQuery } from \"convex/react\";\nimport { useAtom, useAtomValue } from \"jotai\";\nimport { api } from \"@/convex/_generated/api\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\nimport type { LocalVideoTrack } from \"livekit-client\";\nimport type { ClientMachineMessage } from \"@projectors/core/client\";\nimport {\n attachmentKind,\n MAX_ATTACHMENT_BYTES,\n MAX_ATTACHMENTS_PER_MESSAGE,\n normalizeFileContentType,\n} from \"@/src/attachments\";\nimport {\n activeAgentTabAtom,\n inputAtom,\n isLoadingAtom,\n messageTransportAtom,\n scanlinesEnabledAtom,\n timeTravelFrameIdAtom,\n type MessageTransport,\n} from \"@/src/atoms\";\nimport { useKeyboardFocus, useSessionId } from \"@/src/hooks\";\nimport { ProjectorProvider, useProjector } from \"@/src/projector/ProjectorProvider\";\nimport type { SandboxAttachment, SandboxClientInstance, SandboxClientSnapshot, SandboxMessage } from \"@/src/types/display\";\nimport { LiveVoiceClient } from \"@/src/voice/LiveVoiceClient\";\nimport { TerminalPane } from \"./components/terminal/TerminalPane\";\nimport { AgentPane } from \"./components/agent/AgentPane\";\n\nexport function HomeClient({ initialSessionId }: { initialSessionId: Id<\"sessions\"> | null }) {\n const [sessionId, setSessionId] = useSessionId(initialSessionId);\n const [connectionError, setConnectionError] = useState(null);\n const createSession = useAction(api.sessionActions.createSession);\n const sendMessage = useAction(api.sessionActions.sendMessage);\n const generateAttachmentUploadUrl = useMutation(api.attachments.generateUploadUrl);\n const convex = useConvex();\n const resolveAttachmentUrls = useCallback(\n (args: { attachments: StoredAttachmentInput[] }) => convex.query(api.attachments.resolveUrls, args),\n [convex],\n );\n const sendClientMessageToConvex = useAction(api.sessionActions.sendClientMessage);\n const ensureAgentDispatched = useAction(api.livekitAgentActions.ensureAgentDispatched);\n const cloneFromFrame = useMutation(api.sessions.cloneFromFrame);\n const [timeTravelFrameId, setTimeTravelFrameId] = useAtom(timeTravelFrameIdAtom);\n const [sendLiveKitCommand, setSendLiveKitCommand] = useState<((message: ClientMachineMessage) => Promise) | null>(null);\n const handleLiveKitCommandSenderChange = useCallback(\n (sender: ((message: ClientMachineMessage) => Promise) | null) => {\n setSendLiveKitCommand(sender ? () => sender : null);\n },\n [],\n );\n const sendClientMessage = useCallback(\n async (args: { sessionId: Id<\"sessions\">; message: ClientMachineMessage }) => {\n if (sendLiveKitCommand) {\n return await sendLiveKitCommand(args.message);\n }\n return await sendClientMessageToConvex(args);\n },\n [sendClientMessageToConvex, sendLiveKitCommand],\n );\n\n const session = useQuery(\n api.sessions.get,\n sessionId\n ? {\n id: sessionId,\n ...(timeTravelFrameId ? { timetravelFrameId: timeTravelFrameId } : {}),\n }\n : \"skip\",\n );\n const serverMessages = useQuery(\n api.messages.listForFramePath,\n sessionId\n ? {\n sessionId,\n ...(timeTravelFrameId ? { upToFrameId: timeTravelFrameId } : {}),\n }\n : \"skip\",\n );\n const liveKitWorkerStatus = useQuery(api.livekitAgent.getAgentWorkerStatus, sessionId ? { sessionId } : \"skip\");\n\n useEffect(() => {\n if (sessionId && session === undefined) return;\n if (sessionId && session === null) {\n if (timeTravelFrameId) {\n setTimeTravelFrameId(null);\n return;\n }\n setSessionId(null);\n return;\n }\n if (!sessionId) {\n void createSession()\n .then((id) => {\n setConnectionError(null);\n setSessionId(id);\n })\n .catch((error) => {\n setConnectionError(error instanceof Error ? error.message : \"Unable to create session\");\n });\n }\n }, [createSession, session, sessionId, setSessionId, setTimeTravelFrameId, timeTravelFrameId]);\n\n const previousSessionIdRef = useRef | null>(sessionId);\n useEffect(() => {\n if (previousSessionIdRef.current !== sessionId) {\n previousSessionIdRef.current = sessionId;\n setTimeTravelFrameId(null);\n }\n }, [sessionId, setTimeTravelFrameId]);\n\n const latestFrameId = timeTravelFrameId ? null : (session?.frameId ?? null);\n\n return (\n \n \n \n );\n}\n\nfunction HomeClientContent({\n sessionId,\n setSessionId,\n createSession,\n sendMessage,\n generateAttachmentUploadUrl,\n resolveAttachmentUrls,\n cloneFromFrame,\n ensureAgentDispatched,\n serverMessages,\n liveKitWorkerStatus,\n onLiveKitCommandSenderChange,\n connectionError,\n onConnectionErrorChange,\n latestFrameId,\n currentContextEpoch,\n timeTravelFrameId,\n onTimeTravelFrameChange,\n}: {\n sessionId: Id<\"sessions\"> | null;\n setSessionId: (id: Id<\"sessions\"> | null) => void;\n createSession: () => Promise>;\n sendMessage: (args: { sessionId: Id<\"sessions\">; content: string; attachments?: StoredAttachmentInput[] }) => Promise;\n generateAttachmentUploadUrl: () => Promise;\n resolveAttachmentUrls: (args: { attachments: StoredAttachmentInput[] }) => Promise;\n cloneFromFrame: (args: {\n sourceSessionId: Id<\"sessions\">;\n targetFrameId: Id<\"frames\">;\n }) => Promise>;\n ensureAgentDispatched: (args: { sessionId: Id<\"sessions\">; reason: string }) => Promise;\n serverMessages: SandboxMessage[] | undefined;\n liveKitWorkerStatus:\n | {\n status: string;\n ready: boolean;\n agentWorkerHeartbeatAt?: number;\n agentWorkerLeaseExpiresAt?: number;\n agentNextDispatchAt?: number;\n agentDispatchLockExpiresAt?: number;\n agentReconnectAttempt?: number;\n agentLastDispatchError?: string;\n }\n | null\n | undefined;\n onLiveKitCommandSenderChange: (sender: ((message: ClientMachineMessage) => Promise) | null) => void;\n connectionError: string | null;\n onConnectionErrorChange: (error: string | null) => void;\n latestFrameId: Id<\"frames\"> | null;\n currentContextEpoch: number | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrameChange: (frameId: Id<\"frames\"> | null) => void;\n}) {\n const [input, setInput] = useAtom(inputAtom);\n const [isLoading, setIsLoading] = useAtom(isLoadingAtom);\n const [messageTransport, setMessageTransport] = useAtom(messageTransportAtom);\n const [voiceStatus, setVoiceStatus] = useState<{ status: string; detail?: string }>({ status: \"idle\" });\n const [sendLiveKitMessage, setSendLiveKitMessage] = useState<((message: { content: string; attachments: SandboxAttachment[] }) => Promise) | null>(null);\n const [localCameraTrack, setLocalCameraTrack] = useState(null);\n const [pendingAttachments, setPendingAttachments] = useState([]);\n const [attachmentsUploading, setAttachmentsUploading] = useState(false);\n const pendingAttachmentsRef = useRef([]);\n const [agentDocked, setAgentDocked] = useState(false);\n const [agentVisible, setAgentVisible] = useState(true);\n const [statusClock, setStatusClock] = useState(() => Date.now());\n const [messageHistoryCursor, setMessageHistoryCursor] = useState(null);\n const scanlinesEnabled = useAtomValue(scanlinesEnabledAtom);\n const activeTab = useAtomValue(activeAgentTabAtom);\n const { instances: clientInstances, snapshot } = useProjector();\n const isTimeTraveling = Boolean(timeTravelFrameId);\n const persistedAgentControls = getAgentControlsState(snapshot.instance ? [snapshot.instance] : []);\n const persistedVoiceEnabled = Boolean(persistedAgentControls?.liveMode);\n const persistedCameraEnabled = Boolean(persistedAgentControls?.cameraEnabled);\n const activeMessageTransport: MessageTransport = persistedVoiceEnabled ? \"livekit\" : messageTransport;\n\n const terminalRef = useRef(null);\n const agentRef = useRef(null);\n const toggleAgentDock = useCallback(() => {\n setAgentDocked((docked) => !docked);\n }, []);\n const toggleAgentVisible = useCallback(() => {\n setAgentVisible((visible) => !visible);\n }, []);\n useKeyboardFocus(terminalRef, agentRef, toggleAgentVisible);\n const sortedMessages = ((serverMessages ?? []) as SandboxMessage[])\n .slice()\n .sort((a, b) => a.createdAt - b.createdAt);\n const messages = sortedMessages.filter(\n (message) => messageHistoryCursor === null || message.createdAt > messageHistoryCursor,\n );\n\n useEffect(() => {\n setMessageHistoryCursor(null);\n }, [sessionId]);\n\n useEffect(() => {\n pendingAttachmentsRef.current = pendingAttachments;\n }, [pendingAttachments]);\n\n useEffect(() => {\n return () => {\n for (const attachment of pendingAttachmentsRef.current) {\n if (attachment.previewUrl) URL.revokeObjectURL(attachment.previewUrl);\n }\n };\n }, []);\n\n const handleAttachmentFiles = useCallback(\n async (files: File[]) => {\n if (attachmentsUploading) return;\n const uploadableFiles = files.filter((file) => file.size > 0);\n if (uploadableFiles.length === 0) return;\n if (pendingAttachments.length + uploadableFiles.length > MAX_ATTACHMENTS_PER_MESSAGE) {\n onConnectionErrorChange(`Attach up to ${MAX_ATTACHMENTS_PER_MESSAGE} files per message`);\n return;\n }\n const oversized = uploadableFiles.find((file) => file.size > MAX_ATTACHMENT_BYTES);\n if (oversized) {\n onConnectionErrorChange(`${oversized.name} is larger than 25 MB`);\n return;\n }\n\n setAttachmentsUploading(true);\n const previewUrls: string[] = [];\n try {\n const uploaded = await Promise.all(\n uploadableFiles.map(async (file): Promise => {\n const contentType = normalizeFileContentType(file);\n const kind = attachmentKind(contentType);\n const previewUrl = kind === \"image\" ? URL.createObjectURL(file) : undefined;\n const dataUrl = kind === \"image\" ? await readFileAsDataUrl(file) : undefined;\n if (previewUrl) previewUrls.push(previewUrl);\n const uploadUrl = await generateAttachmentUploadUrl();\n const response = await fetch(uploadUrl, {\n method: \"POST\",\n headers: { \"Content-Type\": contentType },\n body: file,\n });\n if (!response.ok) {\n throw new Error(`Upload failed for ${file.name}`);\n }\n const { storageId } = await response.json() as { storageId?: string };\n if (!storageId) {\n throw new Error(`Upload did not return a storage id for ${file.name}`);\n }\n return {\n storageId: storageId as Id<\"_storage\">,\n url: null,\n name: file.name,\n contentType,\n size: file.size,\n kind,\n dataUrl,\n previewUrl,\n };\n }),\n );\n const resolved = await resolveAttachmentUrls({\n attachments: uploaded.map(toStoredAttachmentInput),\n });\n const previewUrlByStorageId = new Map(uploaded.map((attachment) => [attachment.storageId, attachment.previewUrl]));\n const dataUrlByStorageId = new Map(uploaded.map((attachment) => [attachment.storageId, attachment.dataUrl]));\n setPendingAttachments((attachments) => [\n ...attachments,\n ...resolved.map((attachment) => ({\n ...attachment,\n previewUrl: previewUrlByStorageId.get(attachment.storageId),\n dataUrl: dataUrlByStorageId.get(attachment.storageId),\n })),\n ]);\n onConnectionErrorChange(null);\n } catch (error) {\n for (const previewUrl of previewUrls) URL.revokeObjectURL(previewUrl);\n onConnectionErrorChange(error instanceof Error ? error.message : \"Unable to upload attachment\");\n } finally {\n setAttachmentsUploading(false);\n }\n },\n [attachmentsUploading, generateAttachmentUploadUrl, onConnectionErrorChange, pendingAttachments.length, resolveAttachmentUrls],\n );\n\n const handleRemoveAttachment = useCallback((storageId: string) => {\n setPendingAttachments((attachments) => {\n const removed = attachments.find((attachment) => attachment.storageId === storageId);\n if (removed?.previewUrl) URL.revokeObjectURL(removed.previewUrl);\n return attachments.filter((attachment) => attachment.storageId !== storageId);\n });\n }, []);\n\n const clearPendingAttachments = useCallback(() => {\n setPendingAttachments((attachments) => {\n for (const attachment of attachments) {\n if (attachment.previewUrl) URL.revokeObjectURL(attachment.previewUrl);\n }\n return [];\n });\n }, []);\n\n const handleSend = useCallback(async () => {\n const content = input.trim();\n const liveKitAttachments = pendingAttachments.map(({ previewUrl: _previewUrl, ...attachment }) => attachment);\n const storedAttachments = pendingAttachments.map(toStoredAttachmentInput);\n if ((!content && liveKitAttachments.length === 0) || !sessionId || isLoading || isTimeTraveling || attachmentsUploading) return;\n\n if (activeMessageTransport === \"livekit\" && !sendLiveKitMessage) {\n onConnectionErrorChange(\"LiveKit agent is not ready\");\n return;\n }\n\n const liveKitSender = sendLiveKitMessage;\n setInput(\"\");\n setIsLoading(true);\n try {\n if (activeMessageTransport === \"livekit\") {\n if (!liveKitSender) throw new Error(\"LiveKit agent is not ready\");\n await liveKitSender({ content, attachments: liveKitAttachments });\n } else {\n await sendMessage({ sessionId, content, attachments: storedAttachments });\n }\n clearPendingAttachments();\n onConnectionErrorChange(null);\n } catch (error) {\n onConnectionErrorChange(error instanceof Error ? error.message : \"Unable to send message\");\n } finally {\n setIsLoading(false);\n }\n }, [\n input,\n attachmentsUploading,\n activeMessageTransport,\n clearPendingAttachments,\n isLoading,\n onConnectionErrorChange,\n pendingAttachments,\n sendLiveKitMessage,\n sendMessage,\n sessionId,\n setInput,\n setIsLoading,\n isTimeTraveling,\n ]);\n\n const handleTimeTravelFrame = useCallback(\n (frameId: Id<\"frames\">) => {\n onTimeTravelFrameChange(frameId === latestFrameId ? null : frameId);\n },\n [latestFrameId, onTimeTravelFrameChange],\n );\n\n const handleReturnToLatest = useCallback(() => {\n onTimeTravelFrameChange(null);\n }, [onTimeTravelFrameChange]);\n\n const handleForkSession = useCallback(async () => {\n if (!sessionId || !timeTravelFrameId || isLoading) return;\n setIsLoading(true);\n try {\n const nextSessionId = await cloneFromFrame({\n sourceSessionId: sessionId,\n targetFrameId: timeTravelFrameId,\n });\n setInput(\"\");\n onConnectionErrorChange(null);\n onTimeTravelFrameChange(null);\n setSessionId(nextSessionId);\n } catch (error) {\n onConnectionErrorChange(error instanceof Error ? error.message : \"Unable to fork session\");\n } finally {\n setIsLoading(false);\n }\n }, [\n cloneFromFrame,\n isLoading,\n onConnectionErrorChange,\n onTimeTravelFrameChange,\n sessionId,\n setInput,\n setIsLoading,\n setSessionId,\n timeTravelFrameId,\n ]);\n\n const handleReset = useCallback(async () => {\n try {\n const id = await createSession();\n onConnectionErrorChange(null);\n setSessionId(id);\n } catch (error) {\n onConnectionErrorChange(error instanceof Error ? error.message : \"Unable to reset session\");\n }\n }, [createSession, onConnectionErrorChange, setSessionId]);\n\n const handleVoiceStatus = useCallback((status: string, detail?: string) => {\n setVoiceStatus({ status, detail });\n }, []);\n\n const handleLiveKitSenderChange = useCallback((sender: ((message: { content: string; attachments: SandboxAttachment[] }) => Promise) | null) => {\n setSendLiveKitMessage(sender ? () => sender : null);\n if (sender) {\n onConnectionErrorChange(null);\n }\n }, [onConnectionErrorChange]);\n\n const handleLiveKitCommandSenderChange = useCallback((sender: ((message: ClientMachineMessage) => Promise) | null) => {\n onLiveKitCommandSenderChange(sender);\n if (sender) {\n onConnectionErrorChange(null);\n }\n }, [onConnectionErrorChange, onLiveKitCommandSenderChange]);\n\n const themeHue = getThemeHue(clientInstances);\n const liveKitEnabled = persistedVoiceEnabled || persistedCameraEnabled || activeMessageTransport === \"livekit\";\n const liveKitWorkerReady = Boolean(\n liveKitWorkerStatus?.agentWorkerLeaseExpiresAt && liveKitWorkerStatus.agentWorkerLeaseExpiresAt > statusClock,\n );\n const combinedLiveKitStatus = formatLiveKitStatus(voiceStatus, liveKitWorkerStatus, liveKitEnabled, statusClock);\n\n useEffect(() => {\n if (persistedVoiceEnabled && messageTransport !== \"livekit\") {\n setMessageTransport(\"livekit\");\n }\n }, [messageTransport, persistedVoiceEnabled, setMessageTransport]);\n\n useEffect(() => {\n if (!liveKitEnabled) return;\n setStatusClock(Date.now());\n const interval = window.setInterval(() => setStatusClock(Date.now()), 1_000);\n return () => window.clearInterval(interval);\n }, [liveKitEnabled]);\n\n useEffect(() => {\n if (!liveKitEnabled || !sessionId || !liveKitWorkerStatus || liveKitWorkerReady) return;\n\n const now = statusClock;\n const nextDispatchAt = liveKitWorkerStatus.agentNextDispatchAt ?? 0;\n const lockExpiresAt = liveKitWorkerStatus.agentDispatchLockExpiresAt ?? 0;\n const leaseExpiresAt = liveKitWorkerStatus.agentWorkerLeaseExpiresAt ?? 0;\n const retryAt = Math.max(nextDispatchAt, lockExpiresAt, now);\n if (leaseExpiresAt > now) return;\n\n const timeout = window.setTimeout(() => {\n void ensureAgentDispatched({ sessionId, reason: \"client_reconcile\" }).catch((error) => {\n onConnectionErrorChange(error instanceof Error ? error.message : \"Unable to reconnect LiveKit agent\");\n });\n }, Math.max(0, retryAt - now));\n\n return () => window.clearTimeout(timeout);\n }, [ensureAgentDispatched, liveKitEnabled, liveKitWorkerReady, liveKitWorkerStatus, onConnectionErrorChange, sessionId, statusClock]);\n\n useEffect(() => {\n if (agentDocked && agentVisible) {\n agentRef.current?.focus();\n }\n }, [agentDocked, agentVisible]);\n\n useEffect(() => {\n const handleKeyDown = (event: KeyboardEvent) => {\n if (!event.metaKey || event.key.toLowerCase() !== \"k\") return;\n event.preventDefault();\n setMessageHistoryCursor((cursor) => messages.at(-1)?.createdAt ?? cursor);\n };\n\n window.addEventListener(\"keydown\", handleKeyDown);\n return () => window.removeEventListener(\"keydown\", handleKeyDown);\n }, [messages]);\n\n const agentPane = (\n \n );\n\n return (\n \n
\n \n {!agentDocked && agentVisible && agentPane}\n {agentDocked && agentVisible && (\n \n
\n {agentPane}\n
\n
\n )}\n \n \n \n );\n}\n\ntype PendingAttachment = SandboxAttachment & {\n previewUrl?: string;\n};\n\ntype StoredAttachmentInput = Omit & {\n storageId: Id<\"_storage\">;\n};\n\nfunction toStoredAttachmentInput(attachment: SandboxAttachment): StoredAttachmentInput {\n return {\n storageId: attachment.storageId as Id<\"_storage\">,\n name: attachment.name,\n contentType: attachment.contentType,\n size: attachment.size,\n kind: attachment.kind,\n };\n}\n\nfunction readFileAsDataUrl(file: File): Promise {\n return new Promise((resolve, reject) => {\n const reader = new FileReader();\n reader.addEventListener(\"load\", () => {\n if (typeof reader.result === \"string\") {\n resolve(reader.result);\n } else {\n reject(new Error(`Unable to read ${file.name} as a data URL`));\n }\n });\n reader.addEventListener(\"error\", () => {\n reject(reader.error ?? new Error(`Unable to read ${file.name}`));\n });\n reader.readAsDataURL(file);\n });\n}\n\nfunction getAgentControlsState(instances: SandboxClientInstance[]) {\n return findState(instances, \"agentControls\")?.value as\n | {\n liveMode?: boolean;\n cameraEnabled?: boolean;\n streamingEnabled?: boolean;\n testCounter?: number;\n }\n | undefined;\n}\n\nfunction getThemeHue(instances: SandboxClientInstance[]): number {\n const state = findState(instances, \"sandbox\")?.value as\n | { themeHue?: number }\n | undefined;\n return state?.themeHue ?? 126;\n}\n\nfunction formatLiveKitStatus(\n roomStatus: { status: string; detail?: string },\n workerStatus:\n | {\n status: string;\n ready: boolean;\n agentWorkerHeartbeatAt?: number;\n agentWorkerLeaseExpiresAt?: number;\n agentNextDispatchAt?: number;\n agentReconnectAttempt?: number;\n agentLastDispatchError?: string;\n }\n | null\n | undefined,\n enabled: boolean,\n now: number,\n): { status: string; detail?: string } {\n if (!enabled) return { status: \"idle\" };\n if (!workerStatus) return { status: roomStatus.status === \"connected\" ? \"connecting\" : roomStatus.status, detail: roomStatus.detail };\n const ready = Boolean(workerStatus.agentWorkerLeaseExpiresAt && workerStatus.agentWorkerLeaseExpiresAt > now);\n if (ready) {\n const heartbeat = workerStatus.agentWorkerHeartbeatAt\n ? `heartbeat ${Math.max(0, Math.round((now - workerStatus.agentWorkerHeartbeatAt) / 1000))}s ago`\n : roomStatus.detail;\n return { status: \"ready\", detail: heartbeat };\n }\n\n const status = workerStatus.status === \"ready\" ? \"stale\" : workerStatus.status;\n const attempt = workerStatus.agentReconnectAttempt ?? 0;\n const reconnectDetail = workerStatus.agentLastDispatchError\n ? `attempt ${attempt}: ${workerStatus.agentLastDispatchError}`\n : workerStatus.agentNextDispatchAt && workerStatus.agentNextDispatchAt > now\n ? `attempt ${attempt}; retry in ${Math.ceil((workerStatus.agentNextDispatchAt - now) / 1000)}s`\n : roomStatus.detail;\n return { status, detail: reconnectDetail };\n}\n\nfunction findState(instances: SandboxClientInstance[], key: string) {\n for (const instance of instances) {\n const state = findStateInInstance(instance, key);\n if (state) return state;\n }\n return undefined;\n}\n\nfunction findStateInInstance(instance: SandboxClientInstance, key: string): SandboxClientInstance[\"states\"][number] | undefined {\n const state = instance.states.find(\n (item: SandboxClientInstance[\"states\"][number]) => item.key === key,\n );\n if (state) return state;\n for (const member of instance.members) {\n const found = findStateInInstance(member, key);\n if (found) return found;\n }\n for (const child of instance.children) {\n const found = findStateInInstance(child, key);\n if (found) return found;\n }\n return undefined;\n}\n","/apps/sandbox/app/components/agent/AgentPane.tsx":"\"use client\";\n\nimport {\n forwardRef,\n useEffect,\n useLayoutEffect,\n useRef,\n useState,\n type ReactNode,\n} from \"react\";\nimport { useMutation, useQuery } from \"convex/react\";\nimport { useAtom, useAtomValue } from \"jotai\";\nimport type {\n CompiledContributorView,\n CompiledContributor,\n CompiledProjectionTree,\n} from \"@projectors/core\";\nimport type { OptimisticEffigy } from \"@projectors/core/client\";\nimport { api } from \"@/convex/_generated/api\";\nimport type { Doc, Id } from \"@/convex/_generated/dataModel\";\nimport {\n activeAgentTabAtom,\n activeHistorySubtabAtom,\n activeTreeSubtabAtom,\n type AgentTab,\n type HistorySubtab,\n type MessageTransport,\n type TreeSubtab,\n} from \"@/src/atoms\";\nimport { useProjector } from \"@/src/projector/ProjectorProvider\";\nimport type {\n SandboxClientInstance,\n SandboxClientSnapshot,\n SandboxMessage,\n} from \"@/src/types/display\";\n\ntype AgentPaneProps = {\n sessionId: Id<\"sessions\"> | null;\n activeTab: AgentTab;\n docked: boolean;\n onToggleDock: () => void;\n onResetSession: () => void;\n latestFrameId: Id<\"frames\"> | null;\n currentContextEpoch: number | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n onReturnToLatest: () => void;\n onSwitchSession: (sessionId: Id<\"sessions\">) => void;\n messageTransport: MessageTransport;\n onMessageTransportChange: (transport: MessageTransport) => void;\n liveKitStatus: { status: string; detail?: string };\n liveKitReady: boolean;\n};\n\nexport const AgentPane = forwardRef(\n function AgentPane(\n {\n sessionId,\n activeTab,\n docked,\n onToggleDock,\n onResetSession,\n latestFrameId,\n currentContextEpoch,\n timeTravelFrameId,\n onTimeTravelFrame,\n onReturnToLatest,\n onSwitchSession,\n messageTransport,\n onMessageTransportChange,\n liveKitStatus,\n liveKitReady,\n },\n ref,\n ) {\n const [, setActiveTab] = useAtom(activeAgentTabAtom);\n const activeTreeSubtab = useAtomValue(activeTreeSubtabAtom);\n const activeHistorySubtab = useAtomValue(activeHistorySubtabAtom);\n const { effigy, instances, snapshot, readOnly } = useProjector();\n const contentScrollRef = useRef(null);\n const shouldAutoScrollHistoryRef = useRef(true);\n const activeSubtab =\n activeTab === \"tree\"\n ? activeTreeSubtab\n : activeTab === \"history\"\n ? activeHistorySubtab\n : null;\n\n useLayoutEffect(() => {\n if (!contentScrollRef.current) return;\n shouldAutoScrollHistoryRef.current = true;\n contentScrollRef.current.scrollTop =\n activeTab === \"history\" ? contentScrollRef.current.scrollHeight : 0;\n contentScrollRef.current.scrollLeft = 0;\n }, [activeTab, activeSubtab]);\n\n useLayoutEffect(() => {\n const scrollNode = contentScrollRef.current;\n if (!scrollNode || activeTab !== \"history\") return;\n\n const scrollToBottomIfPinned = () => {\n if (shouldAutoScrollHistoryRef.current) {\n scrollNode.scrollTop = scrollNode.scrollHeight;\n }\n };\n const contentNode = scrollNode.firstElementChild;\n const observer = new ResizeObserver(scrollToBottomIfPinned);\n observer.observe(scrollNode);\n if (contentNode) observer.observe(contentNode);\n scrollToBottomIfPinned();\n return () => observer.disconnect();\n }, [activeTab, activeSubtab]);\n\n return (\n \n
\n

\n AGENT\n

\n \n {docked ? \"inline\" : \"dock\"}\n \n
\n \n {\n if (activeTab === \"history\") {\n shouldAutoScrollHistoryRef.current = isWithinBottom(event.currentTarget, 10);\n }\n }}\n className=\"terminal-scrollbar min-h-0 flex-1 overflow-y-auto p-4\"\n >\n {activeTab === \"tree\" && (\n \n )}\n {activeTab === \"state\" && }\n {activeTab === \"history\" && (\n \n )}\n {activeTab === \"commands\" && (\n \n )}\n {activeTab === \"playground\" && (\n \n )}\n {activeTab === \"dev\" && (\n \n )}\n \n \n );\n },\n);\n\nfunction TreeTab({\n sessionId,\n instances,\n projectionTree,\n}: {\n sessionId: Id<\"sessions\"> | null;\n instances: SandboxClientInstance[];\n projectionTree?: CompiledProjectionTree;\n}) {\n const [activeSubtab, setActiveSubtab] = useAtom(activeTreeSubtabAtom);\n const switchSubtab = (tab: TreeSubtab) => {\n if (tab === activeSubtab) return;\n setActiveSubtab(tab);\n };\n\n return (\n
\n
\n {([\"instance\", \"projection\", \"ir\", \"realized\"] as TreeSubtab[]).map(\n (tab) => (\n switchSubtab(tab)}\n className={`px-3 py-2 ${\n activeSubtab === tab\n ? \"border-b border-terminal-green text-terminal-green\"\n : \"text-terminal-green-dim hover:text-terminal-green\"\n }`}\n >\n {tab}\n \n ),\n )}\n
\n {activeSubtab === \"instance\" && }\n {activeSubtab === \"projection\" && (\n \n )}\n {activeSubtab === \"ir\" && }\n {activeSubtab === \"realized\" && (\n \n )}\n
\n );\n}\n\nfunction InstanceTree({ instances }: { instances: SandboxClientInstance[] }) {\n if (instances.length === 0) {\n return No instances;\n }\n\n return (\n
\n {instances.map((instance) => (\n \n ))}\n
\n );\n}\n\nfunction ContributorTree({ tree }: { tree?: CompiledProjectionTree }) {\n if (!tree) {\n return No contributor tree in snapshot;\n }\n if (tree.roots.length === 0) {\n return No contributors;\n }\n return (\n
\n {tree.roots.map((node) => (\n \n ))}\n
\n );\n}\n\ntype RuntimeInspectionBase = {\n generatorId: string;\n kind: \"generator\";\n nodeKey: string;\n name?: string;\n};\n\ntype CompiledIrResult = {\n runtimes: Array<\n RuntimeInspectionBase & {\n inference: {\n preamble: string[];\n recency: string[];\n tools: string[];\n retrievableStates: unknown[];\n };\n }\n >;\n};\n\ntype RealizedPromptsResult = {\n runtimes: Array<\n RuntimeInspectionBase & {\n prompt: {\n provider: string;\n input: unknown;\n };\n }\n >;\n};\n\nfunction CompiledIrTree({ sessionId }: { sessionId: Id<\"sessions\"> | null }) {\n const result = useQuery(\n api.sessions.getCompiledIr,\n sessionId ? { sessionId } : \"skip\",\n ) as CompiledIrResult | null | undefined;\n\n if (!sessionId) return No session;\n if (result === undefined) return Loading IR...;\n if (!result || result.runtimes.length === 0)\n return No generator runtimes;\n\n return (\n
\n {result.runtimes.map((runtime, index) => (\n \n \n \n \n \n \n \n \n ))}\n
\n );\n}\n\nfunction RealizedPromptTree({\n sessionId,\n}: {\n sessionId: Id<\"sessions\"> | null;\n}) {\n const result = useQuery(\n api.sessions.getRealizedPrompts,\n sessionId ? { sessionId } : \"skip\",\n ) as RealizedPromptsResult | null | undefined;\n\n if (!sessionId) return No session;\n if (result === undefined)\n return Loading realized prompts...;\n if (!result || result.runtimes.length === 0)\n return No generator runtimes;\n\n return (\n
\n {result.runtimes.map((runtime, index) => (\n \n \n \n \n \n ))}\n
\n );\n}\n\nfunction RealizedPromptInput({ value }: { value: unknown }) {\n const input = objectRecord(value);\n if (!input) {\n return ;\n }\n\n const system = typeof input.system === \"string\" ? input.system : undefined;\n const messages = Array.isArray(input.messages) ? input.messages : undefined;\n const config = omitKeys(input, [\"system\", \"messages\"]);\n\n return (\n
\n {system !== undefined && (\n \n {system.trim() ? (\n
\n {system}\n
\n ) : (\n system empty\n )}\n \n )}\n {messages !== undefined && (\n \n {messages.length > 0 ? (\n
\n {messages.map((message, index) => (\n \n \n [{messageRole(message)}]:\n \n \n {messageText(message)}\n \n
\n ))}\n
\n ) : (\n messages empty\n )}\n \n )}\n \n {Object.keys(config).length > 0 ? (\n \n ) : (\n config empty\n )}\n \n \n );\n}\n\nfunction RuntimeInspectionBlock({\n runtime,\n summary,\n defaultExpanded = false,\n children,\n}: {\n runtime: RuntimeInspectionBase;\n summary: string;\n defaultExpanded?: boolean;\n children: ReactNode;\n}) {\n const [expanded, setExpanded] = useState(defaultExpanded);\n\n return (\n
\n setExpanded((value) => !value)}\n className=\"flex w-full items-center gap-2 px-2 py-2 text-left hover:bg-terminal-bg focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n \n {expanded ? \"-\" : \"+\"}\n \n {runtime.kind}\n {runtime.nodeKey}\n {runtime.name && (\n {runtime.name}\n )}\n \n {runtime.generatorId}\n \n \n {summary}\n \n \n {expanded && (\n
\n \n {children}\n
\n )}\n
\n );\n}\n\nfunction objectRecord(value: unknown): Record | undefined {\n return value && typeof value === \"object\" && !Array.isArray(value)\n ? (value as Record)\n : undefined;\n}\n\nfunction omitKeys(\n value: Record,\n keys: string[],\n): Record {\n const omitted = new Set(keys);\n return Object.fromEntries(\n Object.entries(value).filter(([key]) => !omitted.has(key)),\n );\n}\n\nfunction messageRole(message: unknown): string {\n const record = objectRecord(message);\n const role = record?.role;\n return typeof role === \"string\" && role ? role : \"message\";\n}\n\nfunction messageText(message: unknown): string {\n const record = objectRecord(message);\n if (!record) return stringifyCompact(message);\n\n const content = record.content;\n if (typeof content === \"string\") return content;\n if (content !== undefined) return renderCompactContent(content);\n\n const text = record.text;\n if (typeof text === \"string\") return text;\n\n return stringifyCompact(message);\n}\n\nfunction renderCompactContent(value: unknown): string {\n if (typeof value === \"string\") return value;\n if (Array.isArray(value)) {\n return value.map(renderCompactContent).filter(Boolean).join(\" \");\n }\n const record = objectRecord(value);\n if (record) {\n if (typeof record.text === \"string\") return record.text;\n if (typeof record.type === \"string\") return `[${record.type}]`;\n }\n return stringifyCompact(value);\n}\n\nfunction stringifyCompact(value: unknown): string {\n if (value === undefined) return \"\";\n if (typeof value === \"string\") return value;\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nfunction NameList({ title, names }: { title: string; names: string[] }) {\n if (names.length === 0) {\n return {title} empty;\n }\n\n return (\n \n
\n {names.map((name, index) => (\n \n {name}\n \n ))}\n
\n \n );\n}\n\ntype FrameDoc = Doc<\"frames\"> & {\n contextEpoch?: number;\n};\n\ntype FrameMessage = {\n type?: string;\n role?: string;\n text?: string;\n value?: unknown;\n name?: string;\n [key: string]: unknown;\n};\n\ntype FrameHistoryView = {\n frame: FrameDoc;\n index: number;\n messages: FrameMessage[];\n frameType: FrameType;\n};\n\ntype FrameType = \"instance\" | \"work\" | \"actor\";\n\ntype HistoryFilterOption = {\n value: string;\n label: string;\n count: number;\n detail?: string;\n title?: string;\n};\n\nfunction HistoryTab({\n sessionId,\n projectionTree,\n latestFrameId,\n currentContextEpoch,\n timeTravelFrameId,\n onTimeTravelFrame,\n onReturnToLatest,\n onSwitchSession,\n}: {\n sessionId: Id<\"sessions\"> | null;\n projectionTree?: CompiledProjectionTree;\n latestFrameId: Id<\"frames\"> | null;\n currentContextEpoch: number | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n onReturnToLatest: () => void;\n onSwitchSession: (sessionId: Id<\"sessions\">) => void;\n}) {\n const [activeSubtab, setActiveSubtab] = useAtom(activeHistorySubtabAtom);\n const stickyHeaderRef = useRef(null);\n const [filterStickyTop, setFilterStickyTop] = useState(0);\n const isTimeTraveling = Boolean(\n timeTravelFrameId && latestFrameId && timeTravelFrameId !== latestFrameId,\n );\n const switchSubtab = (tab: HistorySubtab) => {\n if (tab === activeSubtab) return;\n setActiveSubtab(tab);\n };\n\n useLayoutEffect(() => {\n const header = stickyHeaderRef.current;\n if (!header) {\n setFilterStickyTop(0);\n return;\n }\n const topBleed = Number.parseFloat(getComputedStyle(header).paddingTop) || 0;\n setFilterStickyTop(Math.max(0, header.offsetHeight - topBleed));\n }, [activeSubtab, isTimeTraveling]);\n\n return (\n
\n \n
\n {([\"frames\", \"messages\", \"branches\", \"controls\"] as HistorySubtab[]).map(\n (tab) => (\n switchSubtab(tab)}\n className={`px-3 py-2 ${\n activeSubtab === tab\n ? \"border-b border-terminal-green text-terminal-green\"\n : \"text-terminal-green-dim hover:text-terminal-green\"\n }`}\n >\n {tab}\n \n ),\n )}\n
\n {isTimeTraveling && (\n
\n
\n viewing{\" \"}\n {timeTravelFrameId ? shortId(timeTravelFrameId) : \"history\"}\n
\n \n live latest\n \n
\n )}\n
\n
\n {activeSubtab === \"frames\" && (\n \n )}\n {activeSubtab === \"messages\" && (\n \n )}\n {activeSubtab === \"branches\" && (\n \n )}\n {activeSubtab === \"controls\" && (\n \n )}\n
\n \n );\n}\n\nfunction HistoryControls({\n sessionId,\n timeTravelFrameId,\n onReturnToLatest,\n}: {\n sessionId: Id<\"sessions\"> | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onReturnToLatest: () => void;\n}) {\n const incrementContextEpoch = useMutation(api.sessions.incrementContextEpoch);\n const { effigy, readOnly } = useProjector();\n const [isIncrementing, setIsIncrementing] = useState(false);\n const [error, setError] = useState(null);\n const blockedByHistory = Boolean(timeTravelFrameId || readOnly);\n\n if (!sessionId) return No session;\n\n const handleIncrement = async () => {\n if (blockedByHistory || isIncrementing) return;\n\n setIsIncrementing(true);\n setError(null);\n try {\n await incrementContextEpoch({ sessionId });\n effigy.clearPending();\n } catch (nextError) {\n setError(nextError instanceof Error ? nextError.message : \"Unable to increment epoch\");\n } finally {\n setIsIncrementing(false);\n }\n };\n\n return (\n
\n {blockedByHistory && (\n
\n
viewing historical frame
\n
\n Return to latest before changing the machine context epoch.\n
\n \n live latest\n \n
\n )}\n
\n
\n
\n
context epoch
\n
\n future machine runs will ignore prior epoch frames\n
\n
\n \n {isIncrementing ? \"incrementing...\" : \"increment epoch\"}\n \n
\n {error && (\n
\n {error}\n
\n )}\n
\n
\n );\n}\n\nfunction FramesHistory({\n sessionId,\n projectionTree,\n filterStickyTop,\n latestFrameId,\n currentContextEpoch,\n timeTravelFrameId,\n onTimeTravelFrame,\n}: {\n sessionId: Id<\"sessions\"> | null;\n projectionTree?: CompiledProjectionTree;\n filterStickyTop: number;\n latestFrameId: Id<\"frames\"> | null;\n currentContextEpoch: number | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n}) {\n const frames = useQuery(\n api.frames.list,\n sessionId ? { sessionId } : \"skip\",\n ) as FrameDoc[] | undefined;\n const [expandedFrameIds, setExpandedFrameIds] = useState>(\n new Set(),\n );\n const [selectedFrameTypes, setSelectedFrameTypes] = useState>(\n () => new Set(),\n );\n const [nodeGeneratorFilters, setNodeRuntimeFilters] = useState>(\n () => new Set(),\n );\n\n useEffect(() => {\n setExpandedFrameIds(new Set());\n setSelectedFrameTypes(new Set());\n setNodeRuntimeFilters(new Set());\n }, [sessionId]);\n\n if (!sessionId) return No session;\n if (!frames) return Loading...;\n\n const orderedFrames = frames\n .slice()\n .sort((a, b) => a.createdAt - b.createdAt);\n if (orderedFrames.length === 0)\n return No frames yet;\n const frameViews = orderedFrames.map((frame, index) => {\n const messages = normalizeFrameMessages(frame.messages);\n return {\n frame,\n index,\n messages,\n frameType: frameTypeForFrame(frame, messages),\n };\n });\n const activationGeneratorIds = activationGeneratorIdsFromFrameViews(frameViews);\n const frameTypeOptions = frameTypeFilterOptions(frameViews);\n const nodeOptions = nodeFilterOptions(\n projectionTree,\n frameViews,\n activationGeneratorIds,\n );\n const frameTypeValues = frameTypeOptions.map((option) => option.value);\n const nodeRuntimeValues = nodeOptions.map((option) => option.value);\n const filteredFrameViews = frameViews.filter(\n (view) =>\n (selectedFrameTypes.size === 0 || selectedFrameTypes.has(view.frameType)) &&\n (nodeGeneratorFilters.size === 0 ||\n Array.from(nodeGeneratorFilters).some((generatorId) =>\n frameMatchesRuntime(\n view.frame,\n view.messages,\n activationGeneratorIds,\n generatorId,\n ),\n )),\n );\n const selectedFrameTypeValues = new Set(\n selectedFrameTypes.size === 0 ? frameTypeValues : selectedFrameTypes,\n );\n const selectedNodeRuntimeIds = new Set(\n nodeGeneratorFilters.size === 0 ? nodeRuntimeValues : nodeGeneratorFilters,\n );\n const activeFrameTypeFilters =\n selectedFrameTypes.size === 0\n ? []\n : frameTypeOptions.filter((option) => selectedFrameTypes.has(option.value));\n const activeNodeFilters =\n nodeGeneratorFilters.size === 0 ? [] : nodeOptions.filter((option) => nodeGeneratorFilters.has(option.value));\n const activeFilterCount = activeFrameTypeFilters.length + activeNodeFilters.length;\n const toggleFrameTypeFilter = (value: string) => {\n setSelectedFrameTypes((current) =>\n toggleInclusiveFilter(current, value, frameTypeValues),\n );\n };\n const toggleNodeRuntimeFilter = (value: string) => {\n setNodeRuntimeFilters((current) =>\n toggleInclusiveFilter(current, value, nodeRuntimeValues),\n );\n };\n const clearFilters = () => {\n setSelectedFrameTypes(new Set());\n setNodeRuntimeFilters(new Set());\n };\n\n return (\n
\n \n {filteredFrameViews.length === 0 ? (\n
\n No frames match the current filters\n
\n ) : (\n
\n {filteredFrameViews.map(({ frame, index, messages, frameType }) => {\n const expanded = expandedFrameIds.has(frame._id);\n const isLatest = frame._id === latestFrameId;\n const isSelected = frame._id === timeTravelFrameId && !isLatest;\n const frameEpoch =\n typeof frame.contextEpoch === \"number\" ? frame.contextEpoch : null;\n const isBeforeCurrentEpoch =\n currentContextEpoch !== null &&\n frameEpoch !== null &&\n frameEpoch < currentContextEpoch;\n return (\n \n
\n {\n setExpandedFrameIds((current) => {\n const next = new Set(current);\n if (next.has(frame._id)) next.delete(frame._id);\n else next.add(frame._id);\n return next;\n });\n }}\n aria-expanded={expanded}\n className=\"block min-w-0 flex-1 p-3 text-left hover:bg-terminal-bg focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n
\n
\n
\n \n frame #{index + 1}\n \n \n {shortId(frame._id)}\n \n \n {frameType}\n \n {frameEpoch !== null && (\n \n epoch {frameEpoch}\n \n )}\n {isLatest && (\n latest\n )}\n {isSelected && (\n viewing\n )}\n
\n \n
\n
\n
\n {new Date(frame.createdAt).toLocaleTimeString()}\n
\n
{messages.length} msgs
\n
\n
\n \n onTimeTravelFrame(frame._id)}\n aria-label={`time travel to frame ${index + 1}`}\n title=\"time travel\"\n className={`w-10 shrink-0 border-l border-terminal-green-dimmer hover:bg-terminal-bg focus:outline-none focus:ring-1 focus:ring-terminal-green ${\n isSelected\n ? \"text-terminal-cyan\"\n : \"text-terminal-green-dim hover:text-terminal-green\"\n }`}\n >\n ⑂\n \n
\n {expanded && (\n
\n {messages.length > 0 ? (\n
\n {messages.map((message, messageIndex) => (\n \n ))}\n
\n ) : (\n
\n No frame messages\n
\n )}\n 
\n                      {JSON.stringify(frame, null, 2)}\n
\n
\n )}\n \n );\n })}\n
\n )}\n
\n );\n}\n\nfunction FrameHistoryFilters({\n frameTypeOptions,\n nodeOptions,\n selectedFrameTypes,\n selectedNodeRuntimeIds,\n activeFrameTypeFilters,\n activeNodeFilters,\n activeFilterCount,\n stickyTop,\n onToggleFrameType,\n onToggleNodeRuntime,\n onClear,\n}: {\n frameTypeOptions: HistoryFilterOption[];\n nodeOptions: HistoryFilterOption[];\n selectedFrameTypes: Set;\n selectedNodeRuntimeIds: Set;\n activeFrameTypeFilters: HistoryFilterOption[];\n activeNodeFilters: HistoryFilterOption[];\n activeFilterCount: number;\n stickyTop: number;\n onToggleFrameType: (value: string) => void;\n onToggleNodeRuntime: (value: string) => void;\n onClear: () => void;\n}) {\n const [open, setOpen] = useState(false);\n\n return (\n <>\n \n
\n setOpen(true)}\n className=\"shrink-0 rounded border border-terminal-green-dimmer bg-terminal-bg px-2 py-1 text-terminal-green hover:border-terminal-green focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n filters{activeFilterCount > 0 ? ` ${activeFilterCount}` : \"\"}\n \n {activeFilterCount > 0 && (\n <>\n \n \n \n clear\n \n \n )}\n
\n \n {open && (\n
\n setOpen(false)}\n />\n \n
\n
filters
\n setOpen(false)}\n className=\"text-terminal-green-dim hover:text-terminal-green focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n close\n \n
\n
\n \n \n
\n
\n {activeFilterCount > 0 && (\n \n clear\n \n )}\n setOpen(false)}\n className=\"rounded border border-terminal-green-dimmer bg-terminal-bg px-3 py-1 text-terminal-green hover:border-terminal-green focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n done\n \n
\n \n
\n )}\n \n );\n}\n\nfunction AppliedFilterChips({\n label,\n options,\n onRemove,\n}: {\n label: string;\n options: HistoryFilterOption[];\n onRemove: (value: string) => void;\n}) {\n if (options.length === 0) return null;\n\n return (\n
\n {options.map((option) => (\n onRemove(option.value)}\n className=\"flex max-w-full shrink-0 items-center gap-1 rounded border border-terminal-cyan bg-terminal-bg px-2 py-1 text-terminal-cyan hover:border-terminal-green hover:text-terminal-green focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n {label}\n {option.label}\n x\n \n ))}\n
\n );\n}\n\nfunction FilterChipGroup({\n label,\n options,\n selectedValues,\n emptyLabel,\n onToggle,\n}: {\n label: string;\n options: HistoryFilterOption[];\n selectedValues: Set;\n emptyLabel: string;\n onToggle: (value: string) => void;\n}) {\n return (\n
\n
\n {label}\n
\n {options.length > 0 ? (\n
\n {options.map((option) => {\n const selected = selectedValues.has(option.value);\n return (\n onToggle(option.value)}\n className={`flex max-w-full items-center gap-1 rounded border px-2 py-1 focus:outline-none focus:ring-1 focus:ring-terminal-green ${\n selected\n ? \"border-terminal-cyan bg-terminal-bg text-terminal-cyan\"\n : \"border-terminal-green-dimmer bg-terminal-bg text-terminal-green-dim hover:border-terminal-green hover:text-terminal-green\"\n }`}\n >\n \n {option.label}\n \n {option.detail && (\n \n {option.detail}\n \n )}\n \n {option.count}\n \n \n );\n })}\n
\n ) : (\n
{emptyLabel}
\n )}\n
\n );\n}\nfunction MessagesHistory({\n sessionId,\n latestFrameId,\n timeTravelFrameId,\n onTimeTravelFrame,\n}: {\n sessionId: Id<\"sessions\"> | null;\n latestFrameId: Id<\"frames\"> | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n}) {\n const messages = useQuery(\n api.messages.listForFramePath,\n sessionId\n ? {\n sessionId,\n ...(timeTravelFrameId ? { upToFrameId: timeTravelFrameId } : {}),\n }\n : \"skip\",\n ) as SandboxMessage[] | undefined;\n\n if (!sessionId) return No session;\n if (!messages) return Loading...;\n\n const orderedMessages = messages\n .slice()\n .sort((a, b) => a.createdAt - b.createdAt);\n if (orderedMessages.length === 0)\n return No messages yet;\n\n return (\n
\n {orderedMessages.map((message) => {\n const frameId =\n \"frameId\" in message && typeof message.frameId === \"string\"\n ? (message.frameId as Id<\"frames\">)\n : null;\n const isLatest = frameId === latestFrameId;\n const isSelected = frameId === timeTravelFrameId && !isLatest;\n return (\n {\n if (frameId) onTimeTravelFrame(frameId);\n }}\n className={`block w-full rounded border bg-terminal-bg-lighter p-3 text-left text-xs focus:outline-none focus:ring-1 focus:ring-terminal-green disabled:cursor-default ${\n isSelected\n ? \"border-terminal-cyan\"\n : frameId\n ? \"border-terminal-green-dimmer hover:border-terminal-green\"\n : \"border-terminal-green-dimmer\"\n }`}\n >\n
\n
\n \n {message.role}\n \n {message.mode && (\n \n {message.mode}\n \n )}\n {isLatest && latest}\n {isSelected && (\n viewing\n )}\n {frameId && (\n \n {shortId(frameId)}\n \n )}\n
\n \n {new Date(message.createdAt).toLocaleTimeString()}\n \n
\n
\n {message.content}\n
\n {(message.attachments ?? []).length > 0 && (\n
\n {(message.attachments ?? []).map((attachment) => (\n \n {attachment.name}\n \n ))}\n
\n )}\n \n );\n })}\n
\n );\n}\n\ntype BranchSession = {\n sessionId: Id<\"sessions\">;\n latestFrameId?: Id<\"frames\">;\n contextEpoch: number;\n forkedFromSessionId?: Id<\"sessions\">;\n forkedFromFrameId?: Id<\"frames\">;\n frames: FrameDoc[];\n};\n\ntype FamilyTimeline = {\n familyRootSessionId: Id<\"sessions\">;\n sessions: BranchSession[];\n edges: Array<{\n fromSessionId?: Id<\"sessions\">;\n fromFrameId?: Id<\"frames\">;\n toSessionId: Id<\"sessions\">;\n }>;\n};\n\nfunction BranchesHistory({\n sessionId,\n latestFrameId,\n timeTravelFrameId,\n onTimeTravelFrame,\n onSwitchSession,\n}: {\n sessionId: Id<\"sessions\"> | null;\n latestFrameId: Id<\"frames\"> | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n onSwitchSession: (sessionId: Id<\"sessions\">) => void;\n}) {\n const timeline = useQuery(\n api.sessions.getFamilyTimeline,\n sessionId ? { sessionId } : \"skip\",\n ) as FamilyTimeline | null | undefined;\n const [expandedSessionIds, setExpandedSessionIds] = useState>(\n new Set(),\n );\n\n if (!sessionId) return No session;\n if (timeline === undefined)\n return Loading branches...;\n if (!timeline || timeline.sessions.length === 0)\n return No branch family yet;\n\n const orderedSessions = timeline.sessions\n .slice()\n .sort(\n (a, b) =>\n firstFrameTime(a) - firstFrameTime(b) ||\n a.sessionId.localeCompare(b.sessionId),\n );\n const childrenByParent = new Map();\n for (const branch of orderedSessions) {\n if (!branch.forkedFromSessionId) continue;\n const children = childrenByParent.get(branch.forkedFromSessionId) ?? [];\n children.push(branch);\n childrenByParent.set(branch.forkedFromSessionId, children);\n }\n const rootBranches = orderedSessions.filter(\n (branch) =>\n branch.sessionId === timeline.familyRootSessionId ||\n !branch.forkedFromSessionId,\n );\n const branchNumber = new Map(\n orderedSessions.map((branch, index) => [branch.sessionId, index + 1]),\n );\n const forksByFrame = new Map();\n for (const edge of timeline.edges) {\n if (!edge.fromFrameId) continue;\n forksByFrame.set(\n edge.fromFrameId,\n (forksByFrame.get(edge.fromFrameId) ?? 0) + 1,\n );\n }\n const totalFrames = orderedSessions.reduce(\n (total, branch) => total + branch.frames.length,\n 0,\n );\n const expandedIds =\n expandedSessionIds.size === 0\n ? new Set(orderedSessions.map((branch) => branch.sessionId))\n : expandedSessionIds;\n\n return (\n
\n
\n \n \n \n
\n
\n {rootBranches.map((branch) => (\n {\n setExpandedSessionIds((current) => {\n const source =\n current.size === 0\n ? new Set(orderedSessions.map((item) => item.sessionId))\n : current;\n const next = new Set(source);\n if (next.has(id)) next.delete(id);\n else next.add(id);\n return next;\n });\n }}\n onTimeTravelFrame={onTimeTravelFrame}\n onSwitchSession={onSwitchSession}\n />\n ))}\n
\n
\n );\n}\n\nfunction BranchNode({\n branch,\n depth,\n currentSessionId,\n rootSessionId,\n latestFrameId,\n timeTravelFrameId,\n branchNumber,\n childrenByParent,\n forksByFrame,\n expandedSessionIds,\n onToggleSession,\n onTimeTravelFrame,\n onSwitchSession,\n}: {\n branch: BranchSession;\n depth: number;\n currentSessionId: Id<\"sessions\">;\n rootSessionId: Id<\"sessions\">;\n latestFrameId: Id<\"frames\"> | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n branchNumber: Map, number>;\n childrenByParent: Map;\n forksByFrame: Map;\n expandedSessionIds: Set;\n onToggleSession: (sessionId: Id<\"sessions\">) => void;\n onTimeTravelFrame: (frameId: Id<\"frames\">) => void;\n onSwitchSession: (sessionId: Id<\"sessions\">) => void;\n}) {\n const childBranches = childrenByParent.get(branch.sessionId) ?? [];\n const isCurrent = branch.sessionId === currentSessionId;\n const isRoot = branch.sessionId === rootSessionId;\n const expanded = expandedSessionIds.has(branch.sessionId);\n const orderedFrames = branch.frames\n .slice()\n .sort((a, b) => a.createdAt - b.createdAt);\n const branchLabel = isRoot\n ? \"root\"\n : `branch ${String(branchNumber.get(branch.sessionId) ?? 1).padStart(2, \"0\")}`;\n\n return (\n
\n {depth > 0 && (\n \n )}\n
\n \n
\n onSwitchSession(branch.sessionId)}\n className=\"min-w-0 flex-1 p-3 text-left hover:bg-terminal-bg focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n
\n
\n
\n \n {branchLabel}\n \n {isCurrent && (\n current\n )}\n {isRoot && (\n \n family root\n \n )}\n \n {shortId(branch.sessionId)}\n \n
\n
\n latest{\" \"}\n {branch.latestFrameId ? shortId(branch.latestFrameId) : \"none\"}{\" \"}\n / epoch {branch.contextEpoch}\n
\n
\n
\n
{orderedFrames.length} frames
\n
{childBranches.length} forks
\n
\n
\n \n {childBranches.length > 0 && (\n onToggleSession(branch.sessionId)}\n className=\"w-10 shrink-0 border-l border-terminal-green-dimmer text-terminal-green-dim hover:bg-terminal-bg hover:text-terminal-green focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n {expanded ? \"-\" : \"+\"}\n \n )}\n
\n {\n if (isCurrent) {\n onTimeTravelFrame(frameId);\n } else {\n onSwitchSession(branch.sessionId);\n }\n }}\n />\n {branch.forkedFromSessionId && branch.forkedFromFrameId && (\n
\n forked from {shortId(branch.forkedFromSessionId)} at{\" \"}\n {shortId(branch.forkedFromFrameId)}\n
\n )}\n
\n {expanded && childBranches.length > 0 && (\n
\n {childBranches.map((child) => (\n \n ))}\n
\n )}\n \n \n );\n}\n\nfunction BranchFrameRail({\n frames,\n currentSession,\n latestFrameId,\n timeTravelFrameId,\n forksByFrame,\n onFrameClick,\n}: {\n frames: FrameDoc[];\n currentSession: boolean;\n latestFrameId: Id<\"frames\"> | null;\n timeTravelFrameId: Id<\"frames\"> | null;\n forksByFrame: Map;\n onFrameClick: (frameId: Id<\"frames\">) => void;\n}) {\n if (frames.length === 0) {\n return (\n
\n no frames\n
\n );\n }\n\n return (\n
\n
\n \n {frames.map((frame, index) => {\n const messages = normalizeFrameMessages(frame.messages);\n const isLatest = frame._id === latestFrameId;\n const isSelected =\n currentSession && frame._id === timeTravelFrameId && !isLatest;\n const forkCount = forksByFrame.get(frame._id) ?? 0;\n const label = String(index + 1).padStart(2, \"0\");\n return (\n onFrameClick(frame._id)}\n title={frameSummary(frame, messages)}\n className=\"relative z-[1] flex w-12 shrink-0 flex-col items-center gap-1 text-center focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n 0\n ? \"border-terminal-cyan bg-terminal-bg text-terminal-cyan\"\n : \"border-terminal-green-dimmer bg-terminal-bg text-terminal-green-dim hover:border-terminal-green hover:text-terminal-green\"\n }`}\n >\n {label}\n \n \n {forkCount > 0\n ? `${forkCount} fork${forkCount === 1 ? \"\" : \"s\"}`\n : messageKind(messages)}\n \n \n );\n })}\n
\n
\n );\n}\n\nfunction BranchMetric({ label, value }: { label: string; value: number }) {\n return (\n
\n
{label}
\n
{value}
\n
\n );\n}\n\nfunction firstFrameTime(branch: BranchSession) {\n return branch.frames.reduce(\n (earliest, frame) => Math.min(earliest, frame.createdAt),\n Number.POSITIVE_INFINITY,\n );\n}\n\nfunction messageKind(messages: FrameMessage[]) {\n const first = messages[0];\n if (!first) return \"state\";\n return first.type ?? first.role ?? \"msg\";\n}\n\nfunction HistoryFrameMessage({ message }: { message: FrameMessage }) {\n const type = message.type ?? message.role ?? \"message\";\n return (\n
\n
\n \n {type}\n \n {message.name && (\n {message.name}\n )}\n
\n {typeof message.text === \"string\" ? (\n
\n {message.text}\n
\n ) : (\n 
\n          {JSON.stringify(message, null, 2)}\n
\n )}\n
\n );\n}\n\nfunction CollapsedFrameMessages({\n frame,\n messages,\n}: {\n frame: FrameDoc;\n messages: FrameMessage[];\n}) {\n if (messages.length === 0) {\n return (\n
no messages
\n );\n }\n\n return (\n
\n {messages.map((message, index) => {\n const preview = collapsedFrameMessagePreview(frame, message);\n return (\n
\n \n {preview.label}:\n \n \n {preview.detail}\n \n
\n );\n })}\n
\n );\n}\n\nfunction EmptyHistory({ children }: { children: ReactNode }) {\n return (\n
{children}
\n );\n}\n\nfunction normalizeFrameMessages(messages: unknown): FrameMessage[] {\n return Array.isArray(messages) ? (messages as FrameMessage[]) : [];\n}\n\nfunction toggleSetValue(current: Set, value: string): Set {\n const next = new Set(current);\n if (next.has(value)) next.delete(value);\n else next.add(value);\n return next;\n}\n\nfunction toggleInclusiveFilter(\n current: Set,\n value: string,\n allValues: string[],\n): Set {\n if (allValues.length === 0) return new Set();\n\n const source = current.size === 0 ? new Set(allValues) : current;\n const next = toggleSetValue(source, value);\n return next.size === 0 || next.size === allValues.length ? new Set() : next;\n}\n\nfunction frameTypeFilterOptions(\n frameViews: FrameHistoryView[],\n): HistoryFilterOption[] {\n const counts = new Map();\n for (const view of frameViews) {\n counts.set(view.frameType, (counts.get(view.frameType) ?? 0) + 1);\n }\n return ([\"instance\", \"work\", \"actor\"] as FrameType[])\n .map((frameType) => ({\n value: frameType,\n label: frameType,\n count: counts.get(frameType) ?? 0,\n }))\n .filter((option) => option.count > 0);\n}\n\nfunction nodeFilterOptions(\n projectionTree: CompiledProjectionTree | undefined,\n frameViews: FrameHistoryView[],\n activationGeneratorIds: Map,\n): HistoryFilterOption[] {\n if (!projectionTree) return [];\n\n return collectContributorFilterOptions(projectionTree)\n .map((option) => ({\n ...option,\n count: frameViews.filter((view) =>\n frameMatchesRuntime(\n view.frame,\n view.messages,\n activationGeneratorIds,\n option.value,\n ),\n ).length,\n }))\n .filter((option) => option.count > 0);\n}\n\nfunction collectContributorFilterOptions(\n tree: CompiledProjectionTree,\n): HistoryFilterOption[] {\n const options: HistoryFilterOption[] = [];\n const seen = new Set();\n const visit = (node: CompiledContributor) => {\n if (!seen.has(node.id)) {\n seen.add(node.id);\n const label = node.name ?? node.nodeKey;\n const detailParts = [node.kind, node.runtime.concurrency].filter(Boolean);\n options.push({\n value: node.id,\n label,\n detail: detailParts.join(\" \"),\n count: 0,\n title: `${label} ${node.id}`,\n });\n }\n node.children.forEach(visit);\n };\n tree.roots.forEach(visit);\n return options;\n}\n\nfunction activationGeneratorIdsFromFrameViews(\n frameViews: FrameHistoryView[],\n): Map {\n const runtimeIds = new Map();\n for (const view of frameViews) {\n const frameActivationId = stringValue(view.frame.activationId);\n const frameGeneratorId = stringValue(view.frame.generatorId);\n if (frameActivationId && frameGeneratorId) {\n runtimeIds.set(frameActivationId, frameGeneratorId);\n }\n\n for (const message of view.messages) {\n if (message.type !== \"work\" || message.kind !== \"activation\") continue;\n const activationId = stringValue(message.activationId);\n const generatorId = stringValue(message.generatorId);\n if (activationId && generatorId) {\n runtimeIds.set(activationId, generatorId);\n }\n }\n }\n return runtimeIds;\n}\n\nfunction frameMatchesRuntime(\n frame: FrameDoc,\n messages: FrameMessage[],\n activationGeneratorIds: Map,\n generatorId: string,\n): boolean {\n if (stringValue(frame.generatorId) === generatorId) return true;\n if (\n generatorMatchesGenerator(stringValue(frame.generatorId), generatorId)\n )\n return true;\n const frameActivationId = stringValue(frame.activationId);\n if (\n frameActivationId &&\n activationGeneratorIds.get(frameActivationId) === generatorId\n )\n return true;\n\n return messages.some((message) => {\n if (stringValue(message.generatorId) === generatorId)\n return true;\n if (\n generatorMatchesGenerator(\n stringValue(message.generatorId),\n generatorId,\n )\n )\n return true;\n const activationId = stringValue(message.activationId);\n return Boolean(\n activationId &&\n activationGeneratorIds.get(activationId) === generatorId,\n );\n });\n}\n\nfunction generatorMatchesGenerator(\n frameGeneratorId: string | undefined,\n generatorId: string,\n): boolean {\n return frameGeneratorId === generatorId;\n}\n\nfunction frameTypeForFrame(frame: FrameDoc, messages: FrameMessage[]): FrameType {\n if (messages.some((message) => message.type === \"work\")) return \"work\";\n if (messages.length === 0) return \"instance\";\n if (messages.some((message) => message.type === \"instance\")) return \"instance\";\n return \"actor\";\n}\n\nfunction frameSummary(frame: FrameDoc, messages: FrameMessage[]) {\n const preview = messages.map(messagePreview).find(Boolean);\n if (preview) return preview;\n return `state frame`;\n}\n\nfunction messagePreview(message: FrameMessage) {\n if (typeof message.text === \"string\" && message.text.trim()) {\n return message.text.replace(/\\s+/g, \" \").trim();\n }\n if (typeof message.value === \"string\" && message.value.trim()) {\n return message.value.replace(/\\s+/g, \" \").trim();\n }\n if (message.name) return `${message.type ?? \"tool\"} ${message.name}`;\n return \"\";\n}\n\nfunction collapsedFrameMessagePreview(frame: FrameDoc, message: FrameMessage) {\n if (isToolCallMessage(message)) {\n return {\n label: \"tool call\",\n detail:\n stringValue(message.name) ??\n stringValue(message.toolName) ??\n stringValue(message.tool_name) ??\n \"unknown\",\n };\n }\n\n if (message.type === \"action\") {\n const name = stringValue(message.name);\n const kind = stringValue(message.kind);\n const action = stringValue(message.action);\n const label = `${action ?? \"action\"} ${kind ?? \"message\"}`;\n const payload =\n kind === \"request\"\n ? message.input\n : message.error ?? message.value ?? message.outputMessageIndices;\n const result = truncatePreview(renderUnknown(payload));\n return {\n label,\n detail: [name, result].filter(Boolean).join(\" \"),\n };\n }\n\n if (message.type === \"user\" || message.role === \"user\") {\n return {\n label: \"user\",\n detail: truncatePreview(renderMessageText(message)),\n };\n }\n\n if (message.type === \"assistant\" || message.role === \"assistant\") {\n return {\n label: \"assistant\",\n detail: truncatePreview(renderMessageText(message)),\n };\n }\n\n if (message.type === \"instance\") {\n return { label: \"instance\", detail: renderInstanceMessage(message) };\n }\n\n if (message.type === \"work\") {\n return { label: \"work\", detail: renderWorkMessage(frame, message) };\n }\n\n return {\n label: stringValue(message.type) ?? stringValue(message.role) ?? \"message\",\n detail: truncatePreview(renderUnknown(message)),\n };\n}\n\nfunction isToolCallMessage(message: FrameMessage) {\n return (\n message.type === \"tool_call\" ||\n message.type === \"tool-call\" ||\n message.kind === \"tool.call\" ||\n message.kind === \"tool_call\"\n );\n}\n\nfunction renderMessageText(message: FrameMessage) {\n return renderUnknown(message.text ?? message.content ?? message.value);\n}\n\nfunction renderInstanceMessage(message: FrameMessage) {\n const kind = stringValue(message.kind);\n if (kind === \"state.update\") {\n const update = recordValue(message.update);\n const op = stringValue(update?.op) ?? \"update\";\n const value = op === \"append\" ? update?.values : update?.value;\n return `${op} ${formatKeyList(keysFromRecord(recordValue(value)), stringValue(message.stateKey))}`;\n }\n if (kind) return truncatePreview(kind.replace(/^state\\./, \"\"));\n return \"update\";\n}\n\nfunction renderWorkMessage(frame: FrameDoc, message: FrameMessage) {\n const kind = stringValue(message.kind) ?? \"work\";\n const generatorId =\n stringValue(message.generatorId) ?? frame.generatorId ?? \"\";\n return [kind, generatorId].filter(Boolean).join(\" \");\n}\n\nfunction formatKeyList(keys: string[], fallback?: string) {\n const values = keys.length > 0 ? keys : fallback ? [fallback] : [];\n return values.length > 0 ? values.join(\", \") : \"unknown\";\n}\n\nfunction keysFromRecord(record: Record | undefined) {\n return record ? Object.keys(record) : [];\n}\n\nfunction recordValue(value: unknown) {\n return value && typeof value === \"object\" && !Array.isArray(value)\n ? (value as Record)\n : undefined;\n}\n\nfunction stringValue(value: unknown) {\n return typeof value === \"string\" && value ? value : undefined;\n}\n\nfunction renderUnknown(value: unknown) {\n if (value === undefined || value === null) return \"\";\n if (typeof value === \"string\") return value;\n if (typeof value === \"number\" || typeof value === \"boolean\")\n return String(value);\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nfunction truncatePreview(value: string, maxLength = 160) {\n const compact = value.replace(/\\s+/g, \" \").trim();\n if (compact.length <= maxLength) return compact;\n return `${compact.slice(0, maxLength - 3)}...`;\n}\n\nfunction shortId(id: string) {\n return id.slice(-8);\n}\n\nfunction InstanceNode({\n instance,\n depth,\n defaultExpanded = false,\n}: {\n instance: SandboxClientInstance;\n depth: number;\n defaultExpanded?: boolean;\n}) {\n const [expanded, setExpanded] = useState(defaultExpanded);\n const details =\n instance.states.length + instance.tools.length + instance.commands.length;\n const childCount = instance.members.length + instance.children.length;\n\n return (\n
\n setExpanded((value) => !value)}\n aria-expanded={expanded}\n className=\"flex w-full items-center gap-2 rounded px-1 py-1 text-left hover:bg-terminal-bg-lighter focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n style={{ paddingLeft: depth * 16 + 4 }}\n >\n \n {expanded ? \"-\" : \"+\"}\n \n {instance.kind}\n {instance.nodeKey}\n {instance.name && (\n {instance.name}\n )}\n {instance.contributor.runtimeType}\n \n {instance.contributor.id}\n \n \n {details} meta / {childCount} child\n \n \n {expanded && (\n \n \n )\n : []),\n ]}\n />\n \n \n \n \n {instance.members.length > 0 ? (\n \n {instance.members.map((member) => (\n \n ))}\n \n ) : (\n members empty\n )}\n {instance.children.length > 0 ? (\n \n {instance.children.map((child) => (\n \n ))}\n \n ) : (\n children empty\n )}\n
\n )}\n \n );\n}\n\nfunction Contributor({\n node,\n depth,\n}: {\n node: CompiledContributor;\n depth: number;\n}) {\n const [expanded, setExpanded] = useState(true);\n\n return (\n
\n setExpanded((value) => !value)}\n aria-expanded={expanded}\n className=\"flex w-full items-center gap-2 rounded px-1 py-1 text-left hover:bg-terminal-bg-lighter focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n style={{ paddingLeft: depth * 16 + 4 }}\n >\n \n {expanded ? \"-\" : \"+\"}\n \n {node.kind}\n {node.nodeKey}\n {node.name && (\n {node.name}\n )}\n \n {(Array.isArray(node.runtime.trigger)\n ? node.runtime.trigger\n : [node.runtime.trigger]\n )\n .map((trigger) => trigger.type)\n .join(\", \")}\n \n \n {node.id}\n \n \n system {node.compiled.preamble.length} / dynamic{\" \"}\n {node.compiled.recency.length} / tools{\" \"}\n {node.compiled.tools.length}\n \n \n {expanded && (\n \n )\n : []),\n ]}\n />\n \n \n {node.children.length > 0 ? (\n \n {node.children.map((child) => (\n \n ))}\n \n ) : (\n child generators empty\n )}\n
\n )}\n \n );\n}\n\nfunction CompiledPayload({ node }: { node: CompiledContributor }) {\n const empty =\n node.compiled.preamble.length === 0 &&\n node.compiled.recency.length === 0 &&\n node.compiled.tools.length === 0 &&\n node.compiled.retrievableStates.length === 0;\n\n if (empty) {\n return compiled output empty;\n }\n\n return (\n \n \n \n \n \n \n );\n}\n\nfunction ContributorList({\n contributors,\n}: {\n contributors: CompiledContributorView[];\n}) {\n if (contributors.length === 0) {\n return contributors empty;\n }\n\n return (\n \n
\n {contributors.map((contributor) => (\n \n
\n {contributor.kind}\n {contributor.nodeKey}\n {contributor.name && (\n \n {contributor.name}\n \n )}\n \n {contributor.id}\n \n
\n \n \n \n \n
\n ))}\n \n \n );\n}\n\ntype StateProjectionMeta = {\n slot?: string;\n region?: string;\n exposure?: string;\n};\n\nfunction stateProjectionLabel(projection: StateProjectionMeta): string {\n return [\n projection.exposure,\n projection.slot ? `slot:${projection.slot}` : undefined,\n projection.region ? `region:${projection.region}` : undefined,\n ]\n .filter(Boolean)\n .join(\" \");\n}\n\nfunction StateList({\n states,\n}: {\n states: Array<{\n key: string;\n address: unknown;\n projection?: StateProjectionMeta;\n value: unknown;\n }>;\n}) {\n if (states.length === 0) {\n return states empty;\n }\n\n return (\n \n
\n {states.map((state) => (\n \n ))}\n
\n \n );\n}\n\nfunction StateTreeItem({\n state,\n}: {\n state: {\n key: string;\n address: unknown;\n projection?: StateProjectionMeta;\n value: unknown;\n };\n}) {\n return (\n \n {state.key}\n {state.projection && (\n \n {stateProjectionLabel(state.projection)}\n \n )}\n \n {addressLabel(state.address)}\n \n \n }\n preview={jsonInline(state.value)}\n >\n \n \n );\n}\n\nfunction ActionList({\n title,\n actions,\n}: {\n title: string;\n actions: Array<{\n name: string;\n description?: string;\n target?: unknown;\n inputSchema?: unknown;\n }>;\n}) {\n if (actions.length === 0) {\n return {title} empty;\n }\n\n return (\n \n
\n {actions.map((action, index) => (\n \n ))}\n
\n \n );\n}\n\nfunction ActionTreeItem({\n action,\n}: {\n action: {\n name: string;\n description?: string;\n target?: unknown;\n inputSchema?: unknown;\n };\n}) {\n const hasDetails = action.description || action.inputSchema !== undefined;\n\n return (\n \n {action.name}\n {action.target !== undefined && (\n \n {addressLabel(action.target)}\n \n )}\n \n }\n preview={\n action.description ??\n (action.inputSchema !== undefined ? \"input schema\" : \"\")\n }\n >\n
\n {action.description && (\n
{action.description}
\n )}\n {action.inputSchema !== undefined && (\n \n )}\n {!hasDetails && no action details}\n
\n \n );\n}\n\nfunction JsonDisclosure({ title, value }: { title: string; value: unknown }) {\n const [expanded, setExpanded] = useState(false);\n return (\n \n setExpanded((current) => !current)}\n aria-expanded={expanded}\n className=\"flex w-full items-center gap-2 px-2 py-1 text-left hover:bg-terminal-bg-lighter focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n \n {expanded ? \"-\" : \"+\"}\n \n {title}\n \n {jsonInline(value)}\n \n \n {expanded && (\n 
\n          {JSON.stringify(value, null, 2)}\n
\n )}\n \n );\n}\n\nfunction JsonPreview({\n value,\n className = \"bg-terminal-bg\",\n}: {\n value: unknown;\n className?: string;\n}) {\n return (\n \n {JSON.stringify(value, null, 2)}\n \n );\n}\n\nfunction TreeDisclosure({\n title,\n preview,\n expandedClassName = \"bg-lightener\",\n expandedBodyClassName = \"\",\n children,\n}: {\n title: ReactNode;\n preview?: ReactNode;\n expandedClassName?: string;\n expandedBodyClassName?: string;\n children: ReactNode;\n}) {\n const [expanded, setExpanded] = useState(false);\n return (\n \n setExpanded((current) => !current)}\n aria-expanded={expanded}\n className=\"flex w-full min-w-0 items-center gap-2 px-2 py-1 text-left hover:bg-terminal-bg-lighter focus:outline-none focus:ring-1 focus:ring-terminal-green\"\n >\n \n {expanded ? \"-\" : \"+\"}\n \n
{title}
\n {preview && (\n \n {preview}\n \n )}\n \n {expanded && (\n \n {children}\n \n )}\n \n );\n}\n\nfunction TreeSection({\n title,\n children,\n}: {\n title: string;\n children: ReactNode;\n}) {\n return (\n
\n
\n {title}\n
\n {children}\n
\n );\n}\n\nfunction KeyValueRows({\n rows,\n plain = false,\n}: {\n rows: Array<[string, ReactNode]>;\n plain?: boolean;\n}) {\n return (\n \n {rows.map(([key, value]) => (\n
\n {key}\n \n {value}\n \n
\n ))}\n \n );\n}\n\nfunction EmptyTree({ children }: { children: ReactNode }) {\n return (\n
{children}
\n );\n}\n\nfunction MutedLine({ children }: { children: ReactNode }) {\n return (\n
{children}
\n );\n}\n\nfunction addressLabel(value: unknown): string {\n if (!value || typeof value !== \"object\") {\n return String(value);\n }\n const record = value as Record;\n if (record.type === \"instance\") {\n return `instance:${String(record.instanceId)}`;\n }\n if (record.type === \"member\" && Array.isArray(record.memberPath)) {\n return `member:${String(record.ownerInstanceId)}/${record.memberPath.join(\"/\")}`;\n }\n if (\"instanceId\" in record && \"stateKey\" in record) {\n return `${String(record.instanceId)}:${String(record.stateKey)}`;\n }\n return JSON.stringify(value);\n}\n\nfunction jsonInline(value: unknown): string {\n const text = JSON.stringify(value);\n if (!text) {\n return \"\";\n }\n return text.length > 96 ? `${text.slice(0, 93)}...` : text;\n}\n\nfunction StateTab({ instances }: { instances: SandboxClientInstance[] }) {\n const states = collectStates(instances);\n return (\n
\n {states.map((state) => (\n \n
\n {state.key}\n
\n 
\n            {JSON.stringify(state.value, null, 2)}\n
\n \n ))}\n
\n );\n}\n\nfunction collectStates(instances: SandboxClientInstance[]) {\n const states: SandboxClientInstance[\"states\"] = [];\n const visit = (instance: SandboxClientInstance) => {\n states.push(...instance.states);\n instance.members.forEach(visit);\n instance.children.forEach(visit);\n };\n instances.forEach(visit);\n return states;\n}\n\nfunction CommandsTab({\n instances,\n effigy,\n readOnly,\n}: {\n instances: SandboxClientInstance[];\n effigy: OptimisticEffigy;\n readOnly: boolean;\n}) {\n const commands = collectCommands(instances);\n const sandboxAddress = instances[0]?.states.find(\n (item) => item.key === \"sandbox\",\n )?.address;\n const run = async (commandMeta: SandboxClientInstance[\"commands\"][number]) => {\n if (readOnly) return;\n const name = commandMeta.name;\n const input =\n name === \"setThemeHue\"\n ? { hue: Math.round(Math.random() * 360) }\n : name.startsWith(\"set\")\n ? { enabled: true }\n : {};\n const command = effigy.getCommand(name as never, {\n target: commandMeta.target,\n ...(name === \"setThemeHue\"\n ? {\n optimistic: (ctx) => {\n if (sandboxAddress && \"hue\" in input) {\n ctx.patchAt(sandboxAddress, { themeHue: input.hue });\n }\n },\n }\n : {}),\n });\n await command.run(input as never);\n };\n return (\n
\n {commands.map((command) => (\n run(command)}\n disabled={readOnly}\n className=\"block w-full rounded border border-terminal-green-dimmer px-3 py-2 text-left text-sm text-terminal-green hover:border-terminal-green disabled:cursor-not-allowed disabled:opacity-40\"\n >\n
{command.name}
\n
\n {readOnly\n ? \"fork session to run commands\"\n : (command.description ?? \"client command\")}\n
\n \n ))}\n
\n );\n}\n\nfunction PlaygroundTab({\n instances,\n canonicalInstances,\n effigy,\n readOnly,\n}: {\n instances: SandboxClientInstance[];\n canonicalInstances: SandboxClientInstance[];\n effigy: OptimisticEffigy;\n readOnly: boolean;\n}) {\n const optimisticState = findState(instances, \"agentControls\");\n const canonicalState = findState(canonicalInstances, \"agentControls\");\n const optimisticControls = optimisticState?.value as\n | { testCounter?: number }\n | undefined;\n const canonicalControls = canonicalState?.value as\n | { testCounter?: number }\n | undefined;\n const optimisticValue = optimisticControls?.testCounter ?? 0;\n const canonicalValue = canonicalControls?.testCounter ?? 0;\n const incrementCommand = findCommand(instances, \"incrementTestCounter\");\n const recentCommandResidue = effigy.getRecentCommandResidue();\n const pendingCommands = effigy.getPendingCommands();\n\n const increment = async () => {\n if (readOnly || !optimisticState || !incrementCommand) return;\n const command = effigy.getCommand(\"incrementTestCounter\" as never, {\n target: incrementCommand.target,\n optimistic: (ctx) => {\n ctx.patchAt(optimisticState.address, {\n testCounter: optimisticValue + 1,\n });\n },\n });\n await command.run({ amount: 1 } as never);\n };\n\n return (\n
\n
\n
\n
\n test counter\n
\n
\n \n \n
\n
\n void increment()}\n disabled={readOnly || !optimisticState || !incrementCommand}\n className=\"rounded border border-terminal-green-dimmer px-3 py-2 text-left text-terminal-green hover:border-terminal-green disabled:cursor-not-allowed disabled:opacity-50\"\n >\n {readOnly ? \"fork session to edit\" : \"increment counter\"}\n \n
\n \n \n [\n command.name,\n command.callId,\n command.target ? addressLabel(command.target) : undefined,\n ]\n .filter(Boolean)\n .join(\" / \"),\n )}\n />\n
\n
\n
\n );\n}\n\nfunction Metric({ label, value }: { label: string; value: number }) {\n return (\n
\n
{label}
\n
{value}
\n
\n );\n}\n\nfunction DebugList({ label, values }: { label: string; values: readonly string[] }) {\n return (\n
\n
{label}
\n {values.length > 0 ? (\n
\n {values.map((value) => (\n
\n {value}\n
\n ))}\n
\n ) : (\n
empty
\n )}\n
\n );\n}\n\nfunction DevTab({\n snapshot,\n onResetSession,\n messageTransport,\n onMessageTransportChange,\n liveKitStatus,\n liveKitReady,\n}: {\n snapshot: SandboxClientSnapshot;\n onResetSession: () => void;\n messageTransport: MessageTransport;\n onMessageTransportChange: (transport: MessageTransport) => void;\n liveKitStatus: { status: string; detail?: string };\n liveKitReady: boolean;\n}) {\n return (\n
\n
\n
\n
\n message transport\n
\n
\n {([\"convex\", \"livekit\"] as MessageTransport[]).map((transport) => (\n onMessageTransportChange(transport)}\n className={`rounded border px-3 py-2 text-left text-sm ${\n messageTransport === transport\n ? \"border-terminal-green text-terminal-green\"\n : \"border-terminal-green-dimmer text-terminal-green-dim hover:border-terminal-green hover:text-terminal-green\"\n }`}\n >\n {transport}\n \n ))}\n
\n
\n
\n
\n livekit rpc\n \n {liveKitReady ? \"ready\" : liveKitStatus.status}\n \n
\n {liveKitStatus.detail && (\n \n {liveKitStatus.detail}\n
\n )}\n
\n \n\n \n reset session\n \n 
\n        {JSON.stringify(snapshot, null, 2)}\n
\n \n );\n}\n\nfunction collectCommands(instances: SandboxClientInstance[]) {\n const commands: SandboxClientInstance[\"commands\"] = [];\n const visit = (instance: SandboxClientInstance) => {\n commands.push(...instance.commands);\n instance.members.forEach(visit);\n instance.children.forEach(visit);\n };\n instances.forEach(visit);\n return commands;\n}\n\nfunction findState(\n instances: SandboxClientInstance[],\n key: string,\n): SandboxClientInstance[\"states\"][number] | undefined {\n for (const instance of instances) {\n const state = findStateInInstance(instance, key);\n if (state) return state;\n }\n return undefined;\n}\n\nfunction findCommand(\n instances: SandboxClientInstance[],\n name: string,\n): SandboxClientInstance[\"commands\"][number] | undefined {\n for (const instance of instances) {\n const command = findCommandInInstance(instance, name);\n if (command) return command;\n }\n return undefined;\n}\n\nfunction findCommandInInstance(\n instance: SandboxClientInstance,\n name: string,\n): SandboxClientInstance[\"commands\"][number] | undefined {\n const command = instance.commands.find((item) => item.name === name);\n if (command) return command;\n for (const member of instance.members) {\n const found = findCommandInInstance(member, name);\n if (found) return found;\n }\n for (const child of instance.children) {\n const found = findCommandInInstance(child, name);\n if (found) return found;\n }\n return undefined;\n}\n\nfunction isWithinBottom(element: HTMLElement, threshold: number) {\n return element.scrollHeight - element.scrollTop - element.clientHeight <= threshold;\n}\n\nfunction findStateInInstance(\n instance: SandboxClientInstance,\n key: string,\n): SandboxClientInstance[\"states\"][number] | undefined {\n const state = instance.states.find((item) => item.key === key);\n if (state) return state;\n for (const member of instance.members) {\n const found = findStateInInstance(member, key);\n if (found) return found;\n }\n for (const child of instance.children) {\n const found = findStateInInstance(child, key);\n if (found) return found;\n }\n return undefined;\n}\n","/apps/sandbox/app/components/terminal/TerminalPane.tsx":"\"use client\";\n\nimport { forwardRef, useCallback, useEffect, useLayoutEffect, useRef, useState, type DragEvent } from \"react\";\nimport type { ForwardedRef } from \"react\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\nimport type { SandboxAttachment, SandboxClientInstance, SandboxMessage } from \"@/src/types/display\";\nimport type { OptimisticContext } from \"@projectors/core/client\";\nimport type { LocalVideoTrack } from \"livekit-client\";\nimport { scanlinesEnabledAtom, type MessageTransport } from \"@/src/atoms\";\nimport { useAtom } from \"jotai\";\nimport { useProjector } from \"@/src/projector/ProjectorProvider\";\n\ntype TerminalPaneProps = {\n messages: SandboxMessage[];\n input: string;\n attachments: TerminalAttachment[];\n attachmentsUploading: boolean;\n onInputChange: (value: string) => void;\n onAttachmentFiles: (files: File[]) => void;\n onRemoveAttachment: (storageId: string) => void;\n onSend: () => void;\n onForkSession: () => void;\n onReturnToLatest: () => void;\n isLoading: boolean;\n isTimeTraveling: boolean;\n timeTravelFrameId: Id<\"frames\"> | null;\n connectionError?: string | null;\n voiceStatus?: { status: string; detail?: string };\n messageTransport: MessageTransport;\n liveKitEnabled?: boolean;\n liveKitReady?: boolean;\n liveKitStatus?: { status: string; detail?: string };\n localCameraTrack?: LocalVideoTrack | null;\n};\n\nexport const TerminalPane = forwardRef(\n function TerminalPane(\n {\n messages,\n input,\n attachments,\n attachmentsUploading,\n onInputChange,\n onAttachmentFiles,\n onRemoveAttachment,\n onSend,\n onForkSession,\n onReturnToLatest,\n isLoading,\n isTimeTraveling,\n timeTravelFrameId,\n connectionError,\n voiceStatus,\n messageTransport,\n liveKitEnabled = false,\n liveKitReady = false,\n liveKitStatus,\n localCameraTrack,\n },\n ref,\n ) {\n const scrollRef = useRef(null);\n const shouldAutoScrollRef = useRef(true);\n const textareaRef = useRef(null);\n const [liveKitDiagnosticsOpen, setLiveKitDiagnosticsOpen] = useState(false);\n const [dragDepth, setDragDepth] = useState(0);\n const [scanlinesEnabled, setScanlinesEnabled] = useAtom(scanlinesEnabledAtom);\n const { effigy, instances, readOnly } = useProjector();\n const state = getAgentControlsState(instances);\n const voiceEnabled = Boolean(state?.liveMode);\n const cameraEnabled = Boolean(state?.cameraEnabled);\n const memoryEnabled = Boolean(state?.memoryEnabled);\n const streamingEnabled = state?.streamingEnabled !== false;\n const liveKitTransportActive = messageTransport === \"livekit\";\n const liveKitUnavailable = liveKitTransportActive && liveKitEnabled && !liveKitReady;\n const hasAttachments = attachments.length > 0;\n const isDraggingFiles = dragDepth > 0;\n const sendDisabled = isLoading || attachmentsUploading || (!input.trim() && !hasAttachments) || liveKitUnavailable;\n const handleSubmit = () => {\n if (liveKitUnavailable) {\n setLiveKitDiagnosticsOpen(true);\n return;\n }\n onSend();\n };\n const handleDragEnter = (event: DragEvent) => {\n if (!hasDraggedFiles(event) || isTimeTraveling) return;\n event.preventDefault();\n setDragDepth((depth) => depth + 1);\n };\n const handleDragOver = (event: DragEvent) => {\n if (!hasDraggedFiles(event) || isTimeTraveling) return;\n event.preventDefault();\n event.dataTransfer.dropEffect = \"copy\";\n };\n const handleDragLeave = (event: DragEvent) => {\n if (!hasDraggedFiles(event) || isTimeTraveling) return;\n event.preventDefault();\n setDragDepth((depth) => Math.max(0, depth - 1));\n };\n const handleDrop = (event: DragEvent) => {\n if (!hasDraggedFiles(event) || isTimeTraveling) return;\n event.preventDefault();\n setDragDepth(0);\n const files = Array.from(event.dataTransfer.files).filter((file) => file.size > 0);\n if (files.length > 0) onAttachmentFiles(files);\n };\n const setTextareaRefs = useCallback(\n (node: HTMLTextAreaElement | null) => {\n textareaRef.current = node;\n setForwardedRef(ref, node);\n },\n [ref],\n );\n\n useLayoutEffect(() => {\n const node = scrollRef.current;\n if (node && shouldAutoScrollRef.current) {\n node.scrollTop = node.scrollHeight;\n }\n }, [messages.length, isLoading, input]);\n\n useLayoutEffect(() => {\n const scrollNode = scrollRef.current;\n if (!scrollNode) return;\n\n const scrollToBottomIfPinned = () => {\n if (shouldAutoScrollRef.current) {\n scrollNode.scrollTop = scrollNode.scrollHeight;\n }\n };\n const contentNode = scrollNode.firstElementChild;\n const observer = new ResizeObserver(scrollToBottomIfPinned);\n observer.observe(scrollNode);\n if (contentNode) observer.observe(contentNode);\n scrollToBottomIfPinned();\n return () => observer.disconnect();\n }, []);\n\n useEffect(() => {\n const node = textareaRef.current;\n if (!node) return;\n node.style.height = \"32px\";\n node.style.height = `${Math.min(node.scrollHeight, 160)}px`;\n }, [input]);\n\n const runToggle = async (name: string, enabled: boolean) => {\n if (readOnly) return;\n const commandMeta = findCommand(instances, name);\n const command = effigy.getCommand(name as never, {\n target: commandMeta?.target,\n optimistic: (ctx: OptimisticContext) => {\n const address = findState(instances, \"agentControls\")?.address;\n if (!address) return;\n const field =\n name === \"setVoiceEnabled\"\n ? \"liveMode\"\n : name === \"setCameraEnabled\"\n ? \"cameraEnabled\"\n : name === \"setMemoryEnabled\"\n ? \"memoryEnabled\"\n : \"streamingEnabled\";\n ctx.patchAt(address, { [field]: enabled });\n },\n });\n await command.run({ enabled } as never);\n };\n\n return (\n \n
\n

MESSAGES

\n
\n runToggle(\"setStreamingEnabled\", !streamingEnabled)}\n />\n runToggle(\"setVoiceEnabled\", !voiceEnabled)}\n />\n runToggle(\"setCameraEnabled\", !cameraEnabled)}\n />\n runToggle(\"setMemoryEnabled\", !memoryEnabled)}\n />\n setScanlinesEnabled(!scanlinesEnabled)}\n >\n scanlines\n \n
\n
\n\n {cameraEnabled && (\n \n )}\n\n {\n shouldAutoScrollRef.current = isWithinBottom(event.currentTarget, 10);\n }}\n className={`terminal-scrollbar min-h-0 flex-1 space-y-4 overflow-y-auto p-4 ${\n cameraEnabled ? \"sm:pr-52\" : \"\"\n }`}\n >\n
\n {messages.length === 0 ? (\n
\n

\n Start with a message like{\" \"}\n remember this sandbox uses projector.\n

\n {connectionError && (\n

\n Convex is not connected: {connectionError}\n

\n )}\n {voiceEnabled && voiceStatus && (\n

\n voice {voiceStatus.status}\n {voiceStatus.detail ? `: ${voiceStatus.detail}` : \"\"}\n

\n )}\n
\n ) : (\n <>\n {messages.map((message) => )}\n {connectionError && (\n

\n {connectionError}\n

\n )}\n \n )}\n {voiceEnabled && messages.length > 0 && voiceStatus && (\n
\n voice {voiceStatus.status}\n {voiceStatus.detail ? `: ${voiceStatus.detail}` : \"\"}\n
\n )}\n {isLoading &&
agent is updating projector state...
}\n
\n \n\n {isTimeTraveling ? (\n
\n
\n
\n
historical frame
\n
\n {timeTravelFrameId ? shortId(timeTravelFrameId) : \"selected\"}\n
\n
\n
\n \n live latest\n \n \n fork session\n \n
\n
\n
\n ) : (\n {\n event.preventDefault();\n handleSubmit();\n }}\n >\n {(attachments.length > 0 || attachmentsUploading) && (\n
\n {attachments.map((attachment) => (\n onRemoveAttachment(attachment.storageId)}\n />\n ))}\n {attachmentsUploading && (\n
\n uploading...\n
\n )}\n
\n )}\n
\n >\n onInputChange(event.target.value)}\n onKeyDown={(event) => {\n if (event.key === \"Enter\" && !event.shiftKey) {\n event.preventDefault();\n handleSubmit();\n }\n }}\n rows={1}\n className=\"max-h-40 min-h-8 flex-1 resize-none overflow-y-auto bg-transparent px-1 py-1 text-sm leading-6 text-terminal-green outline-none placeholder:text-terminal-green-dimmer\"\n placeholder=\"Message the projector sandbox...\"\n />\n {liveKitTransportActive ? (\n setLiveKitDiagnosticsOpen(true)}\n className={`flex h-8 w-4 shrink-0 items-center justify-center text-[10px] leading-none ${liveKitStatusColor(liveKitReady, liveKitStatus?.status)}`}\n aria-label=\"Show LiveKit RPC diagnostics\"\n >\n ●\n \n ) : (\n \n ∴\n \n )}\n \n send\n \n
\n \n )}\n {liveKitDiagnosticsOpen && (\n setLiveKitDiagnosticsOpen(false)}\n />\n )}\n \n );\n },\n);\n\nfunction LiveKitDiagnosticsModal({\n ready,\n status,\n onClose,\n}: {\n ready: boolean;\n status?: { status: string; detail?: string };\n onClose: () => void;\n}) {\n return (\n \n event.stopPropagation()}\n >\n
\n
\n

\n LIVEKIT RPC\n

\n
\n session request transport\n
\n
\n \n close\n \n
\n
\n
\n
ready for requests
\n
\n {ready ? \"yes\" : \"no\"}\n
\n
\n
\n
status
\n
\n {status?.status ?? \"connecting\"}\n
\n
\n
\n
detail
\n
\n {status?.detail ?? \"Waiting for the LiveKit room and agent worker to become available.\"}\n
\n
\n
\n \n \n );\n}\n\nfunction liveKitStatusColor(ready: boolean, status?: string): string {\n if (ready) return \"text-terminal-green\";\n return status === \"disconnected\" || status === \"error\" ? \"text-terminal-red\" : \"text-terminal-yellow\";\n}\n\nfunction setForwardedRef(ref: ForwardedRef, value: T | null) {\n if (typeof ref === \"function\") {\n ref(value);\n return;\n }\n if (ref) {\n ref.current = value;\n }\n}\n\nfunction isWithinBottom(element: HTMLElement, threshold: number) {\n return element.scrollHeight - element.scrollTop - element.clientHeight <= threshold;\n}\n\nfunction CameraPreview({ track }: { track: LocalVideoTrack | null }) {\n const videoRef = useRef(null);\n\n useEffect(() => {\n const video = videoRef.current;\n if (!video || !track) return;\n track.attach(video);\n void video.play().catch(() => undefined);\n return () => {\n track.detach(video);\n video.srcObject = null;\n };\n }, [track]);\n\n return (\n
\n
\n {track ? (\n \n ) : (\n
\n camera pending\n
\n )}\n
\n
\n camera {track ? \"publishing\" : \"connecting\"}\n
\n
\n );\n}\n\nfunction Toggle({\n label,\n enabled,\n disabled = false,\n onClick,\n}: {\n label: string;\n enabled: boolean;\n disabled?: boolean;\n onClick: () => void;\n}) {\n return (\n \n {label}\n \n );\n}\n\nfunction TerminalMessage({ message }: { message: SandboxMessage }) {\n const isUser = message.role === \"user\";\n const attachments = message.attachments ?? [];\n return (\n
\n
{isUser ? \"user\" : \"agent\"}
\n
\n {message.content && (\n
{message.content}
\n )}\n {attachments.length > 0 && (\n
\n {attachments.map((attachment) => (\n \n ))}\n
\n )}\n
\n
\n );\n}\n\ntype TerminalAttachment = SandboxAttachment & {\n previewUrl?: string;\n};\n\nfunction PendingAttachmentPreview({\n attachment,\n onRemove,\n}: {\n attachment: TerminalAttachment;\n onRemove: () => void;\n}) {\n if (attachment.kind === \"image\") {\n return (\n
\n \n {attachment.name}\n \n
\n );\n }\n\n return (\n
\n {attachment.name}\n \n
\n );\n}\n\nfunction SentAttachmentPreview({ attachment }: { attachment: SandboxAttachment }) {\n const label = attachment.name || \"attachment\";\n if (attachment.kind === \"image\" && attachment.url) {\n return (\n \n {label}\n \n );\n }\n\n const className = \"max-w-64 truncate rounded border border-terminal-green-dimmer px-2 py-1 text-xs text-terminal-green-dim\";\n if (attachment.url) {\n return (\n \n {label}\n \n );\n }\n return {label};\n}\n\nfunction RemoveAttachmentButton({ onClick }: { onClick: () => void }) {\n return (\n \n x\n \n );\n}\n\nfunction hasDraggedFiles(event: DragEvent): boolean {\n return Array.from(event.dataTransfer.types).includes(\"Files\");\n}\n\nfunction shortId(id: string) {\n return id.slice(-8);\n}\n\nfunction getAgentControlsState(instances: SandboxClientInstance[]) {\n return findState(instances, \"agentControls\")?.value as\n | {\n liveMode?: boolean;\n cameraEnabled?: boolean;\n memoryEnabled?: boolean;\n streamingEnabled?: boolean;\n }\n | undefined;\n}\n\nfunction findState(instances: SandboxClientInstance[], key: string) {\n for (const instance of instances) {\n const state = findStateInInstance(instance, key);\n if (state) return state;\n }\n return undefined;\n}\n\nfunction findCommand(instances: SandboxClientInstance[], name: string) {\n for (const instance of instances) {\n const command = findCommandInInstance(instance, name);\n if (command) return command;\n }\n return undefined;\n}\n\nfunction findCommandInInstance(instance: SandboxClientInstance, name: string): SandboxClientInstance[\"commands\"][number] | undefined {\n const command = instance.commands.find(\n (item: SandboxClientInstance[\"commands\"][number]) => item.name === name,\n );\n if (command) return command;\n for (const member of instance.members) {\n const found = findCommandInInstance(member, name);\n if (found) return found;\n }\n for (const child of instance.children) {\n const found = findCommandInInstance(child, name);\n if (found) return found;\n }\n return undefined;\n}\n\nfunction findStateInInstance(instance: SandboxClientInstance, key: string): SandboxClientInstance[\"states\"][number] | undefined {\n const state = instance.states.find(\n (item: SandboxClientInstance[\"states\"][number]) => item.key === key,\n );\n if (state) return state;\n for (const member of instance.members) {\n const found = findStateInInstance(member, key);\n if (found) return found;\n }\n for (const child of instance.children) {\n const found = findStateInInstance(child, key);\n if (found) return found;\n }\n return undefined;\n}\n","/apps/sandbox/app/globals.css":"@import \"tailwindcss\";\n\n:root {\n --terminal-green: #00ff00;\n --terminal-green-dim: #00aa00;\n --terminal-green-dimmer: #007700;\n --terminal-bg: #0a0a0a;\n --terminal-bg-lighter: #141414;\n --terminal-cyan: #00ffff;\n --terminal-red: #ff4444;\n --terminal-yellow: #ffff00;\n}\n\n@theme inline {\n --color-terminal-green: var(--terminal-green);\n --color-terminal-green-dim: var(--terminal-green-dim);\n --color-terminal-green-dimmer: var(--terminal-green-dimmer);\n --color-terminal-bg: var(--terminal-bg);\n --color-terminal-bg-lighter: var(--terminal-bg-lighter);\n --color-terminal-cyan: var(--terminal-cyan);\n --color-terminal-red: var(--terminal-red);\n --color-terminal-yellow: var(--terminal-yellow);\n --font-sans: var(--font-geist-sans);\n --font-mono: var(--font-geist-mono);\n}\n\nbody {\n background: var(--terminal-bg);\n color: var(--terminal-green);\n font-family: var(--font-geist-mono), monospace;\n /* Ensure content is composited correctly for blend mode */\n isolation: isolate;\n}\n\n.bg-lightener {\n background-color: color-mix(in srgb, var(--terminal-green) 3%, transparent);\n}\n\n/* Gradient overlay that affects text/borders via blend mode */\nbody::after {\n content: \"\";\n position: fixed;\n inset: 0;\n background: linear-gradient(\n 135deg,\n hsl(var(--gradient-hue-1, 180), 100%, 50%) 0%,\n hsl(var(--gradient-hue-2, 280), 100%, 50%) 100%\n );\n mix-blend-mode: multiply;\n pointer-events: none;\n z-index: 9999;\n opacity: var(--gradient-intensity, 0);\n transition: opacity 0.3s ease;\n}\n\n/* Terminal scanlines effect */\n.terminal-scanlines::before {\n content: \"\";\n position: absolute;\n inset: 0;\n background: repeating-linear-gradient(\n 0deg,\n rgba(0, 0, 0, 0.15) 0px,\n rgba(0, 0, 0, 0.15) 1px,\n transparent 1px,\n transparent 2px\n );\n pointer-events: none;\n z-index: 10;\n}\n\n/* Terminal glow effect - uses dynamic color */\n.terminal-glow {\n text-shadow: 0 0 5px var(--glow-color, rgba(0, 255, 0, 0.3));\n}\n\n/* Stronger glow for active elements */\n.terminal-glow-strong {\n text-shadow: 0 0 8px var(--glow-color, rgba(0, 255, 0, 0.5)),\n 0 0 15px var(--glow-color, rgba(0, 255, 0, 0.3));\n}\n\n/* Flicker animation for the cursor */\n@keyframes cursor-blink {\n 0%,\n 50% {\n opacity: 1;\n }\n 51%,\n 100% {\n opacity: 0;\n }\n}\n\n.terminal-cursor {\n animation: cursor-blink 1s step-end infinite;\n}\n\n/* Custom scrollbar for terminal feel */\n.terminal-scrollbar::-webkit-scrollbar {\n width: 8px;\n}\n\n.terminal-scrollbar::-webkit-scrollbar-track {\n background: var(--terminal-bg);\n}\n\n.terminal-scrollbar::-webkit-scrollbar-thumb {\n background: var(--terminal-green-dimmer);\n border-radius: 4px;\n}\n\n.terminal-scrollbar::-webkit-scrollbar-thumb:hover {\n background: var(--terminal-green-dim);\n}\n\n.hide-horizontal-scrollbar {\n scrollbar-width: none;\n -ms-overflow-style: none;\n}\n\n.terminal-scrollbar.hide-horizontal-scrollbar::-webkit-scrollbar,\n.hide-horizontal-scrollbar::-webkit-scrollbar {\n width: 0;\n height: 0;\n display: none;\n}\n\n.terminal-scrollbar.hide-horizontal-scrollbar::-webkit-scrollbar-track,\n.terminal-scrollbar.hide-horizontal-scrollbar::-webkit-scrollbar-thumb,\n.hide-horizontal-scrollbar::-webkit-scrollbar-track,\n.hide-horizontal-scrollbar::-webkit-scrollbar-thumb {\n background: transparent;\n}\n\n/* Input styling */\ninput::placeholder,\ntextarea::placeholder {\n color: var(--terminal-green-dimmer);\n}\n\n/* Focus ring override */\n*:focus {\n outline: 1px solid var(--terminal-green-dim);\n outline-offset: 2px;\n}\n\n/* Pane focus glow - subtle top glow without box outline */\n.pane-focus {\n position: relative;\n}\n\n.pane-focus::after {\n content: \"\";\n position: absolute;\n top: 0;\n left: 0;\n right: 0;\n height: 20px;\n background: linear-gradient(\n to bottom,\n var(--glow-color, rgba(0, 255, 0, 1)),\n transparent\n );\n pointer-events: none;\n opacity: 0;\n transition: opacity 0.15s ease;\n}\n\n.pane-focus:focus,\n.pane-focus:focus-within {\n outline: none;\n}\n\n.pane-focus:focus::after,\n.pane-focus:focus-within::after {\n opacity: 0.4;\n}\n\n/* Tab underline indicator */\n.tab-indicator {\n position: relative;\n}\n\n.tab-indicator::after {\n content: \"\";\n position: absolute;\n bottom: -2px;\n left: 0;\n right: 0;\n height: 2px;\n background: var(--terminal-green);\n transform: scaleX(0);\n transition: transform 0.2s ease;\n}\n\n.tab-indicator.active::after {\n transform: scaleX(1);\n}\n\n/* Live mode mic button glow */\n@keyframes mic-glow {\n 0%, 100% {\n filter: drop-shadow(0 0 3px rgba(0, 255, 0, 0.5));\n }\n 50% {\n filter: drop-shadow(0 0 8px rgba(0, 255, 0, 0.8));\n }\n}\n\n.mic-live {\n animation: mic-glow 2s ease-in-out infinite;\n color: var(--terminal-green);\n}\n\n.cam-live {\n animation: mic-glow 2s ease-in-out infinite;\n color: var(--terminal-green);\n}\n","/apps/sandbox/app/layout.tsx":"import type { Metadata } from \"next\";\nimport { Geist_Mono } from \"next/font/google\";\nimport \"./globals.css\";\nimport { ConvexClientProvider } from \"./ConvexClientProvider\";\n\nconst geistMono = Geist_Mono({\n variable: \"--font-geist-mono\",\n subsets: [\"latin\"],\n});\n\nexport const metadata: Metadata = {\n title: \"Projector Sandbox\",\n description: \"A projector agent sandbox with a retro terminal UI\",\n};\n\nexport default function RootLayout({\n children,\n}: Readonly<{\n children: React.ReactNode;\n}>) {\n return (\n \n \n {children}\n \n \n );\n}\n","/apps/sandbox/app/page.tsx":"import { cookies } from \"next/headers\";\nimport { HomeClient } from \"./HomeClient\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\n\nconst SESSION_COOKIE_KEY = \"sandbox-sessionId\";\n\nexport default async function Home() {\n const cookieStore = await cookies();\n const sessionId = cookieStore.get(SESSION_COOKIE_KEY)?.value as Id<\"sessions\"> | undefined;\n\n return ;\n}\n","/apps/sandbox/convex.json":"{\n \"node\": {\n \"externalPackages\": [\n \"@livekit/rtc-node\",\n \"@livekit/rtc-node-darwin-arm64\"\n ]\n }\n}","/apps/sandbox/convex/attachments.ts":"import { v } from \"convex/values\";\nimport { mutation, query } from \"./_generated/server\";\nimport type { Id } from \"./_generated/dataModel\";\nimport {\n attachmentSummary,\n storedAttachmentsFromContentParts as storedAttachmentsFromContentPartsBase,\n userContentPartsForAttachments,\n} from \"@projectors/sandbox-agent/src/attachments.js\";\n\nexport const attachmentValidator = v.object({\n storageId: v.id(\"_storage\"),\n name: v.string(),\n contentType: v.string(),\n size: v.number(),\n kind: v.union(v.literal(\"image\"), v.literal(\"file\")),\n});\n\nexport type StoredSandboxAttachment = {\n storageId: Id<\"_storage\">;\n name: string;\n contentType: string;\n size: number;\n kind: \"image\" | \"file\";\n};\n\nexport type ResolvedSandboxAttachment = StoredSandboxAttachment & {\n url: string | null;\n};\n\nexport const generateUploadUrl = mutation({\n args: {},\n handler: async (ctx) => {\n return await ctx.storage.generateUploadUrl();\n },\n});\n\nexport const resolveUrls = query({\n args: {\n attachments: v.array(attachmentValidator),\n },\n handler: async (ctx, { attachments }) => {\n return await resolveAttachmentUrls(ctx, attachments) ?? [];\n },\n});\n\nexport function normalizeAttachment(attachment: StoredSandboxAttachment): StoredSandboxAttachment {\n return {\n storageId: attachment.storageId,\n name: attachment.name,\n contentType: attachment.contentType || \"application/octet-stream\",\n size: attachment.size,\n kind: attachment.kind === \"image\" ? \"image\" : \"file\",\n };\n}\n\nexport async function resolveAttachmentUrls(\n ctx: { storage: { getUrl(storageId: Id<\"_storage\">): Promise } },\n attachments: readonly StoredSandboxAttachment[] | undefined,\n): Promise {\n if (!attachments?.length) return undefined;\n return await Promise.all(\n attachments.map(async (attachment) => ({\n ...normalizeAttachment(attachment),\n url: await ctx.storage.getUrl(attachment.storageId),\n })),\n );\n}\n\nexport function userContentPartsForFrame(\n content: string,\n attachments: readonly ResolvedSandboxAttachment[],\n): ReturnType {\n return userContentPartsForAttachments(\n content,\n attachments.map((attachment) => ({\n storageId: attachment.storageId,\n name: attachment.name,\n contentType: attachment.contentType,\n size: attachment.size,\n kind: attachment.kind,\n url: attachment.url,\n })),\n );\n}\n\nexport function storedAttachmentsFromContentParts(\n parts: Parameters[0],\n): StoredSandboxAttachment[] {\n return storedAttachmentsFromContentPartsBase(parts).map((attachment) => ({\n ...attachment,\n storageId: attachment.storageId as Id<\"_storage\">,\n }));\n}\n\nexport { attachmentSummary };\n","/apps/sandbox/convex/branching.ts":"import { mutation } from \"./_generated/server\";\n\nexport const createBranch = mutation({\n args: {},\n handler: async () => null,\n});\n","/apps/sandbox/convex/convexJson.ts":"const ESCAPE_PREFIX = \"__projector_convex_escaped_key__\";\nconst DOLLAR_PREFIX = `${ESCAPE_PREFIX}dollar__`;\nconst PREFIX_PREFIX = `${ESCAPE_PREFIX}prefix__`;\n\nexport function escapeConvexJson(value: T): T {\n if (Array.isArray(value)) {\n return value.map((item) => escapeConvexJson(item)) as T;\n }\n\n if (!value || typeof value !== \"object\") {\n return value;\n }\n\n return Object.fromEntries(\n Object.entries(value).map(([key, entryValue]) => [\n escapeConvexKey(key),\n escapeConvexJson(entryValue),\n ]),\n ) as T;\n}\n\nexport function restoreConvexJson(value: T): T {\n if (Array.isArray(value)) {\n return value.map((item) => restoreConvexJson(item)) as T;\n }\n\n if (!value || typeof value !== \"object\") {\n return value;\n }\n\n return Object.fromEntries(\n Object.entries(value).map(([key, entryValue]) => [\n restoreConvexKey(key),\n restoreConvexJson(entryValue),\n ]),\n ) as T;\n}\n\nexport function stripClientSchemas(value: T): T {\n if (Array.isArray(value)) {\n return value.map((item) => stripClientSchemas(item)) as T;\n }\n\n if (!value || typeof value !== \"object\") {\n return value;\n }\n\n return Object.fromEntries(\n Object.entries(value)\n .filter(([, entryValue]) => entryValue !== undefined)\n .map(([key, entryValue]) => [\n key,\n key === \"schema\" || key === \"inputSchema\"\n ? stripJsonSchemaKeys(entryValue)\n : stripClientSchemas(entryValue),\n ]),\n ) as T;\n}\n\nfunction stripJsonSchemaKeys(value: T): T {\n if (Array.isArray(value)) {\n return value.map((item) => stripJsonSchemaKeys(item)) as T;\n }\n\n if (!value || typeof value !== \"object\") {\n return value;\n }\n\n return Object.fromEntries(\n Object.entries(value)\n .filter(([key, entryValue]) => !key.startsWith(\"$\") && entryValue !== undefined)\n .map(([key, entryValue]) => [key, stripJsonSchemaKeys(entryValue)]),\n ) as T;\n}\n\nfunction escapeConvexKey(key: string): string {\n if (key.startsWith(ESCAPE_PREFIX)) {\n return `${PREFIX_PREFIX}${key.slice(ESCAPE_PREFIX.length)}`;\n }\n if (key.startsWith(\"$\")) {\n return `${DOLLAR_PREFIX}${key.slice(1)}`;\n }\n return key;\n}\n\nfunction restoreConvexKey(key: string): string {\n if (key.startsWith(PREFIX_PREFIX)) {\n return `${ESCAPE_PREFIX}${key.slice(PREFIX_PREFIX.length)}`;\n }\n if (key.startsWith(DOLLAR_PREFIX)) {\n return `$${key.slice(DOLLAR_PREFIX.length)}`;\n }\n return key;\n}\n","/apps/sandbox/convex/frameHistory.ts":"import type { MutationCtx, QueryCtx } from \"./_generated/server\";\nimport type { Doc, Id } from \"./_generated/dataModel\";\nimport { restoreConvexJson } from \"./convexJson\";\n\ntype DbCtx = MutationCtx | QueryCtx;\ntype FrameDoc = Doc<\"frames\">;\ntype SessionDoc = Doc<\"sessions\">;\n\nexport async function listSessionFrameDocs(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n): Promise {\n const rows = await ctx.db\n .query(\"frameIndex\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .collect();\n return await framesForIndexRows(ctx, rows);\n}\n\nexport async function getLatestSessionFrameDoc(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n): Promise {\n const frames = await listSessionFrameDocs(ctx, sessionId);\n return frames.at(-1) ?? null;\n}\n\nexport async function listSessionContextFrameDocs(\n ctx: DbCtx,\n session: SessionDoc,\n): Promise {\n const rows = await ctx.db\n .query(\"frameIndex\")\n .withIndex(\"by_session_context\", (q) =>\n q.eq(\"sessionId\", session._id).eq(\"contextEpoch\", session.contextEpoch),\n )\n .collect();\n return await framesForIndexRows(ctx, rows);\n}\n\nexport async function collectSessionFramePath(\n ctx: DbCtx,\n session: SessionDoc,\n frameId: Id<\"frames\">,\n): Promise {\n const frames = await listSessionFrameDocs(ctx, session._id);\n const targetIndex = frames.findIndex((frame) => frame._id === frameId);\n if (targetIndex < 0) {\n throw new Error(\"Frame is not indexed for session\");\n }\n return frames.slice(0, targetIndex + 1);\n}\n\nexport async function getFrameIndexForSession(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n frameId: Id<\"frames\">,\n) {\n return await ctx.db\n .query(\"frameIndex\")\n .withIndex(\"by_session_frame\", (q) => q.eq(\"sessionId\", sessionId).eq(\"frameId\", frameId))\n .first();\n}\n\nexport function restoreFrame(frame: FrameDoc) {\n const metadata = frame.metadata ? restoreConvexJson(frame.metadata) : undefined;\n const projectorFrameId =\n metadata &&\n typeof metadata === \"object\" &&\n typeof (metadata as Record).projectorFrameId === \"string\"\n ? ((metadata as Record).projectorFrameId as string)\n : frame._id;\n return {\n ...frame,\n id: projectorFrameId,\n messages: restoreConvexJson(frame.messages),\n ...(metadata ? { metadata } : {}),\n ...(frame.provenance ? { provenance: restoreConvexJson(frame.provenance) } : {}),\n };\n}\n\nasync function framesForIndexRows(\n ctx: DbCtx,\n rows: Doc<\"frameIndex\">[],\n): Promise {\n const frames = await Promise.all(rows.map((row) => ctx.db.get(row.frameId)));\n return frames\n .filter((frame): frame is FrameDoc => frame !== null)\n .sort(compareFrames);\n}\n\nfunction compareFrames(a: FrameDoc, b: FrameDoc): number {\n return a.createdAt - b.createdAt || a._id.localeCompare(b._id);\n}\n","/apps/sandbox/convex/frames.ts":"import { v } from \"convex/values\";\nimport { query } from \"./_generated/server\";\nimport { restoreFrame } from \"./frameHistory\";\n\nexport const list = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const session = await ctx.db.get(sessionId);\n if (!session) return [];\n const rows = await ctx.db\n .query(\"frameIndex\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .collect();\n const frames = await Promise.all(\n rows.map(async (row) => {\n const frame = await ctx.db.get(row.frameId);\n return frame ? { ...restoreFrame(frame), contextEpoch: row.contextEpoch } : null;\n }),\n );\n return frames\n .filter((frame) => frame !== null)\n .sort((a, b) => a.createdAt - b.createdAt || a._id.localeCompare(b._id));\n },\n});\n","/apps/sandbox/convex/http.ts":"import { httpRouter } from \"convex/server\";\n\nconst http = httpRouter();\n\nexport default http;\n","/apps/sandbox/convex/livekitAgent.ts":"import { v } from \"convex/values\";\nimport { internalMutation, internalQuery, mutation, query } from \"./_generated/server\";\nimport type { Id } from \"./_generated/dataModel\";\nimport { restoreConvexJson, stripClientSchemas } from \"./convexJson\";\nimport { createSandboxClientSnapshot } from \"@projectors/sandbox-agent/src/projector-sandbox.js\";\nimport type { MachineSyncState } from \"@projectors/core/client\";\nimport { collectSessionFramePath, getLatestSessionFrameDoc } from \"./frameHistory\";\nimport { listMessagesForSession } from \"./messages\";\n\nconst DEFAULT_WORKER_LEASE_TTL_MS = 15_000;\n\nexport const getSession = internalQuery({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => await ctx.db.get(sessionId),\n});\n\nexport const upsertAgentWorkerRoom = internalMutation({\n args: { sessionId: v.id(\"sessions\"), roomName: v.string() },\n handler: async (ctx, { sessionId, roomName }) => {\n const now = Date.now();\n const existing = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n\n if (existing) {\n await ctx.db.patch(existing._id, {\n roomName,\n createdAt: now,\n });\n return existing._id;\n }\n\n return await ctx.db.insert(\"agentWorkerRooms\", { sessionId, roomName, createdAt: now });\n },\n});\n\nexport const getAgentWorkerRoom = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) =>\n await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId)).first(),\n});\n\nexport const getAgentWorkerStatus = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const room = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n if (!room) {\n return {\n status: \"idle\" as const,\n ready: false,\n now: Date.now(),\n };\n }\n\n const now = Date.now();\n const leaseExpiresAt = room.agentWorkerLeaseExpiresAt ?? 0;\n const ready = leaseExpiresAt > now;\n const lockExpiresAt = room.agentDispatchLockExpiresAt ?? 0;\n const nextDispatchAt = room.agentNextDispatchAt ?? 0;\n const hasDispatch = Boolean(room.agentDispatchId);\n const waitingForDispatch = !ready && hasDispatch && now < nextDispatchAt;\n\n const status = ready\n ? \"ready\"\n : lockExpiresAt > now\n ? \"connecting\"\n : waitingForDispatch\n ? \"reconnecting\"\n : hasDispatch || room.agentReconnectAttempt\n ? \"stale\"\n : \"idle\";\n\n return {\n status,\n ready,\n now,\n sessionId: room.sessionId,\n roomName: room.roomName,\n agentDispatchId: room.agentDispatchId,\n agentDispatchCreatedAt: room.agentDispatchCreatedAt,\n agentDispatchLockExpiresAt: room.agentDispatchLockExpiresAt,\n agentReconnectAttempt: room.agentReconnectAttempt ?? 0,\n agentNextDispatchAt: room.agentNextDispatchAt,\n agentLastDispatchError: room.agentLastDispatchError,\n agentLastStatusAt: room.agentLastStatusAt,\n agentWorkerId: room.agentWorkerId,\n agentWorkerHeartbeatAt: room.agentWorkerHeartbeatAt,\n agentWorkerLeaseExpiresAt: room.agentWorkerLeaseExpiresAt,\n };\n },\n});\n\nexport const getSessionIdByRoom = query({\n args: { roomName: v.string() },\n handler: async (ctx, { roomName }) => {\n const room = await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_room\", (q) => q.eq(\"roomName\", roomName)).first();\n return room?.sessionId ?? null;\n },\n});\n\nexport const getAgentInit = query({\n args: { roomName: v.string() },\n handler: async (ctx, { roomName }) => {\n const room = await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_room\", (q) => q.eq(\"roomName\", roomName)).first();\n if (!room) return null;\n\n const session = await ctx.db.get(room.sessionId);\n if (!session) return null;\n const latestFrame = await getLatestSessionFrameDoc(ctx, room.sessionId);\n if (!latestFrame) return null;\n\n const logs = await ctx.db\n .query(\"projectorInstanceLog\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", room.sessionId))\n .collect();\n const framePath = await collectSessionFramePath(ctx, session, latestFrame._id);\n const ancestors = new Set(framePath.map((frame) => frame._id));\n const latestLog = logs\n .filter((entry) => !entry.frameId || ancestors.has(entry.frameId))\n .sort((a, b) => b.createdAt - a.createdAt)[0];\n if (!latestLog) return null;\n\n const instance = restoreConvexJson(latestLog.instance);\n const syncState = restoreConvexJson(\n session.syncState ?? { recentCommandResidue: [] },\n ) as MachineSyncState;\n\n return {\n sessionId: room.sessionId,\n frameId: latestFrame._id,\n instance,\n clientSnapshot: stripClientSchemas(createSandboxClientSnapshot(instance, room.sessionId, syncState)),\n syncState,\n messages: await listMessagesForSession(ctx, room.sessionId),\n };\n },\n});\n\nexport const claimAgentDispatchLock = internalMutation({\n args: { sessionId: v.id(\"sessions\"), lockTtlMs: v.optional(v.number()) },\n handler: async (ctx, { sessionId, lockTtlMs }) => {\n const room = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n if (!room) return null;\n\n const now = Date.now();\n const expiresAt = room.agentDispatchLockExpiresAt ?? 0;\n if (expiresAt > now) return null;\n\n await ctx.db.patch(room._id, { agentDispatchLockExpiresAt: now + (lockTtlMs ?? 20_000) });\n return {\n agentWorkerRoomId: room._id,\n roomName: room.roomName,\n agentDispatchId: room.agentDispatchId ?? null,\n agentDispatchCreatedAt: room.agentDispatchCreatedAt ?? null,\n agentReconnectAttempt: room.agentReconnectAttempt ?? 0,\n };\n },\n});\n\nexport const hasLiveAgentWorkerLease = internalQuery({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const room = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n return (room?.agentWorkerLeaseExpiresAt ?? 0) > Date.now();\n },\n});\n\nexport const getAgentDispatchSnapshot = internalQuery({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const room = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n if (!room) return null;\n\n const now = Date.now();\n return {\n agentWorkerRoomId: room._id,\n roomName: room.roomName,\n hasLiveWorkerLease: (room.agentWorkerLeaseExpiresAt ?? 0) > now,\n agentDispatchId: room.agentDispatchId ?? null,\n agentDispatchCreatedAt: room.agentDispatchCreatedAt ?? null,\n agentNextDispatchAt: room.agentNextDispatchAt ?? null,\n agentReconnectAttempt: room.agentReconnectAttempt ?? 0,\n };\n },\n});\n\nexport const claimAgentWorkerLease = mutation({\n args: {\n roomName: v.string(),\n workerId: v.string(),\n leaseToken: v.string(),\n leaseTtlMs: v.optional(v.number()),\n },\n handler: async (ctx, { roomName, workerId, leaseToken, leaseTtlMs }) => {\n const room = await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_room\", (q) => q.eq(\"roomName\", roomName)).first();\n if (!room) return null;\n\n const now = Date.now();\n const expiresAt = room.agentWorkerLeaseExpiresAt ?? 0;\n if (expiresAt > now && room.agentWorkerLeaseToken !== leaseToken) {\n return null;\n }\n\n await ctx.db.patch(room._id, {\n agentWorkerId: workerId,\n agentWorkerLeaseToken: leaseToken,\n agentWorkerLeaseExpiresAt: now + (leaseTtlMs ?? 15_000),\n agentWorkerHeartbeatAt: now,\n agentReconnectAttempt: 0,\n agentNextDispatchAt: 0,\n agentLastDispatchError: undefined,\n agentLastStatusAt: now,\n });\n\n return {\n agentWorkerRoomId: room._id,\n sessionId: room.sessionId,\n roomName: room.roomName,\n };\n },\n});\n\nexport const renewAgentWorkerLease = mutation({\n args: {\n roomName: v.string(),\n leaseToken: v.string(),\n leaseTtlMs: v.optional(v.number()),\n },\n handler: async (ctx, { roomName, leaseToken, leaseTtlMs }) => {\n const room = await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_room\", (q) => q.eq(\"roomName\", roomName)).first();\n if (!room || room.agentWorkerLeaseToken !== leaseToken) return false;\n\n const now = Date.now();\n if ((room.agentWorkerLeaseExpiresAt ?? 0) <= now) return false;\n\n await ctx.db.patch(room._id, {\n agentWorkerLeaseExpiresAt: now + (leaseTtlMs ?? DEFAULT_WORKER_LEASE_TTL_MS),\n agentWorkerHeartbeatAt: now,\n agentLastStatusAt: now,\n });\n return true;\n },\n});\n\nexport const releaseAgentWorkerLease = mutation({\n args: {\n roomName: v.string(),\n leaseToken: v.string(),\n },\n handler: async (ctx, { roomName, leaseToken }) => {\n const room = await ctx.db.query(\"agentWorkerRooms\").withIndex(\"by_room\", (q) => q.eq(\"roomName\", roomName)).first();\n if (!room || room.agentWorkerLeaseToken !== leaseToken) return false;\n\n await ctx.db.patch(room._id, {\n agentWorkerId: undefined,\n agentWorkerLeaseToken: undefined,\n agentWorkerLeaseExpiresAt: 0,\n agentWorkerHeartbeatAt: Date.now(),\n agentLastStatusAt: Date.now(),\n });\n return true;\n },\n});\n\nexport const releaseAgentDispatchLock = internalMutation({\n args: { agentWorkerRoomId: v.id(\"agentWorkerRooms\") },\n handler: async (ctx, { agentWorkerRoomId }) => {\n await ctx.db.patch(agentWorkerRoomId, { agentDispatchLockExpiresAt: 0 });\n },\n});\n\nexport const recordAgentDispatch = internalMutation({\n args: {\n agentWorkerRoomId: v.id(\"agentWorkerRooms\"),\n agentDispatchId: v.string(),\n agentDispatchCreatedAt: v.number(),\n nextDispatchAt: v.optional(v.number()),\n reconnectAttempt: v.optional(v.number()),\n },\n handler: async (ctx, { agentWorkerRoomId, agentDispatchId, agentDispatchCreatedAt, nextDispatchAt, reconnectAttempt }) => {\n await ctx.db.patch(agentWorkerRoomId, {\n agentDispatchId,\n agentDispatchCreatedAt,\n agentNextDispatchAt: nextDispatchAt ?? agentDispatchCreatedAt,\n agentReconnectAttempt: reconnectAttempt ?? 0,\n agentLastDispatchError: undefined,\n agentLastStatusAt: Date.now(),\n });\n },\n});\n\nexport const recordAgentDispatchFailure = internalMutation({\n args: {\n agentWorkerRoomId: v.id(\"agentWorkerRooms\"),\n error: v.string(),\n nextDispatchAt: v.number(),\n reconnectAttempt: v.number(),\n },\n handler: async (ctx, { agentWorkerRoomId, error, nextDispatchAt, reconnectAttempt }) => {\n await ctx.db.patch(agentWorkerRoomId, {\n agentLastDispatchError: error,\n agentNextDispatchAt: nextDispatchAt,\n agentReconnectAttempt: reconnectAttempt,\n agentLastStatusAt: Date.now(),\n });\n },\n});\n\nexport const clearAgentDispatch = internalMutation({\n args: { agentWorkerRoomId: v.id(\"agentWorkerRooms\") },\n handler: async (ctx, { agentWorkerRoomId }) => {\n await ctx.db.patch(agentWorkerRoomId, {\n agentDispatchId: undefined,\n agentDispatchCreatedAt: undefined,\n agentLastStatusAt: Date.now(),\n });\n },\n});\n","/apps/sandbox/convex/livekitAgentActions.ts":"\"use node\";\n\nimport { action } from \"./_generated/server\";\nimport { internal } from \"./_generated/api\";\nimport { v } from \"convex/values\";\nimport { ParticipantInfo_Kind } from \"@livekit/protocol\";\nimport { AccessToken, AgentDispatchClient, RoomServiceClient } from \"livekit-server-sdk\";\nimport { randomUUID } from \"node:crypto\";\n\nconst AGENT_NAME = \"sandbox-agent\";\nconst DISPATCH_LOCK_TTL_MS = 20_000;\nconst DISPATCH_STALE_GRACE_MS = 10_000;\nconst DISPATCH_BACKOFF_BASE_MS = DISPATCH_STALE_GRACE_MS;\nconst DISPATCH_BACKOFF_MAX_MS = 60_000;\n\ntype ListedAgentDispatch = Awaited>[number];\ntype ListedParticipant = Awaited>[number];\n\nexport const getToken = action({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }): Promise<{ token: string; roomName: string; url: string }> => {\n const session = await ctx.runQuery(internal.livekitAgent.getSession, { sessionId });\n if (!session) {\n throw new Error(\"Session not found\");\n }\n\n const apiKey = process.env.LIVEKIT_API_KEY;\n const apiSecret = process.env.LIVEKIT_API_SECRET;\n const url = process.env.LIVEKIT_URL;\n if (!apiKey || !apiSecret || !url) {\n throw new Error(\"LIVEKIT_API_KEY, LIVEKIT_API_SECRET, and LIVEKIT_URL are required\");\n }\n\n const roomName = `sandbox-${sessionId}`;\n const identity = userParticipantIdentity(sessionId);\n\n const token = new AccessToken(apiKey, apiSecret, {\n identity,\n ttl: \"15m\",\n });\n token.addGrant({\n room: roomName,\n roomJoin: true,\n canPublish: true,\n canSubscribe: true,\n });\n\n await ctx.runMutation(internal.livekitAgent.upsertAgentWorkerRoom, { sessionId, roomName });\n await ensureAgentDispatchedImpl(ctx, {\n apiKey,\n apiSecret,\n liveKitUrl: url,\n sessionId,\n roomName,\n reason: \"token_request\",\n });\n\n return {\n token: await token.toJwt(),\n roomName,\n url,\n };\n },\n});\n\nexport const ensureAgentDispatched = action({\n args: { sessionId: v.id(\"sessions\"), reason: v.optional(v.string()) },\n handler: async (ctx, { sessionId, reason }): Promise => {\n const session = await ctx.runQuery(internal.livekitAgent.getSession, { sessionId });\n if (!session) throw new Error(\"Session not found\");\n\n const apiKey = process.env.LIVEKIT_API_KEY;\n const apiSecret = process.env.LIVEKIT_API_SECRET;\n const url = process.env.LIVEKIT_URL;\n if (!apiKey || !apiSecret || !url) {\n throw new Error(\"LIVEKIT_API_KEY, LIVEKIT_API_SECRET, and LIVEKIT_URL are required\");\n }\n\n const roomName = `sandbox-${sessionId}`;\n await ctx.runMutation(internal.livekitAgent.upsertAgentWorkerRoom, { sessionId, roomName });\n await ensureAgentDispatchedImpl(ctx, {\n apiKey,\n apiSecret,\n liveKitUrl: url,\n sessionId,\n roomName,\n reason: reason ?? \"reconcile\",\n });\n },\n});\n\nasync function ensureAgentDispatchedImpl(\n ctx: any,\n {\n apiKey,\n apiSecret,\n liveKitUrl,\n sessionId,\n roomName,\n reason,\n }: {\n apiKey: string;\n apiSecret: string;\n liveKitUrl: string;\n sessionId: string;\n roomName: string;\n reason: string;\n },\n): Promise {\n const snapshot = await ctx.runQuery(internal.livekitAgent.getAgentDispatchSnapshot, { sessionId });\n const now = Date.now();\n if (!snapshot?.hasLiveWorkerLease && snapshot?.agentNextDispatchAt && snapshot.agentNextDispatchAt > now) {\n return;\n }\n\n const lock = await ctx.runMutation(internal.livekitAgent.claimAgentDispatchLock, {\n sessionId,\n lockTtlMs: DISPATCH_LOCK_TTL_MS,\n });\n if (!lock) return;\n\n try {\n const hasLiveWorkerAfterLock = await ctx.runQuery(internal.livekitAgent.hasLiveAgentWorkerLease, { sessionId });\n\n const now = Date.now();\n const httpUrl = liveKitHttpUrl(liveKitUrl);\n const dispatchClient = new AgentDispatchClient(httpUrl, apiKey, apiSecret);\n const roomService = new RoomServiceClient(httpUrl, apiKey, apiSecret);\n let activeDispatches = await listActiveDispatches(dispatchClient, roomName);\n activeDispatches = await dedupeActiveDispatches(dispatchClient, roomName, activeDispatches);\n\n if (hasLiveWorkerAfterLock) {\n await removeDuplicateAgentParticipants(roomService, roomName, activeDispatches);\n return;\n }\n\n const { hasAgent, removedAgent } = await reconcileAgentParticipants(roomService, roomName, now, lock.agentDispatchCreatedAt, activeDispatches);\n\n if (activeDispatches.length > 0) {\n const dispatchAgeMs = lock.agentDispatchCreatedAt ? now - lock.agentDispatchCreatedAt : Number.POSITIVE_INFINITY;\n if (hasAgent && !removedAgent) {\n const active = activeDispatches[0];\n if (active && lock.agentDispatchId !== active.id) {\n await ctx.runMutation(internal.livekitAgent.recordAgentDispatch, {\n agentWorkerRoomId: lock.agentWorkerRoomId,\n agentDispatchId: active.id,\n agentDispatchCreatedAt: now,\n nextDispatchAt: nextDispatchAt(now, lock.agentReconnectAttempt ?? 0),\n reconnectAttempt: lock.agentReconnectAttempt ?? 0,\n });\n }\n return;\n }\n\n if (dispatchAgeMs < DISPATCH_STALE_GRACE_MS) {\n return;\n }\n\n for (const stale of activeDispatches) {\n await deleteDispatchBestEffort(dispatchClient, roomName, stale.id);\n }\n await ctx.runMutation(internal.livekitAgent.clearAgentDispatch, {\n agentWorkerRoomId: lock.agentWorkerRoomId,\n });\n }\n\n const reconnectAttempt = (lock.agentReconnectAttempt ?? 0) + 1;\n const dispatch = await dispatchClient.createDispatch(roomName, AGENT_NAME, {\n metadata: JSON.stringify({ sessionId, dispatchedAt: now, reason, reconnectAttempt }),\n });\n await ctx.runMutation(internal.livekitAgent.recordAgentDispatch, {\n agentWorkerRoomId: lock.agentWorkerRoomId,\n agentDispatchId: dispatch.id,\n agentDispatchCreatedAt: now,\n nextDispatchAt: nextDispatchAt(now, reconnectAttempt),\n reconnectAttempt,\n });\n } catch (error) {\n const attempt = (lock.agentReconnectAttempt ?? 0) + 1;\n await ctx.runMutation(internal.livekitAgent.recordAgentDispatchFailure, {\n agentWorkerRoomId: lock.agentWorkerRoomId,\n error: error instanceof Error ? error.message : \"Unable to dispatch LiveKit agent\",\n nextDispatchAt: now + backoffDelayMs(attempt),\n reconnectAttempt: attempt,\n });\n } finally {\n await ctx.runMutation(internal.livekitAgent.releaseAgentDispatchLock, {\n agentWorkerRoomId: lock.agentWorkerRoomId,\n });\n }\n}\n\nasync function dedupeActiveDispatches(\n dispatchClient: AgentDispatchClient,\n roomName: string,\n activeDispatches: ListedAgentDispatch[],\n): Promise {\n if (activeDispatches.length <= 1) return activeDispatches;\n\n activeDispatches.sort((a, b) => compareBigIntDesc(a.state?.createdAt ?? 0n, b.state?.createdAt ?? 0n));\n const [keep, ...duplicates] = activeDispatches;\n for (const duplicate of duplicates) {\n await deleteDispatchBestEffort(dispatchClient, roomName, duplicate.id);\n }\n return keep ? [keep] : [];\n}\n\nasync function reconcileAgentParticipants(\n roomService: RoomServiceClient,\n roomName: string,\n now: number,\n lastDispatchAt: number | null,\n activeDispatches: ListedAgentDispatch[],\n): Promise<{ hasAgent: boolean; removedAgent: boolean }> {\n try {\n const participants = await roomService.listParticipants(roomName);\n const agents = orderedAgentParticipants(participants, activeDispatches);\n if (agents.length === 0) return { hasAgent: false, removedAgent: false };\n\n const [keep, ...duplicates] = agents;\n for (const duplicate of duplicates) {\n await removeParticipantBestEffort(roomService, roomName, duplicate.identity);\n }\n\n const agentAgeMs = lastDispatchAt ? now - lastDispatchAt : Number.POSITIVE_INFINITY;\n if (keep && agentAgeMs >= DISPATCH_STALE_GRACE_MS) {\n await removeParticipantBestEffort(roomService, roomName, keep.identity);\n return { hasAgent: false, removedAgent: true };\n }\n\n return { hasAgent: Boolean(keep), removedAgent: false };\n } catch (error) {\n console.warn(`[livekit] Failed to list participants for room ${roomName}:`, error);\n return { hasAgent: false, removedAgent: false };\n }\n}\n\nasync function removeDuplicateAgentParticipants(\n roomService: RoomServiceClient,\n roomName: string,\n activeDispatches: ListedAgentDispatch[],\n): Promise {\n try {\n const participants = await roomService.listParticipants(roomName);\n const agents = orderedAgentParticipants(participants, activeDispatches);\n const [, ...duplicates] = agents;\n for (const duplicate of duplicates) {\n await removeParticipantBestEffort(roomService, roomName, duplicate.identity);\n }\n } catch (error) {\n console.warn(`[livekit] Failed to reconcile duplicate agents for room ${roomName}:`, error);\n }\n}\n\nfunction orderedAgentParticipants(\n participants: ListedParticipant[],\n activeDispatches: ListedAgentDispatch[],\n): ListedParticipant[] {\n const agents = participants.filter((participant) => participant.kind === ParticipantInfo_Kind.AGENT);\n const preferredIdentities = new Set(\n activeDispatches\n .flatMap((dispatch) => dispatch.state?.jobs ?? [])\n .map((job) => job.participant?.identity)\n .filter((identity): identity is string => Boolean(identity)),\n );\n\n return agents.sort((a, b) => {\n const aPreferred = preferredIdentities.has(a.identity);\n const bPreferred = preferredIdentities.has(b.identity);\n if (aPreferred !== bPreferred) return aPreferred ? -1 : 1;\n return compareBigIntDesc(a.joinedAtMs || a.joinedAt, b.joinedAtMs || b.joinedAt);\n });\n}\n\nasync function listActiveDispatches(dispatchClient: AgentDispatchClient, roomName: string) {\n try {\n const dispatches = await dispatchClient.listDispatch(roomName);\n return dispatches.filter((dispatch) => dispatch.agentName === AGENT_NAME && (dispatch.state?.deletedAt ?? 0n) === 0n);\n } catch (error) {\n console.warn(`[livekit] Failed to list dispatches for room ${roomName}:`, error);\n return [];\n }\n}\n\nasync function deleteDispatchBestEffort(dispatchClient: AgentDispatchClient, roomName: string, dispatchId: string) {\n try {\n await dispatchClient.deleteDispatch(dispatchId, roomName);\n } catch (error) {\n console.warn(`[livekit] Failed to delete dispatch ${dispatchId} for room ${roomName}:`, error);\n }\n}\n\nasync function removeParticipantBestEffort(roomService: RoomServiceClient, roomName: string, identity: string) {\n try {\n await roomService.removeParticipant(roomName, identity);\n } catch (error) {\n console.warn(`[livekit] Failed to remove participant ${identity} from room ${roomName}:`, error);\n }\n}\n\nfunction userParticipantIdentity(sessionId: string): string {\n return `user-${sessionId}-${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;\n}\n\nfunction liveKitHttpUrl(url: string): string {\n return url.replace(\"wss://\", \"https://\").replace(\"ws://\", \"http://\");\n}\n\nfunction nextDispatchAt(now: number, reconnectAttempt: number): number {\n return now + backoffDelayMs(reconnectAttempt);\n}\n\nfunction backoffDelayMs(attempt: number): number {\n const exponent = Math.max(0, attempt - 1);\n return Math.min(DISPATCH_BACKOFF_MAX_MS, DISPATCH_BACKOFF_BASE_MS * 2 ** exponent);\n}\n\nfunction compareBigIntDesc(a: bigint, b: bigint): number {\n if (a === b) return 0;\n return a > b ? -1 : 1;\n}\n","/apps/sandbox/convex/messages.ts":"import { v } from \"convex/values\";\nimport { mutation, query } from \"./_generated/server\";\nimport type { MutationCtx, QueryCtx } from \"./_generated/server\";\nimport type { Doc, Id } from \"./_generated/dataModel\";\nimport { collectSessionFramePath, getFrameIndexForSession, getLatestSessionFrameDoc } from \"./frameHistory\";\nimport { attachmentValidator, resolveAttachmentUrls } from \"./attachments\";\n\ntype DbCtx = MutationCtx | QueryCtx;\ntype MessageDoc = Doc<\"messages\">;\n\nexport const listForFramePath = query({\n args: {\n sessionId: v.id(\"sessions\"),\n upToFrameId: v.optional(v.id(\"frames\")),\n },\n handler: async (ctx, { sessionId, upToFrameId }) => {\n const session = await ctx.db.get(sessionId);\n if (!session) return [];\n\n const latestFrame = await getLatestSessionFrameDoc(ctx, sessionId);\n const frameId = upToFrameId ?? latestFrame?._id;\n if (!frameId) return [];\n const framePath = await collectSessionFramePath(ctx, session, frameId);\n const frameIds = new Set(framePath.map((frame) => frame._id));\n const messages = await listMessagesForSession(ctx, sessionId);\n const visibleMessages = messages.filter((message) => frameIds.has(message.frameId));\n return await Promise.all(\n visibleMessages.map(async (message) => ({\n ...message,\n attachments: await resolveAttachmentUrls(ctx, message.attachments),\n })),\n );\n },\n});\n\nexport const add = mutation({\n args: {\n sessionId: v.id(\"sessions\"),\n role: v.union(v.literal(\"user\"), v.literal(\"assistant\")),\n content: v.string(),\n attachments: v.optional(v.array(attachmentValidator)),\n frameId: v.optional(v.id(\"frames\")),\n mode: v.optional(v.union(v.literal(\"text\"), v.literal(\"voice\"))),\n idempotencyKey: v.optional(v.string()),\n streamState: v.optional(v.union(v.literal(\"streaming\"), v.literal(\"complete\"), v.literal(\"error\"))),\n streamSeq: v.optional(v.number()),\n },\n handler: async (ctx, args) => {\n const session = await ctx.db.get(args.sessionId);\n if (!session) throw new Error(\"Session not found\");\n\n const latestFrame = await getLatestSessionFrameDoc(ctx, args.sessionId);\n const frameId = args.frameId ?? latestFrame?._id;\n if (!frameId) throw new Error(\"Session has no frames\");\n const frameIndex = await getFrameIndexForSession(ctx, args.sessionId, frameId);\n if (!frameIndex) throw new Error(\"Message frame is not indexed for session\");\n\n if (args.idempotencyKey) {\n const existingIndex = await ctx.db\n .query(\"messageIndex\")\n .withIndex(\"by_session_idempotency_key\", (q) =>\n q.eq(\"sessionId\", args.sessionId).eq(\"idempotencyKey\", args.idempotencyKey),\n )\n .first();\n const existing = existingIndex ? await ctx.db.get(existingIndex.messageId) : null;\n\n if (existing) {\n // Streaming updates can arrive out of order; never regress visible message content.\n const existingSeq = typeof existing.streamSeq === \"number\" ? existing.streamSeq : -1;\n const nextSeq = typeof args.streamSeq === \"number\" ? args.streamSeq : existingSeq;\n if (nextSeq < existingSeq) {\n return existing._id;\n }\n\n const patch: Partial = {\n content: args.content,\n frameId,\n };\n if (args.attachments !== undefined) patch.attachments = args.attachments;\n if (args.mode !== undefined) patch.mode = args.mode;\n if (args.streamState !== undefined) patch.streamState = args.streamState;\n if (args.streamSeq !== undefined) patch.streamSeq = args.streamSeq;\n await ctx.db.patch(existing._id, patch);\n return existing._id;\n }\n }\n\n const messageId = await ctx.db.insert(\"messages\", {\n role: args.role,\n content: args.content,\n attachments: args.attachments,\n frameId,\n mode: args.mode,\n idempotencyKey: args.idempotencyKey,\n streamState: args.streamState,\n streamSeq: args.streamSeq,\n createdAt: Date.now(),\n });\n await ctx.db.insert(\"messageIndex\", {\n sessionId: args.sessionId,\n messageId,\n idempotencyKey: args.idempotencyKey,\n });\n\n return messageId;\n },\n});\n\nexport async function listMessagesForSession(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n): Promise {\n const rows = await ctx.db\n .query(\"messageIndex\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .collect();\n const messages = await Promise.all(rows.map((row) => ctx.db.get(row.messageId)));\n return messages\n .filter((message): message is MessageDoc => message !== null)\n .sort((a, b) => a.createdAt - b.createdAt || a._id.localeCompare(b._id));\n}\n","/apps/sandbox/convex/schema.ts":"import { defineSchema, defineTable } from \"convex/server\";\nimport { v } from \"convex/values\";\n\nexport default defineSchema({\n sessions: defineTable({\n contextEpoch: v.number(),\n familyRootSessionId: v.optional(v.id(\"sessions\")),\n forkedFromSessionId: v.optional(v.id(\"sessions\")),\n forkedFromFrameId: v.optional(v.id(\"frames\")),\n syncState: v.optional(v.any()),\n })\n .index(\"by_family_root\", [\"familyRootSessionId\"])\n .index(\"by_fork_source\", [\"forkedFromSessionId\"]),\n\n frames: defineTable({\n referenceFrameId: v.optional(v.id(\"frames\")),\n generatorId: v.optional(v.string()),\n activationId: v.optional(v.string()),\n inert: v.optional(v.boolean()),\n metadata: v.optional(v.any()),\n provenance: v.optional(v.any()),\n messages: v.array(v.any()),\n createdAt: v.number(),\n }).index(\"by_reference\", [\"referenceFrameId\"]),\n\n frameIndex: defineTable({\n sessionId: v.id(\"sessions\"),\n frameId: v.id(\"frames\"),\n contextEpoch: v.number(),\n })\n .index(\"by_session\", [\"sessionId\"])\n .index(\"by_session_context\", [\"sessionId\", \"contextEpoch\"])\n .index(\"by_session_frame\", [\"sessionId\", \"frameId\"]),\n\n projectorInstanceLog: defineTable({\n sessionId: v.id(\"sessions\"),\n instanceId: v.string(),\n frameId: v.optional(v.id(\"frames\")),\n message: v.any(),\n instance: v.any(),\n createdAt: v.number(),\n })\n .index(\"by_session\", [\"sessionId\"])\n .index(\"by_session_instance\", [\"sessionId\", \"instanceId\"])\n .index(\"by_frame\", [\"frameId\"]),\n\n messages: defineTable({\n frameId: v.id(\"frames\"),\n role: v.union(v.literal(\"user\"), v.literal(\"assistant\")),\n content: v.string(),\n attachments: v.optional(v.array(v.object({\n storageId: v.id(\"_storage\"),\n name: v.string(),\n contentType: v.string(),\n size: v.number(),\n kind: v.union(v.literal(\"image\"), v.literal(\"file\")),\n }))),\n createdAt: v.number(),\n // Voice mode fields\n mode: v.optional(v.union(v.literal(\"text\"), v.literal(\"voice\"))),\n idempotencyKey: v.optional(v.string()),\n // Streaming fields (best-effort; Convex is the durable source of truth)\n streamState: v.optional(v.union(v.literal(\"streaming\"), v.literal(\"complete\"), v.literal(\"error\"))),\n streamSeq: v.optional(v.number()),\n })\n .index(\"by_frame\", [\"frameId\"])\n .index(\"by_idempotency_key\", [\"idempotencyKey\"])\n .index(\"by_frame_idempotency_key\", [\"frameId\", \"idempotencyKey\"]),\n\n messageIndex: defineTable({\n sessionId: v.id(\"sessions\"),\n messageId: v.id(\"messages\"),\n idempotencyKey: v.optional(v.string()),\n })\n .index(\"by_session\", [\"sessionId\"])\n .index(\"by_session_message\", [\"sessionId\", \"messageId\"])\n .index(\"by_session_idempotency_key\", [\"sessionId\", \"idempotencyKey\"]),\n\n // Ephemeral session state (processing indicators, etc.)\n sessionEphemera: defineTable({\n sessionId: v.id(\"sessions\"),\n isProcessing: v.boolean(),\n processingStartedAt: v.optional(v.number()),\n }).index(\"by_session\", [\"sessionId\"]),\n\n // LiveKit room state for tracking dispatched agent workers.\n agentWorkerRooms: defineTable({\n sessionId: v.id(\"sessions\"),\n roomName: v.string(),\n createdAt: v.number(),\n // Last explicit agent dispatch created for this room (best-effort; used for de-duping dispatches).\n agentDispatchId: v.optional(v.string()),\n agentDispatchCreatedAt: v.optional(v.number()),\n // Simple in-DB lock to prevent concurrent dispatch attempts from creating duplicates.\n agentDispatchLockExpiresAt: v.optional(v.number()),\n // Reconnect bookkeeping for stale/missing workers.\n agentReconnectAttempt: v.optional(v.number()),\n agentNextDispatchAt: v.optional(v.number()),\n agentLastDispatchError: v.optional(v.string()),\n agentLastStatusAt: v.optional(v.number()),\n // Active worker lease. Only the holder may run the room's machine loop.\n agentWorkerId: v.optional(v.string()),\n agentWorkerLeaseToken: v.optional(v.string()),\n agentWorkerLeaseExpiresAt: v.optional(v.number()),\n agentWorkerHeartbeatAt: v.optional(v.number()),\n })\n .index(\"by_session\", [\"sessionId\"])\n .index(\"by_room\", [\"roomName\"]),\n});\n","/apps/sandbox/convex/sessionActions.ts":"\"use node\";\n\nimport { action } from \"./_generated/server\";\nimport { api } from \"./_generated/api\";\nimport type { ActionCtx } from \"./_generated/server\";\nimport { v } from \"convex/values\";\nimport type { Id } from \"./_generated/dataModel\";\nimport { openai as aiSdkOpenAI } from \"@ai-sdk/openai\";\nimport { AiSdkExecutor } from \"@projectors/aisdk-executor\";\nimport {\n ROOT_GENERATOR_ID,\n createMachine,\n executeCommand,\n runMachine,\n type Executor,\n type Frame,\n} from \"@projectors/core\";\nimport {\n attachmentSummary,\n attachmentValidator,\n resolveAttachmentUrls,\n storedAttachmentsFromContentParts,\n userContentPartsForFrame,\n} from \"./attachments\";\nimport type { SandboxAttachmentData } from \"@projectors/sandbox-agent/src/attachments.js\";\nimport { escapeConvexJson } from \"./convexJson\";\nimport {\n createInitialSerializedInstance,\n createSandboxCharter,\n hydrateSandboxInstance,\n} from \"@projectors/sandbox-agent/src/projector-sandbox.js\";\nimport type { ClientMachineMessage } from \"@projectors/core/client\";\n\nconst DISCRETE_MODEL = process.env.OPENAI_DISCRETE_MODEL ?? \"gpt-5.5\";\n\nconst discreteExecutor = new AiSdkExecutor({\n model: aiSdkOpenAI(DISCRETE_MODEL),\n maxOutputTokens: 4096,\n});\n\nexport const createSession = action({\n args: {},\n handler: async (ctx): Promise> => {\n const instance = createInitialSerializedInstance();\n return await ctx.runMutation(api.sessions.create, {\n instanceId: instance.id,\n instance: escapeConvexJson(instance),\n syncState: escapeConvexJson({ recentCommandResidue: [] }),\n });\n },\n});\n\nexport const sendMessage = action({\n args: {\n sessionId: v.id(\"sessions\"),\n content: v.string(),\n attachments: v.optional(v.array(attachmentValidator)),\n },\n handler: async (ctx, { sessionId, content, attachments }): Promise<{ success: true }> => {\n const session = await ctx.runQuery(api.sessions.get, { id: sessionId });\n if (!session) {\n throw new Error(\"Session not found\");\n }\n\n const rootInstance = hydrateSandboxInstance(session.instance, sessionId);\n const contextFrames = (await ctx.runQuery(api.sessions.listMachineContextFrames, { sessionId })) as Frame[];\n const memoryExecutor = new AiSdkExecutor({\n model: aiSdkOpenAI(DISCRETE_MODEL),\n maxOutputTokens: 1024,\n });\n const isMemoryRequest = (request: { inference: { tools: Array<{ name: string }> } }) =>\n request.inference.tools.some((tool) => tool.name === \"saveMemories\");\n const executor: Executor = {\n run: (request) =>\n isMemoryRequest(request)\n ? memoryExecutor.run(request)\n : discreteExecutor.run(request),\n realizePrompt: (request) =>\n isMemoryRequest(request)\n ? memoryExecutor.realizePrompt(request)\n : discreteExecutor.realizePrompt(request),\n };\n const machine = createMachine({\n id: sessionId,\n instance: rootInstance,\n charter: createSandboxCharter(),\n executor,\n frames: contextFrames,\n });\n\n const storedAttachments = attachments ?? [];\n const resolvedAttachments = (await resolveAttachmentUrls(ctx, storedAttachments)) ?? [];\n const userText = content.trim() || (storedAttachments.length ? attachmentSummary(storedAttachments) : \"\");\n const userContent = userContentPartsForFrame(content, resolvedAttachments);\n if (!userText || userContent.length === 0) {\n throw new Error(\"Message requires text or attachments\");\n }\n\n machine.enqueueFrame({\n messages: [{ type: \"user\", content: userContent, text: userText }],\n });\n\n const producedFrames: Frame[] = [];\n for await (const frame of runMachine(machine)) {\n producedFrames.push(frame);\n }\n\n const durableFrames = producedFrames.map((frame) =>\n prepareMachineFrame(frame, ROOT_GENERATOR_ID)\n );\n const frameIds = await ctx.runMutation(api.sessions.appendMachineFrameSequence, {\n sessionId,\n referenceFrameId: session.frameId,\n frames: durableFrames,\n }) as Id<\"frames\">[];\n\n for (const [index, frame] of durableFrames.entries()) {\n const frameId = frameIds[index];\n if (!frameId) continue;\n await persistFrameMessages(ctx, {\n sessionId,\n frame,\n frameId,\n rootGeneratorId: ROOT_GENERATOR_ID,\n });\n }\n\n return { success: true };\n },\n});\n\nexport const sendClientMessage = action({\n args: {\n sessionId: v.id(\"sessions\"),\n message: v.any(),\n },\n handler: async (ctx, { sessionId, message }): Promise => {\n const session = await ctx.runQuery(api.sessions.get, { id: sessionId });\n if (!session) {\n throw new Error(\"Session not found\");\n }\n\n const rootInstance = hydrateSandboxInstance(session.instance, sessionId);\n const contextFrames = (await ctx.runQuery(api.sessions.listMachineContextFrames, { sessionId })) as Frame[];\n const machine = createMachine({\n id: sessionId,\n instance: rootInstance,\n charter: createSandboxCharter(),\n frames: contextFrames,\n });\n const command = message as ClientMachineMessage;\n const result = await executeCommand(machine, command);\n\n const producedFrames: Frame[] = [];\n for await (const frame of runMachine(machine, { scheduleWork: false })) {\n producedFrames.push(frame);\n }\n\n const frameIds = await ctx.runMutation(api.sessions.appendMachineFrameSequence, {\n sessionId,\n referenceFrameId: session.frameId,\n frames: producedFrames,\n }) as Id<\"frames\">[];\n await persistClientCommandAssistantMessages(ctx, {\n sessionId,\n message: command,\n frames: producedFrames,\n frameIds,\n });\n return result;\n },\n});\n\nexport const createSessionAtFoo = createSession;\n\ntype DurableFrame = Frame & { metadata?: Record };\n\nfunction prepareMachineFrame(frame: Frame, rootGeneratorId: string): DurableFrame {\n const mode =\n frame.generatorId === undefined || frame.generatorId === rootGeneratorId\n ? \"text\"\n : \"memory\";\n return {\n ...frame,\n metadata: {\n mode,\n transport: \"convex\",\n },\n };\n}\n\nasync function persistFrameMessages(\n ctx: ActionCtx,\n {\n sessionId,\n frame,\n frameId,\n rootGeneratorId,\n }: {\n sessionId: Id<\"sessions\">;\n frame: Frame;\n frameId: Id<\"frames\">;\n rootGeneratorId: string;\n },\n): Promise {\n for (const message of frame.messages) {\n const text = typeof message.text === \"string\" ? message.text : \"\";\n if (message.type === \"user\" && text.trim()) {\n const attachments = storedAttachmentsFromContentParts(message.content);\n await ctx.runMutation(api.messages.add, {\n sessionId,\n role: \"user\",\n content: text,\n ...(attachments.length ? { attachments } : {}),\n frameId,\n mode: \"text\",\n idempotencyKey: idempotencyKey(\"user\", message),\n });\n }\n\n if (\n message.type === \"assistant\" &&\n shouldPersistAssistantMessage(frame, message, rootGeneratorId) &&\n text.trim()\n ) {\n await ctx.runMutation(api.messages.add, {\n sessionId,\n role: \"assistant\",\n content: text,\n frameId,\n mode: \"text\",\n idempotencyKey: idempotencyKey(\"assistant\", message),\n });\n }\n }\n}\n\nfunction shouldPersistAssistantMessage(\n frame: Frame,\n message: Frame[\"messages\"][number],\n rootGeneratorId: string,\n): boolean {\n if (message.audience === \"self\") return false;\n return frame.generatorId === rootGeneratorId;\n}\n\nasync function persistClientCommandAssistantMessages(\n ctx: ActionCtx,\n {\n sessionId,\n message,\n frames,\n frameIds,\n }: {\n sessionId: Id<\"sessions\">;\n message: ClientMachineMessage;\n frames: Frame[];\n frameIds: Id<\"frames\">[];\n },\n): Promise {\n for (const [frameIndex, frame] of frames.entries()) {\n const frameId = frameIds[frameIndex];\n if (!frameId) continue;\n\n for (const [messageIndex, frameMessage] of frame.messages.entries()) {\n if (frameMessage.type !== \"assistant\") continue;\n const text = typeof frameMessage.text === \"string\" ? frameMessage.text.trim() : \"\";\n if (!text) continue;\n\n await ctx.runMutation(api.messages.add, {\n sessionId,\n role: \"assistant\",\n content: text,\n frameId,\n mode: \"text\",\n idempotencyKey: message.callId\n ? `command:${message.callId}:assistant:${frameIndex}:${messageIndex}`\n : undefined,\n });\n }\n }\n}\n\nfunction idempotencyKey(prefix: string, source: unknown): string {\n if (source && typeof source === \"object\") {\n const record = source as Record;\n const messageId = record.messageId;\n if (typeof messageId === \"string\" && messageId) return `${prefix}:${messageId}`;\n const createdAt = record.createdAt;\n const text = record.text;\n if (createdAt !== undefined && typeof text === \"string\") return `${prefix}:${createdAt}:${text}`;\n }\n return `${prefix}:${crypto.randomUUID()}`;\n}\n","/apps/sandbox/convex/sessionEphemera.ts":"import { v } from \"convex/values\";\nimport { mutation, query } from \"./_generated/server\";\n\nexport const get = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async () => ({ isProcessing: false }),\n});\n\nexport const setProcessing = mutation({\n args: { sessionId: v.id(\"sessions\"), isProcessing: v.boolean() },\n handler: async () => null,\n});\n","/apps/sandbox/convex/sessions.ts":"import { v } from \"convex/values\";\nimport { mutation, query } from \"./_generated/server\";\nimport type { MutationCtx, QueryCtx } from \"./_generated/server\";\nimport type { Doc, Id } from \"./_generated/dataModel\";\nimport { AiSdkExecutor } from \"@projectors/aisdk-executor\";\nimport { escapeConvexJson, restoreConvexJson, stripClientSchemas } from \"./convexJson\";\nimport {\n createSandboxClientSnapshot,\n createSandboxCharter,\n hydrateSandboxInstance,\n hydrateSandboxSourceInstance,\n serializeSandboxSourceInstance,\n} from \"@projectors/sandbox-agent/src/projector-sandbox.js\";\nimport {\n applyInstanceMessage,\n compileProjection,\n createMachine,\n inspectCompiledProjectionTree,\n type Charter,\n type CompiledInference,\n type CompiledContributor,\n type Executor,\n type ExecutorRealizedPrompt,\n type Frame,\n type FrameDraft,\n type InstanceMessage,\n} from \"@projectors/core\";\nimport {\n recordCommandResidue,\n type MachineSyncState,\n} from \"@projectors/core/client\";\nimport {\n collectSessionFramePath,\n getLatestSessionFrameDoc,\n getFrameIndexForSession,\n listSessionContextFrameDocs,\n listSessionFrameDocs,\n restoreFrame,\n} from \"./frameHistory\";\nimport { listMessagesForSession } from \"./messages\";\nimport type { SandboxAttachmentData } from \"@projectors/sandbox-agent/src/attachments.js\";\n\nconst DISCRETE_MODEL =\n typeof process !== \"undefined\"\n ? process.env.OPENAI_DISCRETE_MODEL ?? \"gpt-5.5\"\n : \"gpt-5.5\";\n\ntype DbCtx = MutationCtx | QueryCtx;\ntype SessionDoc = Doc<\"sessions\">;\ntype FrameDoc = Doc<\"frames\">;\n\nexport const create = mutation({\n args: {\n instanceId: v.string(),\n instance: v.any(),\n syncState: v.optional(v.any()),\n },\n handler: async (ctx, { instanceId, instance, syncState }) => {\n const now = Date.now();\n const sessionId = await ctx.db.insert(\"sessions\", {\n contextEpoch: 0,\n familyRootSessionId: undefined,\n forkedFromSessionId: undefined,\n forkedFromFrameId: undefined,\n syncState,\n });\n\n const frameId = await ctx.db.insert(\"frames\", {\n referenceFrameId: undefined,\n metadata: escapeConvexJson({ type: \"init\" }),\n messages: [],\n createdAt: now,\n });\n await ctx.db.insert(\"frameIndex\", {\n sessionId,\n frameId,\n contextEpoch: 0,\n });\n\n await ctx.db.insert(\"projectorInstanceLog\", {\n sessionId,\n instanceId,\n frameId,\n message: escapeConvexJson({ type: \"init\" }),\n instance,\n createdAt: now,\n });\n\n await ctx.db.patch(sessionId, {\n familyRootSessionId: sessionId,\n });\n\n return sessionId;\n },\n});\n\nexport const get = query({\n args: {\n id: v.id(\"sessions\"),\n timetravelFrameId: v.optional(v.id(\"frames\")),\n },\n handler: async (ctx, { id, timetravelFrameId }) => {\n const session = await ctx.db.get(id);\n if (!session) return null;\n\n const latestFrame = await getLatestSessionFrameDoc(ctx, id);\n const effectiveFrameId = timetravelFrameId ?? latestFrame?._id;\n if (!effectiveFrameId) return null;\n if (!(await getFrameIndexForSession(ctx, id, effectiveFrameId))) return null;\n const effectiveFrame = await ctx.db.get(effectiveFrameId);\n if (!effectiveFrame) return null;\n const instance = await getInstanceForFramePath(ctx, session, effectiveFrameId);\n if (!instance) return null;\n const syncState = restoreConvexJson(\n session.syncState ?? { recentCommandResidue: [] },\n ) as MachineSyncState;\n const clientSnapshot = createSandboxClientSnapshot(instance, id, syncState);\n\n return {\n sessionId: id,\n frameId: effectiveFrameId,\n contextEpoch: session.contextEpoch,\n familyRootSessionId: session.familyRootSessionId ?? id,\n forkedFromSessionId: session.forkedFromSessionId,\n forkedFromFrameId: session.forkedFromFrameId,\n instanceId: instance.id,\n instance,\n clientSnapshot: stripClientSchemas(clientSnapshot),\n syncState,\n recentCommandResidue:\n (syncState.recentCommandResidue as string[] | undefined) ?? [],\n createdAt: effectiveFrame.createdAt,\n };\n },\n});\n\nexport const getCompiledIr = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const state = await getObservabilityState(ctx, sessionId);\n if (!state) return null;\n\n return {\n sessionId,\n runtimes: state.runtimes.map((runtime) => {\n const inference = compileRuntimeInference(state, runtime);\n return {\n ...runtime,\n inference: {\n preamble: inference.preamble,\n recency: inference.recency,\n tools: inference.tools.map((tool) => tool.name),\n retrievableStates: inference.retrievableStates,\n },\n };\n }),\n };\n },\n});\n\nexport const getRealizedPrompts = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const state = await getObservabilityState(ctx, sessionId);\n if (!state) return null;\n\n const runtimes = await Promise.all(\n state.runtimes.map(async (runtime) => {\n const inference = compileRuntimeInference(state, runtime);\n let prompt: ExecutorRealizedPrompt;\n try {\n prompt = await state.executor.realizePrompt({\n generatorId: runtime.generatorId,\n activationId: \"\",\n inference,\n });\n } catch (error) {\n prompt = {\n provider: \"error\",\n input: { message: error instanceof Error ? error.message : String(error) },\n };\n }\n\n return {\n ...runtime,\n prompt: stripClientSchemas(prompt),\n };\n }),\n );\n\n return { sessionId, runtimes };\n },\n});\n\ntype RuntimeInspectionTarget = {\n generatorId: string;\n kind: \"generator\";\n nodeKey: string;\n name?: string;\n};\n\ntype ObservabilityState = {\n root: ReturnType;\n charter: Charter;\n frames: Frame[];\n executor: Executor;\n runtimes: RuntimeInspectionTarget[];\n};\n\nasync function getObservabilityState(\n ctx: QueryCtx,\n sessionId: Id<\"sessions\">,\n): Promise {\n const session = await ctx.db.get(sessionId);\n if (!session) return null;\n const latestFrame = await getLatestSessionFrameDoc(ctx, sessionId);\n if (!latestFrame) return null;\n\n const instance = await getInstanceForFramePath(ctx, session, latestFrame._id);\n if (!instance) return null;\n\n const root = hydrateSandboxInstance(instance, sessionId);\n const frames = await getMachineContextFrames(ctx, session);\n const executor = createObservabilityExecutor();\n const charter = createSandboxCharter();\n const tree = inspectCompiledProjectionTree(root, {\n charter,\n });\n const runtimes = collectRuntimeInspectionTargets(tree.roots);\n\n return {\n root,\n charter,\n frames,\n executor,\n runtimes,\n };\n}\n\nfunction collectRuntimeInspectionTargets(\n nodes: CompiledContributor[],\n): RuntimeInspectionTarget[] {\n const targets: RuntimeInspectionTarget[] = [];\n const visit = (node: CompiledContributor) => {\n targets.push({\n generatorId: node.id,\n kind: node.kind,\n nodeKey: node.nodeKey,\n ...(node.name ? { name: node.name } : {}),\n });\n node.children.forEach(visit);\n };\n nodes.forEach(visit);\n return targets;\n}\n\nfunction compileRuntimeInference(\n state: ObservabilityState,\n runtime: RuntimeInspectionTarget,\n): CompiledInference {\n return compileProjection(state.root, {\n charter: state.charter,\n targetGeneratorId: runtime.generatorId,\n frameHistory: state.frames,\n });\n}\n\nfunction createObservabilityExecutor(): Executor {\n const discreteExecutor = new AiSdkExecutor({\n model: modelRef(DISCRETE_MODEL),\n maxOutputTokens: 4096,\n });\n const memoryExecutor = new AiSdkExecutor({\n model: modelRef(DISCRETE_MODEL),\n maxOutputTokens: 1024,\n });\n const isMemoryRequest = (request: { inference: { tools: Array<{ name: string }> } }) =>\n request.inference.tools.some((tool) => tool.name === \"saveMemories\");\n\n return {\n run: async () => ({ completionReason: \"done\" }),\n realizePrompt: (request) =>\n isMemoryRequest(request)\n ? memoryExecutor.realizePrompt(request)\n : discreteExecutor.realizePrompt(request),\n };\n}\n\nfunction modelRef(modelId: string): never {\n return {\n provider: \"openai\",\n modelId,\n } as never;\n}\n\nexport const getFamilyTimeline = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const selectedSession = await ctx.db.get(sessionId);\n if (!selectedSession) return null;\n\n const familyRootSessionId = selectedSession.familyRootSessionId ?? selectedSession._id;\n const familySessions = await ctx.db\n .query(\"sessions\")\n .withIndex(\"by_family_root\", (q) => q.eq(\"familyRootSessionId\", familyRootSessionId))\n .collect();\n if (!familySessions.some((session) => session._id === familyRootSessionId)) {\n const rootSession = await ctx.db.get(familyRootSessionId);\n if (rootSession) familySessions.push(rootSession);\n }\n\n const sessions = await Promise.all(\n familySessions.map(async (session) => {\n const frames = await listSessionFrameDocs(ctx, session._id);\n return {\n sessionId: session._id,\n contextEpoch: session.contextEpoch,\n forkedFromSessionId: session.forkedFromSessionId,\n forkedFromFrameId: session.forkedFromFrameId,\n frames: frames.map(restoreFrame),\n };\n }),\n );\n\n return {\n familyRootSessionId,\n sessions,\n edges: familySessions\n .filter((session) => session.forkedFromSessionId && session.forkedFromFrameId)\n .map((session) => ({\n fromSessionId: session.forkedFromSessionId,\n fromFrameId: session.forkedFromFrameId,\n toSessionId: session._id,\n })),\n };\n },\n});\n\nexport const listMachineContextFrames = query({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const session = await ctx.db.get(sessionId);\n if (!session) return [];\n return await getMachineContextFrames(ctx, session);\n },\n});\n\nexport const incrementContextEpoch = mutation({\n args: { sessionId: v.id(\"sessions\") },\n handler: async (ctx, { sessionId }) => {\n const session = await getSessionOrThrow(ctx, sessionId);\n const nextEpoch = session.contextEpoch + 1;\n await ctx.db.patch(sessionId, { contextEpoch: nextEpoch });\n const workerRoom = await ctx.db\n .query(\"agentWorkerRooms\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .first();\n if (workerRoom) {\n await ctx.db.patch(workerRoom._id, {\n agentWorkerId: undefined,\n agentWorkerLeaseToken: undefined,\n agentWorkerLeaseExpiresAt: 0,\n agentWorkerHeartbeatAt: Date.now(),\n agentLastStatusAt: Date.now(),\n });\n }\n return { contextEpoch: nextEpoch };\n },\n});\n\nexport const appendMachineFrame = mutation({\n args: {\n sessionId: v.id(\"sessions\"),\n referenceFrameId: v.optional(v.id(\"frames\")),\n frame: v.any(),\n },\n handler: async (ctx, { sessionId, referenceFrameId, frame }) => {\n const session = await getSessionOrThrow(ctx, sessionId);\n return await appendMachineFrameInternal(ctx, {\n sessionId,\n session,\n frame: frame as FrameDraft,\n referenceFrameId,\n });\n },\n});\n\nexport const appendMachineFrameSequence = mutation({\n args: {\n sessionId: v.id(\"sessions\"),\n referenceFrameId: v.optional(v.id(\"frames\")),\n frames: v.array(v.any()),\n },\n handler: async (ctx, { sessionId, referenceFrameId, frames }) => {\n const session = await getSessionOrThrow(ctx, sessionId);\n return await appendMachineFrameSequenceInternal(ctx, {\n sessionId,\n session,\n referenceFrameId,\n frames: frames as Frame[],\n });\n },\n});\n\nexport const cloneFromFrame = mutation({\n args: {\n sourceSessionId: v.id(\"sessions\"),\n targetFrameId: v.id(\"frames\"),\n },\n handler: async (ctx, { sourceSessionId, targetFrameId }) => {\n const sourceSession = await ctx.db.get(sourceSessionId);\n if (!sourceSession) throw new Error(\"Source session not found\");\n\n const targetFrameIndex = await getFrameIndexForSession(ctx, sourceSessionId, targetFrameId);\n if (!targetFrameIndex) throw new Error(\"Target frame not found in source session\");\n\n const sourceFrames = await collectSessionFramePath(ctx, sourceSession, targetFrameId);\n const sourceFrameIds = new Set(sourceFrames.map((frame) => frame._id));\n const instance = await getInstanceForFramePath(ctx, sourceSession, targetFrameId);\n if (!instance) throw new Error(\"Source session has no instance log for target frame\");\n\n const newSessionId = await ctx.db.insert(\"sessions\", {\n contextEpoch: targetFrameIndex.contextEpoch,\n familyRootSessionId: sourceSession.familyRootSessionId ?? sourceSessionId,\n forkedFromSessionId: sourceSessionId,\n forkedFromFrameId: targetFrameId,\n syncState: escapeConvexJson({ recentCommandResidue: [] }),\n });\n\n const sourceFrameIndexRows = await ctx.db\n .query(\"frameIndex\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sourceSessionId))\n .collect();\n for (const row of sourceFrameIndexRows) {\n if (!sourceFrameIds.has(row.frameId)) continue;\n await ctx.db.insert(\"frameIndex\", {\n sessionId: newSessionId,\n frameId: row.frameId,\n contextEpoch: row.contextEpoch,\n });\n }\n\n const sourceLogs = await ctx.db\n .query(\"projectorInstanceLog\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sourceSessionId))\n .collect();\n for (const log of sourceLogs) {\n if (log.frameId && !sourceFrameIds.has(log.frameId)) continue;\n await ctx.db.insert(\"projectorInstanceLog\", {\n sessionId: newSessionId,\n instanceId: log.instanceId,\n frameId: log.frameId,\n message: log.message,\n instance: log.instance,\n createdAt: log.createdAt,\n });\n }\n\n const sourceMessages = await listMessagesForSession(ctx, sourceSessionId);\n for (const message of sourceMessages) {\n if (!message || !sourceFrameIds.has(message.frameId)) continue;\n await ctx.db.insert(\"messageIndex\", {\n sessionId: newSessionId,\n messageId: message._id,\n idempotencyKey: message.idempotencyKey,\n });\n }\n\n return newSessionId;\n },\n});\n\nasync function getSessionOrThrow(ctx: MutationCtx, sessionId: Id<\"sessions\">) {\n const session = await ctx.db.get(sessionId);\n if (!session) {\n throw new Error(\"Session not found\");\n }\n return session;\n}\n\nasync function appendMachineFrameInternal(\n ctx: MutationCtx,\n {\n sessionId,\n session,\n frame,\n referenceFrameId,\n }: {\n sessionId: Id<\"sessions\">;\n session: SessionDoc;\n frame: (FrameDraft | Frame) & { metadata?: Record };\n referenceFrameId?: Id<\"frames\">;\n },\n) {\n const effectiveReferenceFrameId = referenceFrameId ?? (await getLatestSessionFrameDoc(ctx, sessionId))?._id;\n if (effectiveReferenceFrameId && !(await getFrameIndexForSession(ctx, sessionId, effectiveReferenceFrameId))) {\n throw new Error(\"Reference frame is not indexed for session\");\n }\n\n const metadata =\n \"id\" in frame && typeof frame.id === \"string\"\n ? { ...(frame.metadata ?? {}), projectorFrameId: frame.id }\n : frame.metadata;\n const frameId = await ctx.db.insert(\"frames\", {\n referenceFrameId: effectiveReferenceFrameId,\n ...(frame.generatorId !== undefined ? { generatorId: frame.generatorId } : {}),\n ...(frame.activationId !== undefined ? { activationId: frame.activationId } : {}),\n ...(frame.inert !== undefined ? { inert: frame.inert } : {}),\n ...(metadata !== undefined ? { metadata: escapeConvexJson(metadata) } : {}),\n ...(frame.provenance !== undefined ? { provenance: escapeConvexJson(frame.provenance) } : {}),\n messages: escapeConvexJson(frame.messages),\n createdAt: Date.now(),\n });\n await ctx.db.insert(\"frameIndex\", {\n sessionId,\n frameId,\n contextEpoch: session.contextEpoch,\n });\n await applyFrameInstanceMessages(ctx, sessionId, frameId, frame.messages);\n await recordFrameCommandResidue(ctx, sessionId, frame.messages);\n return frameId;\n}\n\nasync function recordFrameCommandResidue(\n ctx: MutationCtx,\n sessionId: Id<\"sessions\">,\n messages: readonly FrameDraft[\"messages\"][number][],\n): Promise {\n const callIds = messages\n .filter(\n (message) =>\n message.type === \"action\" &&\n message.kind === \"result\" &&\n message.action === \"command\" &&\n typeof message.callId === \"string\" &&\n message.callId.length > 0,\n )\n .map((message) => message.callId as string);\n if (callIds.length === 0) return;\n\n const session = await ctx.db.get(sessionId);\n if (!session) return;\n\n const syncState = restoreConvexJson(\n session.syncState ?? { recentCommandResidue: [] },\n ) as MachineSyncState;\n const nextSyncState = callIds.reduce(\n (state, callId) => recordCommandResidue(state, callId, { limit: 20 }),\n syncState,\n );\n await ctx.db.patch(sessionId, {\n syncState: escapeConvexJson(nextSyncState),\n });\n}\n\nasync function appendMachineFrameSequenceInternal(\n ctx: MutationCtx,\n {\n sessionId,\n session,\n referenceFrameId,\n frames,\n }: {\n sessionId: Id<\"sessions\">;\n session: SessionDoc;\n referenceFrameId?: Id<\"frames\">;\n frames: Frame[];\n },\n) {\n const frameIds: Id<\"frames\">[] = [];\n let currentReferenceFrameId = referenceFrameId;\n\n for (const frame of frames) {\n const frameId = await appendMachineFrameInternal(ctx, {\n sessionId,\n session,\n referenceFrameId: currentReferenceFrameId,\n frame,\n });\n frameIds.push(frameId);\n currentReferenceFrameId = frameId;\n }\n\n return frameIds;\n}\n\nasync function applyFrameInstanceMessages(\n ctx: MutationCtx,\n sessionId: Id<\"sessions\">,\n frameId: Id<\"frames\">,\n messages: Frame[\"messages\"],\n): Promise {\n for (const message of messages) {\n if (!isInstanceMessage(message)) continue;\n await applyFrameInstanceMessage(ctx, sessionId, frameId, message);\n }\n}\n\nasync function applyFrameInstanceMessage(\n ctx: MutationCtx,\n sessionId: Id<\"sessions\">,\n frameId: Id<\"frames\">,\n message: InstanceMessage,\n): Promise {\n const targetInstanceId = instanceMessageTargetId(message);\n const source = await getLatestSourceForInstanceMessage(ctx, sessionId, message, targetInstanceId);\n if (!source) {\n throw new Error(`No source instance contains target instance \"${targetInstanceId}\"`);\n }\n\n applyInstanceMessage(source, message, createSandboxCharter());\n\n await ctx.db.insert(\"projectorInstanceLog\", {\n sessionId,\n instanceId: source.id,\n frameId,\n message: escapeConvexJson(message),\n instance: escapeConvexJson(serializeSandboxSourceInstance(source)),\n createdAt: Date.now(),\n });\n}\n\nfunction instanceMessageTargetId(message: InstanceMessage): string {\n if (message.kind === \"spawn\" || message.kind === \"attach\") {\n return message.parentInstanceId;\n }\n return message.instanceId;\n}\n\nasync function getLatestSourceForInstanceMessage(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n message: InstanceMessage,\n targetInstanceId: string,\n) {\n const source = await getLatestSourceContainingInstance(ctx, sessionId, targetInstanceId);\n if (source || message.kind !== \"remove\") {\n return source;\n }\n\n return await getLatestSource(ctx, sessionId);\n}\n\nfunction isInstanceMessage(message: unknown): message is InstanceMessage {\n if (!message || typeof message !== \"object\") return false;\n const record = message as Record;\n return record.type === \"instance\" &&\n (record.kind === \"state.update\" ||\n record.kind === \"transition\" ||\n record.kind === \"spawn\" ||\n record.kind === \"attach\" ||\n record.kind === \"remove\");\n}\n\nasync function getLatestSourceContainingInstance(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n targetInstanceId: string,\n) {\n const latestLogs = await getLatestSourceLogs(ctx, sessionId);\n for (const log of latestLogs) {\n const source = hydrateSandboxSourceInstance(restoreConvexJson(log.instance));\n if (containsInstance(source, targetInstanceId)) {\n return source;\n }\n }\n return null;\n}\n\nasync function getLatestSource(\n ctx: DbCtx,\n sessionId: Id<\"sessions\">,\n) {\n const [latestLog] = await getLatestSourceLogs(ctx, sessionId);\n return latestLog ? hydrateSandboxSourceInstance(restoreConvexJson(latestLog.instance)) : null;\n}\n\nasync function getLatestSourceLogs(ctx: DbCtx, sessionId: Id<\"sessions\">) {\n const logs = await ctx.db\n .query(\"projectorInstanceLog\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", sessionId))\n .collect();\n const latestByInstance = new Map>();\n for (const log of logs.sort((a, b) => a.createdAt - b.createdAt || a._id.localeCompare(b._id))) {\n latestByInstance.set(log.instanceId, log);\n }\n return [...latestByInstance.values()].sort((a, b) => b.createdAt - a.createdAt || b._id.localeCompare(a._id));\n}\n\nfunction containsInstance(instance: ReturnType, targetInstanceId: string): boolean {\n if (instance.id === targetInstanceId) return true;\n return (instance.children ?? []).some((child) => containsInstance(child, targetInstanceId));\n}\n\nasync function getInstanceForFramePath(\n ctx: DbCtx,\n session: SessionDoc,\n frameId: Id<\"frames\">,\n) {\n const log = await getLatestInstanceLogForFramePath(ctx, session, frameId);\n return log ? restoreConvexJson(log.instance) : null;\n}\n\nasync function getLatestInstanceLogForFramePath(\n ctx: DbCtx,\n session: SessionDoc,\n frameId: Id<\"frames\">,\n) {\n const path = await collectSessionFramePath(ctx, session, frameId);\n const frameIds = new Set(path.map((frame) => frame._id));\n const logs = await ctx.db\n .query(\"projectorInstanceLog\")\n .withIndex(\"by_session\", (q) => q.eq(\"sessionId\", session._id))\n .collect();\n return logs\n .filter((entry) => !entry.frameId || frameIds.has(entry.frameId))\n .sort((a, b) => b.createdAt - a.createdAt || b._id.localeCompare(a._id))[0];\n}\n\nasync function getMachineContextFrames(ctx: DbCtx, session: SessionDoc): Promise {\n return (await listSessionContextFrameDocs(ctx, session)).map(restoreFrame) as Frame[];\n}\n","/apps/sandbox/convex/tsconfig.json":"{\n /* This TypeScript project config describes the environment that\n * Convex functions run in and is used to typecheck them.\n * You can modify it, but some settings are required to use Convex.\n */\n \"compilerOptions\": {\n /* These settings are not required by Convex and can be modified. */\n \"allowJs\": true,\n \"strict\": true,\n \"moduleResolution\": \"Bundler\",\n \"jsx\": \"react-jsx\",\n \"skipLibCheck\": true,\n \"allowSyntheticDefaultImports\": true,\n\n /* These compiler options are required by Convex */\n \"target\": \"ESNext\",\n \"lib\": [\"ES2021\", \"dom\"],\n \"forceConsistentCasingInFileNames\": true,\n \"module\": \"ESNext\",\n \"isolatedModules\": true,\n \"noEmit\": true\n },\n \"include\": [\"./**/*\"],\n \"exclude\": [\"./_generated\"]\n}\n","/apps/sandbox/next.config.ts":"import type { NextConfig } from \"next\";\n\nconst nextConfig: NextConfig = {\n transpilePackages: [\"@projectors/core\", \"@projectors/sandbox-agent\"],\n};\n\nexport default nextConfig;\n","/apps/sandbox/package.json":"{\n \"name\": \"@projectors/sandbox\",\n \"version\": \"0.1.0\",\n \"private\": true,\n \"license\": \"MIT\",\n \"scripts\": {\n \"dev\": \"next dev\",\n \"build\": \"next build\",\n \"start\": \"next start\",\n \"lint\": \"eslint\",\n \"convex:truncate:all\": \"bun run ../../scripts/convex-truncate.ts --all\",\n \"typecheck\": \"tsgo\"\n },\n \"dependencies\": {\n \"@ai-sdk/openai\": \"catalog:\",\n \"@livekit/components-react\": \"^2.9.19\",\n \"@projectors/aisdk-executor\": \"workspace:*\",\n \"cli-spinners\": \"^3.4.0\",\n \"convex\": \"^1.31.2\",\n \"jotai\": \"^2.16.1\",\n \"js-cookie\": \"^3.0.5\",\n \"json5\": \"^2.2.3\",\n \"livekit-client\": \"^2.17.0\",\n \"livekit-server-sdk\": \"^2.15.0\",\n \"@projectors/sandbox-agent\": \"workspace:*\",\n \"@projectors/core\": \"workspace:*\",\n \"next\": \"16.1.1\",\n \"react\": \"19.2.3\",\n \"react-dom\": \"19.2.3\",\n \"react-simple-code-editor\": \"^0.14.1\",\n \"zod\": \"catalog:\"\n },\n \"devDependencies\": {\n \"@tailwindcss/postcss\": \"^4\",\n \"@types/cli-spinners\": \"^2.0.0\",\n \"@types/js-cookie\": \"^3.0.6\",\n \"@types/node\": \"^20\",\n \"@types/react\": \"^19\",\n \"@types/react-dom\": \"^19\",\n \"eslint\": \"^9\",\n \"eslint-config-next\": \"16.1.1\",\n \"tailwindcss\": \"^4\",\n \"typescript\": \"^5\"\n }\n}\n","/apps/sandbox/postcss.config.mjs":"const config = {\n plugins: {\n \"@tailwindcss/postcss\": {},\n },\n};\n\nexport default config;\n","/apps/sandbox/src/atoms.ts":"import { atom } from \"jotai\";\nimport { atomWithStorage } from \"jotai/utils\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\n\nexport const inputAtom = atom(\"\");\nexport const isLoadingAtom = atom(false);\nexport const timeTravelFrameIdAtom = atom | null>(null);\nexport const scanlinesEnabledAtom = atomWithStorage(\"sandbox-scanlines\", true);\nexport type MessageTransport = \"convex\" | \"livekit\";\nexport const messageTransportAtom = atomWithStorage(\"sandbox-message-transport\", \"convex\");\n\nexport type AgentTab = \"tree\" | \"state\" | \"history\" | \"commands\" | \"playground\" | \"dev\";\nexport const activeAgentTabAtom = atomWithStorage(\"sandbox-agent-tab\", \"tree\");\n\nexport type TreeSubtab = \"instance\" | \"projection\" | \"ir\" | \"realized\";\nexport const activeTreeSubtabAtom = atomWithStorage(\"sandbox-tree-subtab\", \"instance\");\n\nexport type HistorySubtab = \"frames\" | \"messages\" | \"branches\" | \"controls\";\nexport const activeHistorySubtabAtom = atomWithStorage(\"sandbox-history-subtab\", \"frames\");\n","/apps/sandbox/src/attachments.ts":"import type { SandboxAttachment } from \"./types/display\";\n\nexport const MAX_ATTACHMENTS_PER_MESSAGE = 8;\nexport const MAX_ATTACHMENT_BYTES = 25 * 1024 * 1024;\n\nexport function isImageContentType(contentType: string): boolean {\n return contentType.toLowerCase().startsWith(\"image/\");\n}\n\nexport function attachmentKind(contentType: string): SandboxAttachment[\"kind\"] {\n return isImageContentType(contentType) ? \"image\" : \"file\";\n}\n\nexport function attachmentSummary(attachments: readonly Pick[]): string {\n return `Attached: ${attachments.map((attachment) => attachment.name).join(\", \")}`;\n}\n\nexport function normalizeFileContentType(file: File): string {\n return file.type || \"application/octet-stream\";\n}\n","/apps/sandbox/src/hooks.ts":"import { useCallback, useEffect, useState } from \"react\";\nimport Cookies from \"js-cookie\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\n\nconst SESSION_COOKIE_KEY = \"sandbox-sessionId\";\n\nfunction getStoredSessionId(): Id<\"sessions\"> | null {\n const stored = Cookies.get(SESSION_COOKIE_KEY);\n return stored ? (stored as Id<\"sessions\">) : null;\n}\n\nexport function useSessionId(initialSessionId: Id<\"sessions\"> | null) {\n const [sessionId, setSessionIdState] = useState | null>(\n () => initialSessionId ?? getStoredSessionId(),\n );\n\n const setSessionId = useCallback((id: Id<\"sessions\"> | null) => {\n if (id) {\n Cookies.set(SESSION_COOKIE_KEY, id, { expires: 365 });\n } else {\n Cookies.remove(SESSION_COOKIE_KEY);\n }\n setSessionIdState(id);\n }, []);\n\n return [sessionId, setSessionId] as const;\n}\n\nexport function useKeyboardFocus(\n messageRef: React.RefObject,\n agentRef: React.RefObject,\n onAgentShortcut?: () => void,\n) {\n useEffect(() => {\n const onKeyDown = (event: KeyboardEvent) => {\n const target = event.target as HTMLElement | null;\n const typing =\n target?.tagName === \"INPUT\" ||\n target?.tagName === \"TEXTAREA\" ||\n target?.isContentEditable;\n if (typing || event.metaKey || event.ctrlKey || event.altKey) return;\n if (event.key.toLowerCase() === \"m\") {\n event.preventDefault();\n messageRef.current?.focus();\n }\n if (event.key.toLowerCase() === \"a\") {\n event.preventDefault();\n if (onAgentShortcut) {\n onAgentShortcut();\n } else {\n agentRef.current?.focus();\n }\n }\n };\n window.addEventListener(\"keydown\", onKeyDown);\n return () => window.removeEventListener(\"keydown\", onKeyDown);\n }, [agentRef, messageRef, onAgentShortcut]);\n}\n","/apps/sandbox/src/projector/ProjectorProvider.tsx":"\"use client\";\n\nimport {\n createContext,\n useContext,\n useEffect,\n useRef,\n useState,\n type ReactNode,\n} from \"react\";\nimport {\n createMachineEffigy,\n createOptimisticEffigy,\n type ClientMachineMessage,\n type OptimisticEffigy,\n} from \"@projectors/core/client\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\nimport type { SandboxClientInstance, SandboxClientSnapshot } from \"@/src/types/display\";\n\ntype SendClientMessageAction = (args: {\n sessionId: Id<\"sessions\">;\n message: ClientMachineMessage;\n}) => Promise;\n\nconst EMPTY_CLIENT_SNAPSHOT: SandboxClientSnapshot = {\n instance: null,\n recentCommandResidue: [],\n};\n\ntype ProjectorContextValue = {\n effigy: OptimisticEffigy;\n instances: SandboxClientInstance[];\n snapshot: SandboxClientSnapshot;\n readOnly: boolean;\n};\n\nconst ProjectorContext = createContext(null);\n\nexport function ProjectorProvider({\n children,\n sessionId,\n sendClientMessage,\n snapshot,\n readOnly = false,\n}: {\n children: ReactNode;\n sessionId: Id<\"sessions\"> | null;\n sendClientMessage: SendClientMessageAction;\n snapshot?: SandboxClientSnapshot | null;\n readOnly?: boolean;\n}) {\n const sessionIdRef = useRef | null>(null);\n const sendClientMessageRef = useRef(null);\n const readOnlyRef = useRef(readOnly);\n const effigyRef = useRef | null>(null);\n\n if (!effigyRef.current) {\n effigyRef.current = createOptimisticEffigy(\n createMachineEffigy(async (message) => {\n const activeSessionId = sessionIdRef.current;\n const activeSendClientMessage = sendClientMessageRef.current;\n if (readOnlyRef.current) {\n throw new Error(\"Fork this historical frame before sending commands\");\n }\n if (!activeSessionId || !activeSendClientMessage) {\n throw new Error(\"No active session\");\n }\n return await activeSendClientMessage({ sessionId: activeSessionId, message });\n }),\n );\n }\n\n const effigy = effigyRef.current;\n const currentSnapshot = snapshot ?? EMPTY_CLIENT_SNAPSHOT;\n const residue = currentSnapshot.recentCommandResidue ?? [];\n const [, rerenderOptimisticState] = useState(0);\n\n useEffect(() => {\n sendClientMessageRef.current = sendClientMessage;\n }, [sendClientMessage]);\n\n useEffect(() => {\n readOnlyRef.current = readOnly;\n if (readOnly) {\n effigy.clearPending();\n }\n }, [effigy, readOnly]);\n\n useEffect(() => {\n if (sessionIdRef.current !== sessionId) {\n effigy.clearPending();\n sessionIdRef.current = sessionId;\n }\n }, [effigy, sessionId]);\n\n useEffect(() => {\n effigy.setRecentCommandResidue(residue);\n effigy.setInstances(currentSnapshot.instance ? [currentSnapshot.instance] : []);\n }, [effigy, residue, currentSnapshot.instance]);\n\n useEffect(\n () => effigy.subscribe(() => rerenderOptimisticState((version) => version + 1)),\n [effigy],\n );\n\n return (\n \n {children}\n \n );\n}\n\nexport function useProjector() {\n const context = useContext(ProjectorContext);\n if (!context) {\n throw new Error(\"useProjector must be used within ProjectorProvider\");\n }\n return context;\n}\n","/apps/sandbox/src/types/css.d.ts":"// tsgo (unlike tsc with Next's bundler resolution) checks side-effect CSS\n// imports; declare them so `import \"./globals.css\"` typechecks.\ndeclare module \"*.css\";\n","/apps/sandbox/src/types/display.ts":"import type { CompiledProjectionTree } from \"@projectors/core\";\nimport type { ClientInstance, MachineClientSnapshot } from \"@projectors/core/client\";\n\nexport type SandboxClientInstance = ClientInstance;\nexport type SandboxClientSnapshot = MachineClientSnapshot & {\n projectionTree?: CompiledProjectionTree;\n};\n\nexport type SandboxAttachment = {\n storageId: string;\n url: string | null;\n dataUrl?: string;\n name: string;\n contentType: string;\n size: number;\n kind: \"image\" | \"file\";\n};\n\nexport type SandboxMessage = {\n _id: string;\n role: \"user\" | \"assistant\";\n content: string;\n createdAt: number;\n frameId: string;\n mode?: \"text\" | \"voice\";\n attachments?: SandboxAttachment[];\n};\n","/apps/sandbox/src/voice/LiveVoiceClient.tsx":"\"use client\";\n\nimport { useAction } from \"convex/react\";\nimport { useCallback, useEffect, useRef } from \"react\";\nimport {\n ConnectionState,\n ParticipantKind,\n Room,\n RoomEvent,\n Track,\n type LocalVideoTrack,\n type RemoteParticipant,\n type RemoteTrackPublication,\n} from \"livekit-client\";\nimport type { ClientMachineMessage } from \"@projectors/core/client\";\nimport { api } from \"@/convex/_generated/api\";\nimport type { Id } from \"@/convex/_generated/dataModel\";\nimport type { SandboxAttachment } from \"@/src/types/display\";\n\ntype VoiceStatus = \"idle\" | \"connecting\" | \"connected\" | \"disconnected\" | \"error\";\ntype SendLiveKitMessage = (message: { content: string; attachments: SandboxAttachment[] }) => Promise;\ntype SendLiveKitCommand = (message: ClientMachineMessage) => Promise;\nconst MESSAGE_TOPIC = \"sandbox.message.v1\";\nconst COMMAND_RPC_METHOD = \"sandbox.command.v1\";\nconst CAMERA_CAPTURE_OPTIONS = {\n resolution: { width: 1280, height: 720 },\n frameRate: 30,\n} as const;\n\ntype LiveVoiceClientProps = {\n sessionId: Id<\"sessions\"> | null;\n liveKitEnabled: boolean;\n liveKitWorkerReady: boolean;\n voiceEnabled: boolean;\n cameraEnabled: boolean;\n onStatusChange?: (status: VoiceStatus, detail?: string) => void;\n onSendMessageChange?: (sendMessage: SendLiveKitMessage | null) => void;\n onSendCommandChange?: (sendCommand: SendLiveKitCommand | null) => void;\n onLocalCameraTrackChange?: (track: LocalVideoTrack | null) => void;\n};\n\nexport function LiveVoiceClient({\n sessionId,\n liveKitEnabled,\n liveKitWorkerReady,\n voiceEnabled,\n cameraEnabled,\n onStatusChange,\n onSendMessageChange,\n onSendCommandChange,\n onLocalCameraTrackChange,\n}: LiveVoiceClientProps) {\n const getToken = useAction(api.livekitAgentActions.getToken);\n const roomRef = useRef(null);\n const roomSessionIdRef = useRef | null>(null);\n const audioRef = useRef(null);\n const connectingRef = useRef(false);\n const cameraEnabledRef = useRef(cameraEnabled);\n const voiceEnabledRef = useRef(voiceEnabled);\n const publishSenderRef = useRef<(room: Room | null) => void>(() => undefined);\n\n useEffect(() => {\n cameraEnabledRef.current = cameraEnabled;\n }, [cameraEnabled]);\n\n useEffect(() => {\n voiceEnabledRef.current = voiceEnabled;\n }, [voiceEnabled]);\n\n const attachAgentAudioPublication = useCallback((publication: RemoteTrackPublication) => {\n const audio = audioRef.current;\n if (!audio || !publication.track) return;\n publication.track.attach(audio);\n void audio.play().catch((error) => {\n console.info(\"[sandbox/livekit] unable to start agent audio playback\", error);\n });\n }, []);\n\n const reconcileAgentAudioSubscriptions = useCallback((room: Room, enabled: boolean) => {\n for (const participant of room.remoteParticipants.values()) {\n if (!isAgentParticipant(participant)) continue;\n for (const publication of participant.audioTrackPublications.values()) {\n publication.setSubscribed(enabled);\n if (!enabled) {\n publication.track?.detach();\n } else {\n attachAgentAudioPublication(publication);\n }\n }\n }\n }, [attachAgentAudioPublication]);\n\n const setAudioElement = useCallback((element: HTMLAudioElement | null) => {\n audioRef.current = element;\n const room = roomRef.current;\n if (element && room?.state === ConnectionState.Connected && voiceEnabledRef.current) {\n reconcileAgentAudioSubscriptions(room, true);\n }\n }, [reconcileAgentAudioSubscriptions]);\n\n const publishSender = useCallback((room: Room | null) => {\n const roomSessionId = roomSessionIdRef.current;\n if (\n !liveKitWorkerReady ||\n !sessionId ||\n roomSessionId !== sessionId ||\n !room ||\n room.state !== ConnectionState.Connected\n ) {\n onSendMessageChange?.(null);\n onSendCommandChange?.(null);\n return;\n }\n\n const agentParticipant = findAgentParticipant(room);\n if (!agentParticipant) {\n onSendMessageChange?.(null);\n onSendCommandChange?.(null);\n return;\n }\n\n onSendMessageChange?.(async ({ content, attachments }) => {\n if (\n roomRef.current !== room ||\n roomSessionIdRef.current !== sessionId ||\n room.state !== ConnectionState.Connected\n ) {\n onSendMessageChange?.(null);\n throw new Error(\"LiveKit room changed before the message was sent\");\n }\n const payload = new TextEncoder().encode(JSON.stringify({ content, attachments, sentAt: Date.now() }));\n console.info(\"[sandbox/livekit] publish text message\", {\n topic: MESSAGE_TOPIC,\n bytes: payload.byteLength,\n contentChars: content.length,\n attachments: summarizeLiveKitAttachments(attachments),\n participants: [...room.remoteParticipants.values()].map((participant) => participant.identity),\n });\n await room.localParticipant.publishData(payload, {\n reliable: true,\n topic: MESSAGE_TOPIC,\n });\n onStatusChange?.(\"connected\", `sent ${payload.byteLength} bytes`);\n });\n\n onSendCommandChange?.(async (message: ClientMachineMessage) => {\n if (\n roomRef.current !== room ||\n roomSessionIdRef.current !== sessionId ||\n room.state !== ConnectionState.Connected\n ) {\n onSendCommandChange?.(null);\n throw new Error(\"LiveKit room changed before the command was sent\");\n }\n const target = findAgentParticipant(room);\n if (!target) {\n onSendCommandChange?.(null);\n throw new Error(\"LiveKit agent is not ready\");\n }\n const payload = JSON.stringify({ sessionId, message, sentAt: Date.now() });\n const response = await room.localParticipant.performRpc({\n destinationIdentity: target.identity,\n method: COMMAND_RPC_METHOD,\n payload,\n responseTimeout: 15_000,\n });\n return JSON.parse(response) as unknown;\n });\n }, [liveKitWorkerReady, onSendCommandChange, onSendMessageChange, onStatusChange, sessionId]);\n\n useEffect(() => {\n publishSenderRef.current = publishSender;\n }, [publishSender]);\n\n useEffect(() => {\n publishSender(roomRef.current);\n }, [publishSender]);\n\n const disconnect = useCallback(async () => {\n connectingRef.current = false;\n onSendMessageChange?.(null);\n onSendCommandChange?.(null);\n onLocalCameraTrackChange?.(null);\n const room = roomRef.current;\n roomRef.current = null;\n roomSessionIdRef.current = null;\n if (room) {\n await room.disconnect();\n }\n if (audioRef.current) {\n audioRef.current.srcObject = null;\n }\n onStatusChange?.(\"disconnected\");\n }, [onLocalCameraTrackChange, onSendCommandChange, onSendMessageChange, onStatusChange]);\n\n useEffect(() => {\n onSendMessageChange?.(null);\n onSendCommandChange?.(null);\n if (roomRef.current && roomSessionIdRef.current !== sessionId) {\n void disconnect();\n }\n }, [disconnect, onSendCommandChange, onSendMessageChange, sessionId]);\n\n useEffect(() => {\n if (!liveKitEnabled || !sessionId) {\n void disconnect();\n if (!liveKitEnabled) onStatusChange?.(\"idle\");\n return;\n }\n\n if (roomRef.current || connectingRef.current) {\n return;\n }\n\n let cancelled = false;\n connectingRef.current = true;\n onStatusChange?.(\"connecting\");\n\n void (async () => {\n try {\n const { token, url } = await getToken({ sessionId });\n if (cancelled) return;\n\n const room = new Room({ adaptiveStream: true, dynacast: true });\n roomRef.current = room;\n roomSessionIdRef.current = sessionId;\n const reconcileRemoteAudioSubscription = (\n publication: RemoteTrackPublication,\n participant: RemoteParticipant,\n ) => {\n if (publication.kind !== Track.Kind.Audio) return;\n const shouldSubscribe = voiceEnabledRef.current && isAgentParticipant(participant);\n publication.setSubscribed(shouldSubscribe);\n if (!shouldSubscribe) {\n publication.track?.detach();\n } else {\n attachAgentAudioPublication(publication);\n }\n };\n const reconcileRemoteAudioSubscriptions = () => {\n reconcileAgentAudioSubscriptions(room, voiceEnabledRef.current);\n };\n\n room.on(RoomEvent.ConnectionStateChanged, (state) => {\n if (state === ConnectionState.Connected) {\n onStatusChange?.(\"connected\", `room ${room.name}`);\n reconcileRemoteAudioSubscriptions();\n publishSenderRef.current(room);\n }\n if (state === ConnectionState.Disconnected) {\n onStatusChange?.(\"disconnected\");\n onSendMessageChange?.(null);\n onSendCommandChange?.(null);\n }\n });\n\n room.on(RoomEvent.ParticipantConnected, (participant) => {\n onStatusChange?.(\n \"connected\",\n isAgentParticipant(participant)\n ? `agent ${participant.identity}`\n : `${room.remoteParticipants.size} remote participant(s)`,\n );\n for (const publication of participant.audioTrackPublications.values()) {\n reconcileRemoteAudioSubscription(publication, participant);\n }\n publishSenderRef.current(room);\n });\n room.on(RoomEvent.ParticipantDisconnected, () => {\n onStatusChange?.(\"connected\", `${room.remoteParticipants.size} remote participant(s)`);\n publishSenderRef.current(room);\n });\n\n room.on(RoomEvent.TrackPublished, (publication, participant) => {\n reconcileRemoteAudioSubscription(publication, participant);\n });\n\n room.on(RoomEvent.TrackSubscribed, (track, publication, participant) => {\n if (track.kind !== Track.Kind.Audio) return;\n if (!voiceEnabledRef.current || !isAgentParticipant(participant)) {\n publication.setSubscribed(false);\n track.detach();\n return;\n }\n attachAgentAudioPublication(publication);\n });\n\n room.on(RoomEvent.TrackUnsubscribed, (track) => {\n track.detach();\n });\n\n room.on(RoomEvent.LocalTrackPublished, (publication) => {\n if (publication.source === Track.Source.Camera) {\n onLocalCameraTrackChange?.(publication.videoTrack ?? null);\n }\n });\n\n room.on(RoomEvent.LocalTrackUnpublished, (publication) => {\n if (publication.source === Track.Source.Camera) {\n onLocalCameraTrackChange?.(null);\n }\n });\n\n await room.connect(url, token, { autoSubscribe: false });\n if (cancelled || roomRef.current !== room) return;\n await room.localParticipant.setMicrophoneEnabled(voiceEnabledRef.current);\n if (cancelled || roomRef.current !== room) return;\n const cameraPublication = await room.localParticipant.setCameraEnabled(\n cameraEnabledRef.current,\n CAMERA_CAPTURE_OPTIONS,\n );\n if (cancelled || roomRef.current !== room) return;\n onLocalCameraTrackChange?.(cameraPublication?.videoTrack ?? null);\n publishSenderRef.current(room);\n } catch (error) {\n if (cancelled) return;\n onStatusChange?.(\"error\", error instanceof Error ? error.message : \"Unable to connect voice\");\n await disconnect();\n } finally {\n connectingRef.current = false;\n }\n })();\n\n return () => {\n cancelled = true;\n void disconnect();\n };\n }, [attachAgentAudioPublication, disconnect, getToken, liveKitEnabled, onLocalCameraTrackChange, onSendCommandChange, onSendMessageChange, onStatusChange, reconcileAgentAudioSubscriptions, sessionId]);\n\n useEffect(() => {\n const room = roomRef.current;\n if (!room || room.state !== ConnectionState.Connected) return;\n reconcileAgentAudioSubscriptions(room, voiceEnabled);\n void room.localParticipant.setMicrophoneEnabled(voiceEnabled).catch((error) => {\n onStatusChange?.(\"error\", error instanceof Error ? error.message : \"Unable to toggle microphone\");\n });\n void room.localParticipant\n .setCameraEnabled(cameraEnabled, CAMERA_CAPTURE_OPTIONS)\n .then((publication) => {\n if (roomRef.current !== room || room.state !== ConnectionState.Connected) return;\n onLocalCameraTrackChange?.(cameraEnabled ? publication?.videoTrack ?? null : null);\n })\n .catch((error) => {\n if (roomRef.current !== room) return;\n onLocalCameraTrackChange?.(null);\n onStatusChange?.(\"error\", error instanceof Error ? error.message : \"Unable to toggle camera\");\n });\n }, [cameraEnabled, onLocalCameraTrackChange, onStatusChange, reconcileAgentAudioSubscriptions, voiceEnabled]);\n\n return