diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 7a6837a..b4135ac 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -6,10 +6,8 @@ import { type UIMessage, } from "ai"; -import { publishedGameKey } from "~/lib/sandbox-paths"; import { isValidThreadId } from "~/lib/thread-id"; import { recordGameVersion } from "~/server/db/games"; -import { publicObjectUrl } from "~/server/r2"; import { prepareLessonSandbox } from "~/server/sandbox/prepare"; import { createLessonAgent } from "~/mastra/agents/lesson-agent"; import { @@ -76,15 +74,28 @@ export async function POST(req: Request) { // inside a tool, if the model ever calls one. const trace = { id: traceId, log }; const sandboxPromise = prepareLessonSandbox({ threadId, userId, trace }); + // Handed to the tools for when the container dies between the lifecycle + // check and a command (auto-stop/archive racing the request). A full + // re-prepare rather than a bare restart, because the R2 mount does not + // survive the container; prepareLessonSandbox is idempotent. + const recoverSandbox = () => { + log("sandbox.recover", { threadId }); + return prepareLessonSandbox({ threadId, userId, trace }); + }; const agent = await createLessonAgent({ threadId, userId, model, sandboxPromise, - // Stable for the life of the thread: every publish overwrites this one key, - // so the teacher's link never changes. - publishedUrl: publicObjectUrl(publishedGameKey(userId, threadId)), + recoverSandbox, + // Stable for the life of the thread: /play proxies current/index.html, + // the one key every publish overwrites, so the teacher's link never + // changes. The app route rather than the bucket URL on purpose — the raw + // key embeds the Clerk user id, and this URL is what the model pastes + // into chat. Origin comes from the request so dev and preview deploys + // hand out links on their own host. + publishedUrl: new URL(`/play/${threadId}`, req.url).toString(), // Bound to this thread and user here, so the tool cannot record a version // against anyone else's game — the model never supplies either id. recordVersion: ({ version, label }) => diff --git a/src/app/play/[threadId]/route.ts b/src/app/play/[threadId]/route.ts new file mode 100644 index 0000000..7e3ab84 --- /dev/null +++ b/src/app/play/[threadId]/route.ts @@ -0,0 +1,50 @@ +import type { NextRequest } from "next/server"; + +import { publishedGameKey } from "~/lib/sandbox-paths"; +import { isValidThreadId } from "~/lib/thread-id"; +import { latestGameVersionByThread } from "~/server/db/games"; +import { publicObjectUrl } from "~/server/r2"; + +/** + * The teacher's share link: `/play/`. + * + * The published game physically lives at + * `games///current/index.html` in the bucket — a path that + * bakes the owner's Clerk user id into every URL. Handing that URL out leaks + * the id to whole classrooms, so this route is the one the app shares instead: + * it resolves the owner from the publish index and streams the game through, + * keeping the bucket layout (and the user id) server-side. + * + * Public on purpose (see `middleware.ts`): a share link is opened by students + * who hold no session. The thread id is the only capability, same as the + * bucket's unguessable-path model, and it serves nothing but the built game. + */ +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ threadId: string }> }, +) { + const { threadId } = await params; + if (!isValidThreadId(threadId)) return new Response(null, { status: 404 }); + + const row = await latestGameVersionByThread(threadId); + if (!row) return new Response(null, { status: 404 }); + + // `current/index.html`, not the versioned key — the share link must keep + // tracking the newest publish, exactly like the raw current/ URL did. + const url = publicObjectUrl(publishedGameKey(row.userId, threadId)); + if (!url) return new Response(null, { status: 503 }); + + const upstream = await fetch(url, { cache: "no-store" }); + if (!upstream.ok || !upstream.body) return new Response(null, { status: 404 }); + + return new Response(upstream.body, { + headers: { + "Content-Type": "text/html; charset=utf-8", + // A class of thirty opening the link at once should hit the CDN, but a + // republish should show up within a minute — current/ is mutable. + "Cache-Control": "public, max-age=0, s-maxage=60", + // Share links are for people who hold them, not search results. + "X-Robots-Tag": "noindex", + }, + }); +} diff --git a/src/components/chat/chat-header.tsx b/src/components/chat/chat-header.tsx index ab0deea..78a208d 100644 --- a/src/components/chat/chat-header.tsx +++ b/src/components/chat/chat-header.tsx @@ -9,9 +9,11 @@ import { api } from "~/trpc/react"; /** * Thread title, plus the teacher's share link once a game exists. * - * The link is `current/index.html` — the one key every publish overwrites — so - * it keeps working as the game is revised and can be handed to a class once. - * The preview pane deliberately uses the versioned URL instead; see + * The link is `/play/` — the app route that proxies + * `current/index.html`, the one key every publish overwrites — so it keeps + * working as the game is revised, can be handed to a class once, and never + * exposes the bucket path (which embeds the owner's Clerk user id). The + * preview pane deliberately uses the versioned bucket URL instead; see * `GamePreview`. */ export function ChatHeader({ @@ -23,12 +25,16 @@ export function ChatHeader({ }) { const latest = api.games.latest.useQuery({ threadId }); const [copied, setCopied] = useState(false); - const shareUrl = latest.data?.shareUrl ?? null; + const sharePath = latest.data?.sharePath ?? null; const copy = async () => { - if (!shareUrl) return; + if (!sharePath) return; try { - await navigator.clipboard.writeText(shareUrl); + // Absolute at copy time — the clipboard leaves this origin, the href + // below doesn't have to. + await navigator.clipboard.writeText( + new URL(sharePath, window.location.origin).toString(), + ); setCopied(true); setTimeout(() => setCopied(false), 1500); } catch { @@ -45,7 +51,7 @@ export function ChatHeader({ {title} - {shareUrl ? ( + {sharePath ? (
v{latest.data?.version} @@ -54,7 +60,7 @@ export function ChatHeader({ {copied ? "Copied" : "Copy link"} diff --git a/src/mastra/agents/lesson-agent.ts b/src/mastra/agents/lesson-agent.ts index 544117b..8150fe6 100644 --- a/src/mastra/agents/lesson-agent.ts +++ b/src/mastra/agents/lesson-agent.ts @@ -55,6 +55,12 @@ export type CreateLessonAgentOptions = { * tools and stays a pure planner — it keeps the skill tools. */ sandboxPromise?: Promise; + /** + * Re-runs `prepareLessonSandbox` when the sandbox container dies mid- + * request (auto-stop/archive racing a tool call). Optional, like the + * sandbox itself. + */ + recoverSandbox?: () => Promise; /** * Public URL this thread's published game will live at. Resolved by the * route because it needs `~/env`; passed down so neither this factory nor @@ -109,6 +115,7 @@ export async function createLessonAgent(opts: CreateLessonAgentOptions) { tools: opts.sandboxPromise ? createSandboxTools({ sandboxPromise: opts.sandboxPromise, + recoverSandbox: opts.recoverSandbox, publishedUrl: opts.publishedUrl, recordVersion: opts.recordVersion, trace: opts.trace, diff --git a/src/mastra/tools/bash-tool.ts b/src/mastra/tools/bash-tool.ts index 765cd96..c651f76 100644 --- a/src/mastra/tools/bash-tool.ts +++ b/src/mastra/tools/bash-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { ENGINE_ROOT, GAME_ROOT } from "~/lib/sandbox-paths"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; /** Beyond this the output goes to a file and the model gets a pointer. */ @@ -18,6 +19,8 @@ export type CreateBashToolOptions = { * still booting/mounting; the first `bash` call pays whatever is left. */ sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; /** Extra env for every command (on top of the sandbox's own envVars). */ env?: Record; trace?: LessonTrace; @@ -39,6 +42,7 @@ async function spillToFile(sandbox: Sandbox, output: string): Promise { */ export function createBashTool({ sandboxPromise, + recoverSandbox, env, trace, }: CreateBashToolOptions) { @@ -92,38 +96,43 @@ export function createBashTool({ try { // The only await on the sandbox — by now it is usually already warm. - const sandbox = await sandboxPromise; - const res = await runCommand(sandbox, command, { - cwd: GAME_ROOT, - env, - timeoutSeconds, - }); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const res = await runCommand(sandbox, command, { + cwd: GAME_ROOT, + env, + timeoutSeconds, + }); - const truncated = res.stdout.length > MAX_OUTPUT_CHARS; - const outputPath = truncated - ? await spillToFile(sandbox, res.stdout) - : undefined; - const durationMs = Math.round(performance.now() - startedAt); + const truncated = res.stdout.length > MAX_OUTPUT_CHARS; + const outputPath = truncated + ? await spillToFile(sandbox, res.stdout) + : undefined; + const durationMs = Math.round(performance.now() - startedAt); - trace?.log("tool.bash.end", { - command, - exitCode: res.exitCode, - outputChars: res.stdout.length, - truncated, - outputPath, - durationMs, - }); + trace?.log("tool.bash.end", { + command, + exitCode: res.exitCode, + outputChars: res.stdout.length, + truncated, + outputPath, + durationMs, + }); - return { - output: truncated - ? `${res.stdout.slice(0, MAX_OUTPUT_CHARS)}\n\n[truncated: ${res.stdout.length} chars total; full output at ${outputPath}]` - : res.stdout, - exitCode: res.exitCode, - durationMs, - truncated, - outputChars: res.stdout.length, - outputPath, - }; + return { + output: truncated + ? `${res.stdout.slice(0, MAX_OUTPUT_CHARS)}\n\n[truncated: ${res.stdout.length} chars total; full output at ${outputPath}]` + : res.stdout, + exitCode: res.exitCode, + durationMs, + truncated, + outputChars: res.stdout.length, + outputPath, + }; + }, + ); } catch (err) { // Sandbox never came up, or the SDK call itself failed. Hand the model // a readable failure instead of killing the step. diff --git a/src/mastra/tools/edit-file-tool.ts b/src/mastra/tools/edit-file-tool.ts index 5c77aca..f4cb612 100644 --- a/src/mastra/tools/edit-file-tool.ts +++ b/src/mastra/tools/edit-file-tool.ts @@ -4,9 +4,12 @@ import { z } from "zod"; import { expandSandboxPath } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; export type CreateEditFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -23,6 +26,7 @@ function errorMessage(err: unknown): string { */ export function createEditFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateEditFileToolOptions) { return createTool({ @@ -59,43 +63,47 @@ export function createEditFileTool({ } try { - const sandbox = await sandboxPromise; + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const details = await sandbox.fs.getFileDetails(path).catch(() => null); + if (!details || details.isDir) { + return { + ok: false, + path, + error: `No file at "${input}". Check the path, or use write to create it.`, + }; + } - const details = await sandbox.fs.getFileDetails(path).catch(() => null); - if (!details || details.isDir) { - return { - ok: false, - path, - error: `No file at "${input}". Check the path, or use write to create it.`, - }; - } + const content = (await sandbox.fs.downloadFile(path)).toString("utf-8"); + const occurrences = content.split(old_string).length - 1; - const content = (await sandbox.fs.downloadFile(path)).toString("utf-8"); - const occurrences = content.split(old_string).length - 1; + if (occurrences === 0) { + return { + ok: false, + path, + error: `old_string was not found in "${input}". Read the file and copy the exact text — including indentation — you want to replace.`, + }; + } + if (occurrences > 1 && !replace_all) { + return { + ok: false, + path, + error: `old_string matches ${occurrences} times in "${input}", so the edit is ambiguous. Add surrounding context to make it unique, or set replace_all=true.`, + }; + } - if (occurrences === 0) { - return { - ok: false, - path, - error: `old_string was not found in "${input}". Read the file and copy the exact text — including indentation — you want to replace.`, - }; - } - if (occurrences > 1 && !replace_all) { - return { - ok: false, - path, - error: `old_string matches ${occurrences} times in "${input}", so the edit is ambiguous. Add surrounding context to make it unique, or set replace_all=true.`, - }; - } + const updated = replace_all + ? content.split(old_string).join(new_string) + : content.replace(old_string, new_string); + await sandbox.fs.uploadFile(Buffer.from(updated, "utf-8"), path); - const updated = replace_all - ? content.split(old_string).join(new_string) - : content.replace(old_string, new_string); - await sandbox.fs.uploadFile(Buffer.from(updated, "utf-8"), path); - - const replacements = replace_all ? occurrences : 1; - trace?.log("tool.edit", { path, replacements }); - return { ok: true, path, replacements }; + const replacements = replace_all ? occurrences : 1; + trace?.log("tool.edit", { path, replacements }); + return { ok: true, path, replacements }; + }, + ); } catch (err) { const message = errorMessage(err); trace?.log("tool.edit.error", { path, error: message }); diff --git a/src/mastra/tools/index.ts b/src/mastra/tools/index.ts index c9a1beb..886e93e 100644 --- a/src/mastra/tools/index.ts +++ b/src/mastra/tools/index.ts @@ -11,6 +11,12 @@ import { createWriteFileTool } from "./write-file-tool"; export type SandboxToolOptions = { /** Un-awaited on purpose — see `~/server/sandbox/prepare`. */ sandboxPromise: Promise; + /** + * Re-runs the full sandbox preparation (start + R2 mount + hydrate) when + * the container dies under a tool call — auto-stop or archive racing a + * request. Supplied by the route; omitted in tests. + */ + recoverSandbox?: () => Promise; /** * Where this thread's published game will be readable. Computed by the * caller (it needs env) and handed down, so nothing under `tools/` has to diff --git a/src/mastra/tools/publish-tool.ts b/src/mastra/tools/publish-tool.ts index 1179b78..5cada6c 100644 --- a/src/mastra/tools/publish-tool.ts +++ b/src/mastra/tools/publish-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { GAME_ROOT, PUBLISHED_FILE } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import { publishScript, validateScript } from "~/server/sandbox/scripts"; /** Tests plus build. Long, because a cold vite build is not quick. */ @@ -19,6 +20,8 @@ const MAX_GATE_OUTPUT_CHARS = 6_000; export type CreatePublishToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; /** * Public URL of the published file, precomputed by the caller from * `publishedGameKey`. Null when R2_PUBLIC_BASE_URL is unset — publishing @@ -58,6 +61,7 @@ function tail(output: string): string { */ export function createPublishTool({ sandboxPromise, + recoverSandbox, publishedUrl, recordVersion, trace, @@ -99,111 +103,114 @@ export function createPublishTool({ message, }); - let sandbox: Sandbox; try { - sandbox = await sandboxPromise; - } catch (err) { - log("tool.publish.sandbox_unavailable"); - return fail( - `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, - ); - } - - // The engine's own gate, first and by absolute path out of ~/engine. - // It is seconds rather than minutes, so a game that cannot be finished - // fails here instead of after a cold vite build. It also runs on data the - // model cannot reach: `npm test` is a file in the working tree, and the - // model owns that file — "the tests passed" only means what the tests - // happen to assert today. - const validated = await runCommand(sandbox, validateScript(), { - cwd: GAME_ROOT, - timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, - }); - if (!validated.success) { - log("tool.publish.validate_failed", { exitCode: validated.exitCode }); - // Never publish a game the gate could not read. A validator that passes - // when it finds nothing would certify an unplayable game. - const preamble = - validated.exitCode === EXIT_GAME_NOT_FOUND - ? "Not published — the validator could not find this game's data, so nothing was checked." - : "Not published — the game did not pass validation."; - return fail(`${preamble} Fix this and call publish again:\n${tail(validated.stdout)}`); - } - log("tool.publish.validated", { - durationMs: Math.round(performance.now() - startedAt), - }); - - const gate = await runCommand( - sandbox, - "npm test --silent && npm run build --silent", - { cwd: GAME_ROOT, timeoutSeconds: GATE_TIMEOUT_SECONDS }, - ); - if (!gate.success) { - log("tool.publish.gate_failed", { exitCode: gate.exitCode }); - return fail( - `Not published — tests or build failed (exit ${gate.exitCode}). Fix this and call publish again:\n${tail(gate.stdout)}`, - ); - } - log("tool.publish.gate_passed", { - durationMs: Math.round(performance.now() - startedAt), - }); - - const res = await runCommand(sandbox, publishScript(), { - timeoutSeconds: COPY_TIMEOUT_SECONDS, - }); - const out = res.stdout.trim(); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + // The engine's own gate, first and by absolute path out of ~/engine. + // It is seconds rather than minutes, so a game that cannot be finished + // fails here instead of after a cold vite build. It also runs on data the + // model cannot reach: `npm test` is a file in the working tree, and the + // model owns that file — "the tests passed" only means what the tests + // happen to assert today. + const validated = await runCommand(sandbox, validateScript(), { + cwd: GAME_ROOT, + timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, + }); + if (!validated.success) { + log("tool.publish.validate_failed", { exitCode: validated.exitCode }); + // Never publish a game the gate could not read. A validator that passes + // when it finds nothing would certify an unplayable game. + const preamble = + validated.exitCode === EXIT_GAME_NOT_FOUND + ? "Not published — the validator could not find this game's data, so nothing was checked." + : "Not published — the game did not pass validation."; + return fail(`${preamble} Fix this and call publish again:\n${tail(validated.stdout)}`); + } + log("tool.publish.validated", { + durationMs: Math.round(performance.now() - startedAt), + }); - if (!res.success) { - log("tool.publish.copy_failed", { exitCode: res.exitCode }); - if (out.startsWith("NO_BUILD")) { - return fail( - `The build succeeded but ${GAME_ROOT}/dist/${PUBLISHED_FILE} is missing. Check the build's output directory.`, - ); - } - if (out.startsWith("NOT_SELF_CONTAINED")) { - return fail( - `dist/ holds files besides ${PUBLISHED_FILE}, so the game would load assets that are not published:\n${out}\n` + - "The build must inline everything into one HTML file — check vite-plugin-singlefile is enabled in vite.config.ts.", + const gate = await runCommand( + sandbox, + "npm test --silent && npm run build --silent", + { cwd: GAME_ROOT, timeoutSeconds: GATE_TIMEOUT_SECONDS }, ); - } - // The published file is written last, so the live game is untouched. - return fail( - `Publish failed (exit ${res.exitCode}); the previously published game is unchanged:\n${tail(out)}`, - ); - } + if (!gate.success) { + log("tool.publish.gate_failed", { exitCode: gate.exitCode }); + return fail( + `Not published — tests or build failed (exit ${gate.exitCode}). Fix this and call publish again:\n${tail(gate.stdout)}`, + ); + } + log("tool.publish.gate_passed", { + durationMs: Math.round(performance.now() - startedAt), + }); - const version = Number(out.split("\n").pop()?.replace("PUBLISHED ", "")); - log("tool.publish.ok", { - version, - durationMs: Math.round(performance.now() - startedAt), - }); + const res = await runCommand(sandbox, publishScript(), { + timeoutSeconds: COPY_TIMEOUT_SECONDS, + }); + const out = res.stdout.trim(); + + if (!res.success) { + log("tool.publish.copy_failed", { exitCode: res.exitCode }); + if (out.startsWith("NO_BUILD")) { + return fail( + `The build succeeded but ${GAME_ROOT}/dist/${PUBLISHED_FILE} is missing. Check the build's output directory.`, + ); + } + if (out.startsWith("NOT_SELF_CONTAINED")) { + return fail( + `dist/ holds files besides ${PUBLISHED_FILE}, so the game would load assets that are not published:\n${out}\n` + + "The build must inline everything into one HTML file — check vite-plugin-singlefile is enabled in vite.config.ts.", + ); + } + // The published file is written last, so the live game is untouched. + return fail( + `Publish failed (exit ${res.exitCode}); the previously published game is unchanged:\n${tail(out)}`, + ); + } - // Index the new version for the app. Deliberately after the bucket write - // and deliberately non-fatal: the game IS published at this point, and - // telling the model otherwise would send it round the whole gate again to - // fix a database it cannot reach. A dropped row costs the version list - // one entry; the game itself is still live and the next publish indexes - // normally. - if (recordVersion && Number.isFinite(version)) { - try { - await recordVersion({ version, label: intent }); - } catch (err) { - log("tool.publish.record_failed", { + const version = Number(out.split("\n").pop()?.replace("PUBLISHED ", "")); + log("tool.publish.ok", { version, - error: err instanceof Error ? err.message : String(err), + durationMs: Math.round(performance.now() - startedAt), }); - } - } - return { - ok: true, - version: Number.isFinite(version) ? version : null, - url: publishedUrl ?? null, - key: publishedUrl ? null : `current/${PUBLISHED_FILE}`, - message: publishedUrl - ? `Published version ${version}. The teacher can play it at ${publishedUrl}.` - : `Published version ${version}. No public URL is configured yet, so share it once R2_PUBLIC_BASE_URL is set.`, - }; + // Index the new version for the app. Deliberately after the bucket write + // and deliberately non-fatal: the game IS published at this point, and + // telling the model otherwise would send it round the whole gate again to + // fix a database it cannot reach. A dropped row costs the version list + // one entry; the game itself is still live and the next publish indexes + // normally. + if (recordVersion && Number.isFinite(version)) { + try { + await recordVersion({ version, label: intent }); + } catch (err) { + log("tool.publish.record_failed", { + version, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + return { + ok: true, + version: Number.isFinite(version) ? version : null, + url: publishedUrl ?? null, + key: publishedUrl ? null : `current/${PUBLISHED_FILE}`, + message: publishedUrl + ? `Published version ${version}. The teacher can play it at ${publishedUrl}.` + : `Published version ${version}. No public URL is configured yet, so share it once R2_PUBLIC_BASE_URL is set.`, + }; + }, + ); + } catch (err) { + log("tool.publish.sandbox_unavailable"); + return fail( + `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, + ); + } }, }); } diff --git a/src/mastra/tools/read-file-tool.ts b/src/mastra/tools/read-file-tool.ts index fbf8b63..236da5c 100644 --- a/src/mastra/tools/read-file-tool.ts +++ b/src/mastra/tools/read-file-tool.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { ENGINE_ROOT, expandSandboxPath, GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; /** Above this, downloading the whole file into the server is wasteful — `bash` with head/grep is the right tool. */ const MAX_FILE_BYTES = 2_000_000; @@ -13,6 +14,8 @@ const MAX_DIR_ENTRIES = 200; export type CreateReadFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -42,6 +45,7 @@ function sliceLines(text: string, offset?: number, limit?: number): string { */ export function createReadFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateReadFileToolOptions) { return createTool({ @@ -77,62 +81,67 @@ export function createReadFileTool({ execute: async ({ path: input, offset, limit }) => { const path = expandSandboxPath(input); try { - const sandbox = await sandboxPromise; - const details = await sandbox.fs.getFileDetails(path); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const details = await sandbox.fs.getFileDetails(path); - if (details.isDir) { - const entries = await sandbox.fs.listFiles(path); - const shown = entries.slice(0, MAX_DIR_ENTRIES); - trace?.log("tool.read.dir", { path, entries: entries.length }); - return { - ok: true, - path, - kind: "directory" as const, - size: entries.length, - truncated: entries.length > shown.length, - content: shown - .map((e) => (e.isDir ? `${e.name}/` : `${e.name} (${e.size} B)`)) - .join("\n"), - }; - } + if (details.isDir) { + const entries = await sandbox.fs.listFiles(path); + const shown = entries.slice(0, MAX_DIR_ENTRIES); + trace?.log("tool.read.dir", { path, entries: entries.length }); + return { + ok: true, + path, + kind: "directory" as const, + size: entries.length, + truncated: entries.length > shown.length, + content: shown + .map((e) => (e.isDir ? `${e.name}/` : `${e.name} (${e.size} B)`)) + .join("\n"), + }; + } - if (details.size > MAX_FILE_BYTES) { - return { - ok: false, - path, - error: `File is ${details.size} bytes, too large to read whole. Use bash with head/tail/grep to pull out the part you need.`, - }; - } + if (details.size > MAX_FILE_BYTES) { + return { + ok: false, + path, + error: `File is ${details.size} bytes, too large to read whole. Use bash with head/tail/grep to pull out the part you need.`, + }; + } - const buffer = await sandbox.fs.downloadFile(path); - if (looksBinary(buffer)) { - return { - ok: false, - path, - kind: "file" as const, - size: details.size, - error: - "This is a binary file, so there is nothing readable to return. Describe it to the teacher by name/size instead of trying to read it.", - }; - } + const buffer = await sandbox.fs.downloadFile(path); + if (looksBinary(buffer)) { + return { + ok: false, + path, + kind: "file" as const, + size: details.size, + error: + "This is a binary file, so there is nothing readable to return. Describe it to the teacher by name/size instead of trying to read it.", + }; + } - const text = sliceLines(buffer.toString("utf-8"), offset, limit); - const truncated = text.length > MAX_CHARS; - trace?.log("tool.read.file", { - path, - bytes: details.size, - truncated, - }); - return { - ok: true, - path, - kind: "file" as const, - size: details.size, - truncated, - content: truncated - ? `${text.slice(0, MAX_CHARS)}\n\n[truncated — re-read with offset/limit, or use bash grep to find the part you need]` - : text, - }; + const text = sliceLines(buffer.toString("utf-8"), offset, limit); + const truncated = text.length > MAX_CHARS; + trace?.log("tool.read.file", { + path, + bytes: details.size, + truncated, + }); + return { + ok: true, + path, + kind: "file" as const, + size: details.size, + truncated, + content: truncated + ? `${text.slice(0, MAX_CHARS)}\n\n[truncated — re-read with offset/limit, or use bash grep to find the part you need]` + : text, + }; + }, + ); } catch (err) { const message = errorMessage(err); trace?.log("tool.read.error", { path, error: message }); diff --git a/src/mastra/tools/validate-tool.ts b/src/mastra/tools/validate-tool.ts index 1293951..93127b1 100644 --- a/src/mastra/tools/validate-tool.ts +++ b/src/mastra/tools/validate-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import { validateScript } from "~/server/sandbox/scripts"; /** Pure functions over the authored data — fast, unlike the test+build gate. */ @@ -15,6 +16,8 @@ const EXIT_GAME_NOT_FOUND = 2; export type CreateValidateToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -44,6 +47,7 @@ const levelReport = z.object({ */ export function createValidateTool({ sandboxPromise, + recoverSandbox, trace, }: CreateValidateToolOptions) { return createTool({ @@ -80,67 +84,70 @@ export function createValidateTool({ message, }); - let sandbox: Sandbox; try { - sandbox = await sandboxPromise; - } catch (err) { - log("tool.validate.sandbox_unavailable"); - return fail( - `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, - ); - } + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const res = await runCommand(sandbox, validateScript(), { + cwd: GAME_ROOT, + timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, + }); + const durationMs = Math.round(performance.now() - startedAt); - const res = await runCommand(sandbox, validateScript(), { - cwd: GAME_ROOT, - timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, - }); - const durationMs = Math.round(performance.now() - startedAt); + const parsed = parseReport(res.stdout); + if (!parsed) { + log("tool.validate.unparseable", { + exitCode: res.exitCode, + durationMs, + }); + return fail( + `The validator did not return a readable report (exit ${res.exitCode}):\n${res.stdout.slice(-2000)}`, + ); + } - const parsed = parseReport(res.stdout); - if (!parsed) { - log("tool.validate.unparseable", { - exitCode: res.exitCode, - durationMs, - }); - return fail( - `The validator did not return a readable report (exit ${res.exitCode}):\n${res.stdout.slice(-2000)}`, - ); - } + if (res.exitCode === EXIT_GAME_NOT_FOUND) { + log("tool.validate.not_found", { durationMs }); + return fail( + [ + `The validator could not find the game: ${parsed.error ?? "unknown reason"}`, + parsed.hint, + "Nothing was checked, so do not treat this as a pass.", + ] + .filter(Boolean) + .join(" "), + ); + } - if (res.exitCode === EXIT_GAME_NOT_FOUND) { - log("tool.validate.not_found", { durationMs }); - return fail( - [ - `The validator could not find the game: ${parsed.error ?? "unknown reason"}`, - parsed.hint, - "Nothing was checked, so do not treat this as a pass.", - ] - .filter(Boolean) - .join(" "), - ); - } + const levels = parsed.items?.flatMap((item) => item.levels ?? []); + const errors = parsed.errors ?? []; - const levels = parsed.items?.flatMap((item) => item.levels ?? []); - const errors = parsed.errors ?? []; + if (res.exitCode === EXIT_GAME_HAS_ERRORS || parsed.ok === false) { + log("tool.validate.failed", { errorCount: errors.length, durationMs }); + return { + ok: false, + errors, + levels, + message: `The game is not ready: ${errors.length} problem(s) found. Fix these and run validate again.`, + }; + } - if (res.exitCode === EXIT_GAME_HAS_ERRORS || parsed.ok === false) { - log("tool.validate.failed", { errorCount: errors.length, durationMs }); - return { - ok: false, - errors, - levels, - message: `The game is not ready: ${errors.length} problem(s) found. Fix these and run validate again.`, - }; + log("tool.validate.ok", { durationMs }); + return { + ok: true, + errors: [], + levels, + message: + "The game is coherent, winnable by reasoning, and plays through to a win. Safe to publish.", + }; + }, + ); + } catch (err) { + log("tool.validate.sandbox_unavailable"); + return fail( + `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, + ); } - - log("tool.validate.ok", { durationMs }); - return { - ok: true, - errors: [], - levels, - message: - "The game is coherent, winnable by reasoning, and plays through to a win. Safe to publish.", - }; }, }); } diff --git a/src/mastra/tools/write-file-tool.ts b/src/mastra/tools/write-file-tool.ts index cf53edd..3ceb7a2 100644 --- a/src/mastra/tools/write-file-tool.ts +++ b/src/mastra/tools/write-file-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { expandSandboxPath, GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { quoteShellArg, runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; function parentDir(path: string): string { const i = path.lastIndexOf("/"); @@ -13,6 +14,8 @@ function parentDir(path: string): string { export type CreateWriteFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -26,6 +29,7 @@ export type CreateWriteFileToolOptions = { */ export function createWriteFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateWriteFileToolOptions) { return createTool({ @@ -44,26 +48,30 @@ export function createWriteFileTool({ execute: async ({ path: input, content }) => { const path = expandSandboxPath(input); try { - const sandbox = await sandboxPromise; + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const dir = parentDir(path); + const mkdir = await runCommand( + sandbox, + `mkdir -p ${quoteShellArg(dir)}`, + ); + if (!mkdir.success) { + return { + ok: false, + path, + error: `Could not create parent directory "${dir}" (exit ${mkdir.exitCode}): ${mkdir.stdout.trim() || "no output"}. Check the path is valid and writable.`, + }; + } - const dir = parentDir(path); - const mkdir = await runCommand( - sandbox, - `mkdir -p ${quoteShellArg(dir)}`, - ); - if (!mkdir.success) { - return { - ok: false, - path, - error: `Could not create parent directory "${dir}" (exit ${mkdir.exitCode}): ${mkdir.stdout.trim() || "no output"}. Check the path is valid and writable.`, - }; - } - - const buffer = Buffer.from(content, "utf-8"); - await sandbox.fs.uploadFile(buffer, path); - trace?.log("tool.write", { path, bytes: buffer.byteLength }); + const buffer = Buffer.from(content, "utf-8"); + await sandbox.fs.uploadFile(buffer, path); + trace?.log("tool.write", { path, bytes: buffer.byteLength }); - return { ok: true, path, bytesWritten: buffer.byteLength }; + return { ok: true, path, bytesWritten: buffer.byteLength }; + }, + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); trace?.log("tool.write.error", { path, error: message }); diff --git a/src/middleware.ts b/src/middleware.ts index f51903f..5464cfd 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -12,6 +12,9 @@ const isPublicRoute = createRouteMatcher([ "/", "/privacy", "/terms", + // The share link. Students opening a teacher's game hold no session, and the + // route serves nothing but the published build — see app/play/[threadId]. + "/play(.*)", "/sign-in(.*)", "/sign-up(.*)", "/robots.txt", diff --git a/src/server/api/routers/games.ts b/src/server/api/routers/games.ts index 928b475..74af595 100644 --- a/src/server/api/routers/games.ts +++ b/src/server/api/routers/games.ts @@ -1,6 +1,5 @@ import { z } from "zod"; -import { publishedGameKey } from "~/lib/sandbox-paths"; import { isValidThreadId } from "~/lib/thread-id"; import { latestGameVersion, listGameVersions } from "~/server/db/games"; import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc"; @@ -31,9 +30,14 @@ export const gamesRouter = createTRPCRouter({ // The VERSIONED key, not current/index.html. Two reasons: this URL is // immutable, so the CDN can cache it forever and no cache-buster is // needed; and the pane can show any version, not only the newest. - // current/ stays the stable link a teacher shares — see `shareUrl`. + // current/ stays the stable target a teacher shares — see `sharePath`. url: publicObjectUrl(row.htmlKey), - shareUrl: publicObjectUrl(publishedGameKey(ctx.userId, input.threadId)), + // A same-origin path, NOT the bucket URL: the raw key embeds the + // Clerk user id, and a share link travels to whole classrooms. The + // /play route proxies current/index.html and keeps the id server-side. + // A path (not an absolute URL) so localhost and preview deploys hand + // out links on their own origin. + sharePath: `/play/${input.threadId}`, }; }), diff --git a/src/server/db/games.ts b/src/server/db/games.ts index aed7239..b2c69a4 100644 --- a/src/server/db/games.ts +++ b/src/server/db/games.ts @@ -62,6 +62,22 @@ export async function latestGameVersion(threadId: string, userId: string) { return row ?? null; } +/** + * The newest version by thread id alone — for the public /play route, where + * there is no signed-in user to filter on. The thread id is an unguessable + * UUID and the row is only ever used to locate the built game in the bucket, + * so this widens what a share link serves, not what a stranger can enumerate. + */ +export async function latestGameVersionByThread(threadId: string) { + const [row] = await db + .select() + .from(gameVersions) + .where(eq(gameVersions.threadId, threadId)) + .orderBy(desc(gameVersions.version)) + .limit(1); + return row ?? null; +} + /** Every version of a thread's game, newest first. */ export async function listGameVersions(threadId: string, userId: string) { return db diff --git a/src/server/sandbox/__tests__/lifecycle.test.ts b/src/server/sandbox/__tests__/lifecycle.test.ts new file mode 100644 index 0000000..3f419e8 --- /dev/null +++ b/src/server/sandbox/__tests__/lifecycle.test.ts @@ -0,0 +1,141 @@ +import type { Sandbox } from "@daytonaio/sdk"; +import { SandboxState } from "@daytonaio/sdk"; +import { describe, expect, it, vi } from "vitest"; + +import { + isSandboxUnreachableError, + planSandboxStep, + withRecoveredSandbox, +} from "../lifecycle"; + +/** The exact message from the production incident (trace 65e76ce8). */ +const CONTAINER_IP_ERROR = + "bad request: failed to resolve container IP after 3 attempts: no IP address found. Is the Sandbox started?"; + +describe("planSandboxStep", () => { + it("uses a started sandbox as-is", () => { + expect(planSandboxStep(SandboxState.STARTED)).toBe("use"); + }); + + it("starts sandboxes at rest", () => { + expect(planSandboxStep(SandboxState.STOPPED)).toBe("start"); + expect(planSandboxStep(SandboxState.ARCHIVED)).toBe("start"); + expect(planSandboxStep(SandboxState.PAUSED)).toBe("start"); + }); + + it("waits on sandboxes already on the way up", () => { + expect(planSandboxStep(SandboxState.STARTING)).toBe("wait-started"); + expect(planSandboxStep(SandboxState.RESTORING)).toBe("wait-started"); + expect(planSandboxStep(SandboxState.RESUMING)).toBe("wait-started"); + }); + + it("polls mid-transition states instead of using them (the incident)", () => { + // A sandbox observed while Daytona archives it has no container; the old + // code returned it as "existing-other" and every tool call failed. + expect(planSandboxStep(SandboxState.ARCHIVING)).toBe("wait"); + expect(planSandboxStep(SandboxState.STOPPING)).toBe("wait"); + expect(planSandboxStep(SandboxState.PAUSING)).toBe("wait"); + }); + + it("never returns 'use' for an unknown future state", () => { + expect(planSandboxStep("some-new-state" as Sandbox["state"])).toBe("wait"); + expect(planSandboxStep(undefined)).toBe("wait"); + }); + + it("fails fast on broken or terminal sandboxes", () => { + expect(planSandboxStep(SandboxState.ERROR)).toBe("fail"); + expect(planSandboxStep(SandboxState.BUILD_FAILED)).toBe("fail"); + expect(planSandboxStep(SandboxState.DESTROYED)).toBe("fail"); + expect(planSandboxStep(SandboxState.DESTROYING)).toBe("fail"); + }); +}); + +describe("isSandboxUnreachableError", () => { + it("matches the production container-IP error", () => { + expect(isSandboxUnreachableError(new Error(CONTAINER_IP_ERROR))).toBe(true); + }); + + it("matches non-Error throws too", () => { + expect(isSandboxUnreachableError(CONTAINER_IP_ERROR)).toBe(true); + }); + + it("ignores unrelated errors", () => { + expect(isSandboxUnreachableError(new Error("ENOENT: no such file"))).toBe( + false, + ); + expect(isSandboxUnreachableError(new Error("npm test failed"))).toBe(false); + }); +}); + +describe("withRecoveredSandbox", () => { + const sandboxA = { id: "a" } as unknown as Sandbox; + const sandboxB = { id: "b" } as unknown as Sandbox; + + it("runs against the resolved sandbox on the happy path", async () => { + const recover = vi.fn(); + const result = await withRecoveredSandbox( + Promise.resolve(sandboxA), + recover, + async (sandbox) => sandbox.id, + ); + expect(result).toBe("a"); + expect(recover).not.toHaveBeenCalled(); + }); + + it("recovers when the preparation promise itself died unreachable", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const result = await withRecoveredSandbox( + Promise.reject(new Error(CONTAINER_IP_ERROR)), + recover, + async (sandbox) => sandbox.id, + ); + expect(result).toBe("b"); + expect(recover).toHaveBeenCalledTimes(1); + }); + + it("recovers when the container dies under the call, and reruns it once", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const fn = vi + .fn<(sandbox: Sandbox) => Promise>() + .mockRejectedValueOnce(new Error(CONTAINER_IP_ERROR)) + .mockImplementation(async (sandbox) => sandbox.id); + const result = await withRecoveredSandbox( + Promise.resolve(sandboxA), + recover, + fn, + ); + expect(result).toBe("b"); + expect(fn).toHaveBeenNthCalledWith(1, sandboxA); + expect(fn).toHaveBeenNthCalledWith(2, sandboxB); + }); + + it("recovers at most once", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const fn = vi.fn().mockRejectedValue(new Error(CONTAINER_IP_ERROR)); + await expect( + withRecoveredSandbox(Promise.resolve(sandboxA), recover, fn), + ).rejects.toThrow(/container IP/); + expect(recover).toHaveBeenCalledTimes(1); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it("rethrows unrelated errors without recovering", async () => { + const recover = vi.fn(); + await expect( + withRecoveredSandbox(Promise.resolve(sandboxA), recover, async () => { + throw new Error("old_string was not found"); + }), + ).rejects.toThrow(/old_string/); + expect(recover).not.toHaveBeenCalled(); + }); + + it("does nothing special when no recover function is supplied", async () => { + await expect( + withRecoveredSandbox( + Promise.reject(new Error(CONTAINER_IP_ERROR)), + undefined, + async (sandbox) => sandbox.id, + ), + ).rejects.toThrow(/container IP/); + }); +}); diff --git a/src/server/sandbox/daytona.ts b/src/server/sandbox/daytona.ts index 7cd082e..c8db03c 100644 --- a/src/server/sandbox/daytona.ts +++ b/src/server/sandbox/daytona.ts @@ -6,6 +6,7 @@ import { } from "@daytonaio/sdk"; import { env } from "~/env"; +import { planSandboxStep } from "./lifecycle"; let client: Daytona | null = null; @@ -24,7 +25,8 @@ export type SandboxLifecycleStatus = | "existing-started" | "existing-started-from-stopped" | "existing-waited-from-starting" - | "existing-other"; + /** Found mid-transition (archiving, stopping, …) and walked to started. */ + | "existing-recovered"; /** * Daytona signals a missing sandbox with DaytonaNotFoundError; the message check @@ -37,39 +39,119 @@ function isNotFound(err: unknown): boolean { ); } +/** Generous: restoring an archived ~3 GiB filesystem is minutes, not seconds. */ +const START_TIMEOUT_SECONDS = 120; +/** How long a sandbox may sit in a transitional state before we give up. */ +const SETTLE_TIMEOUT_MS = 180_000; +const SETTLE_POLL_MS = 2_000; +/** start() attempts before concluding the sandbox will not come up. */ +const MAX_START_ATTEMPTS = 3; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Walk the sandbox to STARTED, whatever state it is observed in. + * + * The old version handled exactly four states and silently returned the + * sandbox for every other one — so a thread reopened while Daytona was + * archiving it ("archiving" takes minutes for a 3 GiB disk) got a handle with + * no running container, and every tool call died with "failed to resolve + * container IP". Now: startable states are started, in-flight states are + * waited out (with a deadline), and broken states throw something readable + * instead of a container-IP error five calls later. + */ +async function ensureSandboxStarted(sandbox: Sandbox): Promise { + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + let startAttempts = 0; + + for (;;) { + const state = sandbox.state; + const step = planSandboxStep(state); + + if (step === "use") return; + + if (step === "fail") { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} is "${String(state)}" and cannot be started. ` + + `It needs to be deleted (or recreated) in Daytona before this thread can build again.`, + ); + } + + if (Date.now() >= deadline) { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} did not reach "started" within ${Math.round( + SETTLE_TIMEOUT_MS / 1000, + )}s (still "${String(state)}"). Try again in a minute.`, + ); + } + + if (step === "start") { + if (++startAttempts > MAX_START_ATTEMPTS) { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} fell back to "${String(state)}" after ${MAX_START_ATTEMPTS} start attempts.`, + ); + } + await sandbox.start(START_TIMEOUT_SECONDS); + } else if (step === "wait-started") { + await sandbox.waitUntilStarted(START_TIMEOUT_SECONDS); + } else { + // "wait": mid-transition (stopping, archiving, …). Poll until it lands + // in a state we can act on. + await sleep(SETTLE_POLL_MS); + } + + await sandbox.refreshData(); + } +} + +/** The status the old four-state walk would have reported, for log continuity. */ +function statusForInitialState( + state: Sandbox["state"], +): SandboxLifecycleStatus { + if (state === SandboxState.STARTED) return "existing-started"; + if ( + state === SandboxState.STOPPED || + state === SandboxState.ARCHIVED || + state === SandboxState.PAUSED + ) { + return "existing-started-from-stopped"; + } + if (state === SandboxState.STARTING) return "existing-waited-from-starting"; + return "existing-recovered"; +} + /** * Resolve the sandbox for a stable name, walking its lifecycle: * missing → create * STARTED → use - * STOPPED / ARCHIVED → start, then use - * STARTING → wait, then use + * STOPPED / ARCHIVED / PAUSED → start, then use + * STARTING (and other states already on the way up) → wait, then use + * mid-transition (STOPPING / ARCHIVING / …) → wait until settled, then start + * ERROR / DESTROYED → throw a readable error * Idempotent: safe to call on every request. */ - export async function getOrCreateSandbox( sandboxId: string, opts: { env?: Record }, -): Promise<{ sandbox: Sandbox; status: SandboxLifecycleStatus }> { +): Promise<{ + sandbox: Sandbox; + status: SandboxLifecycleStatus; + /** Raw state the sandbox was first observed in; undefined when created. */ + initialState?: string; +}> { const daytona = getDaytonaClient(); try { const sandbox = await daytona.get(sandboxId); - - if (sandbox.state === SandboxState.STARTED) { - return { sandbox, status: "existing-started" }; - } - if ( - sandbox.state === SandboxState.STOPPED || - sandbox.state === SandboxState.ARCHIVED - ) { - await sandbox.start(); - return { sandbox, status: "existing-started-from-stopped" }; - } - if (sandbox.state === SandboxState.STARTING) { - await sandbox.waitUntilStarted(); - return { sandbox, status: "existing-waited-from-starting" }; - } - return { sandbox, status: "existing-other" }; + const initialState = sandbox.state; + await ensureSandboxStarted(sandbox); + return { + sandbox, + status: statusForInitialState(initialState), + initialState: String(initialState), + }; } catch (err) { // ONLY "no such sandbox" means create one. Swallowing everything here turns // a transient 500 or an auth failure into a create against a name that is @@ -104,8 +186,9 @@ export async function getOrCreateSandbox( // platform maximum of 7 days, and a stopped sandbox holds its disk the // whole time. At ~3 GiB each against a 30 GiB org cap that fills in about // three days, so reclaim has to outrun it. Archiving is lossless (unlike - // autoDelete) and needs no new code path: getOrCreateSandbox already - // start()s an ARCHIVED sandbox. Cost is a slower restore on a cold thread. + // autoDelete) and needs no new code path: getOrCreateSandbox walks an + // ARCHIVED (or mid-ARCHIVING) sandbox back to started. Cost is a slower + // restore on a cold thread. autoArchiveInterval: 60, autoDeleteInterval: -1, // never auto-delete; recreating means a fresh npm ci envVars: opts.env, diff --git a/src/server/sandbox/lifecycle.ts b/src/server/sandbox/lifecycle.ts new file mode 100644 index 0000000..9e29e2d --- /dev/null +++ b/src/server/sandbox/lifecycle.ts @@ -0,0 +1,99 @@ +/** + * Pure sandbox-lifecycle logic — no `~/env`, no SDK client — so it can be + * imported by `~/mastra/tools/*` and unit-tested without Daytona. + * + * Why this exists: Daytona has ~23 sandbox states, and the old lifecycle walk + * in `daytona.ts` handled four of them. A thread reopened while its sandbox + * was mid-archive (state "archiving") got a handle with no running container, + * and every tool call failed with "failed to resolve container IP … Is the + * Sandbox started?". Every state now maps to an explicit step, and unknown + * states are waited on rather than silently used. + */ +import { SandboxState, type Sandbox } from "@daytonaio/sdk"; + +/** What to do next for a sandbox observed in `state`. */ +export type SandboxStep = + /** Running — use it. */ + | "use" + /** At rest but startable (stopped / archived / paused) — call start(). */ + | "start" + /** Already on its way up — wait for started. */ + | "wait-started" + /** Mid-transition (stopping, archiving, …) — poll until it settles. */ + | "wait" + /** Terminal or broken — starting it is impossible; surface a clear error. */ + | "fail"; + +export function planSandboxStep(state: Sandbox["state"]): SandboxStep { + switch (state) { + case SandboxState.STARTED: + return "use"; + case SandboxState.STOPPED: + case SandboxState.ARCHIVED: + case SandboxState.PAUSED: + return "start"; + case SandboxState.STARTING: + case SandboxState.RESUMING: + case SandboxState.RESTORING: + case SandboxState.CREATING: + case SandboxState.PULLING_SNAPSHOT: + return "wait-started"; + case SandboxState.ERROR: + case SandboxState.BUILD_FAILED: + case SandboxState.DESTROYED: + case SandboxState.DESTROYING: + return "fail"; + // stopping, archiving, pausing, snapshotting, resizing, forking, unknown, + // and anything a future SDK adds: in motion (or unreadable) — poll until + // it lands somewhere actionable. The caller owns the deadline. + default: + return "wait"; + } +} + +/** + * Does this error mean "the container under this sandbox handle is gone"? + * + * That is the signature of the sandbox being stopped/archived between our + * state check and the command — the exact failure seen in production as + * "bad request: failed to resolve container IP after 3 attempts: no IP + * address found. Is the Sandbox started?". + */ +export function isSandboxUnreachableError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return /failed to resolve container ip|is the sandbox started\?|sandbox is not (?:started|running)/i.test( + message, + ); +} + +/** + * Run `fn` against the sandbox, recovering ONCE if the container turns out to + * be gone — either the promise itself rejected that way (boot raced an + * archive) or a call inside `fn` did (auto-stop raced a long request). + * + * `recover` is expected to re-run the full preparation (start + R2 mount + + * hydrate) — a bare restart is not enough, because the s3fs mount does not + * survive the container. `prepareLessonSandbox` is idempotent and is the + * intended recover function. `fn` must therefore be safe to re-run from the + * top, which every tool body is: nothing in it completed if the container was + * unreachable. + */ +export async function withRecoveredSandbox( + sandboxPromise: Promise, + recover: (() => Promise) | undefined, + fn: (sandbox: Sandbox) => Promise, +): Promise { + let sandbox: Sandbox; + try { + sandbox = await sandboxPromise; + } catch (err) { + if (!recover || !isSandboxUnreachableError(err)) throw err; + return fn(await recover()); + } + try { + return await fn(sandbox); + } catch (err) { + if (!recover || !isSandboxUnreachableError(err)) throw err; + return fn(await recover()); + } +} diff --git a/src/server/sandbox/prepare.ts b/src/server/sandbox/prepare.ts index 5033b1d..bbd5a5d 100644 --- a/src/server/sandbox/prepare.ts +++ b/src/server/sandbox/prepare.ts @@ -72,12 +72,13 @@ export function prepareLessonSandbox(opts: { const log = opts.trace?.log ?? (() => {}); const promise = (async () => { - const { sandbox, status } = await getOrCreateSandbox( + const { sandbox, status, initialState } = await getOrCreateSandbox( sandboxIdForThread(opts.threadId), { env: SANDBOX_ENV }, ); log("sandbox.acquired", { status, + state: initialState ?? "new", id: sandbox.id, durationMs: Math.round(performance.now() - startedAt), });