From dd37db909ea6076e295d3e64cd70715631c021e0 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Mon, 31 Aug 2026 17:47:56 +0530 Subject: [PATCH 1/2] feat(share): serve share links through /play/ instead of the raw bucket URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Copy link / Open URL was the game's physical R2 path, games///current/index.html — every shared link carried the owner's Clerk user id to whole classrooms. A new public /play/[threadId] route resolves the owner from the lessonplay_game_version index and streams current/index.html through, so the bucket layout (and the user id) stays server-side. The tRPC router now hands the UI a same-origin sharePath, the header builds the absolute URL from the page's own origin at copy time, and the agent's publishedUrl — what the model pastes into chat after a publish — points at /play too, derived from the request origin so dev and preview deploys hand out links on their own host. Co-Authored-By: Claude Fable 5 --- src/app/api/chat/route.ts | 12 ++++--- src/app/play/[threadId]/route.ts | 50 +++++++++++++++++++++++++++++ src/components/chat/chat-header.tsx | 22 ++++++++----- src/middleware.ts | 3 ++ src/server/api/routers/games.ts | 10 ++++-- src/server/db/games.ts | 16 +++++++++ 6 files changed, 97 insertions(+), 16 deletions(-) create mode 100644 src/app/play/[threadId]/route.ts diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 7a6837a..565e95c 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -6,10 +6,8 @@ import { type UIMessage, } from "ai"; -import { publishedGameKey } from "~/lib/sandbox-paths"; import { isValidThreadId } from "~/lib/thread-id"; import { recordGameVersion } from "~/server/db/games"; -import { publicObjectUrl } from "~/server/r2"; import { prepareLessonSandbox } from "~/server/sandbox/prepare"; import { createLessonAgent } from "~/mastra/agents/lesson-agent"; import { @@ -82,9 +80,13 @@ export async function POST(req: Request) { userId, model, sandboxPromise, - // Stable for the life of the thread: every publish overwrites this one key, - // so the teacher's link never changes. - publishedUrl: publicObjectUrl(publishedGameKey(userId, threadId)), + // Stable for the life of the thread: /play proxies current/index.html, + // the one key every publish overwrites, so the teacher's link never + // changes. The app route rather than the bucket URL on purpose — the raw + // key embeds the Clerk user id, and this URL is what the model pastes + // into chat. Origin comes from the request so dev and preview deploys + // hand out links on their own host. + publishedUrl: new URL(`/play/${threadId}`, req.url).toString(), // Bound to this thread and user here, so the tool cannot record a version // against anyone else's game — the model never supplies either id. recordVersion: ({ version, label }) => diff --git a/src/app/play/[threadId]/route.ts b/src/app/play/[threadId]/route.ts new file mode 100644 index 0000000..7e3ab84 --- /dev/null +++ b/src/app/play/[threadId]/route.ts @@ -0,0 +1,50 @@ +import type { NextRequest } from "next/server"; + +import { publishedGameKey } from "~/lib/sandbox-paths"; +import { isValidThreadId } from "~/lib/thread-id"; +import { latestGameVersionByThread } from "~/server/db/games"; +import { publicObjectUrl } from "~/server/r2"; + +/** + * The teacher's share link: `/play/`. + * + * The published game physically lives at + * `games///current/index.html` in the bucket — a path that + * bakes the owner's Clerk user id into every URL. Handing that URL out leaks + * the id to whole classrooms, so this route is the one the app shares instead: + * it resolves the owner from the publish index and streams the game through, + * keeping the bucket layout (and the user id) server-side. + * + * Public on purpose (see `middleware.ts`): a share link is opened by students + * who hold no session. The thread id is the only capability, same as the + * bucket's unguessable-path model, and it serves nothing but the built game. + */ +export async function GET( + _req: NextRequest, + { params }: { params: Promise<{ threadId: string }> }, +) { + const { threadId } = await params; + if (!isValidThreadId(threadId)) return new Response(null, { status: 404 }); + + const row = await latestGameVersionByThread(threadId); + if (!row) return new Response(null, { status: 404 }); + + // `current/index.html`, not the versioned key — the share link must keep + // tracking the newest publish, exactly like the raw current/ URL did. + const url = publicObjectUrl(publishedGameKey(row.userId, threadId)); + if (!url) return new Response(null, { status: 503 }); + + const upstream = await fetch(url, { cache: "no-store" }); + if (!upstream.ok || !upstream.body) return new Response(null, { status: 404 }); + + return new Response(upstream.body, { + headers: { + "Content-Type": "text/html; charset=utf-8", + // A class of thirty opening the link at once should hit the CDN, but a + // republish should show up within a minute — current/ is mutable. + "Cache-Control": "public, max-age=0, s-maxage=60", + // Share links are for people who hold them, not search results. + "X-Robots-Tag": "noindex", + }, + }); +} diff --git a/src/components/chat/chat-header.tsx b/src/components/chat/chat-header.tsx index ab0deea..78a208d 100644 --- a/src/components/chat/chat-header.tsx +++ b/src/components/chat/chat-header.tsx @@ -9,9 +9,11 @@ import { api } from "~/trpc/react"; /** * Thread title, plus the teacher's share link once a game exists. * - * The link is `current/index.html` — the one key every publish overwrites — so - * it keeps working as the game is revised and can be handed to a class once. - * The preview pane deliberately uses the versioned URL instead; see + * The link is `/play/` — the app route that proxies + * `current/index.html`, the one key every publish overwrites — so it keeps + * working as the game is revised, can be handed to a class once, and never + * exposes the bucket path (which embeds the owner's Clerk user id). The + * preview pane deliberately uses the versioned bucket URL instead; see * `GamePreview`. */ export function ChatHeader({ @@ -23,12 +25,16 @@ export function ChatHeader({ }) { const latest = api.games.latest.useQuery({ threadId }); const [copied, setCopied] = useState(false); - const shareUrl = latest.data?.shareUrl ?? null; + const sharePath = latest.data?.sharePath ?? null; const copy = async () => { - if (!shareUrl) return; + if (!sharePath) return; try { - await navigator.clipboard.writeText(shareUrl); + // Absolute at copy time — the clipboard leaves this origin, the href + // below doesn't have to. + await navigator.clipboard.writeText( + new URL(sharePath, window.location.origin).toString(), + ); setCopied(true); setTimeout(() => setCopied(false), 1500); } catch { @@ -45,7 +51,7 @@ export function ChatHeader({ {title} - {shareUrl ? ( + {sharePath ? (
v{latest.data?.version} @@ -54,7 +60,7 @@ export function ChatHeader({ {copied ? "Copied" : "Copy link"} diff --git a/src/middleware.ts b/src/middleware.ts index f51903f..5464cfd 100644 --- a/src/middleware.ts +++ b/src/middleware.ts @@ -12,6 +12,9 @@ const isPublicRoute = createRouteMatcher([ "/", "/privacy", "/terms", + // The share link. Students opening a teacher's game hold no session, and the + // route serves nothing but the published build — see app/play/[threadId]. + "/play(.*)", "/sign-in(.*)", "/sign-up(.*)", "/robots.txt", diff --git a/src/server/api/routers/games.ts b/src/server/api/routers/games.ts index 928b475..74af595 100644 --- a/src/server/api/routers/games.ts +++ b/src/server/api/routers/games.ts @@ -1,6 +1,5 @@ import { z } from "zod"; -import { publishedGameKey } from "~/lib/sandbox-paths"; import { isValidThreadId } from "~/lib/thread-id"; import { latestGameVersion, listGameVersions } from "~/server/db/games"; import { createTRPCRouter, protectedProcedure } from "~/server/api/trpc"; @@ -31,9 +30,14 @@ export const gamesRouter = createTRPCRouter({ // The VERSIONED key, not current/index.html. Two reasons: this URL is // immutable, so the CDN can cache it forever and no cache-buster is // needed; and the pane can show any version, not only the newest. - // current/ stays the stable link a teacher shares — see `shareUrl`. + // current/ stays the stable target a teacher shares — see `sharePath`. url: publicObjectUrl(row.htmlKey), - shareUrl: publicObjectUrl(publishedGameKey(ctx.userId, input.threadId)), + // A same-origin path, NOT the bucket URL: the raw key embeds the + // Clerk user id, and a share link travels to whole classrooms. The + // /play route proxies current/index.html and keeps the id server-side. + // A path (not an absolute URL) so localhost and preview deploys hand + // out links on their own origin. + sharePath: `/play/${input.threadId}`, }; }), diff --git a/src/server/db/games.ts b/src/server/db/games.ts index aed7239..b2c69a4 100644 --- a/src/server/db/games.ts +++ b/src/server/db/games.ts @@ -62,6 +62,22 @@ export async function latestGameVersion(threadId: string, userId: string) { return row ?? null; } +/** + * The newest version by thread id alone — for the public /play route, where + * there is no signed-in user to filter on. The thread id is an unguessable + * UUID and the row is only ever used to locate the built game in the bucket, + * so this widens what a share link serves, not what a stranger can enumerate. + */ +export async function latestGameVersionByThread(threadId: string) { + const [row] = await db + .select() + .from(gameVersions) + .where(eq(gameVersions.threadId, threadId)) + .orderBy(desc(gameVersions.version)) + .limit(1); + return row ?? null; +} + /** Every version of a thread's game, newest first. */ export async function listGameVersions(threadId: string, userId: string) { return db From 0922edf6f4482e075f0cf8185775e0d3db9ed006 Mon Sep 17 00:00:00 2001 From: Rishabh Singh Date: Mon, 31 Aug 2026 17:58:18 +0530 Subject: [PATCH 2/2] fix(sandbox): survive every Daytona lifecycle state, and recover when the container dies mid-request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production incident (trace 65e76ce8): a thread reopened while Daytona was archiving its sandbox got a handle in state "archiving", which the old four-state walk in getOrCreateSandbox silently returned as "existing-other" without starting it. Every tool call then failed with "bad request: failed to resolve container IP after 3 attempts: no IP address found. Is the Sandbox started?". Three guardrails: - src/server/sandbox/lifecycle.ts (new, env-free): planSandboxStep maps ALL ~23 SDK states to an explicit step — use / start / wait-started / wait / fail — so mid-transition states (archiving, stopping, pausing, ...) are polled until they settle instead of being used, unknown future states wait rather than pass, and broken states (error, destroyed) throw something readable up front. - daytona.ts: ensureSandboxStarted walks any observed state to STARTED with a deadline and a start-attempt cap; getOrCreateSandbox now reports the raw initial state, and prepare logs it, so the next novel state shows up in one log line instead of needing to be inferred. - withRecoveredSandbox: every sandbox tool (bash/read/write/edit/validate/ publish) retries once via a route-supplied recoverSandbox when the container dies under a call (auto-stop or archive racing a request). Recovery re-runs the full prepareLessonSandbox — not a bare restart — because the s3fs mount does not survive the container; prepare is already idempotent. Covered by unit tests in src/server/sandbox/__tests__/lifecycle.test.ts, including the exact production error message. Co-Authored-By: Claude Fable 5 --- src/app/api/chat/route.ts | 9 + src/mastra/agents/lesson-agent.ts | 7 + src/mastra/tools/bash-tool.ts | 67 +++--- src/mastra/tools/edit-file-tool.ts | 74 ++++--- src/mastra/tools/index.ts | 6 + src/mastra/tools/publish-tool.ts | 201 +++++++++--------- src/mastra/tools/read-file-tool.ts | 113 +++++----- src/mastra/tools/validate-tool.ts | 115 +++++----- src/mastra/tools/write-file-tool.ts | 44 ++-- .../sandbox/__tests__/lifecycle.test.ts | 141 ++++++++++++ src/server/sandbox/daytona.ts | 129 +++++++++-- src/server/sandbox/lifecycle.ts | 99 +++++++++ src/server/sandbox/prepare.ts | 3 +- 13 files changed, 701 insertions(+), 307 deletions(-) create mode 100644 src/server/sandbox/__tests__/lifecycle.test.ts create mode 100644 src/server/sandbox/lifecycle.ts diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index 565e95c..b4135ac 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -74,12 +74,21 @@ export async function POST(req: Request) { // inside a tool, if the model ever calls one. const trace = { id: traceId, log }; const sandboxPromise = prepareLessonSandbox({ threadId, userId, trace }); + // Handed to the tools for when the container dies between the lifecycle + // check and a command (auto-stop/archive racing the request). A full + // re-prepare rather than a bare restart, because the R2 mount does not + // survive the container; prepareLessonSandbox is idempotent. + const recoverSandbox = () => { + log("sandbox.recover", { threadId }); + return prepareLessonSandbox({ threadId, userId, trace }); + }; const agent = await createLessonAgent({ threadId, userId, model, sandboxPromise, + recoverSandbox, // Stable for the life of the thread: /play proxies current/index.html, // the one key every publish overwrites, so the teacher's link never // changes. The app route rather than the bucket URL on purpose — the raw diff --git a/src/mastra/agents/lesson-agent.ts b/src/mastra/agents/lesson-agent.ts index 544117b..8150fe6 100644 --- a/src/mastra/agents/lesson-agent.ts +++ b/src/mastra/agents/lesson-agent.ts @@ -55,6 +55,12 @@ export type CreateLessonAgentOptions = { * tools and stays a pure planner — it keeps the skill tools. */ sandboxPromise?: Promise; + /** + * Re-runs `prepareLessonSandbox` when the sandbox container dies mid- + * request (auto-stop/archive racing a tool call). Optional, like the + * sandbox itself. + */ + recoverSandbox?: () => Promise; /** * Public URL this thread's published game will live at. Resolved by the * route because it needs `~/env`; passed down so neither this factory nor @@ -109,6 +115,7 @@ export async function createLessonAgent(opts: CreateLessonAgentOptions) { tools: opts.sandboxPromise ? createSandboxTools({ sandboxPromise: opts.sandboxPromise, + recoverSandbox: opts.recoverSandbox, publishedUrl: opts.publishedUrl, recordVersion: opts.recordVersion, trace: opts.trace, diff --git a/src/mastra/tools/bash-tool.ts b/src/mastra/tools/bash-tool.ts index 765cd96..c651f76 100644 --- a/src/mastra/tools/bash-tool.ts +++ b/src/mastra/tools/bash-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { ENGINE_ROOT, GAME_ROOT } from "~/lib/sandbox-paths"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; /** Beyond this the output goes to a file and the model gets a pointer. */ @@ -18,6 +19,8 @@ export type CreateBashToolOptions = { * still booting/mounting; the first `bash` call pays whatever is left. */ sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; /** Extra env for every command (on top of the sandbox's own envVars). */ env?: Record; trace?: LessonTrace; @@ -39,6 +42,7 @@ async function spillToFile(sandbox: Sandbox, output: string): Promise { */ export function createBashTool({ sandboxPromise, + recoverSandbox, env, trace, }: CreateBashToolOptions) { @@ -92,38 +96,43 @@ export function createBashTool({ try { // The only await on the sandbox — by now it is usually already warm. - const sandbox = await sandboxPromise; - const res = await runCommand(sandbox, command, { - cwd: GAME_ROOT, - env, - timeoutSeconds, - }); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const res = await runCommand(sandbox, command, { + cwd: GAME_ROOT, + env, + timeoutSeconds, + }); - const truncated = res.stdout.length > MAX_OUTPUT_CHARS; - const outputPath = truncated - ? await spillToFile(sandbox, res.stdout) - : undefined; - const durationMs = Math.round(performance.now() - startedAt); + const truncated = res.stdout.length > MAX_OUTPUT_CHARS; + const outputPath = truncated + ? await spillToFile(sandbox, res.stdout) + : undefined; + const durationMs = Math.round(performance.now() - startedAt); - trace?.log("tool.bash.end", { - command, - exitCode: res.exitCode, - outputChars: res.stdout.length, - truncated, - outputPath, - durationMs, - }); + trace?.log("tool.bash.end", { + command, + exitCode: res.exitCode, + outputChars: res.stdout.length, + truncated, + outputPath, + durationMs, + }); - return { - output: truncated - ? `${res.stdout.slice(0, MAX_OUTPUT_CHARS)}\n\n[truncated: ${res.stdout.length} chars total; full output at ${outputPath}]` - : res.stdout, - exitCode: res.exitCode, - durationMs, - truncated, - outputChars: res.stdout.length, - outputPath, - }; + return { + output: truncated + ? `${res.stdout.slice(0, MAX_OUTPUT_CHARS)}\n\n[truncated: ${res.stdout.length} chars total; full output at ${outputPath}]` + : res.stdout, + exitCode: res.exitCode, + durationMs, + truncated, + outputChars: res.stdout.length, + outputPath, + }; + }, + ); } catch (err) { // Sandbox never came up, or the SDK call itself failed. Hand the model // a readable failure instead of killing the step. diff --git a/src/mastra/tools/edit-file-tool.ts b/src/mastra/tools/edit-file-tool.ts index 5c77aca..f4cb612 100644 --- a/src/mastra/tools/edit-file-tool.ts +++ b/src/mastra/tools/edit-file-tool.ts @@ -4,9 +4,12 @@ import { z } from "zod"; import { expandSandboxPath } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; export type CreateEditFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -23,6 +26,7 @@ function errorMessage(err: unknown): string { */ export function createEditFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateEditFileToolOptions) { return createTool({ @@ -59,43 +63,47 @@ export function createEditFileTool({ } try { - const sandbox = await sandboxPromise; + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const details = await sandbox.fs.getFileDetails(path).catch(() => null); + if (!details || details.isDir) { + return { + ok: false, + path, + error: `No file at "${input}". Check the path, or use write to create it.`, + }; + } - const details = await sandbox.fs.getFileDetails(path).catch(() => null); - if (!details || details.isDir) { - return { - ok: false, - path, - error: `No file at "${input}". Check the path, or use write to create it.`, - }; - } + const content = (await sandbox.fs.downloadFile(path)).toString("utf-8"); + const occurrences = content.split(old_string).length - 1; - const content = (await sandbox.fs.downloadFile(path)).toString("utf-8"); - const occurrences = content.split(old_string).length - 1; + if (occurrences === 0) { + return { + ok: false, + path, + error: `old_string was not found in "${input}". Read the file and copy the exact text — including indentation — you want to replace.`, + }; + } + if (occurrences > 1 && !replace_all) { + return { + ok: false, + path, + error: `old_string matches ${occurrences} times in "${input}", so the edit is ambiguous. Add surrounding context to make it unique, or set replace_all=true.`, + }; + } - if (occurrences === 0) { - return { - ok: false, - path, - error: `old_string was not found in "${input}". Read the file and copy the exact text — including indentation — you want to replace.`, - }; - } - if (occurrences > 1 && !replace_all) { - return { - ok: false, - path, - error: `old_string matches ${occurrences} times in "${input}", so the edit is ambiguous. Add surrounding context to make it unique, or set replace_all=true.`, - }; - } + const updated = replace_all + ? content.split(old_string).join(new_string) + : content.replace(old_string, new_string); + await sandbox.fs.uploadFile(Buffer.from(updated, "utf-8"), path); - const updated = replace_all - ? content.split(old_string).join(new_string) - : content.replace(old_string, new_string); - await sandbox.fs.uploadFile(Buffer.from(updated, "utf-8"), path); - - const replacements = replace_all ? occurrences : 1; - trace?.log("tool.edit", { path, replacements }); - return { ok: true, path, replacements }; + const replacements = replace_all ? occurrences : 1; + trace?.log("tool.edit", { path, replacements }); + return { ok: true, path, replacements }; + }, + ); } catch (err) { const message = errorMessage(err); trace?.log("tool.edit.error", { path, error: message }); diff --git a/src/mastra/tools/index.ts b/src/mastra/tools/index.ts index c9a1beb..886e93e 100644 --- a/src/mastra/tools/index.ts +++ b/src/mastra/tools/index.ts @@ -11,6 +11,12 @@ import { createWriteFileTool } from "./write-file-tool"; export type SandboxToolOptions = { /** Un-awaited on purpose — see `~/server/sandbox/prepare`. */ sandboxPromise: Promise; + /** + * Re-runs the full sandbox preparation (start + R2 mount + hydrate) when + * the container dies under a tool call — auto-stop or archive racing a + * request. Supplied by the route; omitted in tests. + */ + recoverSandbox?: () => Promise; /** * Where this thread's published game will be readable. Computed by the * caller (it needs env) and handed down, so nothing under `tools/` has to diff --git a/src/mastra/tools/publish-tool.ts b/src/mastra/tools/publish-tool.ts index 1179b78..5cada6c 100644 --- a/src/mastra/tools/publish-tool.ts +++ b/src/mastra/tools/publish-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { GAME_ROOT, PUBLISHED_FILE } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import { publishScript, validateScript } from "~/server/sandbox/scripts"; /** Tests plus build. Long, because a cold vite build is not quick. */ @@ -19,6 +20,8 @@ const MAX_GATE_OUTPUT_CHARS = 6_000; export type CreatePublishToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; /** * Public URL of the published file, precomputed by the caller from * `publishedGameKey`. Null when R2_PUBLIC_BASE_URL is unset — publishing @@ -58,6 +61,7 @@ function tail(output: string): string { */ export function createPublishTool({ sandboxPromise, + recoverSandbox, publishedUrl, recordVersion, trace, @@ -99,111 +103,114 @@ export function createPublishTool({ message, }); - let sandbox: Sandbox; try { - sandbox = await sandboxPromise; - } catch (err) { - log("tool.publish.sandbox_unavailable"); - return fail( - `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, - ); - } - - // The engine's own gate, first and by absolute path out of ~/engine. - // It is seconds rather than minutes, so a game that cannot be finished - // fails here instead of after a cold vite build. It also runs on data the - // model cannot reach: `npm test` is a file in the working tree, and the - // model owns that file — "the tests passed" only means what the tests - // happen to assert today. - const validated = await runCommand(sandbox, validateScript(), { - cwd: GAME_ROOT, - timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, - }); - if (!validated.success) { - log("tool.publish.validate_failed", { exitCode: validated.exitCode }); - // Never publish a game the gate could not read. A validator that passes - // when it finds nothing would certify an unplayable game. - const preamble = - validated.exitCode === EXIT_GAME_NOT_FOUND - ? "Not published — the validator could not find this game's data, so nothing was checked." - : "Not published — the game did not pass validation."; - return fail(`${preamble} Fix this and call publish again:\n${tail(validated.stdout)}`); - } - log("tool.publish.validated", { - durationMs: Math.round(performance.now() - startedAt), - }); - - const gate = await runCommand( - sandbox, - "npm test --silent && npm run build --silent", - { cwd: GAME_ROOT, timeoutSeconds: GATE_TIMEOUT_SECONDS }, - ); - if (!gate.success) { - log("tool.publish.gate_failed", { exitCode: gate.exitCode }); - return fail( - `Not published — tests or build failed (exit ${gate.exitCode}). Fix this and call publish again:\n${tail(gate.stdout)}`, - ); - } - log("tool.publish.gate_passed", { - durationMs: Math.round(performance.now() - startedAt), - }); - - const res = await runCommand(sandbox, publishScript(), { - timeoutSeconds: COPY_TIMEOUT_SECONDS, - }); - const out = res.stdout.trim(); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + // The engine's own gate, first and by absolute path out of ~/engine. + // It is seconds rather than minutes, so a game that cannot be finished + // fails here instead of after a cold vite build. It also runs on data the + // model cannot reach: `npm test` is a file in the working tree, and the + // model owns that file — "the tests passed" only means what the tests + // happen to assert today. + const validated = await runCommand(sandbox, validateScript(), { + cwd: GAME_ROOT, + timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, + }); + if (!validated.success) { + log("tool.publish.validate_failed", { exitCode: validated.exitCode }); + // Never publish a game the gate could not read. A validator that passes + // when it finds nothing would certify an unplayable game. + const preamble = + validated.exitCode === EXIT_GAME_NOT_FOUND + ? "Not published — the validator could not find this game's data, so nothing was checked." + : "Not published — the game did not pass validation."; + return fail(`${preamble} Fix this and call publish again:\n${tail(validated.stdout)}`); + } + log("tool.publish.validated", { + durationMs: Math.round(performance.now() - startedAt), + }); - if (!res.success) { - log("tool.publish.copy_failed", { exitCode: res.exitCode }); - if (out.startsWith("NO_BUILD")) { - return fail( - `The build succeeded but ${GAME_ROOT}/dist/${PUBLISHED_FILE} is missing. Check the build's output directory.`, - ); - } - if (out.startsWith("NOT_SELF_CONTAINED")) { - return fail( - `dist/ holds files besides ${PUBLISHED_FILE}, so the game would load assets that are not published:\n${out}\n` + - "The build must inline everything into one HTML file — check vite-plugin-singlefile is enabled in vite.config.ts.", + const gate = await runCommand( + sandbox, + "npm test --silent && npm run build --silent", + { cwd: GAME_ROOT, timeoutSeconds: GATE_TIMEOUT_SECONDS }, ); - } - // The published file is written last, so the live game is untouched. - return fail( - `Publish failed (exit ${res.exitCode}); the previously published game is unchanged:\n${tail(out)}`, - ); - } + if (!gate.success) { + log("tool.publish.gate_failed", { exitCode: gate.exitCode }); + return fail( + `Not published — tests or build failed (exit ${gate.exitCode}). Fix this and call publish again:\n${tail(gate.stdout)}`, + ); + } + log("tool.publish.gate_passed", { + durationMs: Math.round(performance.now() - startedAt), + }); - const version = Number(out.split("\n").pop()?.replace("PUBLISHED ", "")); - log("tool.publish.ok", { - version, - durationMs: Math.round(performance.now() - startedAt), - }); + const res = await runCommand(sandbox, publishScript(), { + timeoutSeconds: COPY_TIMEOUT_SECONDS, + }); + const out = res.stdout.trim(); + + if (!res.success) { + log("tool.publish.copy_failed", { exitCode: res.exitCode }); + if (out.startsWith("NO_BUILD")) { + return fail( + `The build succeeded but ${GAME_ROOT}/dist/${PUBLISHED_FILE} is missing. Check the build's output directory.`, + ); + } + if (out.startsWith("NOT_SELF_CONTAINED")) { + return fail( + `dist/ holds files besides ${PUBLISHED_FILE}, so the game would load assets that are not published:\n${out}\n` + + "The build must inline everything into one HTML file — check vite-plugin-singlefile is enabled in vite.config.ts.", + ); + } + // The published file is written last, so the live game is untouched. + return fail( + `Publish failed (exit ${res.exitCode}); the previously published game is unchanged:\n${tail(out)}`, + ); + } - // Index the new version for the app. Deliberately after the bucket write - // and deliberately non-fatal: the game IS published at this point, and - // telling the model otherwise would send it round the whole gate again to - // fix a database it cannot reach. A dropped row costs the version list - // one entry; the game itself is still live and the next publish indexes - // normally. - if (recordVersion && Number.isFinite(version)) { - try { - await recordVersion({ version, label: intent }); - } catch (err) { - log("tool.publish.record_failed", { + const version = Number(out.split("\n").pop()?.replace("PUBLISHED ", "")); + log("tool.publish.ok", { version, - error: err instanceof Error ? err.message : String(err), + durationMs: Math.round(performance.now() - startedAt), }); - } - } - return { - ok: true, - version: Number.isFinite(version) ? version : null, - url: publishedUrl ?? null, - key: publishedUrl ? null : `current/${PUBLISHED_FILE}`, - message: publishedUrl - ? `Published version ${version}. The teacher can play it at ${publishedUrl}.` - : `Published version ${version}. No public URL is configured yet, so share it once R2_PUBLIC_BASE_URL is set.`, - }; + // Index the new version for the app. Deliberately after the bucket write + // and deliberately non-fatal: the game IS published at this point, and + // telling the model otherwise would send it round the whole gate again to + // fix a database it cannot reach. A dropped row costs the version list + // one entry; the game itself is still live and the next publish indexes + // normally. + if (recordVersion && Number.isFinite(version)) { + try { + await recordVersion({ version, label: intent }); + } catch (err) { + log("tool.publish.record_failed", { + version, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + return { + ok: true, + version: Number.isFinite(version) ? version : null, + url: publishedUrl ?? null, + key: publishedUrl ? null : `current/${PUBLISHED_FILE}`, + message: publishedUrl + ? `Published version ${version}. The teacher can play it at ${publishedUrl}.` + : `Published version ${version}. No public URL is configured yet, so share it once R2_PUBLIC_BASE_URL is set.`, + }; + }, + ); + } catch (err) { + log("tool.publish.sandbox_unavailable"); + return fail( + `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, + ); + } }, }); } diff --git a/src/mastra/tools/read-file-tool.ts b/src/mastra/tools/read-file-tool.ts index fbf8b63..236da5c 100644 --- a/src/mastra/tools/read-file-tool.ts +++ b/src/mastra/tools/read-file-tool.ts @@ -4,6 +4,7 @@ import { z } from "zod"; import { ENGINE_ROOT, expandSandboxPath, GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; /** Above this, downloading the whole file into the server is wasteful — `bash` with head/grep is the right tool. */ const MAX_FILE_BYTES = 2_000_000; @@ -13,6 +14,8 @@ const MAX_DIR_ENTRIES = 200; export type CreateReadFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -42,6 +45,7 @@ function sliceLines(text: string, offset?: number, limit?: number): string { */ export function createReadFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateReadFileToolOptions) { return createTool({ @@ -77,62 +81,67 @@ export function createReadFileTool({ execute: async ({ path: input, offset, limit }) => { const path = expandSandboxPath(input); try { - const sandbox = await sandboxPromise; - const details = await sandbox.fs.getFileDetails(path); + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const details = await sandbox.fs.getFileDetails(path); - if (details.isDir) { - const entries = await sandbox.fs.listFiles(path); - const shown = entries.slice(0, MAX_DIR_ENTRIES); - trace?.log("tool.read.dir", { path, entries: entries.length }); - return { - ok: true, - path, - kind: "directory" as const, - size: entries.length, - truncated: entries.length > shown.length, - content: shown - .map((e) => (e.isDir ? `${e.name}/` : `${e.name} (${e.size} B)`)) - .join("\n"), - }; - } + if (details.isDir) { + const entries = await sandbox.fs.listFiles(path); + const shown = entries.slice(0, MAX_DIR_ENTRIES); + trace?.log("tool.read.dir", { path, entries: entries.length }); + return { + ok: true, + path, + kind: "directory" as const, + size: entries.length, + truncated: entries.length > shown.length, + content: shown + .map((e) => (e.isDir ? `${e.name}/` : `${e.name} (${e.size} B)`)) + .join("\n"), + }; + } - if (details.size > MAX_FILE_BYTES) { - return { - ok: false, - path, - error: `File is ${details.size} bytes, too large to read whole. Use bash with head/tail/grep to pull out the part you need.`, - }; - } + if (details.size > MAX_FILE_BYTES) { + return { + ok: false, + path, + error: `File is ${details.size} bytes, too large to read whole. Use bash with head/tail/grep to pull out the part you need.`, + }; + } - const buffer = await sandbox.fs.downloadFile(path); - if (looksBinary(buffer)) { - return { - ok: false, - path, - kind: "file" as const, - size: details.size, - error: - "This is a binary file, so there is nothing readable to return. Describe it to the teacher by name/size instead of trying to read it.", - }; - } + const buffer = await sandbox.fs.downloadFile(path); + if (looksBinary(buffer)) { + return { + ok: false, + path, + kind: "file" as const, + size: details.size, + error: + "This is a binary file, so there is nothing readable to return. Describe it to the teacher by name/size instead of trying to read it.", + }; + } - const text = sliceLines(buffer.toString("utf-8"), offset, limit); - const truncated = text.length > MAX_CHARS; - trace?.log("tool.read.file", { - path, - bytes: details.size, - truncated, - }); - return { - ok: true, - path, - kind: "file" as const, - size: details.size, - truncated, - content: truncated - ? `${text.slice(0, MAX_CHARS)}\n\n[truncated — re-read with offset/limit, or use bash grep to find the part you need]` - : text, - }; + const text = sliceLines(buffer.toString("utf-8"), offset, limit); + const truncated = text.length > MAX_CHARS; + trace?.log("tool.read.file", { + path, + bytes: details.size, + truncated, + }); + return { + ok: true, + path, + kind: "file" as const, + size: details.size, + truncated, + content: truncated + ? `${text.slice(0, MAX_CHARS)}\n\n[truncated — re-read with offset/limit, or use bash grep to find the part you need]` + : text, + }; + }, + ); } catch (err) { const message = errorMessage(err); trace?.log("tool.read.error", { path, error: message }); diff --git a/src/mastra/tools/validate-tool.ts b/src/mastra/tools/validate-tool.ts index 1293951..93127b1 100644 --- a/src/mastra/tools/validate-tool.ts +++ b/src/mastra/tools/validate-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; import { validateScript } from "~/server/sandbox/scripts"; /** Pure functions over the authored data — fast, unlike the test+build gate. */ @@ -15,6 +16,8 @@ const EXIT_GAME_NOT_FOUND = 2; export type CreateValidateToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -44,6 +47,7 @@ const levelReport = z.object({ */ export function createValidateTool({ sandboxPromise, + recoverSandbox, trace, }: CreateValidateToolOptions) { return createTool({ @@ -80,67 +84,70 @@ export function createValidateTool({ message, }); - let sandbox: Sandbox; try { - sandbox = await sandboxPromise; - } catch (err) { - log("tool.validate.sandbox_unavailable"); - return fail( - `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, - ); - } + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const res = await runCommand(sandbox, validateScript(), { + cwd: GAME_ROOT, + timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, + }); + const durationMs = Math.round(performance.now() - startedAt); - const res = await runCommand(sandbox, validateScript(), { - cwd: GAME_ROOT, - timeoutSeconds: VALIDATE_TIMEOUT_SECONDS, - }); - const durationMs = Math.round(performance.now() - startedAt); + const parsed = parseReport(res.stdout); + if (!parsed) { + log("tool.validate.unparseable", { + exitCode: res.exitCode, + durationMs, + }); + return fail( + `The validator did not return a readable report (exit ${res.exitCode}):\n${res.stdout.slice(-2000)}`, + ); + } - const parsed = parseReport(res.stdout); - if (!parsed) { - log("tool.validate.unparseable", { - exitCode: res.exitCode, - durationMs, - }); - return fail( - `The validator did not return a readable report (exit ${res.exitCode}):\n${res.stdout.slice(-2000)}`, - ); - } + if (res.exitCode === EXIT_GAME_NOT_FOUND) { + log("tool.validate.not_found", { durationMs }); + return fail( + [ + `The validator could not find the game: ${parsed.error ?? "unknown reason"}`, + parsed.hint, + "Nothing was checked, so do not treat this as a pass.", + ] + .filter(Boolean) + .join(" "), + ); + } - if (res.exitCode === EXIT_GAME_NOT_FOUND) { - log("tool.validate.not_found", { durationMs }); - return fail( - [ - `The validator could not find the game: ${parsed.error ?? "unknown reason"}`, - parsed.hint, - "Nothing was checked, so do not treat this as a pass.", - ] - .filter(Boolean) - .join(" "), - ); - } + const levels = parsed.items?.flatMap((item) => item.levels ?? []); + const errors = parsed.errors ?? []; - const levels = parsed.items?.flatMap((item) => item.levels ?? []); - const errors = parsed.errors ?? []; + if (res.exitCode === EXIT_GAME_HAS_ERRORS || parsed.ok === false) { + log("tool.validate.failed", { errorCount: errors.length, durationMs }); + return { + ok: false, + errors, + levels, + message: `The game is not ready: ${errors.length} problem(s) found. Fix these and run validate again.`, + }; + } - if (res.exitCode === EXIT_GAME_HAS_ERRORS || parsed.ok === false) { - log("tool.validate.failed", { errorCount: errors.length, durationMs }); - return { - ok: false, - errors, - levels, - message: `The game is not ready: ${errors.length} problem(s) found. Fix these and run validate again.`, - }; + log("tool.validate.ok", { durationMs }); + return { + ok: true, + errors: [], + levels, + message: + "The game is coherent, winnable by reasoning, and plays through to a win. Safe to publish.", + }; + }, + ); + } catch (err) { + log("tool.validate.sandbox_unavailable"); + return fail( + `Sandbox unavailable: ${err instanceof Error ? err.message : String(err)}`, + ); } - - log("tool.validate.ok", { durationMs }); - return { - ok: true, - errors: [], - levels, - message: - "The game is coherent, winnable by reasoning, and plays through to a win. Safe to publish.", - }; }, }); } diff --git a/src/mastra/tools/write-file-tool.ts b/src/mastra/tools/write-file-tool.ts index cf53edd..3ceb7a2 100644 --- a/src/mastra/tools/write-file-tool.ts +++ b/src/mastra/tools/write-file-tool.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { expandSandboxPath, GAME_ROOT } from "~/lib/sandbox-paths"; import type { LessonTrace } from "~/mastra/agents/lesson-shared"; import { quoteShellArg, runCommand } from "~/server/sandbox/exec"; +import { withRecoveredSandbox } from "~/server/sandbox/lifecycle"; function parentDir(path: string): string { const i = path.lastIndexOf("/"); @@ -13,6 +14,8 @@ function parentDir(path: string): string { export type CreateWriteFileToolOptions = { sandboxPromise: Promise; + /** Re-runs the full sandbox preparation if the container died. See lifecycle.ts. */ + recoverSandbox?: () => Promise; trace?: LessonTrace; }; @@ -26,6 +29,7 @@ export type CreateWriteFileToolOptions = { */ export function createWriteFileTool({ sandboxPromise, + recoverSandbox, trace, }: CreateWriteFileToolOptions) { return createTool({ @@ -44,26 +48,30 @@ export function createWriteFileTool({ execute: async ({ path: input, content }) => { const path = expandSandboxPath(input); try { - const sandbox = await sandboxPromise; + return await withRecoveredSandbox( + sandboxPromise, + recoverSandbox, + async (sandbox) => { + const dir = parentDir(path); + const mkdir = await runCommand( + sandbox, + `mkdir -p ${quoteShellArg(dir)}`, + ); + if (!mkdir.success) { + return { + ok: false, + path, + error: `Could not create parent directory "${dir}" (exit ${mkdir.exitCode}): ${mkdir.stdout.trim() || "no output"}. Check the path is valid and writable.`, + }; + } - const dir = parentDir(path); - const mkdir = await runCommand( - sandbox, - `mkdir -p ${quoteShellArg(dir)}`, - ); - if (!mkdir.success) { - return { - ok: false, - path, - error: `Could not create parent directory "${dir}" (exit ${mkdir.exitCode}): ${mkdir.stdout.trim() || "no output"}. Check the path is valid and writable.`, - }; - } - - const buffer = Buffer.from(content, "utf-8"); - await sandbox.fs.uploadFile(buffer, path); - trace?.log("tool.write", { path, bytes: buffer.byteLength }); + const buffer = Buffer.from(content, "utf-8"); + await sandbox.fs.uploadFile(buffer, path); + trace?.log("tool.write", { path, bytes: buffer.byteLength }); - return { ok: true, path, bytesWritten: buffer.byteLength }; + return { ok: true, path, bytesWritten: buffer.byteLength }; + }, + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); trace?.log("tool.write.error", { path, error: message }); diff --git a/src/server/sandbox/__tests__/lifecycle.test.ts b/src/server/sandbox/__tests__/lifecycle.test.ts new file mode 100644 index 0000000..3f419e8 --- /dev/null +++ b/src/server/sandbox/__tests__/lifecycle.test.ts @@ -0,0 +1,141 @@ +import type { Sandbox } from "@daytonaio/sdk"; +import { SandboxState } from "@daytonaio/sdk"; +import { describe, expect, it, vi } from "vitest"; + +import { + isSandboxUnreachableError, + planSandboxStep, + withRecoveredSandbox, +} from "../lifecycle"; + +/** The exact message from the production incident (trace 65e76ce8). */ +const CONTAINER_IP_ERROR = + "bad request: failed to resolve container IP after 3 attempts: no IP address found. Is the Sandbox started?"; + +describe("planSandboxStep", () => { + it("uses a started sandbox as-is", () => { + expect(planSandboxStep(SandboxState.STARTED)).toBe("use"); + }); + + it("starts sandboxes at rest", () => { + expect(planSandboxStep(SandboxState.STOPPED)).toBe("start"); + expect(planSandboxStep(SandboxState.ARCHIVED)).toBe("start"); + expect(planSandboxStep(SandboxState.PAUSED)).toBe("start"); + }); + + it("waits on sandboxes already on the way up", () => { + expect(planSandboxStep(SandboxState.STARTING)).toBe("wait-started"); + expect(planSandboxStep(SandboxState.RESTORING)).toBe("wait-started"); + expect(planSandboxStep(SandboxState.RESUMING)).toBe("wait-started"); + }); + + it("polls mid-transition states instead of using them (the incident)", () => { + // A sandbox observed while Daytona archives it has no container; the old + // code returned it as "existing-other" and every tool call failed. + expect(planSandboxStep(SandboxState.ARCHIVING)).toBe("wait"); + expect(planSandboxStep(SandboxState.STOPPING)).toBe("wait"); + expect(planSandboxStep(SandboxState.PAUSING)).toBe("wait"); + }); + + it("never returns 'use' for an unknown future state", () => { + expect(planSandboxStep("some-new-state" as Sandbox["state"])).toBe("wait"); + expect(planSandboxStep(undefined)).toBe("wait"); + }); + + it("fails fast on broken or terminal sandboxes", () => { + expect(planSandboxStep(SandboxState.ERROR)).toBe("fail"); + expect(planSandboxStep(SandboxState.BUILD_FAILED)).toBe("fail"); + expect(planSandboxStep(SandboxState.DESTROYED)).toBe("fail"); + expect(planSandboxStep(SandboxState.DESTROYING)).toBe("fail"); + }); +}); + +describe("isSandboxUnreachableError", () => { + it("matches the production container-IP error", () => { + expect(isSandboxUnreachableError(new Error(CONTAINER_IP_ERROR))).toBe(true); + }); + + it("matches non-Error throws too", () => { + expect(isSandboxUnreachableError(CONTAINER_IP_ERROR)).toBe(true); + }); + + it("ignores unrelated errors", () => { + expect(isSandboxUnreachableError(new Error("ENOENT: no such file"))).toBe( + false, + ); + expect(isSandboxUnreachableError(new Error("npm test failed"))).toBe(false); + }); +}); + +describe("withRecoveredSandbox", () => { + const sandboxA = { id: "a" } as unknown as Sandbox; + const sandboxB = { id: "b" } as unknown as Sandbox; + + it("runs against the resolved sandbox on the happy path", async () => { + const recover = vi.fn(); + const result = await withRecoveredSandbox( + Promise.resolve(sandboxA), + recover, + async (sandbox) => sandbox.id, + ); + expect(result).toBe("a"); + expect(recover).not.toHaveBeenCalled(); + }); + + it("recovers when the preparation promise itself died unreachable", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const result = await withRecoveredSandbox( + Promise.reject(new Error(CONTAINER_IP_ERROR)), + recover, + async (sandbox) => sandbox.id, + ); + expect(result).toBe("b"); + expect(recover).toHaveBeenCalledTimes(1); + }); + + it("recovers when the container dies under the call, and reruns it once", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const fn = vi + .fn<(sandbox: Sandbox) => Promise>() + .mockRejectedValueOnce(new Error(CONTAINER_IP_ERROR)) + .mockImplementation(async (sandbox) => sandbox.id); + const result = await withRecoveredSandbox( + Promise.resolve(sandboxA), + recover, + fn, + ); + expect(result).toBe("b"); + expect(fn).toHaveBeenNthCalledWith(1, sandboxA); + expect(fn).toHaveBeenNthCalledWith(2, sandboxB); + }); + + it("recovers at most once", async () => { + const recover = vi.fn().mockResolvedValue(sandboxB); + const fn = vi.fn().mockRejectedValue(new Error(CONTAINER_IP_ERROR)); + await expect( + withRecoveredSandbox(Promise.resolve(sandboxA), recover, fn), + ).rejects.toThrow(/container IP/); + expect(recover).toHaveBeenCalledTimes(1); + expect(fn).toHaveBeenCalledTimes(2); + }); + + it("rethrows unrelated errors without recovering", async () => { + const recover = vi.fn(); + await expect( + withRecoveredSandbox(Promise.resolve(sandboxA), recover, async () => { + throw new Error("old_string was not found"); + }), + ).rejects.toThrow(/old_string/); + expect(recover).not.toHaveBeenCalled(); + }); + + it("does nothing special when no recover function is supplied", async () => { + await expect( + withRecoveredSandbox( + Promise.reject(new Error(CONTAINER_IP_ERROR)), + undefined, + async (sandbox) => sandbox.id, + ), + ).rejects.toThrow(/container IP/); + }); +}); diff --git a/src/server/sandbox/daytona.ts b/src/server/sandbox/daytona.ts index 7cd082e..c8db03c 100644 --- a/src/server/sandbox/daytona.ts +++ b/src/server/sandbox/daytona.ts @@ -6,6 +6,7 @@ import { } from "@daytonaio/sdk"; import { env } from "~/env"; +import { planSandboxStep } from "./lifecycle"; let client: Daytona | null = null; @@ -24,7 +25,8 @@ export type SandboxLifecycleStatus = | "existing-started" | "existing-started-from-stopped" | "existing-waited-from-starting" - | "existing-other"; + /** Found mid-transition (archiving, stopping, …) and walked to started. */ + | "existing-recovered"; /** * Daytona signals a missing sandbox with DaytonaNotFoundError; the message check @@ -37,39 +39,119 @@ function isNotFound(err: unknown): boolean { ); } +/** Generous: restoring an archived ~3 GiB filesystem is minutes, not seconds. */ +const START_TIMEOUT_SECONDS = 120; +/** How long a sandbox may sit in a transitional state before we give up. */ +const SETTLE_TIMEOUT_MS = 180_000; +const SETTLE_POLL_MS = 2_000; +/** start() attempts before concluding the sandbox will not come up. */ +const MAX_START_ATTEMPTS = 3; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Walk the sandbox to STARTED, whatever state it is observed in. + * + * The old version handled exactly four states and silently returned the + * sandbox for every other one — so a thread reopened while Daytona was + * archiving it ("archiving" takes minutes for a 3 GiB disk) got a handle with + * no running container, and every tool call died with "failed to resolve + * container IP". Now: startable states are started, in-flight states are + * waited out (with a deadline), and broken states throw something readable + * instead of a container-IP error five calls later. + */ +async function ensureSandboxStarted(sandbox: Sandbox): Promise { + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + let startAttempts = 0; + + for (;;) { + const state = sandbox.state; + const step = planSandboxStep(state); + + if (step === "use") return; + + if (step === "fail") { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} is "${String(state)}" and cannot be started. ` + + `It needs to be deleted (or recreated) in Daytona before this thread can build again.`, + ); + } + + if (Date.now() >= deadline) { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} did not reach "started" within ${Math.round( + SETTLE_TIMEOUT_MS / 1000, + )}s (still "${String(state)}"). Try again in a minute.`, + ); + } + + if (step === "start") { + if (++startAttempts > MAX_START_ATTEMPTS) { + throw new Error( + `[LessonPlay] sandbox ${sandbox.id} fell back to "${String(state)}" after ${MAX_START_ATTEMPTS} start attempts.`, + ); + } + await sandbox.start(START_TIMEOUT_SECONDS); + } else if (step === "wait-started") { + await sandbox.waitUntilStarted(START_TIMEOUT_SECONDS); + } else { + // "wait": mid-transition (stopping, archiving, …). Poll until it lands + // in a state we can act on. + await sleep(SETTLE_POLL_MS); + } + + await sandbox.refreshData(); + } +} + +/** The status the old four-state walk would have reported, for log continuity. */ +function statusForInitialState( + state: Sandbox["state"], +): SandboxLifecycleStatus { + if (state === SandboxState.STARTED) return "existing-started"; + if ( + state === SandboxState.STOPPED || + state === SandboxState.ARCHIVED || + state === SandboxState.PAUSED + ) { + return "existing-started-from-stopped"; + } + if (state === SandboxState.STARTING) return "existing-waited-from-starting"; + return "existing-recovered"; +} + /** * Resolve the sandbox for a stable name, walking its lifecycle: * missing → create * STARTED → use - * STOPPED / ARCHIVED → start, then use - * STARTING → wait, then use + * STOPPED / ARCHIVED / PAUSED → start, then use + * STARTING (and other states already on the way up) → wait, then use + * mid-transition (STOPPING / ARCHIVING / …) → wait until settled, then start + * ERROR / DESTROYED → throw a readable error * Idempotent: safe to call on every request. */ - export async function getOrCreateSandbox( sandboxId: string, opts: { env?: Record }, -): Promise<{ sandbox: Sandbox; status: SandboxLifecycleStatus }> { +): Promise<{ + sandbox: Sandbox; + status: SandboxLifecycleStatus; + /** Raw state the sandbox was first observed in; undefined when created. */ + initialState?: string; +}> { const daytona = getDaytonaClient(); try { const sandbox = await daytona.get(sandboxId); - - if (sandbox.state === SandboxState.STARTED) { - return { sandbox, status: "existing-started" }; - } - if ( - sandbox.state === SandboxState.STOPPED || - sandbox.state === SandboxState.ARCHIVED - ) { - await sandbox.start(); - return { sandbox, status: "existing-started-from-stopped" }; - } - if (sandbox.state === SandboxState.STARTING) { - await sandbox.waitUntilStarted(); - return { sandbox, status: "existing-waited-from-starting" }; - } - return { sandbox, status: "existing-other" }; + const initialState = sandbox.state; + await ensureSandboxStarted(sandbox); + return { + sandbox, + status: statusForInitialState(initialState), + initialState: String(initialState), + }; } catch (err) { // ONLY "no such sandbox" means create one. Swallowing everything here turns // a transient 500 or an auth failure into a create against a name that is @@ -104,8 +186,9 @@ export async function getOrCreateSandbox( // platform maximum of 7 days, and a stopped sandbox holds its disk the // whole time. At ~3 GiB each against a 30 GiB org cap that fills in about // three days, so reclaim has to outrun it. Archiving is lossless (unlike - // autoDelete) and needs no new code path: getOrCreateSandbox already - // start()s an ARCHIVED sandbox. Cost is a slower restore on a cold thread. + // autoDelete) and needs no new code path: getOrCreateSandbox walks an + // ARCHIVED (or mid-ARCHIVING) sandbox back to started. Cost is a slower + // restore on a cold thread. autoArchiveInterval: 60, autoDeleteInterval: -1, // never auto-delete; recreating means a fresh npm ci envVars: opts.env, diff --git a/src/server/sandbox/lifecycle.ts b/src/server/sandbox/lifecycle.ts new file mode 100644 index 0000000..9e29e2d --- /dev/null +++ b/src/server/sandbox/lifecycle.ts @@ -0,0 +1,99 @@ +/** + * Pure sandbox-lifecycle logic — no `~/env`, no SDK client — so it can be + * imported by `~/mastra/tools/*` and unit-tested without Daytona. + * + * Why this exists: Daytona has ~23 sandbox states, and the old lifecycle walk + * in `daytona.ts` handled four of them. A thread reopened while its sandbox + * was mid-archive (state "archiving") got a handle with no running container, + * and every tool call failed with "failed to resolve container IP … Is the + * Sandbox started?". Every state now maps to an explicit step, and unknown + * states are waited on rather than silently used. + */ +import { SandboxState, type Sandbox } from "@daytonaio/sdk"; + +/** What to do next for a sandbox observed in `state`. */ +export type SandboxStep = + /** Running — use it. */ + | "use" + /** At rest but startable (stopped / archived / paused) — call start(). */ + | "start" + /** Already on its way up — wait for started. */ + | "wait-started" + /** Mid-transition (stopping, archiving, …) — poll until it settles. */ + | "wait" + /** Terminal or broken — starting it is impossible; surface a clear error. */ + | "fail"; + +export function planSandboxStep(state: Sandbox["state"]): SandboxStep { + switch (state) { + case SandboxState.STARTED: + return "use"; + case SandboxState.STOPPED: + case SandboxState.ARCHIVED: + case SandboxState.PAUSED: + return "start"; + case SandboxState.STARTING: + case SandboxState.RESUMING: + case SandboxState.RESTORING: + case SandboxState.CREATING: + case SandboxState.PULLING_SNAPSHOT: + return "wait-started"; + case SandboxState.ERROR: + case SandboxState.BUILD_FAILED: + case SandboxState.DESTROYED: + case SandboxState.DESTROYING: + return "fail"; + // stopping, archiving, pausing, snapshotting, resizing, forking, unknown, + // and anything a future SDK adds: in motion (or unreadable) — poll until + // it lands somewhere actionable. The caller owns the deadline. + default: + return "wait"; + } +} + +/** + * Does this error mean "the container under this sandbox handle is gone"? + * + * That is the signature of the sandbox being stopped/archived between our + * state check and the command — the exact failure seen in production as + * "bad request: failed to resolve container IP after 3 attempts: no IP + * address found. Is the Sandbox started?". + */ +export function isSandboxUnreachableError(err: unknown): boolean { + const message = err instanceof Error ? err.message : String(err); + return /failed to resolve container ip|is the sandbox started\?|sandbox is not (?:started|running)/i.test( + message, + ); +} + +/** + * Run `fn` against the sandbox, recovering ONCE if the container turns out to + * be gone — either the promise itself rejected that way (boot raced an + * archive) or a call inside `fn` did (auto-stop raced a long request). + * + * `recover` is expected to re-run the full preparation (start + R2 mount + + * hydrate) — a bare restart is not enough, because the s3fs mount does not + * survive the container. `prepareLessonSandbox` is idempotent and is the + * intended recover function. `fn` must therefore be safe to re-run from the + * top, which every tool body is: nothing in it completed if the container was + * unreachable. + */ +export async function withRecoveredSandbox( + sandboxPromise: Promise, + recover: (() => Promise) | undefined, + fn: (sandbox: Sandbox) => Promise, +): Promise { + let sandbox: Sandbox; + try { + sandbox = await sandboxPromise; + } catch (err) { + if (!recover || !isSandboxUnreachableError(err)) throw err; + return fn(await recover()); + } + try { + return await fn(sandbox); + } catch (err) { + if (!recover || !isSandboxUnreachableError(err)) throw err; + return fn(await recover()); + } +} diff --git a/src/server/sandbox/prepare.ts b/src/server/sandbox/prepare.ts index 5033b1d..bbd5a5d 100644 --- a/src/server/sandbox/prepare.ts +++ b/src/server/sandbox/prepare.ts @@ -72,12 +72,13 @@ export function prepareLessonSandbox(opts: { const log = opts.trace?.log ?? (() => {}); const promise = (async () => { - const { sandbox, status } = await getOrCreateSandbox( + const { sandbox, status, initialState } = await getOrCreateSandbox( sandboxIdForThread(opts.threadId), { env: SANDBOX_ENV }, ); log("sandbox.acquired", { status, + state: initialState ?? "new", id: sandbox.id, durationMs: Math.round(performance.now() - startedAt), });