diff --git a/src/lib/server/media-fetch-job.server.ts b/src/lib/server/media-fetch-job.server.ts index 9232e91..3f5a637 100644 --- a/src/lib/server/media-fetch-job.server.ts +++ b/src/lib/server/media-fetch-job.server.ts @@ -521,3 +521,48 @@ export async function abortMediaFetchJob(outputs: Record | nu const state = readState(outputs); if (state?.sandboxId) await deleteSandbox(state.sandboxId); } + + +/** + * Reap orphaned Daytona sandboxes from the synchronous (non-job) media-fetch path used by + * external API callers (see media-fetch.server.ts). That path has no AgentRun tracking it, and + * its `finally { sandbox.delete() }` never runs if the caller's Vercel function is killed on + * timeout instead of throwing — so a failing/retrying external caller (e.g. a Hermes playbook + * step) can leak one sandbox per attempt well before Daytona's own 10-minute autoStopInterval + * catches up, and a burst of leaked sandboxes can exhaust the account's concurrency limit and + * stall unrelated, properly-tracked runs. Called from /api/cron/ops as a backstop. + * + * Only targets `purpose: media-fetch` sandboxes (the untracked sync path). `media-fetch-job` + * sandboxes are already tracked by tickMediaFetchJob/abortMediaFetchJob and have their own + * 60-minute autoStopInterval, so reaping them here would risk killing a run still legitimately + * in progress. + */ +export async function reapStaleMediaFetchSandboxes(maxAgeMinutes = 8): Promise { + let daytona: Awaited>["daytona"]; + try { + ({ daytona } = await daytonaClient()); + } catch { + return 0; + } + const cutoff = Date.now() - maxAgeMinutes * 60_000; + let reaped = 0; + try { + const iter = daytona.list({ labels: { purpose: "media-fetch", app: "clippyos" }, limit: 20 }); + for await (const sandbox of iter as AsyncIterable) { + const state = String(sandbox.state ?? "").toLowerCase(); + if (state === "stopped" || state === "destroyed" || state === "destroying" || state === "archived") continue; + const createdAt = Date.parse(String(sandbox.createdAt ?? "")); + if (!Number.isFinite(createdAt) || createdAt > cutoff) continue; + try { + if (sandbox.delete) await sandbox.delete(); + else if (sandbox.stop) await sandbox.stop(); + reaped += 1; + } catch { + /* best-effort; auto-stop still applies */ + } + } + } catch { + /* Daytona list unavailable this tick; try again next cron run */ + } + return reaped; +} diff --git a/src/routes/api/cron/ops.ts b/src/routes/api/cron/ops.ts index 1cda074..e4767cd 100644 --- a/src/routes/api/cron/ops.ts +++ b/src/routes/api/cron/ops.ts @@ -35,6 +35,13 @@ export const Route = createFileRoute("/api/cron/ops")({ } catch { mediaFetchTicked = 0; } + let mediaFetchSandboxesReaped = 0; + try { + const { reapStaleMediaFetchSandboxes } = await import("@/lib/server/media-fetch-job.server"); + mediaFetchSandboxesReaped = await reapStaleMediaFetchSandboxes(); + } catch { + mediaFetchSandboxesReaped = 0; + } let machineState = "unknown"; try { const { getSocialMachineStatus } = await import("@/lib/server/daytona.server"); @@ -47,6 +54,7 @@ export const Route = createFileRoute("/api/cron/ops")({ ok: true, linearSwept, mediaFetchTicked, + mediaFetchSandboxesReaped, machineState, startedMachine: false, note: "Cron never starts the Social Machine. Idle pause is owned by Daytona. Hibernate is pause, not destroy.",