Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions src/lib/server/media-fetch-job.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -521,3 +521,48 @@ export async function abortMediaFetchJob(outputs: Record<string, JsonValue> | nu
const state = readState(outputs);
if (state?.sandboxId) await deleteSandbox(state.sandboxId);
}


/**
* Reap orphaned Daytona sandboxes from the synchronous (non-job) media-fetch path used by
* external API callers (see media-fetch.server.ts). That path has no AgentRun tracking it, and
* its `finally { sandbox.delete() }` never runs if the caller's Vercel function is killed on
* timeout instead of throwing — so a failing/retrying external caller (e.g. a Hermes playbook
* step) can leak one sandbox per attempt well before Daytona's own 10-minute autoStopInterval
* catches up, and a burst of leaked sandboxes can exhaust the account's concurrency limit and
* stall unrelated, properly-tracked runs. Called from /api/cron/ops as a backstop.
*
* Only targets `purpose: media-fetch` sandboxes (the untracked sync path). `media-fetch-job`
* sandboxes are already tracked by tickMediaFetchJob/abortMediaFetchJob and have their own
* 60-minute autoStopInterval, so reaping them here would risk killing a run still legitimately
* in progress.
*/
export async function reapStaleMediaFetchSandboxes(maxAgeMinutes = 8): Promise<number> {
let daytona: Awaited<ReturnType<typeof daytonaClient>>["daytona"];
try {
({ daytona } = await daytonaClient());
} catch {
return 0;
}
const cutoff = Date.now() - maxAgeMinutes * 60_000;
let reaped = 0;
try {
const iter = daytona.list({ labels: { purpose: "media-fetch", app: "clippyos" }, limit: 20 });
for await (const sandbox of iter as AsyncIterable<AnySandbox>) {
const state = String(sandbox.state ?? "").toLowerCase();
if (state === "stopped" || state === "destroyed" || state === "destroying" || state === "archived") continue;
const createdAt = Date.parse(String(sandbox.createdAt ?? ""));
if (!Number.isFinite(createdAt) || createdAt > cutoff) continue;
try {
if (sandbox.delete) await sandbox.delete();
else if (sandbox.stop) await sandbox.stop();
reaped += 1;
} catch {
/* best-effort; auto-stop still applies */
}
}
} catch {
/* Daytona list unavailable this tick; try again next cron run */
}
return reaped;
}
8 changes: 8 additions & 0 deletions src/routes/api/cron/ops.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@ export const Route = createFileRoute("/api/cron/ops")({
} catch {
mediaFetchTicked = 0;
}
let mediaFetchSandboxesReaped = 0;
try {
const { reapStaleMediaFetchSandboxes } = await import("@/lib/server/media-fetch-job.server");
mediaFetchSandboxesReaped = await reapStaleMediaFetchSandboxes();
} catch {
mediaFetchSandboxesReaped = 0;
}
let machineState = "unknown";
try {
const { getSocialMachineStatus } = await import("@/lib/server/daytona.server");
Expand All @@ -47,6 +54,7 @@ export const Route = createFileRoute("/api/cron/ops")({
ok: true,
linearSwept,
mediaFetchTicked,
mediaFetchSandboxesReaped,
machineState,
startedMachine: false,
note: "Cron never starts the Social Machine. Idle pause is owned by Daytona. Hibernate is pause, not destroy.",
Expand Down
Loading