From 0c9f37850d0fc86485e5b793d46727ad51f09247 Mon Sep 17 00:00:00 2001 From: om952 Date: Fri, 26 Jun 2026 17:39:34 +0530 Subject: [PATCH 1/2] =?UTF-8?q?feat(heartbeat):=20port=20upstream=20PR=20#?= =?UTF-8?q?6736=20=E2=80=94=20autocompact=5Fthrash=20stop=20reason=20+=20s?= =?UTF-8?q?ession=20rotation=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tracks upstream paperclipai/paperclip#6736. Changes: - Add `autocompact_thrash` as a first-class HeartbeatRunStopReason - Export `isAutocompactThrashError()` to detect thrash error strings - Update `inferHeartbeatRunStopReason()` to classify thrash before adapter_failed catch-all - Restructure `evaluateSessionCompaction()` to always fetch >=1 prior run, then force session rotation when the most-recent run was a thrash failure (covers both new typed stopReason and legacy error string detection) - Fix `finalizeAgentStatus()` to accept subprocess exit code; transition agent from error -> idle when subprocess exits 0 even if run was pre-terminated as failed - Filter mentioned skill IDs through `isUuidLike()` to avoid slug-mention false positives - Unit tests for thrash detection classifier and process-recovery exit-code-0 override Closes OpenScanAI/Levi#29 --- .../heartbeat-process-recovery.test.ts | 72 ++++++++++++++++ .../services/heartbeat-stop-metadata.test.ts | 44 ++++++++++ .../src/services/heartbeat-stop-metadata.ts | 11 +++ server/src/services/heartbeat.ts | 83 ++++++++++++++++--- 4 files changed, 198 insertions(+), 12 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 8930b073059..b2af58f3672 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -2950,4 +2950,76 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); expect(runs).toHaveLength(1); }); + + it("clears agent error status to idle when subprocess exits 0 even if run was pre-terminated as failed by recovery", async () => { + // Scenario: process-loss recovery marks a run "failed" before the subprocess + // exits. The subprocess then exits cleanly (exit code 0). Without the fix, + // the latestRun.status === "failed" check causes outcome="failed" and the + // agent stays stuck in "error". With the fix, subprocessExitCode=0 overrides + // the transition to "idle". + const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); + + // Put the agent into "error" to simulate a previous failure. + await db + .update(agents) + .set({ status: "error", updatedAt: new Date() }) + .where(eq(agents.id, agentId)); + + // Set the issue to "in_review" so that releaseIssueExecutionAndPromote does not + // enqueue a recovery run when the simulated run is marked failed. If the issue + // stays "in_progress", the recovery loop starts a new run immediately, putting + // the agent into "running" before finalizeAgentStatus fires — which prevents the + // exit-code-0 override from being reached. + await db + .update(issues) + .set({ status: "in_review", updatedAt: new Date() }) + .where(eq(issues.id, issueId)); + + // Mock: simulate process-loss recovery pre-terminating the run as "failed" + // while the subprocess is still executing, then returning exit code 0. + mockAdapterExecute.mockImplementationOnce(async (ctx: { runId: string }) => { + await db + .update(heartbeatRuns) + .set({ + status: "failed", + error: "process_lost: pid not alive (simulated recovery)", + errorCode: "process_lost", + finishedAt: new Date(), + updatedAt: new Date(), + }) + .where(eq(heartbeatRuns.id, ctx.runId)); + return { + exitCode: 0, + signal: null, + timedOut: false, + errorMessage: null, + summary: "Subprocess exited cleanly despite run pre-termination.", + provider: "test", + model: "test-model", + }; + }); + + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await waitForHeartbeatIdle(db, 5_000); + + // Wait for finalizeAgentStatus to fire after the run settles. executeRun sets + // agent.status = "running" before calling the adapter; finalizeAgentStatus + // resets it afterward. waitForHeartbeatIdle returns as soon as all runs are + // terminal, which can race with finalizeAgentStatus on the agent row. + const agent = await waitForValue( + () => + db + .select({ status: agents.status }) + .from(agents) + .where(eq(agents.id, agentId)) + .then((rows) => { + const row = rows[0]; + return row && row.status === "idle" ? row : null; + }), + 5_000, + ); + expect(agent?.status).toBe("idle"); + }); }); diff --git a/server/src/services/heartbeat-stop-metadata.test.ts b/server/src/services/heartbeat-stop-metadata.test.ts index fc6d54c82e9..652a0d18e0c 100644 --- a/server/src/services/heartbeat-stop-metadata.test.ts +++ b/server/src/services/heartbeat-stop-metadata.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildHeartbeatRunStopMetadata, + isAutocompactThrashError, mergeHeartbeatRunStopMetadata, resolveHeartbeatRunTimeoutPolicy, } from "./heartbeat-stop-metadata.js"; @@ -98,6 +99,49 @@ describe("heartbeat stop metadata", () => { ).toBe("completed"); }); + it("classifies autocompact-thrash error strings as autocompact_thrash", () => { + for (const message of ["Autocompact is thrashing", "Autocompact is thrashing after 3 attempts"]) { + expect(isAutocompactThrashError(message)).toBe(true); + expect( + buildHeartbeatRunStopMetadata({ + adapterType: "claude_local", + adapterConfig: {}, + outcome: "failed", + errorCode: "adapter_failed", + errorMessage: message, + }).stopReason, + ).toBe("autocompact_thrash"); + } + + for (const message of ["context refilled to the limit", "ERROR: context refilled to the limit after compaction"]) { + expect(isAutocompactThrashError(message)).toBe(true); + expect( + buildHeartbeatRunStopMetadata({ + adapterType: "claude_local", + adapterConfig: {}, + outcome: "failed", + errorCode: "adapter_failed", + errorMessage: message, + }).stopReason, + ).toBe("autocompact_thrash"); + } + }); + + it("does not classify non-thrash failures as autocompact_thrash", () => { + for (const message of [null, undefined, "", "process exited with code 1", "Unknown error from adapter"]) { + expect(isAutocompactThrashError(message)).toBe(false); + expect( + buildHeartbeatRunStopMetadata({ + adapterType: "claude_local", + adapterConfig: {}, + outcome: "failed", + errorCode: "adapter_failed", + errorMessage: message, + }).stopReason, + ).toBe("adapter_failed"); + } + }); + it("preserves existing result fields when merging stop metadata", () => { const result = mergeHeartbeatRunStopMetadata( { summary: "done" }, diff --git a/server/src/services/heartbeat-stop-metadata.ts b/server/src/services/heartbeat-stop-metadata.ts index de80a32bba1..b624b3e92da 100644 --- a/server/src/services/heartbeat-stop-metadata.ts +++ b/server/src/services/heartbeat-stop-metadata.ts @@ -8,6 +8,7 @@ export type HeartbeatRunStopReason = | "paused" | "max_turns_exhausted" | "process_lost" + | "autocompact_thrash" | "adapter_failed"; export interface HeartbeatRunTimeoutPolicy { @@ -77,6 +78,13 @@ export function resolveHeartbeatRunTimeoutPolicy( }; } +/** Detects autocompact-thrash failures by the known error strings emitted by the adapter. */ +export function isAutocompactThrashError(errorMessage: string | null | undefined): boolean { + if (!errorMessage) return false; + const lower = errorMessage.toLowerCase(); + return lower.includes("autocompact is thrashing") || lower.includes("context refilled to the limit"); +} + export function inferHeartbeatRunStopReason(input: { outcome: HeartbeatRunOutcome; errorCode?: string | null; @@ -87,6 +95,9 @@ export function inferHeartbeatRunStopReason(input: { if (maxTurnStopReason) return maxTurnStopReason; if (input.outcome === "timed_out") return "timeout"; if (input.outcome === "failed" && input.errorCode === "process_lost") return "process_lost"; + if (input.outcome === "failed" && isAutocompactThrashError(input.errorMessage)) { + return "autocompact_thrash"; + } if (input.outcome === "cancelled") { const message = (input.errorMessage ?? "").toLowerCase(); if (message.includes("budget")) return "budget_paused"; diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 8c34e99233a..3068b1d19b6 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -76,6 +76,7 @@ import { } from "./heartbeat-run-summary.js"; import { buildHeartbeatRunStopMetadata, + isAutocompactThrashError, mergeHeartbeatRunStopMetadata, normalizeMaxTurnStopReason, } from "./heartbeat-stop-metadata.js"; @@ -524,7 +525,7 @@ async function resolveRunScopedMentionedSkillKeys(input: { issue.title, issue.description ?? "", ...comments.map((comment) => comment.body), - ]); + ]).filter((id) => isUuidLike(id)); if (mentionedSkillIds.length === 0) return []; const skillRows = await input.db @@ -3342,22 +3343,18 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const policy = parseSessionCompactionPolicy(agent); - if (!policy.enabled || !hasSessionCompactionThresholds(policy)) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: null, - }; - } + const policyEnabled = policy.enabled && hasSessionCompactionThresholds(policy); - const fetchLimit = Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4); + // Always fetch at least 1 run: needed for the autocompact-thrash guard even when + // session compaction policy is disabled. + const fetchLimit = policyEnabled ? Math.max(policy.maxSessionRuns > 0 ? policy.maxSessionRuns + 1 : 0, 4) : 1; const runs = await db .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt, usageJson: heartbeatRuns.usageJson, error: heartbeatRuns.error, + resultStopReason: sql`${heartbeatRuns.resultJson} ->> 'stopReason'`.as("resultStopReason"), ...heartbeatRunListResultColumns, }) .from(heartbeatRuns) @@ -3375,6 +3372,58 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } const latestRun = runs[0] ?? null; + + // Autocompact-thrash guard: force session rotation regardless of compaction policy. + // Detects both the typed stopReason (runs after this fix) and legacy error strings + // (runs before this fix that were stored as adapter_failed). + if ( + latestRun && + (latestRun.resultStopReason === "autocompact_thrash" || isAutocompactThrashError(latestRun.error)) + ) { + const thrashReason = "prior run hit autocompact context thrash; resuming would deterministically fail again"; + const latestSummaryForThrash = summarizeHeartbeatRunListResultJson({ + summary: latestRun.resultSummary, + result: latestRun.resultResult, + message: latestRun.resultMessage, + error: latestRun.resultError, + totalCostUsd: latestRun.resultTotalCostUsd, + costUsd: latestRun.resultCostUsd, + costUsdCamel: latestRun.resultCostUsdCamel, + }); + const latestTextSummaryForThrash = + readNonEmptyString(latestSummaryForThrash?.summary) ?? + readNonEmptyString(latestSummaryForThrash?.result) ?? + readNonEmptyString(latestSummaryForThrash?.message) ?? + readNonEmptyString(latestRun.error); + const thrashHandoffMarkdown = [ + "Paperclip session handoff:", + `- Previous session: ${sessionId}`, + issueId ? `- Issue: ${issueId}` : "", + `- Rotation reason: ${thrashReason}`, + latestTextSummaryForThrash ? `- Last run summary: ${latestTextSummaryForThrash}` : "", + input.continuationSummaryBody + ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` + : "", + "Continue from the current task state. Rebuild only the minimum context you need.", + ] + .filter(Boolean) + .join("\n"); + return { + rotate: true, + reason: thrashReason, + handoffMarkdown: thrashHandoffMarkdown, + previousRunId: latestRun.id, + }; + } + + if (!policyEnabled) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: latestRun?.id ?? null, + }; + } const oldestRun = policy.maxSessionAgeHours > 0 ? await getOldestRunForSession(agent.id, sessionId) @@ -6295,6 +6344,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) async function finalizeAgentStatus( agentId: string, outcome: "succeeded" | "failed" | "cancelled" | "timed_out", + opts?: { subprocessExitCode?: number | null }, ) { const existing = await getAgent(agentId); if (!existing) return; @@ -6306,13 +6356,22 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const isFirstHeartbeat = !existing.lastHeartbeatAt; const runningCount = await countRunningRunsForAgent(agentId); - const nextStatus = + let nextStatus: "running" | "idle" | "error" = runningCount > 0 ? "running" : outcome === "succeeded" || outcome === "cancelled" ? "idle" : "error"; + // If the subprocess exited cleanly (exit code 0) but the run was pre-terminated + // as "failed" by process-loss recovery before the subprocess finished, still + // transition the agent from "error" to "idle" so the successful work is recognised. + // The gate on existing.status === "error" prevents this from masking a first-run + // failure where the agent was "running" when finalizeAgentStatus was called. + if (nextStatus === "error" && opts?.subprocessExitCode === 0 && existing.status === "error") { + nextStatus = "idle"; + } + const updated = await db .update(agents) .set({ @@ -8077,7 +8136,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) } } } - await finalizeAgentStatus(agent.id, outcome); + await finalizeAgentStatus(agent.id, outcome, { subprocessExitCode: adapterResult.exitCode ?? null }); } catch (err) { const message = redactCurrentUserText( err instanceof Error ? err.message : "Unknown adapter failure", From f0cb4f195b4b53c20ad87a829f91c48ee6338553 Mon Sep 17 00:00:00 2001 From: om952 Date: Thu, 2 Jul 2026 10:51:56 +0530 Subject: [PATCH 2/2] fix(heartbeat): rotate to fresh session on autocompact thrash (#29) Ports upstream PR #6736 behavior: when a heartbeat run fails because the adapter session is autocompact-thrashing, force a session rotation instead of resuming the poisoned session. Detects both the typed `resultStopReason` and legacy error-string markers. - Move `evaluateSessionCompactionFromRuns` to module top-level so it can be exported cleanly and unit-tested. - Remove unrelated `maxConsecutiveAdapterFailed` session-compaction branch. - Add regression coverage in `heartbeat-process-recovery.test.ts` that seeds a known task session, simulates the thrash failure, and proves the recovery run receives a fresh session and does not retry the poisoned one. Closes #29 --- .../heartbeat-process-recovery.test.ts | 138 ++++++++++++- server/src/services/heartbeat.ts | 192 +++++++++++------- 2 files changed, 261 insertions(+), 69 deletions(-) diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index b2af58f3672..5dc39f81404 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -6,6 +6,7 @@ import { activityLog, agents, agentRuntimeState, + agentTaskSessions, agentWakeupRequests, budgetPolicies, companySkills, @@ -321,6 +322,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { await db.delete(agentRuntimeState); await db.delete(companySkills); await db.delete(costEvents); + await db.delete(agentTaskSessions); await db.delete(environmentLeases); await db.delete(environments); await db.delete(issueWorkProducts); @@ -3020,6 +3022,140 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { }), 5_000, ); + expect(agent?.status).toBe("idle"); }); -}); + + it("rotates to a fresh session and avoids a second wasted compaction after autocompact thrash", async () => { + const sessionId = "session-thrash-" + randomUUID(); + const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); + + // Pre-seed the agent task session so the first run resumes from a known session. + await db.insert(agentTaskSessions).values({ + companyId, + agentId, + adapterType: "codex_local", + taskKey: issueId, + sessionDisplayId: sessionId, + sessionParamsJson: { sessionId }, + lastError: null, + }); + + // Remove the issue_assigned wakeReason so the seeded task session is not reset on wake. + await db + .update(heartbeatRuns) + .set({ + contextSnapshot: { + issueId, + taskId: issueId, + }, + }) + .where(eq(heartbeatRuns.id, runId)); + + // First adapter invocation: simulate an autocompact-thrash failure with a cost event. + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 1, + signal: null, + timedOut: false, + errorCode: "adapter_failed", + errorMessage: "Autocompact is thrashing after 3 attempts", + provider: "test", + model: "test-model", + usage: { inputTokens: 100_000, outputTokens: 10_000, cachedInputTokens: 0 }, + costUsd: 0.5, + }); + + // Second adapter invocation (the recovery run) should succeed with a fresh session. + mockAdapterExecute.mockResolvedValueOnce({ + exitCode: 0, + signal: null, + timedOut: false, + errorMessage: null, + summary: "Recovered after session rotation.", + provider: "test", + model: "test-model", + usage: { inputTokens: 1_000, outputTokens: 500, cachedInputTokens: 0 }, + costUsd: 0.01, + }); + + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await waitForRunToSettle(heartbeat, runId, 5_000); + await waitForHeartbeatIdle(db, 5_000); + + const runs = await db + .select() + .from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, agentId)) + .orderBy(heartbeatRuns.createdAt); + expect(runs.length).toBeGreaterThanOrEqual(2); + + const thrashedRun = runs.find((r) => r.id === runId); + const recoveryRun = runs.find((r) => r.retryOfRunId === runId); + if (!thrashedRun) throw new Error("Expected original thrashed run to exist"); + if (!recoveryRun) throw new Error("Expected recovery run to exist"); + + expect(thrashedRun.status).toBe("failed"); + expect(thrashedRun.errorCode).toBe("adapter_failed"); + expect((thrashedRun.resultJson as Record | null)?.stopReason).toBe("autocompact_thrash"); + expect(thrashedRun.sessionIdAfter).toBe(sessionId); + + // Locate the adapter calls that correspond to the thrashed and recovery runs. + // The exact number of calls can include an optional successful-run handoff; what matters + // is that the thrashed run used the poisoned session and the recovery run used a fresh one. + const adapterCalls = mockAdapterExecute.mock.calls; + const thrashedCallIndex = adapterCalls.findIndex( + (c) => + ((c[0] as { runtime?: { sessionId?: string | null } } | undefined)?.runtime?.sessionId ?? null) === + sessionId, + ); + const recoveryCallIndex = adapterCalls.findIndex( + (c, idx) => + idx > thrashedCallIndex && + ((c[0] as { runtime?: { sessionId?: string | null } } | undefined)?.runtime?.sessionId ?? null) === null, + ); + const thrashedCall = + thrashedCallIndex >= 0 + ? (adapterCalls[thrashedCallIndex][0] as { runtime?: { sessionId: string | null } } | undefined) + : undefined; + const recoveryCall = + recoveryCallIndex >= 0 + ? (adapterCalls[recoveryCallIndex][0] as { runtime?: { sessionId: string | null } } | undefined) + : undefined; + + expect(thrashedCall?.runtime?.sessionId).toBe(sessionId); + expect(recoveryCall?.runtime?.sessionId).toBeNull(); + + // The recovery run must resume with a fresh session ID, not the poisoned thrashed session. + expect(recoveryRun.sessionIdBefore).not.toBe(sessionId); + + // No second wasted compaction attempt should be recorded: only one run failed with the + // autocompact_thrash stop reason, and the recovery run succeeds without another + // adapter_failed / autocompact_thrash outcome. + const autocompactThrashRuns = runs.filter( + (r) => + (r.resultJson as Record | null)?.stopReason === "autocompact_thrash" || + r.errorCode === "adapter_failed" || + String(r.error ?? "").toLowerCase().includes("autocompact"), + ); + expect(autocompactThrashRuns).toHaveLength(1); + + const costEventRows = await db + .select() + .from(costEvents) + .where(eq(costEvents.agentId, agentId)) + .orderBy(costEvents.occurredAt); + + const thrashCostEvent = costEventRows.find((e) => e.heartbeatRunId === thrashedRun.id); + expect(thrashCostEvent).toBeTruthy(); + + // No duplicate cost event should be associated with the thrashed run. + expect(costEventRows.filter((e) => e.heartbeatRunId === thrashedRun.id)).toHaveLength(1); + + // The recovery run's context snapshot must carry the rotation signal and reason. + const recoveryContext = recoveryRun.contextSnapshot as Record | null; + expect(recoveryContext?.paperclipSessionRotationReason).toContain("autocompact"); + expect(recoveryContext?.paperclipPreviousSessionId).toBe(sessionId); + expect(recoveryRun.status).toBe("succeeded"); + }); +}); \ No newline at end of file diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 3068b1d19b6..5ba28f07ea5 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -1034,7 +1034,7 @@ type UsageTotals = { outputTokens: number; }; -type SessionCompactionDecision = { +export type SessionCompactionDecision = { rotate: boolean; reason: string | null; handoffMarkdown: string | null; @@ -1501,6 +1501,117 @@ function formatCount(value: number | null | undefined) { return value.toLocaleString("en-US"); } +export function evaluateSessionCompactionFromRuns(input: { + agent: typeof agents.$inferSelect; + sessionId: string; + issueId: string | null; + policy: SessionCompactionPolicy; + runs: Array<{ + id: string; + createdAt: Date; + usageJson: Record | null; + error: string | null; + errorCode?: string | null; + resultJson?: Record | null; + resultSummary?: string | null; + resultResult?: string | null; + resultMessage?: string | null; + resultError?: string | null; + resultTotalCostUsd?: string | null; + resultCostUsd?: string | null; + resultCostUsdCamel?: string | null; + }>; + oldestRun: { + id: string; + createdAt: Date; + usageJson: Record | null; + error: string | null; + } | null; + continuationSummaryBody?: string | null; +}): SessionCompactionDecision { + const { agent, sessionId, issueId, policy, runs, oldestRun } = input; + const latestRun = runs[0] ?? null; + if (!latestRun) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: null, + }; + } + + const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); + const sessionAgeHours = + latestRun && oldestRun + ? Math.max( + 0, + (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), + ) + : 0; + + let reason: string | null = null; + if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { + reason = `session exceeded ${policy.maxSessionRuns} runs`; + } else if ( + policy.maxRawInputTokens > 0 && + latestRawUsage && + latestRawUsage.inputTokens >= policy.maxRawInputTokens + ) { + reason = + `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + + `(threshold ${formatCount(policy.maxRawInputTokens)})`; + } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { + reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; + } + + if (!reason) { + return { + rotate: false, + reason: null, + handoffMarkdown: null, + previousRunId: latestRun.id, + }; + } + + const latestSummary = summarizeHeartbeatRunListResultJson({ + summary: latestRun?.resultSummary, + result: latestRun?.resultResult, + message: latestRun?.resultMessage, + error: latestRun?.resultError, + totalCostUsd: latestRun?.resultTotalCostUsd, + costUsd: latestRun?.resultCostUsd, + costUsdCamel: latestRun?.resultCostUsdCamel, + }); + const latestTextSummary = + readNonEmptyString(latestSummary?.summary) ?? + readNonEmptyString(latestSummary?.result) ?? + readNonEmptyString(latestSummary?.message) ?? + readNonEmptyString(latestRun.error); + + const handoffMarkdown = [ + "Paperclip session handoff:", + `- Previous session: ${sessionId}`, + issueId ? `- Issue: ${issueId}` : "", + `- Rotation reason: ${reason}`, + latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", + input.continuationSummaryBody + ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` + : "", + "Continue from the current task state. Rebuild only the minimum context you need.", + ] + .filter(Boolean) + .join("\n"); + + return { + rotate: true, + reason, + handoffMarkdown, + previousRunId: latestRun.id, + }; +} + + + export function parseSessionCompactionPolicy(agent: typeof agents.$inferSelect): SessionCompactionPolicy { return resolveSessionCompactionPolicy(agent.adapterType, agent.runtimeConfig).policy; } @@ -3294,6 +3405,8 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) .select({ id: heartbeatRuns.id, createdAt: heartbeatRuns.createdAt, + usageJson: heartbeatRuns.usageJson, + error: heartbeatRuns.error, }) .from(heartbeatRuns) .where(and(eq(heartbeatRuns.agentId, agentId), eq(heartbeatRuns.sessionIdAfter, sessionId))) @@ -3355,6 +3468,7 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) usageJson: heartbeatRuns.usageJson, error: heartbeatRuns.error, resultStopReason: sql`${heartbeatRuns.resultJson} ->> 'stopReason'`.as("resultStopReason"), + errorCode: heartbeatRuns.errorCode, ...heartbeatRunListResultColumns, }) .from(heartbeatRuns) @@ -3427,75 +3541,17 @@ export function heartbeatService(db: Db, options: HeartbeatServiceOptions = {}) const oldestRun = policy.maxSessionAgeHours > 0 ? await getOldestRunForSession(agent.id, sessionId) - : runs[runs.length - 1] ?? latestRun; - const latestRawUsage = readRawUsageTotals(latestRun?.usageJson); - const sessionAgeHours = - latestRun && oldestRun - ? Math.max( - 0, - (new Date(latestRun.createdAt).getTime() - new Date(oldestRun.createdAt).getTime()) / (1000 * 60 * 60), - ) - : 0; - - let reason: string | null = null; - if (policy.maxSessionRuns > 0 && runs.length > policy.maxSessionRuns) { - reason = `session exceeded ${policy.maxSessionRuns} runs`; - } else if ( - policy.maxRawInputTokens > 0 && - latestRawUsage && - latestRawUsage.inputTokens >= policy.maxRawInputTokens - ) { - reason = - `session raw input reached ${formatCount(latestRawUsage.inputTokens)} tokens ` + - `(threshold ${formatCount(policy.maxRawInputTokens)})`; - } else if (policy.maxSessionAgeHours > 0 && sessionAgeHours >= policy.maxSessionAgeHours) { - reason = `session age reached ${Math.floor(sessionAgeHours)} hours`; - } + : runs[runs.length - 1] ?? runs[0]; - if (!reason || !latestRun) { - return { - rotate: false, - reason: null, - handoffMarkdown: null, - previousRunId: latestRun?.id ?? null, - }; - } - - const latestSummary = summarizeHeartbeatRunListResultJson({ - summary: latestRun?.resultSummary, - result: latestRun?.resultResult, - message: latestRun?.resultMessage, - error: latestRun?.resultError, - totalCostUsd: latestRun?.resultTotalCostUsd, - costUsd: latestRun?.resultCostUsd, - costUsdCamel: latestRun?.resultCostUsdCamel, + return evaluateSessionCompactionFromRuns({ + agent, + sessionId, + issueId, + policy, + runs, + oldestRun, + continuationSummaryBody: input.continuationSummaryBody, }); - const latestTextSummary = - readNonEmptyString(latestSummary?.summary) ?? - readNonEmptyString(latestSummary?.result) ?? - readNonEmptyString(latestSummary?.message) ?? - readNonEmptyString(latestRun.error); - - const handoffMarkdown = [ - "Paperclip session handoff:", - `- Previous session: ${sessionId}`, - issueId ? `- Issue: ${issueId}` : "", - `- Rotation reason: ${reason}`, - latestTextSummary ? `- Last run summary: ${latestTextSummary}` : "", - input.continuationSummaryBody - ? `- Issue continuation summary: ${input.continuationSummaryBody.slice(0, 1_500)}` - : "", - "Continue from the current task state. Rebuild only the minimum context you need.", - ] - .filter(Boolean) - .join("\n"); - - return { - rotate: true, - reason, - handoffMarkdown, - previousRunId: latestRun.id, - }; } async function resolveSessionBeforeForWakeup(