fix(heartbeat): rotate to fresh session on autocompact thrash (#29)

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -6,6 +6,7 @@ import {
   activityLog,
   agents,
   agentRuntimeState,
+  agentTaskSessions,
   agentWakeupRequests,
   budgetPolicies,
   companySkills,
@@ -321,6 +322,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     await db.delete(agentRuntimeState);
     await db.delete(companySkills);
     await db.delete(costEvents);
+    await db.delete(agentTaskSessions);
     await db.delete(environmentLeases);
     await db.delete(environments);
     await db.delete(issueWorkProducts);
@@ -2950,4 +2952,210 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => {
     const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
     expect(runs).toHaveLength(1);
   });
-});
+
+  it("clears agent error status to idle when subprocess exits 0 even if run was pre-terminated as failed by recovery", async () => {
+    // Scenario: process-loss recovery marks a run "failed" before the subprocess
+    // exits. The subprocess then exits cleanly (exit code 0). Without the fix,
+    // the latestRun.status === "failed" check causes outcome="failed" and the
+    // agent stays stuck in "error". With the fix, subprocessExitCode=0 overrides
+    // the transition to "idle".
+    const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
+
+    // Put the agent into "error" to simulate a previous failure.
+    await db
+      .update(agents)
+      .set({ status: "error", updatedAt: new Date() })
+      .where(eq(agents.id, agentId));
+
+    // Set the issue to "in_review" so that releaseIssueExecutionAndPromote does not
+    // enqueue a recovery run when the simulated run is marked failed. If the issue
+    // stays "in_progress", the recovery loop starts a new run immediately, putting
+    // the agent into "running" before finalizeAgentStatus fires — which prevents the
+    // exit-code-0 override from being reached.
+    await db
+      .update(issues)
+      .set({ status: "in_review", updatedAt: new Date() })
+      .where(eq(issues.id, issueId));
+
+    // Mock: simulate process-loss recovery pre-terminating the run as "failed"
+    // while the subprocess is still executing, then returning exit code 0.
+    mockAdapterExecute.mockImplementationOnce(async (ctx: { runId: string }) => {
+      await db
+        .update(heartbeatRuns)
+        .set({
+          status: "failed",
+          error: "process_lost: pid not alive (simulated recovery)",
+          errorCode: "process_lost",
+          finishedAt: new Date(),
+          updatedAt: new Date(),
+        })
+        .where(eq(heartbeatRuns.id, ctx.runId));
+      return {
+        exitCode: 0,
+        signal: null,
+        timedOut: false,
+        errorMessage: null,
+        summary: "Subprocess exited cleanly despite run pre-termination.",
+        provider: "test",
+        model: "test-model",
+      };
+    });
+
+    const heartbeat = heartbeatService(db);
+    await heartbeat.resumeQueuedRuns();
+    await waitForRunToSettle(heartbeat, runId, 5_000);
+    await waitForHeartbeatIdle(db, 5_000);
+
+    // Wait for finalizeAgentStatus to fire after the run settles. executeRun sets
+    // agent.status = "running" before calling the adapter; finalizeAgentStatus
+    // resets it afterward. waitForHeartbeatIdle returns as soon as all runs are
+    // terminal, which can race with finalizeAgentStatus on the agent row.
+    const agent = await waitForValue(
+      () =>
+        db
+          .select({ status: agents.status })
+          .from(agents)
+          .where(eq(agents.id, agentId))
+          .then((rows) => {
+            const row = rows[0];
+            return row && row.status === "idle" ? row : null;
+          }),
+      5_000,
+    );
+
+    expect(agent?.status).toBe("idle");
+  });
+
+  it("rotates to a fresh session and avoids a second wasted compaction after autocompact thrash", async () => {
+    const sessionId = "session-thrash-" + randomUUID();
+    const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture();
+
+    // Pre-seed the agent task session so the first run resumes from a known session.
+    await db.insert(agentTaskSessions).values({
+      companyId,
+      agentId,
+      adapterType: "codex_local",
+      taskKey: issueId,
+      sessionDisplayId: sessionId,
+      sessionParamsJson: { sessionId },
+      lastError: null,
+    });
+
+    // Remove the issue_assigned wakeReason so the seeded task session is not reset on wake.
+    await db
+      .update(heartbeatRuns)
+      .set({
+        contextSnapshot: {
+          issueId,
+          taskId: issueId,
+        },
+      })
+      .where(eq(heartbeatRuns.id, runId));
+
+    // First adapter invocation: simulate an autocompact-thrash failure with a cost event.
+    mockAdapterExecute.mockResolvedValueOnce({
+      exitCode: 1,
+      signal: null,
+      timedOut: false,
+      errorCode: "adapter_failed",
+      errorMessage: "Autocompact is thrashing after 3 attempts",
+      provider: "test",
+      model: "test-model",
+      usage: { inputTokens: 100_000, outputTokens: 10_000, cachedInputTokens: 0 },
+      costUsd: 0.5,
+    });
+
+    // Second adapter invocation (the recovery run) should succeed with a fresh session.
+    mockAdapterExecute.mockResolvedValueOnce({
+      exitCode: 0,
+      signal: null,
+      timedOut: false,
+      errorMessage: null,
+      summary: "Recovered after session rotation.",
+      provider: "test",
+      model: "test-model",
+      usage: { inputTokens: 1_000, outputTokens: 500, cachedInputTokens: 0 },
+      costUsd: 0.01,
+    });
+
+    const heartbeat = heartbeatService(db);
+    await heartbeat.resumeQueuedRuns();
+    await waitForRunToSettle(heartbeat, runId, 5_000);
+    await waitForHeartbeatIdle(db, 5_000);
+
+    const runs = await db
+      .select()
+      .from(heartbeatRuns)
+      .where(eq(heartbeatRuns.agentId, agentId))
+      .orderBy(heartbeatRuns.createdAt);
+    expect(runs.length).toBeGreaterThanOrEqual(2);
+
+    const thrashedRun = runs.find((r) => r.id === runId);
+    const recoveryRun = runs.find((r) => r.retryOfRunId === runId);
+    if (!thrashedRun) throw new Error("Expected original thrashed run to exist");
+    if (!recoveryRun) throw new Error("Expected recovery run to exist");
+
+    expect(thrashedRun.status).toBe("failed");
+    expect(thrashedRun.errorCode).toBe("adapter_failed");
+    expect((thrashedRun.resultJson as Record<string, unknown> | null)?.stopReason).toBe("autocompact_thrash");
+    expect(thrashedRun.sessionIdAfter).toBe(sessionId);
+
+    // Locate the adapter calls that correspond to the thrashed and recovery runs.
+    // The exact number of calls can include an optional successful-run handoff; what matters
+    // is that the thrashed run used the poisoned session and the recovery run used a fresh one.
+    const adapterCalls = mockAdapterExecute.mock.calls;
+    const thrashedCallIndex = adapterCalls.findIndex(
+      (c) =>
+        ((c[0] as { runtime?: { sessionId?: string | null } } | undefined)?.runtime?.sessionId ?? null) ===
+        sessionId,
+    );
+    const recoveryCallIndex = adapterCalls.findIndex(
+      (c, idx) =>
+        idx > thrashedCallIndex &&
+        ((c[0] as { runtime?: { sessionId?: string | null } } | undefined)?.runtime?.sessionId ?? null) === null,
+    );
+    const thrashedCall =
+      thrashedCallIndex >= 0
+        ? (adapterCalls[thrashedCallIndex][0] as { runtime?: { sessionId: string | null } } | undefined)
+        : undefined;
+    const recoveryCall =
+      recoveryCallIndex >= 0
+        ? (adapterCalls[recoveryCallIndex][0] as { runtime?: { sessionId: string | null } } | undefined)
+        : undefined;
+
+    expect(thrashedCall?.runtime?.sessionId).toBe(sessionId);
+    expect(recoveryCall?.runtime?.sessionId).toBeNull();
+
+    // The recovery run must resume with a fresh session ID, not the poisoned thrashed session.
+    expect(recoveryRun.sessionIdBefore).not.toBe(sessionId);
+
+    // No second wasted compaction attempt should be recorded: only one run failed with the
+    // autocompact_thrash stop reason, and the recovery run succeeds without another
+    // adapter_failed / autocompact_thrash outcome.
+    const autocompactThrashRuns = runs.filter(
+      (r) =>
+        (r.resultJson as Record<string, unknown> | null)?.stopReason === "autocompact_thrash" ||
+        r.errorCode === "adapter_failed" ||
+        String(r.error ?? "").toLowerCase().includes("autocompact"),
+    );
+    expect(autocompactThrashRuns).toHaveLength(1);
+
+    const costEventRows = await db
+      .select()
+      .from(costEvents)
+      .where(eq(costEvents.agentId, agentId))
+      .orderBy(costEvents.occurredAt);
+
+    const thrashCostEvent = costEventRows.find((e) => e.heartbeatRunId === thrashedRun.id);
+    expect(thrashCostEvent).toBeTruthy();
+
+    // No duplicate cost event should be associated with the thrashed run.
+    expect(costEventRows.filter((e) => e.heartbeatRunId === thrashedRun.id)).toHaveLength(1);
+
+    // The recovery run's context snapshot must carry the rotation signal and reason.
+    const recoveryContext = recoveryRun.contextSnapshot as Record<string, unknown> | null;
+    expect(recoveryContext?.paperclipSessionRotationReason).toContain("autocompact");
+    expect(recoveryContext?.paperclipPreviousSessionId).toBe(sessionId);
+    expect(recoveryRun.status).toBe("succeeded");
+  });
+});

server/src/services/heartbeat-stop-metadata.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "vitest";
 import {
   buildHeartbeatRunStopMetadata,
+  isAutocompactThrashError,
   mergeHeartbeatRunStopMetadata,
   resolveHeartbeatRunTimeoutPolicy,
 } from "./heartbeat-stop-metadata.js";
@@ -98,6 +99,49 @@ describe("heartbeat stop metadata", () => {
     ).toBe("completed");
   });
 
+  it("classifies autocompact-thrash error strings as autocompact_thrash", () => {
+    for (const message of ["Autocompact is thrashing", "Autocompact is thrashing after 3 attempts"]) {
+      expect(isAutocompactThrashError(message)).toBe(true);
+      expect(
+        buildHeartbeatRunStopMetadata({
+          adapterType: "claude_local",
+          adapterConfig: {},
+          outcome: "failed",
+          errorCode: "adapter_failed",
+          errorMessage: message,
+        }).stopReason,
+      ).toBe("autocompact_thrash");
+    }
+
+    for (const message of ["context refilled to the limit", "ERROR: context refilled to the limit after compaction"]) {
+      expect(isAutocompactThrashError(message)).toBe(true);
+      expect(
+        buildHeartbeatRunStopMetadata({
+          adapterType: "claude_local",
+          adapterConfig: {},
+          outcome: "failed",
+          errorCode: "adapter_failed",
+          errorMessage: message,
+        }).stopReason,
+      ).toBe("autocompact_thrash");
+    }
+  });
+
+  it("does not classify non-thrash failures as autocompact_thrash", () => {
+    for (const message of [null, undefined, "", "process exited with code 1", "Unknown error from adapter"]) {
+      expect(isAutocompactThrashError(message)).toBe(false);
+      expect(
+        buildHeartbeatRunStopMetadata({
+          adapterType: "claude_local",
+          adapterConfig: {},
+          outcome: "failed",
+          errorCode: "adapter_failed",
+          errorMessage: message,
+        }).stopReason,
+      ).toBe("adapter_failed");
+    }
+  });
+
   it("preserves existing result fields when merging stop metadata", () => {
     const result = mergeHeartbeatRunStopMetadata(
       { summary: "done" },

server/src/services/heartbeat-stop-metadata.ts

@@ -8,6 +8,7 @@ export type HeartbeatRunStopReason =
   | "paused"
   | "max_turns_exhausted"
   | "process_lost"
+  | "autocompact_thrash"
   | "adapter_failed";
 
 export interface HeartbeatRunTimeoutPolicy {
@@ -77,6 +78,13 @@ export function resolveHeartbeatRunTimeoutPolicy(
   };
 }
 
+/** Detects autocompact-thrash failures by the known error strings emitted by the adapter. */
+export function isAutocompactThrashError(errorMessage: string | null | undefined): boolean {
+  if (!errorMessage) return false;
+  const lower = errorMessage.toLowerCase();
+  return lower.includes("autocompact is thrashing") || lower.includes("context refilled to the limit");
+}
+
 export function inferHeartbeatRunStopReason(input: {
   outcome: HeartbeatRunOutcome;
   errorCode?: string | null;
@@ -87,6 +95,9 @@ export function inferHeartbeatRunStopReason(input: {
   if (maxTurnStopReason) return maxTurnStopReason;
   if (input.outcome === "timed_out") return "timeout";
   if (input.outcome === "failed" && input.errorCode === "process_lost") return "process_lost";
+  if (input.outcome === "failed" && isAutocompactThrashError(input.errorMessage)) {
+    return "autocompact_thrash";
+  }
   if (input.outcome === "cancelled") {
     const message = (input.errorMessage ?? "").toLowerCase();
     if (message.includes("budget")) return "budget_paused";