test(worker): exercise every workflow in a real Temporal sandbox and gate replay determinism

CONTRIBUTING.md

@@ -106,6 +106,37 @@ Keep changes reviewable and include the exact commands and limitations of your
 verification. Never include credentials, private transcripts or local runtime
 state in a contribution.
 
+## Workflow sandbox and replay histories
+
+`apps/worker/tests/workflow-sandbox.test.ts` runs every workflow in `apps/worker/src`
+inside a real `TestWorkflowEnvironment.createTimeSkipping()` worker with stubbed
+activities: no database, no Docker, no provider. The mocked suites next to it keep
+their unit-level coverage; the sandbox suite is what exercises real timers, signal
+ordering and `continueAsNew`.
+
+`apps/worker/tests/workflow-replay.test.ts` replays the JSON histories in
+`apps/worker/tests/fixtures/workflow-histories/` against the current workflow code,
+so a change that is not replay-compatible fails in CI instead of in a user's
+installation. `apps/worker/tests/workflow-replay-gate.test.ts` proves the gate itself
+still fires: it replays one history against unchanged code and against a copy that
+adds a timer, and requires the second to fail with a determinism violation.
+
+### Re-recording a history
+
+```sh
+npm run temporal:record-histories -w @space/worker
+```
+
+The script runs one canonical execution per workflow in the time-skipping sandbox and
+overwrites the fixtures. Treat the output as test data, not as a snapshot to refresh
+when a test is red.
+
+**Re-record only together with a `patched()` change that keeps running executions
+valid.** Never re-record to make a failing replay pass: a failing replay means a
+deployed execution would break when the worker picks it up. If a workflow change is
+intentionally replay-breaking, wrap it in `patched()`, add the fixture that covers the
+pre-patch branch, and say so in the pull request.
+
 ## License of contributions
 
 SpaceApp is licensed under the [MIT License](LICENSE). Contributions are

apps/worker/package.json

@@ -10,7 +10,8 @@
     "start": "node dist/worker.js",
     "build": "tsc -p tsconfig.json",
     "check": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run"
+    "test": "npm run build -w @space/contracts && vitest run",
+    "temporal:record-histories": "node scripts/record-workflow-histories.mjs"
   },
   "dependencies": {
     "@space/codex-app-server": "0.2.5",
@@ -26,6 +27,7 @@
     "undici": "8.10.2"
   },
   "devDependencies": {
+    "@temporalio/testing": "1.24.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.12"
   },

apps/worker/scripts/record-workflow-histories.mjs

@@ -0,0 +1,165 @@
+#!/usr/bin/env node
+/**
+ * Records real Temporal histories for every workflow in apps/worker into
+ * tests/fixtures/workflow-histories/ so `workflow-replay.test.ts` can replay them
+ * against the current workflow code in CI.
+ *
+ * Run with `npm run temporal:record-histories -w @space/worker`.
+ *
+ * Re-record only together with a `patched()` change. Never re-record to make a
+ * failing replay pass: a failing replay means a running execution would break.
+ *
+ * The scenarios here are intentionally minimal — one canonical run per workflow,
+ * plus the supervisor's continue-as-new handover. They exist to produce a replayable
+ * history, not to assert behaviour; behaviour lives in workflow-sandbox.test.ts.
+ */
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { TestWorkflowEnvironment } from "@temporalio/testing";
+import { Worker } from "@temporalio/worker";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const fixtures = join(here, "..", "tests", "fixtures", "workflow-histories");
+
+const MAIN = join(here, "..", "src", "workflows.ts");
+const MAINTENANCE = join(here, "..", "src", "memory-maintenance-workflows.ts");
+const MUTATION = join(here, "..", "src", "memory-mutation-workflows.ts");
+
+const DEFAULT_TASK_QUEUE = "space-agent-turns";
+const MEMORY_MAINTENANCE_TASK_QUEUE = "space-memory-maintenance";
+const MEMORY_MUTATION_TASK_QUEUE = "space-memory-mutations";
+
+const verifiedTurn = () => ({
+  status: "COMPLETED",
+  message: "ok",
+  roomAgentOutcome: { status: "VERIFIED", executedActionCount: 0, statusReason: "Verified." }
+});
+
+const activities = {
+  recordDummyTurnStarted: async () => undefined,
+  recordDummyTurnCompleted: async () => ({ status: "COMPLETED", message: "ok" }),
+  runCodexAppServerTurn: async () => verifiedTurn(),
+  runOpenCodeAgentTurn: async () => verifiedTurn(),
+  runCliAgentTurn: async () => verifiedTurn(),
+  runRoomAgentBackgroundAction: async () => ({ status: "COMPLETED" }),
+  markRoomAgentMissionStarted: async () => undefined,
+  markRoomAgentMissionFinished: async () => undefined,
+  markRoomAgentMissionContinued: async () => undefined,
+  settleRoomAgentBackgroundMission: async () => true,
+  checkRoomAgentSupervisorTurn: async () => "PROCEED",
+  discardRoomAgentSupervisorTurn: async () => undefined,
+  finishRoomAgentSupervisorTurn: async () => undefined,
+  prepareRoomAgentBackgroundContinuation: async () => ({ state: "DONE" }),
+  failRoomAgentBackgroundAction: async () => undefined,
+  dotGoalStep: async () => ({ state: "DONE", runId: null, goalStatus: "COMPLETE" }),
+  dotGoalObserve: async () => ({ run: "SUCCEEDED", goalStatus: "COMPLETE" }),
+  refreshMemoryGraphSnapshot: async (input) => ({
+    status: "REFRESHED", generatedAt: input.scheduledAt, sourceHash: "hash-1",
+    previousSourceHash: null, summary: {}, durationMs: 1
+  }),
+  refreshAllMonthsMemoryGraphSnapshot: async () => undefined,
+  reconcileStaleSpaceAgentSessions: async () => undefined,
+  createScheduledMemoryAudit: async () => ({ id: "run-recorded", workflowId: "space-memory-maintenance:recorded" }),
+  prepareMemoryConsolidation: async () => ({ changeSetId: "cs-recorded", run: { id: "run-recorded", status: "PENDING" } }),
+  finalizeMemoryRepair: async (input) => ({ id: input.runId, status: "REPAIRED" }),
+  executePersistedMemoryChangeSet: async () => ({ status: "APPLIED" })
+};
+
+const turn = { roomId: "room-recorder", paneId: "pane-recorder", prompt: "Record a history", artifactIds: [] };
+
+const scenario = [
+  { name: "main-dummy-turn", bundle: MAIN, queue: DEFAULT_TASK_QUEUE, type: "dummyTurnWorkflow", args: [turn] },
+  { name: "main-codex-turn", bundle: MAIN, queue: DEFAULT_TASK_QUEUE, type: "codexAppServerTurnWorkflow", args: [turn] },
+  {
+    name: "main-room-action", bundle: MAIN, queue: DEFAULT_TASK_QUEUE, type: "roomAgentActionWorkflow",
+    workflowId: "space-room-action:recorded", args: [{ bridge: { roomId: "room-recorder", missionId: "mission-recorder" } }]
+  },
+  { name: "main-dot-goal", bundle: MAIN, queue: DEFAULT_TASK_QUEUE, type: "dotGoalWorkflow", args: [{ goalId: "goal-recorder" }] },
+  {
+    name: "main-supervisor-first-run", bundle: MAIN, queue: DEFAULT_TASK_QUEUE, type: "roomAgentSupervisorWorkflow",
+    args: [{
+      roomId: "room-recorder", monitoringMissionIds: [], interrupted: [], processedCount: 49,
+      pending: [{ missionId: "mission-recorder", turn: { ...turn, agentRunId: "agent-run-recorder", traceId: "trace-recorder-1" } }]
+    }],
+    recordFirstRun: true
+  },
+  {
+    name: "maintenance-run", bundle: MAINTENANCE, queue: MEMORY_MAINTENANCE_TASK_QUEUE, type: "memoryMaintenanceWorkflow",
+    args: [{ scheduledAt: "2026-10-25T00:00:00.000Z", traceId: "trace-recorder" }]
+  },
+  {
+    name: "maintenance-consolidation", bundle: MAINTENANCE, queue: MEMORY_MAINTENANCE_TASK_QUEUE, type: "memoryConsolidationWorkflow",
+    args: [{ runId: "run-recorder", traceId: "trace-recorder" }]
+  },
+  {
+    name: "mutation-run", bundle: MUTATION, queue: MEMORY_MUTATION_TASK_QUEUE, type: "memoryMutationWorkflow",
+    args: [{ changeSetId: "cs-recorder", traceId: "trace-recorder" }]
+  }
+];
+
+const namespace = "default";
+
+/**
+ * Waits until the workflow has handed over to a new run, then returns the id of the
+ * run that was superseded. Waiting on the run id instead of sleeping a fixed interval
+ * keeps the recorded history shape stable across machines.
+ */
+async function waitForContinuation(env, workflowId, firstRunId, name, timeoutMs = 120_000) {
+  const latest = env.client.workflow.getHandle(workflowId);
+  const deadline = Date.now() + timeoutMs;
+  for (;;) {
+    const description = await latest.describe();
+    if (description.runId !== firstRunId) return;
+    if (Date.now() > deadline) throw new Error(`${name}: the run never continued as new`);
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+}
+
+async function main() {
+  const env = await TestWorkflowEnvironment.createTimeSkipping();
+  const workers = await Promise.all(
+    [...new Set(scenario.map((entry) => `${entry.bundle}|${entry.queue}`))].map((key) => {
+      const [bundle, queue] = key.split("|");
+      return Worker.create({ connection: env.nativeConnection, taskQueue: queue, workflowsPath: bundle, activities });
+    })
+  );
+  const runs = workers.map((worker) => worker.run());
+
+  await mkdir(fixtures, { recursive: true });
+
+  try {
+    for (const entry of scenario) {
+      const workflowId = entry.workflowId ?? `${entry.name}-recorder`;
+      const handle = await env.client.workflow.start(entry.type, {
+        taskQueue: entry.queue, workflowId, args: entry.args
+      });
+
+      // A workflow that never returns is recorded from its first run: that is the run
+      // holding the activities, the child workflow and the handover, not the idle run
+      // that continueAsNew leaves behind.
+      const recordedRunId = handle.firstExecutionRunId;
+      if (entry.recordFirstRun) await waitForContinuation(env, workflowId, recordedRunId, entry.name);
+      else await handle.result();
+
+      const response = await env.client.workflowService.getWorkflowExecutionHistory({
+        namespace,
+        execution: { workflowId, runId: recordedRunId }
+      });
+      const events = response.history?.events?.length ?? 0;
+      await writeFile(join(fixtures, `${entry.name}.json`), `${JSON.stringify(response.history, null, 2)}\n`, "utf8");
+      console.log(`recorded ${entry.name}.json (${events} events, run ${recordedRunId.slice(0, 8)})`);
+
+      if (entry.recordFirstRun) await handle.terminate("recorded for the replay gate");
+    }
+  } finally {
+    for (const worker of workers) worker.shutdown();
+    await Promise.allSettled(runs);
+    await env.teardown();
+  }
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});

apps/worker/tests/fixtures/replay/clean-workflows.ts

@@ -0,0 +1,21 @@
+import { proxyActivities, sleep } from "@temporalio/workflow";
+
+/**
+ * Reference workflow for the replay gate: a fixed number of activities separated by
+ * timers. Its recorded history must replay against this file unchanged.
+ *
+ * The sibling `drifted-workflows.ts` deliberately differs by one extra timer; the
+ * replay test asserts that the drift is caught as a determinism violation.
+ */
+const { recordReplayProbeStep } = proxyActivities<{
+  recordReplayProbeStep(input: { step: number }): Promise<void>;
+}>({ startToCloseTimeout: "1 minute" });
+
+export async function replayProbeWorkflow(): Promise<string> {
+  await recordReplayProbeStep({ step: 1 });
+  await sleep("1 minute");
+  await recordReplayProbeStep({ step: 2 });
+  await sleep("1 minute");
+  await recordReplayProbeStep({ step: 3 });
+  return "done";
+}

apps/worker/tests/fixtures/replay/drifted-workflows.ts

@@ -0,0 +1,22 @@
+import { proxyActivities, sleep } from "@temporalio/workflow";
+
+/**
+ * The same workflow as `clean-workflows.ts` with one extra timer at the start.
+ *
+ * It exists only so the replay gate can prove it still catches drift: replaying a
+ * history recorded by the clean file against this file must fail with
+ * `DeterminismViolationError`. Nothing in production references this file.
+ */
+const { recordReplayProbeStep } = proxyActivities<{
+  recordReplayProbeStep(input: { step: number }): Promise<void>;
+}>({ startToCloseTimeout: "1 minute" });
+
+export async function replayProbeWorkflow(): Promise<string> {
+  await sleep("1 minute"); // <- the drift: a timer the recorded history never had
+  await recordReplayProbeStep({ step: 1 });
+  await sleep("1 minute");
+  await recordReplayProbeStep({ step: 2 });
+  await sleep("1 minute");
+  await recordReplayProbeStep({ step: 3 });
+  return "done";
+}