test(worker): run every Temporal workflow in a real time-skipping sandbox and gate replay determinism (#60)

apps/worker/package.json

@@ -10,7 +10,8 @@
     "start": "node dist/worker.js",
     "build": "tsc -p tsconfig.json",
     "check": "tsc -p tsconfig.json --noEmit",
-    "test": "vitest run"
+    "test": "vitest run",
+    "temporal:record-histories": "node --experimental-strip-types tests/workflow-histories.mts --record"
   },
   "dependencies": {
     "@space/codex-app-server": "0.2.5",
@@ -26,6 +27,7 @@
     "undici": "8.10.2"
   },
   "devDependencies": {
+    "@temporalio/testing": "1.24.0",
     "typescript": "^5.9.3",
     "vitest": "^4.1.12"
   },

apps/worker/src/workflows.ts

@@ -140,7 +140,12 @@ export async function roomAgentSupervisorWorkflow(rawInput: RoomAgentSupervisorI
   return continuingRoomAgentSupervisorWorkflow(rawInput);
 }
 
-async function legacyRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervisorInput): Promise<never> {
+/**
+ * The pre-`room-background-continuation-v1` branch. Exported so the real
+ * workflow sandbox can exercise it directly: a fresh execution always takes the
+ * patched branch, so only a historical replay used to reach this code.
+ */
+export async function legacyRoomAgentSupervisorWorkflow(rawInput: RoomAgentSupervisorInput): Promise<never> {
   const input = roomAgentSupervisorInputSchema.parse(rawInput);
   let pending = [...input.pending];
   let interrupted: Array<{ item: RoomAgentSupervisorQueueItem; reason: string }> = [];

apps/worker/tests/fixtures/replay-violation-workflows.ts

@@ -0,0 +1,23 @@
+/**
+ * Test-only workflow module for the replay gate's negative case.
+ *
+ * Same exported name as the recorded history, but the workflow now schedules an
+ * extra activity before the recorded one. Replaying a history with no such
+ * command must fail with a determinism violation, which proves the gate is
+ * wired to real replay.
+ */
+import { proxyActivities } from "@temporalio/workflow";
+
+const { recordDummyTurnStarted, recordDummyTurnCompleted, extraUnrecordedActivity } =
+  proxyActivities<{
+    recordDummyTurnStarted(input: unknown): Promise<{ workflowId: string }>;
+    recordDummyTurnCompleted(input: unknown): Promise<unknown>;
+    extraUnrecordedActivity(input: unknown): Promise<void>;
+  }>({ startToCloseTimeout: "1 minute", retry: { maximumAttempts: 3 } });
+
+export async function dummyTurnWorkflow(input: unknown): Promise<unknown> {
+  // The command the recorded history does not contain.
+  await extraUnrecordedActivity(input);
+  await recordDummyTurnStarted(input);
+  return recordDummyTurnCompleted(input);
+}

apps/worker/tests/fixtures/workflow-histories/README.md

@@ -0,0 +1,50 @@
+# How to re-record workflow histories
+
+The replay gate in `workflow-replay.test.ts` compares the workflow code in
+`apps/worker/src` against the histories in `fixtures/workflow-histories/`.
+Every workflow that ships must have at least one recorded history, and every
+recorded history must replay without a determinism violation.
+
+## Record
+
+```sh
+npm run temporal:record-histories -w @space/worker
+```
+
+This starts a time-skipping Temporal test environment, runs one instance of
+each workflow with stub activities, and writes the closed run's events to
+`tests/fixtures/workflow-histories/<workflowName>.json`. The history is read
+only after `result()` resolves: an earlier read returns the first events of a
+still-open run, which replays trivially and would hide a determinism break.
+
+## When re-recording is allowed
+
+Only together with a `patched()` change, in the same commit. Never re-record to
+make a failing replay pass: a replay failure means a running execution would
+break on the new code, and replacing the history erases exactly the evidence
+that says so.
+
+## Adding a workflow
+
+Add it to `recordHistories()` in `workflow-histories.mts` with stub activities,
+record, and confirm `workflow-replay.test.ts` replays the new file. If the
+workflow lives in its own module (as `memoryMaintenanceWorkflow` and
+`memoryMutationWorkflow` do), add the module to `bundleFor()` in
+`workflow-replay.test.ts` so the history is replayed against the bundle that
+worker actually ships.
+
+A workflow that never returns (the room supervisor parks in `condition()`) is
+closed by cancelling it before recording. `record()` then tolerates the
+cancellation rejection: the run is closed, which is all replay needs.
+
+Both supervisor branches are recorded. `legacyRoomAgentSupervisorWorkflow` is the
+pre-`room-background-continuation-v1` branch: a fresh execution always takes the
+patched branch, so only a history can pin it.
+
+## Proving the gate still bites
+
+`fixtures/replay-violation-workflows.ts` is a test-only variant that schedules an
+extra activity before the recorded one. `workflow-replay.test.ts` replays the
+recorded `dummyTurnWorkflow` history against it and expects a
+`DeterminismViolationError`. If that test ever passes without an error, the gate
+is no longer wired to real replay.

apps/worker/tests/fixtures/workflow-histories/codexAppServerTurnWorkflow.json

@@ -0,0 +1,253 @@
+[
+ {
+  "eventId": "1",
+  "eventTime": {
+   "seconds": "3053954524",
+   "nanos": 360000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED",
+  "workflowExecutionStartedEventAttributes": {
+   "workflowType": {
+    "name": "codexAppServerTurnWorkflow"
+   },
+   "taskQueue": {
+    "name": "space-turns",
+    "kind": "TASK_QUEUE_KIND_NORMAL"
+   },
+   "input": {
+    "payloads": [
+     {
+      "metadata": {
+       "encoding": "anNvbi9wbGFpbg=="
+      },
+      "data": "eyJyb29tSWQiOiJyb29tOnJlY29yZCIsInBhbmVJZCI6InBhbmU6cmVjb3JkIiwicHJvbXB0IjoicmVjb3JkIGEgaGlzdG9yeSIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InJlY29yZCBhIGhpc3RvcnkiLCJtb2RlbElkIjoiZ3B0LTUuNi1zb2wiLCJyZWFzb25pbmdFZmZvcnQiOiJoaWdoIiwiYWdlbnRTZXNzaW9uSWQiOiJzZXNzaW9uOnJlY29yZCIsImFnZW50UnVuSWQiOiJhZ2VudF9ydW46cmVjb3JkIiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnVzZXItcmVjb3JkIiwiYWdlbnRBc3Npc3RhbnRNZXNzYWdlSWQiOiJhZ2VudF9tc2c6YXNzaXN0YW50LXJlY29yZCIsInRyYWNlSWQiOiJ0cmFjZTpyZWNvcmQifQ=="
+     }
+    ]
+   },
+   "workflowExecutionTimeout": {
+    "seconds": "315360000"
+   },
+   "workflowRunTimeout": {
+    "seconds": "315360000"
+   },
+   "workflowTaskTimeout": {
+    "seconds": "10"
+   },
+   "originalExecutionRunId": "aa0f9e59-308e-4608-adab-bbea24598787",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "firstExecutionRunId": "aa0f9e59-308e-4608-adab-bbea24598787",
+   "attempt": 1,
+   "firstWorkflowTaskBackoff": {},
+   "header": {}
+  }
+ },
+ {
+  "eventId": "2",
+  "eventTime": {
+   "seconds": "3053954524",
+   "nanos": 360000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED",
+  "workflowTaskScheduledEventAttributes": {
+   "taskQueue": {
+    "name": "space-turns",
+    "kind": "TASK_QUEUE_KIND_NORMAL"
+   },
+   "startToCloseTimeout": {
+    "seconds": "10"
+   },
+   "attempt": 1
+  }
+ },
+ {
+  "eventId": "3",
+  "eventTime": {
+   "seconds": "3053954524",
+   "nanos": 360000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED",
+  "workflowTaskStartedEventAttributes": {
+   "scheduledEventId": "2",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "historySizeBytes": "634"
+  }
+ },
+ {
+  "eventId": "4",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 305000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED",
+  "workflowTaskCompletedEventAttributes": {
+   "scheduledEventId": "2",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "binaryChecksum": "@temporalio/worker@1.24.0+86d854fc950701a33c6f41ef73ce75a518c298fa916171764a1b9e18166f7477",
+   "sdkMetadata": {
+    "coreUsedFlags": [
+     1,
+     2,
+     3
+    ],
+    "sdkName": "temporal-typescript",
+    "sdkVersion": "1.24.0"
+   },
+   "meteringMetadata": {}
+  }
+ },
+ {
+  "eventId": "5",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 305000000
+  },
+  "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED",
+  "activityTaskScheduledEventAttributes": {
+   "activityId": "1",
+   "activityType": {
+    "name": "runCodexAppServerTurn"
+   },
+   "taskQueue": {
+    "name": "space-turns",
+    "kind": "TASK_QUEUE_KIND_NORMAL"
+   },
+   "header": {},
+   "input": {
+    "payloads": [
+     {
+      "metadata": {
+       "encoding": "anNvbi9wbGFpbg=="
+      },
+      "data": "eyJyb29tSWQiOiJyb29tOnJlY29yZCIsInBhbmVJZCI6InBhbmU6cmVjb3JkIiwicHJvbXB0IjoicmVjb3JkIGEgaGlzdG9yeSIsInJvb21BZ2VudE9wZXJhdG9yUmVxdWVzdCI6InJlY29yZCBhIGhpc3RvcnkiLCJtb2RlbElkIjoiZ3B0LTUuNi1zb2wiLCJyZWFzb25pbmdFZmZvcnQiOiJoaWdoIiwiYWdlbnRTZXNzaW9uSWQiOiJzZXNzaW9uOnJlY29yZCIsImFnZW50UnVuSWQiOiJhZ2VudF9ydW46cmVjb3JkIiwiYWdlbnRVc2VyTWVzc2FnZUlkIjoiYWdlbnRfbXNnOnVzZXItcmVjb3JkIiwiYWdlbnRBc3Npc3RhbnRNZXNzYWdlSWQiOiJhZ2VudF9tc2c6YXNzaXN0YW50LXJlY29yZCIsInRyYWNlSWQiOiJ0cmFjZTpyZWNvcmQifQ=="
+     }
+    ]
+   },
+   "scheduleToCloseTimeout": {
+    "seconds": "315360000"
+   },
+   "scheduleToStartTimeout": {
+    "seconds": "315360000"
+   },
+   "startToCloseTimeout": {
+    "seconds": "315360000"
+   },
+   "heartbeatTimeout": {
+    "seconds": "15"
+   },
+   "workflowTaskCompletedEventId": "3",
+   "retryPolicy": {
+    "initialInterval": {
+     "seconds": "1"
+    },
+    "backoffCoefficient": 2,
+    "maximumInterval": {
+     "seconds": "100"
+    },
+    "maximumAttempts": 5
+   }
+  }
+ },
+ {
+  "eventId": "6",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 306000000
+  },
+  "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED",
+  "activityTaskStartedEventAttributes": {
+   "scheduledEventId": "5",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "attempt": 1
+  }
+ },
+ {
+  "eventId": "7",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 310000000
+  },
+  "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED",
+  "activityTaskCompletedEventAttributes": {
+   "result": {
+    "payloads": [
+     {
+      "metadata": {
+       "encoding": "anNvbi9wbGFpbg=="
+      },
+      "data": "eyJ3b3JrZmxvd0lkIjoid2Y6cmVjb3JkOmNvZGV4LXR1cm4iLCJyb29tSWQiOiJyb29tOnJlY29yZCIsInBhbmVJZCI6InBhbmU6cmVjb3JkIiwidHJhY2VJZCI6InRyYWNlOnJlY29yZCJ9"
+     }
+    ]
+   },
+   "scheduledEventId": "5",
+   "startedEventId": "6",
+   "identity": "32544@DESKTOP-QG1LIN3"
+  }
+ },
+ {
+  "eventId": "8",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 310000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED",
+  "workflowTaskScheduledEventAttributes": {
+   "taskQueue": {
+    "name": "space-turns",
+    "kind": "TASK_QUEUE_KIND_NORMAL"
+   },
+   "startToCloseTimeout": {
+    "seconds": "10"
+   },
+   "attempt": 1
+  }
+ },
+ {
+  "eventId": "9",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 310000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED",
+  "workflowTaskStartedEventAttributes": {
+   "scheduledEventId": "8",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "historySizeBytes": "1651"
+  }
+ },
+ {
+  "eventId": "10",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 317000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED",
+  "workflowTaskCompletedEventAttributes": {
+   "scheduledEventId": "8",
+   "identity": "32544@DESKTOP-QG1LIN3",
+   "binaryChecksum": "@temporalio/worker@1.24.0+86d854fc950701a33c6f41ef73ce75a518c298fa916171764a1b9e18166f7477",
+   "sdkMetadata": {},
+   "meteringMetadata": {}
+  }
+ },
+ {
+  "eventId": "11",
+  "eventTime": {
+   "seconds": "3053954525",
+   "nanos": 317000000
+  },
+  "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED",
+  "workflowExecutionCompletedEventAttributes": {
+   "result": {
+    "payloads": [
+     {
+      "metadata": {
+       "encoding": "anNvbi9wbGFpbg=="
+      },
+      "data": "eyJ3b3JrZmxvd0lkIjoid2Y6cmVjb3JkOmNvZGV4LXR1cm4iLCJyb29tSWQiOiJyb29tOnJlY29yZCIsInBhbmVJZCI6InBhbmU6cmVjb3JkIiwidHJhY2VJZCI6InRyYWNlOnJlY29yZCJ9"
+     }
+    ]
+   },
+   "workflowTaskCompletedEventId": "9"
+  }
+ }
+]