diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 610b9fb18..fd6f041dc 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,13 +6,13 @@ }, "metadata": { "description": "CBEPX fork of the OpenAI Codex plugin for Claude Code: max/ultra effort, per-thread config overrides, gpt-5.6 aliases, rescue agent fixes.", - "version": "1.1.1" + "version": "1.2.0" }, "plugins": [ { "name": "codex", "description": "Use Codex from Claude Code to review code or delegate tasks.", - "version": "1.1.1", + "version": "1.2.0", "author": { "name": "OpenAI" }, diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml new file mode 100644 index 000000000..90f06a82a --- /dev/null +++ b/.github/workflows/release-verify.yml @@ -0,0 +1,66 @@ +name: Verify release + +# Mirrors the release gate of CBEPX/cc-plugin-codex (publish.yml) minus the npm +# publish step: this plugin is installed from the repository through +# .claude-plugin/marketplace.json, so a release only needs to be verified. + +on: + release: + types: + - published + workflow_dispatch: + inputs: + ref: + description: Git ref to verify + required: false + default: main + +permissions: + contents: read + +jobs: + verify: + name: Verify + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Check out release ref + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.release.tag_name || inputs.ref || github.ref_name }} + + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 22 + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Install Codex CLI + run: npm install -g @openai/codex + + - name: Codex version + run: codex --version + + - name: Version metadata matches the tag + run: npm run check-version + + - name: Run test suite + run: npm test + + - name: No leaked test processes + run: | + sleep 10 + if pgrep -f codex-plugin-test- ; then echo "leaked test processes" >&2; exit 1; fi + + - name: Run build + run: npm run build + + - name: Runtime dependency audit + run: npm audit --omit=dev + + - name: Pack (dry run) + run: npm pack --dry-run diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab061eee..8e1756398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,53 @@ # Changelog +## 1.2.0 — 2026-08-28 + +### Merged from upstream pull requests +- #355 `SessionEnd` now terminates only the ending session's own foreground jobs; a workspace with any active job keeps its shared broker alive across `SessionEnd` instead of tearing it down out from under the worker, and the hook only clears its own broker-session record when the endpoint still matches (a replacement broker's record survives). +- #425 a PID-liveness reaper marks a `queued`/`running` job `failed` ("worker exited before completing") once its worker process is confirmed dead (`kill(pid, 0)` reports `ESRCH`), deletes the job's private one-shot request payload, and clears `requestFile`; a `queued` job with no recorded PID yet gets a 30 s grace window before it is reaped, covering a worker that died between spawn and the PID being recorded. + +### Fork changes +- `task --await [--await-timeout-ms ]` launches the same tracked job record as `--background`, then waits for it: exit 0 when the job completed, 1 when it failed or was cancelled, 3 when the default 540000 ms wait elapses while the job is still queued or running (prints a `Re-run: node "" result --wait --timeout-ms 540000` hint). +- `--prompt-stdin` takes the prompt as raw stdin, stripping exactly one trailing newline and nothing else; the mutual exclusion against `--args-stdin`, `--prompt-file`, and a positional prompt is checked on the raw argv before any stdin is read, so a bad combination fails fast instead of blocking on stdin. +- `result [--wait [--timeout-ms ]]` exits 0 for any terminal job record — completed, failed, or cancelled alike, since this is retrieval, not a pass/fail signal — and exits 3 with the same resumable hint when the job is still active and `--wait` was not given; `--json` returns `{ job, storedJob }`. +- Fixed upstream #498/#524: `result ` on a still-`queued`/`running` job used to throw "No job found for ``"; it now reports the job's real status plus the `result … --wait` hint, exit 3. +- `--await-timeout-ms` and `result --timeout-ms` are validated as finite positive integers (rejects `0`, negatives, fractional, and non-numeric values). +- Worker-survival caveat: the detached worker outlives the companion only when the companion returns on its own via exit 3 — a host process-tree kill (e.g. Claude Code's Bash tool timeout) kills the worker too, so keep `--await-timeout-ms` below the host's own timeout. +- The worker's PID is now recorded right after spawn (`updateJobPid`), so a `cancel` issued while a job is still `queued` has a process to signal; `cancel` also releases the job's private one-shot request payload itself, since a cancelled job is terminal and the reaper never revisits it. The parent never rewrites the job JSON after the spawn — that read-modify-write could put a queued snapshot over a record the worker had already completed, losing its `result`, `threadId` and `turnId` while the state index already said terminal. The PID goes into an atomic `jobs/.pid` sidecar (write + rename) plus the existing pid-only index patch; readers (`cancel`, the reaper, `SessionEnd` cleanup) take `job.pid ?? sidecar` and never read a sidecar for a terminal job, and the sidecar is deleted with the job's terminal write, reap, cancel or prune. +- `SessionEnd` now reaps dead background workers before deciding whether an active background job should keep the broker alive, so a hard-killed (e.g. OOM/`SIGKILL`) worker can no longer pin a broker and its job's private payload alive indefinitely. +- `/codex:rescue` and the `codex-rescue` agent are now a single `node … task --await --prompt-stdin <<'CODEX_PROMPT_'` call each, replacing the old two-Bash-call `mktemp`/`trap`/`status --wait`-poll-loop dance; the per-call random delimiter suffix now only has one heredoc to protect (the request prose) since flags travel on the command line and there is no second `--args-stdin` heredoc anymore; on exit 3 the only allowed follow-up is re-running the printed `Re-run:` hint for that same job. +- The resume decision (`task-resume-candidate --json` + one `AskUserQuestion`) is now made once, before the synchronous/`--background` split, so both paths get the same explicit `--resume-last`/`--fresh` flag; a `--background` handoff never resumes silently — the agent runs fresh unless it is handed an explicit `--resume-last` from that shared decision. +- `rescue.md`'s `allowed-tools` is back to `Bash(node:*), AskUserQuestion, Agent` (widened to bare `Bash` in 1.1.0 only to support the removed two-Bash-call flow). +- `--turn-timeout-ms ` (or `CODEX_TURN_TIMEOUT_MS`) on `task`, `review`, and `adversarial-review` bounds a single Codex turn: on expiry it sends `turn/interrupt` and returns a structured failed result ("turn timed out after `` ms") instead of hanging or throwing; default is `0` (unbounded, unchanged behavior); the budget is persisted into `--background`/`--await` job requests so detached workers run under the same limit; the timeout now waits up to 10 s for the turn's terminal notification after `turn/interrupt` instead of declaring the turn dead the moment the RPC answers, and an unacknowledged interrupt is reported as such ("interrupt not acknowledged — the turn may still be running in the shared runtime, check status or cancel") — on a non-broker transport the app-server it owns is closed so the runaway turn dies with the process. +- Documented: with `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0`, a broker kept alive by an active background job across `SessionEnd` never exits on its own — the idle self-terminate safety net from #457 is disabled in that configuration. +- Known limitations: `kill(pid, 0)` reads a zombie process as alive and cannot detect PID reuse, so the reaper can occasionally misjudge a dead worker's liveness (documented in `lib/tracked-jobs.mjs`); a `turn/start` call that never answers is still unbounded — `--turn-timeout-ms` only covers the window after `turn/start` resolves. +- `task --await` jobs are recorded as background jobs (they survive SessionEnd and keep the broker while active, bounded by the reaper and the idle timeout). +- The legacy-request move is limited to `queued` records, and every terminal write releases the job's private payload. A worker consumes its request *before* `runTrackedJob` flips the record to `running`, so staging a payload for a running legacy job would have written plaintext `--config` values that nothing reads and nothing deletes; `runTrackedJob` now drops the payload file alongside the PID sidecar on both its terminal paths, so a payload the worker never consumed cannot outlive the job either. +- Migrating a 1.1.1 record moves its raw request into a fresh 0600 `jobs/.request.json` before the record loses it, for jobs that are still `queued`. 1.1.1 wrote no private payload, so the record was the only copy — and it is exactly what a worker falls back to, which would have started Codex with `"[redacted]"` in place of real `--config` values (auth headers included). No record is left unredacted on disk any more. +- Records written by 1.1.1 are redacted too: `--config` values are stripped at the read boundary every `status`/`result` output crosses (`loadState` and the job-file reader), and the same read rewrites the index and the job file once so the values stop living in long-lived state. Redaction now has a single implementation, `redactConfigValues` in `lib/state.mjs`. +- A tracked job record never stores `--config` **values** any more: it keeps the keys and writes every value as `[redacted]`, because classifying secrets by key name (`/key|token|secret|auth|password/i`) missed real credentials such as a `http_headers.Cookie` override, which then reached `state.json`, the job file, `status --json` and `result --json`. The real values still reach Codex — they live only in the private 0600 `jobs/.request.json` the worker consumes. +- `SessionEnd` no longer shuts the shared broker down while **any** job in the workspace is still `queued`/`running` — the check used to count only jobs flagged `background: true`, so another Claude session's foreground run (which survives this session's own-jobs-only cleanup) had its runtime pulled out from under it. The check runs on the state left behind by that cleanup and, as before, reaps dead workers first so a killed worker cannot pin the broker. +- `state.json`, job files and the private request payload are now written atomically (sibling temp file + `rename`). A reader that caught a plain `writeFileSync` mid-flight parsed a truncated — often zero-byte — file, and since `loadState` answers a parse failure with an empty job list, that read looked exactly like an idle workspace: a `SessionEnd` landing in that window could shut the shared broker down under a live job (and the reaper/`cancel` paths could miss active jobs the same way). +- The reaper now reconciles a terminal job file back into the state index instead of returning it to the current caller only: a worker that died between its terminal `writeJobFile` and its `upsertJob` left an active index entry that `assertThreadIsFree` (which now also reads the reaped list) saw as a phantom running job, blocking every later resume of that thread. +- `close()` on a direct app-server is idempotent as well as bounded: the deadline used to apply to the first call only, and every later call awaited process exit with no bound at all. The turn timeout closes twice by design (once to kill the runaway turn, once on the way out of the run), so the single case the deadline exists for was the one that still hung. +- A bounded turn now always ends in a terminal job record. The acknowledgement window after `turn/interrupt` is a referenced timer raced against the transport's own exit, so an app-server that answers the interrupt and dies can no longer let the companion exit with the job still `running` (and no longer costs the full 10 s wait), and closing a directly owned app-server escalates stdin EOF → `SIGTERM` → `SIGKILL` with a hard 5 s deadline instead of awaiting process exit forever. +- Documented: partial output on a timed-out turn is best-effort — only items Codex had already completed are kept, so a turn interrupted mid-message reports less text than Codex produced. +- The reaper reads the authoritative job file **before** it looks at PID liveness, for every active index entry: `kill(pid, 0)` reads a zombie as alive and cannot see a recycled PID, so a job whose file was already terminal could keep a phantom `running` index entry — blocking resume on its thread and keeping `SessionEnd` from ever releasing the broker — for as long as anything held that PID. Liveness is now only consulted for jobs whose own file still says they are active. +- `review`/`adversarial-review` now persist `--background` on the job record, so a review dispatched to outlive its session (`nohup … --background &`) keeps its record — and with it `/codex:status` and `/codex:result` — after that session ends. +- Shutdown-if-idle counts every connected client, not only those that have already sent a message: a client is a client from the moment it is accepted, and `CodexAppServerClient` connects before it writes `initialize`, so a shutdown landing in that gap killed the broker under it. The readiness probe now closes its connection fully before reporting the broker up (an open probe would otherwise read as a phantom client), and a client whose broker connection is dropped before `initialize` is answered falls back to its own app-server instead of failing the run. +- The `broker/shutdown` handshake is framed by newline and matched by request id, and it is bounded (5 s). A socket carries bytes, not messages, so a `{"busy":true}` reply split across two `data` events used to fail to parse and be read as "not busy" — tearing down a broker in the middle of another session's turn — while a peer that connected and stayed silent blocked `SessionEnd` forever. An unanswered or unparsable handshake is now reported as unknown, and `SessionEnd` treats anything but a confirmed "not busy" as a reason to leave the broker (and its record, pid file and endpoint) alone. +- The state-lock timeout carries a typed `code` (`CODEX_STATE_LOCK_TIMEOUT`), and `SessionEnd` absorbs only that. It used to recognise the timeout by matching "state lock" in the message, which would equally have swallowed the lock's own integrity error — or any filesystem error whose path happens to contain the phrase — reporting a real cleanup failure as a spent budget and exiting 0. Anything that is not the typed timeout now fails the hook as before. +- `CODEX_COMPANION_SESSION_END_BUDGET_MS` can only *shorten* the SessionEnd budget. The hook timeout in `hooks.json` is a fixed number that no environment variable can raise, so an override above the 12 s ceiling would have put the deadline past the point where Claude Code kills the hook — the exact failure the budget exists to prevent. A larger value is now refused with a note naming it and the ceiling. +- `SessionEnd` runs to one absolute budget (`SESSION_END_BUDGET_MS`, 12 s; `CODEX_COMPANION_SESSION_END_BUDGET_MS` can only shorten it): the workspace state lock, each broker handshake, the busy retries and the teardown probe are each clamped to what is left of it, and a spent budget is logged (`budgetExhausted=true`) and the broker left alone. Those bounds add up past any single one of them — the reaper alone takes the lock once per dead job — so a `busy` answer followed by a broker that stopped answering, or a couple of dead jobs behind a wedged lock holder, used to run past the hook timeout and be killed mid-decision. A lock the hook cannot take is now reported in that decision line rather than crashing the hook. `hooks/hooks.json` raises the `SessionEnd` timeout from 5 s to 15 s, above the worst case, and a test asserts it stays above the budget so the two numbers cannot drift. +- `SessionEnd` no longer takes a single `busy` answer as final. The broker counts every connected socket as a client, so a worker this hook has just reaped can still show up as one until its close event is processed; the handshake is now retried every 100 ms for up to 1 s before the broker is left alone. A broker that is genuinely serving another session stays busy for the whole window and keeps its record, endpoint and pid file, as before. Every SessionEnd decision — active jobs, busy, unconfirmed, teardown — is now written to stderr; the ones taken after the handshake carry the retry count. +- The broker has exactly one shutdown and one exit. Both signals, the `broker/shutdown` RPC and the idle timeout now join the same memoized teardown, and the process leaves only once it has finished: a second trigger used to get an immediate `return` and exit out from under the first — orphaning the app-server child mid-kill and leaving the endpoint socket, the pid file and the ownership record behind. That record is now cleared last, with the rest of the cleanup, instead of before the teardown starts. +- The broker can no longer be made to ignore `SIGTERM`. Its shutdown awaited `server.close()`, which fires only once every connection has closed, and `socket.end()` is a graceful half-close — so one client that never answered the FIN (a wedged peer, or one whose process was already gone but whose close had not been processed yet) left the shutdown hanging and the signal handler never reached `process.exit(0)`. A `SessionEnd` that signalled such a broker went on with its teardown while the process stayed alive. Connections now get a 1 s grace to close and are destroyed after it. +- Releases ship as GitHub Releases (title `codex-plugin-cc vX.Y.Z`) with the `npm pack` tarball and its SHA-256 attached; `release-verify.yml` re-runs the gate on the published tag; procedure in `docs/RELEASING.md`. +- The broker itself now refuses `broker/shutdown` while any other client is connected (replying `{ "busy": true }` and continuing to serve), and `SessionEnd` treats that refusal as "leave everything alone". The hook's active-job check is only a snapshot: another session could enqueue a job and connect in the gap before the shutdown RPC, and the broker used to shut down regardless, killing that turn. A broker kept alive this way is reaped by its own idle timeout — except with `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0`, where a client that stays connected keeps the broker alive until the operator stops it. +- The ticket lock fails closed: a directory listing, a read or a `stat` that fails (permissions, I/O) aborts the acquisition instead of being read as an empty queue, an unowned entry or an infinitely old one — only `ENOENT` is an answer, meaning the entry left the queue. Junk entries (content that reads but does not parse; entries are published by rename, so a live holder's is never half-written) are still aged out after 2 s. Every blocker is judged before any is evicted, so a verdict that fails partway leaves the queue exactly as it found it: a failed acquisition removes nothing but its own entries and never runs the callback. The bounded wait is checked before every scan after the first, so a blocker that cannot be cleared ends in the timeout error rather than a hot loop, and the poll pause is skipped only after an eviction that actually happened — an abandoned entry this process may not remove no longer hammers the filesystem for the whole budget — while an uncontended lock is still taken whatever the budget was. +- The workspace state lock is a Lamport bakery on files (`state.lock.d/`): an acquirer creates `choosing.`, takes a number one above the highest ticket on display, creates `..ticket`, drops its `choosing` file, and holds the lock once no foreign `choosing` file remains and no ticket sorts before its own (by number, ties by token). Releasing is one `unlink` of its own ticket. Nothing shared is ever replaced or removed: every name is unique to one acquisition and every file's content is immutable, so a verdict about a file cannot go stale before it is acted on — which is what every previous design (mkdir + holder file, rename-aside takeover, tombstone fences) could not guarantee, since POSIX has no conditional replace. Only entries whose owner is provably gone are cleared, on the unchanged contract: dead PID → at once; unreadable entry → after 2 s; unusable PID → after 30 s; live PID → never, with the wait error naming that PID and its exact ticket file. A pre-1.2.0 `state.lock` directory is left untouched. +- Every read-modify-write of `state.json` now runs under a cross-process workspace lock (the ticket lock above; 25 ms polling and a 5 s bound). Atomic writes only stopped torn reads: two processes could still read the same file, and the one that wrote last would treat every job the other had added as deleted — pruning that job's file, private payload, PID sidecar and log. `updateState`, `upsertJob`, `updateJobPid`, `saveState` (whose prune is a diff of the snapshot it read), the reaper's reconciliation, `cancel` and the `SessionEnd` cleanup all re-read inside the lock; readers (`status`, `result`, `listJobs`) stay lock-free. + ## 1.1.1 — 2026-08-28 - Broker idle self-terminate (upstream #457): the shared Codex runtime exits after 30 minutes without a connected client (`CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS` / `--idle-timeout`), so idle brokers and their app-server children no longer accumulate (#543). diff --git a/README.md b/README.md index 8abdc7296..bdc6ed268 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,14 @@ Ask Codex to redesign the database connection to be more resilient. - if you do not pass `--model` or `--effort`, Codex chooses its own defaults. - `--effort` accepts `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`, and `ultra`. Which of those a given model actually supports is decided by Codex, not by the plugin — run `codex debug models` to see the reasoning levels each model advertises. - model aliases: `spark` -> `gpt-5.3-codex-spark`, `sol` -> `gpt-5.6-sol`, `luna` -> `gpt-5.6-luna`, `terra` -> `gpt-5.6-terra`, `mini` -> `gpt-5.4-mini` -- `--config key=value` (repeatable, also on `/codex:review` and `/codex:adversarial-review`) forwards a `config.toml` override to the Codex thread, e.g. `--config model_provider=ollama`. On `--resume-last` the plugin opens a fresh app-server session (cold resume) so `--config` overrides, sandbox and approval policy take effect; model and effort for the resumed turn are sent on the turn, never on the resume request. +- `--config key=value` (repeatable, also on `/codex:review` and `/codex:adversarial-review`) forwards a `config.toml` override to the Codex thread, e.g. `--config model_provider=ollama`. On `--resume-last` the plugin opens a fresh app-server session (cold resume) so `--config` overrides, sandbox and approval policy take effect; model and effort for the resumed turn are sent on the turn, never on the resume request. In a `--background`/`--await` job record the config **keys** are recorded and the **values** are never stored (they read back as `[redacted]` in `status`/`result`): the real values live only in the job's private 0600 `jobs/.request.json`, which the worker consumes and deletes. - follow-up rescue requests can continue the latest Codex task in the repo +- under the hood, `/codex:rescue` and the `codex-rescue` agent are each a single `scripts/codex-companion.mjs task --await --prompt-stdin ` call: `--await [--await-timeout-ms ]` launches the same tracked background job as `--background`, then waits for it (default 540000 ms), and `--prompt-stdin` reads the prompt as stdin verbatim (so it cannot be combined with `--args-stdin`, `--prompt-file`, or prompt text on the command line). Exit code is 0 when the job completed, 1 when it failed or was cancelled, and 3 when the wait times out while the job is still queued or running — exit 3 prints a `Re-run: node "" result --wait --timeout-ms 540000` hint, which is the only follow-up call the rescue flow makes. +- `result [--wait [--timeout-ms ]]` answers a different question, so it has its own contract: `result` exits 0 for any terminal record (completed, failed or cancelled) and 3 while the job is still active. Its exit code means "a result was retrieved", not "the job succeeded" — unlike `task --await` it never returns 1 for a failed job, so read the rendered record for the outcome. A plain `result ` on a still-running job prints the same `--wait` hint and exits 3 instead of failing (fixes upstream #498/#524, which reported "No job found" for a running job). `--json` on either returns `{ job, storedJob }` (or, on a timeout, the `status --json` snapshot plus a `resumeCommand` field). +- The detached worker outlives the companion only when the companion returns on its own (exit 3); a host process-tree kill — e.g. Claude Code's Bash timeout — also kills the worker, so keep `--await-timeout-ms` below the host limit (default 540000 < 600000). +- `--turn-timeout-ms ` (or `CODEX_TURN_TIMEOUT_MS`, also on `/codex:review` and `/codex:adversarial-review`) bounds a single Codex turn: on expiry it interrupts the turn and returns a structured failed result ("turn timed out after `` ms") instead of hanging. Default is `0` (unbounded). The budget travels with a `--background`/`--await` job, so a detached worker enforces it too. The interrupt is not trusted on its own: the run waits up to 10 s for the turn's terminal notification, and if none arrives the failure says so ("interrupt not acknowledged — the turn may still be running in the shared runtime, check status or cancel"), because a shared broker runtime can keep executing a turn nobody is listening to any more. A run that owns its own app-server (a cold `--resume-last`) closes it in that case, which does stop the turn (stdin EOF, then `SIGTERM`, then `SIGKILL`, so the close is bounded too). Partial output on a timed-out turn is best-effort: only whole items Codex had already completed are kept, so a turn interrupted mid-message reports less text than Codex had produced. +- the `SessionEnd` hook works to one absolute budget (`SESSION_END_BUDGET_MS`, 12 s; `CODEX_COMPANION_SESSION_END_BUDGET_MS` can only *shorten* it — a larger value is ignored with a note, since the hook timeout is fixed), and every bounded step inside it — the workspace state lock, each broker handshake, the busy retries, the teardown probe — is clamped to what is left of that budget. `hooks/hooks.json` gives `SessionEnd` a 15 s timeout, which must stay **above** the budget: below it Claude Code would kill the hook mid-decision instead of letting it report one. A test asserts the pair, so the two numbers cannot drift apart. +- if a background job's session ends while `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0`, the shared broker that keeps running for that job never self-terminates on its own — its normal idle exit is disabled in that configuration, so the broker only goes away once the job finishes (or is reaped as dead) and a later `SessionEnd` runs. ### `/codex:transfer` @@ -208,8 +214,12 @@ Examples: ```bash /codex:result /codex:result task-abc123 +/codex:result task-abc123 --wait +/codex:result task-abc123 --wait --timeout-ms 60000 ``` +On a job that already has a terminal record (completed, failed, or cancelled), `/codex:result` exits 0 and shows it — the exit code reports that a result was retrieved, not whether the job succeeded. On a job that is still queued or running, a plain `/codex:result ` prints a `Re-run: … result --wait` hint and exits 3 instead of failing; add `--wait [--timeout-ms ]` (default 540000 ms) to block until the job reaches a terminal status instead of returning immediately. `--json` returns `{ job, storedJob }` (or, on a `--wait` timeout, the `status --json` snapshot plus a `resumeCommand` field). + ### `/codex:cancel` Cancels an active background Codex job. @@ -328,6 +338,27 @@ That means: Yes. If you already use Codex, the plugin picks up the same [configuration](#common-configurations). +### A command failed with "Timed out … waiting for the Codex state lock" + +Every write to this workspace's job state is serialized by a ticket lock: each +command takes a numbered ticket in `state.lock.d/` and waits for the tickets ahead +of it. A ticket whose process is gone is cleared automatically, so a crash never +wedges the workspace. A ticket whose process is still *running* is never taken +away — a slow writer and a stuck one look the same from outside, and taking the +lock from a process that is mid-write is how state gets corrupted — so the error +names that PID and the exact ticket file. If that process really is stuck, stop it +and the next command goes through; if the PID belongs to something unrelated (PID +reuse), delete the ticket file the error names. + +### A command failed with a raw `EACCES` or `EIO` from the state directory + +The same lock refuses to guess. If a ticket in `state.lock.d/` cannot be listed, +read or `stat`ed, the command fails with that error instead of assuming the entry +is absent or abandoned — guessing there is what would let two commands write the +job state at once. Fix the permissions on the state directory (or remove the entry +the error names, once you know no Codex command is using it) and the next command +goes through. + ### Can I keep using my current API key or base URL setup? Yes. Because the plugin uses your local Codex CLI, your existing sign-in method and config still apply. diff --git a/docs/RELEASING.md b/docs/RELEASING.md new file mode 100644 index 000000000..d1c51f7d2 --- /dev/null +++ b/docs/RELEASING.md @@ -0,0 +1,76 @@ +# Releasing + +Releases follow the same shape as [CBEPX/cc-plugin-codex](https://github.com/CBEPX/cc-plugin-codex/releases): +a git tag, a GitHub Release with hand-written notes, and the `npm pack` tarball (plus its +SHA-256) attached as the release artifact. There is no npm publish (`package.json` is +`private`); users install from this repository through `.claude-plugin/marketplace.json`. + +## 1. Prepare the release branch + +```bash +git checkout -b release/vX.Y.Z main +npm run bump-version -- X.Y.Z # package.json, plugin.json, marketplace.json +npm run check-version # all version metadata matches +``` + +Add a `## X.Y.Z — YYYY-MM-DD` section at the top of `CHANGELOG.md`. + +## 2. Gate (local, then CI) + +```bash +npm run build && npm run check-version && claude plugin validate . --strict +npm test; sleep 10; pgrep -f codex-plugin-test- | wc -l # must print 0 +npm audit --omit=dev +``` + +Open a pull request against `main`; `Pull Request CI` must be green. Merge with a merge +commit (`gh pr merge N --merge`). + +## 3. Tag and build the artifact + +```bash +git checkout main && git pull +git tag -a vX.Y.Z -m "vX.Y.Z" && git push origin vX.Y.Z +npm pack # cbepx-codex-plugin-cc-X.Y.Z.tgz +shasum -a 256 cbepx-codex-plugin-cc-X.Y.Z.tgz > cbepx-codex-plugin-cc-X.Y.Z.tgz.sha256 +``` + +## 4. Publish the GitHub Release + +```bash +gh release create vX.Y.Z \ + --title "codex-plugin-cc vX.Y.Z" \ + --notes-file notes.md \ + cbepx-codex-plugin-cc-X.Y.Z.tgz cbepx-codex-plugin-cc-X.Y.Z.tgz.sha256 +``` + +`notes.md` template: + +```markdown +One-line summary of the release. + +### Highlights +- … + +### Compatibility +- CLI / hook / state-format changes callers must know about (omit if none). + +### Validation +- Exact tag target: `` +- Local gate: N/N tests, 0 leaked test processes, `npm run build`, `npm run check-version`, `claude plugin validate . --strict` +- GitHub CI: +- Review: Codex adversarial review (verdict), Claude review (verdict) +- Runtime dependency audit: `npm audit --omit=dev` reports 0 vulnerabilities + +### Artifact +` cbepx-codex-plugin-cc-X.Y.Z.tgz` +``` + +Publishing the release triggers `.github/workflows/release-verify.yml`, which re-runs the +gate on the tag (tests, leak check, build, version check, audit, pack dry run). + +## 5. Update local installs + +```bash +claude plugin marketplace update cbepx && claude plugin update codex@cbepx +``` diff --git a/docs/superpowers/plans/2026-08-28-codex-plugin-cc-v1.2.0.md b/docs/superpowers/plans/2026-08-28-codex-plugin-cc-v1.2.0.md new file mode 100644 index 000000000..254247b49 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-codex-plugin-cc-v1.2.0.md @@ -0,0 +1,142 @@ +# codex-plugin-cc v1.2.0 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the rescue flow a single `node` call (companion-side await), restore the narrow `Bash(node:*)` grant, close the remaining lifecycle gaps (own-jobs-only SessionEnd, PID liveness reaping, bounded turns), and add the missing live-broker SessionEnd test. + +**Architecture:** New companion subcommand behaviour `task --await` (launch as a tracked background job, poll until terminal or `--await-timeout-ms`, print the result; on timeout print a resumable hint and exit 3) plus `--prompt-stdin` (raw, untokenized stdin = prompt) so a slash-command body is exactly one `node …` invocation with one quoted heredoc. Rescue command/agent bodies collapse to that call; `allowed-tools` goes back to `Bash(node:*)`. Lifecycle: cherry-pick upstream #355 (SessionEnd terminates only jobs it owns), #425 (PID liveness → reap zombie `running` jobs), #376 (bounded `captureTurn` with a configurable turn budget) — resolving against the fork's `disableBroker` cold-resume path and v1.1.1 shutdown changes. + +**Tech Stack:** Node ≥18.18, ESM `.mjs`, `node --test`, fake Codex fixture, `gh`. + +**Spec:** memory `codex-plugin-cc-fork-backlog` (v1.2 items); v1.1.0/v1.1.1 review residuals (random heredoc delimiters instruction-only; `allowed-tools: Bash` too broad; no test that SessionEnd kills a live broker). + +## Global Constraints + +- Repo `/Users/g.mehrenin/project/personal/codex-plugin-cc`, `origin`=CBEPX, `upstream`=openai. Branch `release/v1.2.0` from `main` (6672679 = v1.1.1). +- Gate (zsh reserves `status`, use `st`): `npm test > /tmp/npm-test.log 2>&1; st=$?; rg -e 'ℹ (tests|pass|fail)' -e '^not ok' /tmp/npm-test.log; test "$st" -eq 0` → `fail 0` (150 at base); `sleep 10; pgrep -f codex-plugin-test- | wc -l` → 0; `npm run build`; `npm run check-version`; `claude plugin validate . --strict` before the release commit. +- Tooling rule (user): never `grep` — ripgrep `rg`. No `git add -A`. Trailer `Co-Authored-By: Claude Fable 5 `. No push until the controller says so. +- Upstream PR merges: `git fetch upstream pull/N/head:pr/N && git merge --no-ff --no-edit pr/N`; keep-both on conflicts; the fork's `withAppServer(cwd, fn, clientOptions)`, `disableBroker` cold resume, `assertThreadIsFree`, `buildThreadConfig`, `--args-stdin`, v1.1.1 broker shutdown/ownership code must survive — verify by reading after each merge. +- Shell bodies of commands/agents: exactly one `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" …` invocation per Bash block, prose only via a quoted heredoc on stdin, flags on the command line; `allowed-tools: Bash(node:*)`. + +--- + +### Task 1: `task --await` + `--prompt-stdin` in the companion + +**Files:** +- Modify: `plugins/codex/scripts/codex-companion.mjs` — `handleTask` (new options `await`, `await-timeout-ms`, `prompt-stdin`), `printUsage`, reuse `enqueueBackgroundTask`, `waitForSingleJobSnapshot`/`handleStatus` internals, `handleResult` rendering; `readTaskPrompt` (prompt from raw stdin when `--prompt-stdin`). +- Test: `tests/runtime.test.mjs`, `tests/args.test.mjs` (booleanOptions/valueOptions additions), `tests/commands.test.mjs` untouched here. + +**Interfaces:** +- Consumes: `enqueueBackgroundTask(cwd, job, request)` (returns jobId; job record persisted before spawn since v1.1.0), `waitForSingleJobSnapshot(workspaceRoot, jobId, { timeoutMs, pollIntervalMs })`, `renderTaskResult(job)` / the `result` path, `isActiveJobStatus(status)`, `readStdinIfPiped()`. +- Produces (contract fixed by Codex-review rulings F1–F4, F11 — see ledger): + - `task --await [--await-timeout-ms ] …` → enqueue a background job (record identical to `--background`), then wait via the same snapshot polling `status --wait` uses; terminal → print exactly what `result ` prints (`resolveResultJob` + `readStoredJob` + `renderStoredJobResult`); exit 0 completed, 1 failed/cancelled; timeout (default 540000, must be a finite positive integer; `0`/negative/`NaN`/`Infinity` → usage error) → text: one line `Still running: job . Re-run: node "" result --wait --timeout-ms 540000`, exit **3**; `--json`: `{ job, storedJob }` on terminal, or the `status --json` snapshot plus `"resumeCommand"` on timeout. + - `result [--wait [--timeout-ms ]]` → terminal record retrieved → exit 0 even if `job.status === "failed"` (existing retrieval semantics); still `queued`/`running` without `--wait` → the hint line, exit 3; with `--wait` → same timeout contract. `status --wait` semantics unchanged (exit 0 + `waitTimedOut`). + - `--prompt-stdin` → raw stdin, decoded UTF-8, exactly one trailing `\r?\n` removed, nothing else (no `.trim()`); the check for `--prompt-stdin` happens on the RAW argv before `applyArgsStdin` (which would otherwise consume stdin); `--prompt-stdin`+`--args-stdin`, +`--prompt-file`, +positional prompt, `--await`+`--background`, `--await-timeout-ms` without `--await` → usage errors before any stdin read (fast even with an open, empty stdin). + - Survival caveat (documented): the detached worker survives only when the companion returns by itself (exit 3); a host process-tree kill (Claude's Bash timeout) also kills the worker. + +- [ ] **Step 1: Failing runtime tests** — append to `tests/runtime.test.mjs` (use `seededRepo()`, `installFakeCodex`, `buildEnv`, `SCRIPT` helpers already in the file): + +```js +test("task --await launches a tracked job, waits, and prints the result", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + const statePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir); + const result = run("node", [SCRIPT, "task", "--await", "--json", "--model", "sol", "--effort", "low", "--prompt-stdin"], { + cwd: repo, env: buildEnv(binDir), input: "line one \\d+ \"quoted\" 'single'\nline two\n" + }); + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(result.stdout); + assert.match(out.jobId, /^task-/); + assert.equal(out.status, "completed"); + assert.ok(typeof out.rawOutput === "string" && out.rawOutput.length > 0); + const fakeState = JSON.parse(fs.readFileSync(statePath, "utf8")); + assert.equal(fakeState.lastTurnStart.prompt, "line one \\d+ \"quoted\" 'single'\nline two"); + assert.equal(fakeState.lastTurnStart.effort, "low"); + const status = run("node", [SCRIPT, "status", out.jobId, "--json"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(JSON.parse(status.stdout).job.status, "completed"); +}); + +test("task --await exits 3 with a resumable hint when the await timeout elapses", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "3000" }); // add this fixture knob if absent: delays turn/start completion + const result = run("node", [SCRIPT, "task", "--await", "--await-timeout-ms", "500", "--prompt-stdin"], { cwd: repo, env, input: "slow task\n" }); + assert.equal(result.status, 3); + assert.match(result.stdout, /Still running: job task-[A-Za-z0-9_-]+\. Re-run: node .*result task-[A-Za-z0-9_-]+ --wait --timeout-ms 540000/); + const jobId = result.stdout.match(/job (task-[A-Za-z0-9_-]+)/)[1]; + const done = run("node", [SCRIPT, "result", jobId, "--wait", "--timeout-ms", "20000"], { cwd: repo, env }); + assert.equal(done.status, 0, done.stderr); +}); + +test("task rejects --prompt-stdin combined with --args-stdin or --prompt-file", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const r = run("node", [SCRIPT, "task", "--prompt-stdin", "--args-stdin"], { cwd: repo, env: buildEnv(binDir), input: "x" }); + assert.notEqual(r.status, 0); + assert.match(r.stderr, /--prompt-stdin/); +}); +``` + +- [ ] **Step 2: Run — expect FAIL** (`Unknown option: --await`). + +- [ ] **Step 3: Implement** in `codex-companion.mjs`: add `"await"`, `"prompt-stdin"` to `handleTask`'s `booleanOptions` and `"await-timeout-ms"` to `valueOptions`; in `main()`/`applyArgsStdin` guard the mutual exclusion (`--prompt-stdin` + `--args-stdin` → throw `"--prompt-stdin cannot be combined with --args-stdin; put flags on the command line."`); `readTaskPrompt`: if `options["prompt-stdin"]` → `readStdinIfPiped()` raw (error if empty); `handleTask`: when `options.await`, build the request exactly like the background branch, `enqueueBackgroundTask`, then `await waitForSingleJobSnapshot(workspaceRoot, jobId, { timeoutMs: Number(options["await-timeout-ms"] ?? 540000), pollIntervalMs: 1000 })`; if terminal → print via the `result` renderer (`--json` → the `result --json` object plus `jobId`), exit 0/1 by job status; else print the hint and `process.exitCode = 3`. `handleResult`: add `--wait`/`--timeout-ms` using the same snapshot wait. Fixture: add `FAKE_CODEX_TURN_DELAY_MS` if no delay knob exists (sleep before emitting the turn-completed notification). + +- [ ] **Step 4: Tests pass; gate → `fail 0`; commit** `feat(task): --await and --prompt-stdin; result --wait`. + +--- + +### Task 2: Rescue bodies → single `node` call; `allowed-tools: Bash(node:*)` + +**Files:** `plugins/codex/commands/rescue.md`, `plugins/codex/agents/codex-rescue.md`, `plugins/codex/skills/codex-cli-runtime/SKILL.md`, `tests/commands.test.mjs`, `README.md`, `CHANGELOG.md`. + +**Interfaces:** Consumes Task 1's `task --await --prompt-stdin` (exit 0 completed / 1 failed-or-cancelled / 3 still running with a `Re-run: node "" result --wait --timeout-ms 540000` hint line) and `result --wait`. Codex-review rulings: the resume decision (`task-resume-candidate` + one `AskUserQuestion`) happens in the main command BEFORE the sync/background split, so the agent (which has no `AskUserQuestion`) receives an explicit `--resume-last` or `--fresh`; `--background` without an explicit `--resume` is always fresh; `--write` only when the user explicitly asked to modify files; the payload Bash block is exactly one `node …` call (the `task-resume-candidate` call before it is a separate block); the prompt heredoc keeps a high-entropy delimiter that does not occur as an exact line in the request. Produces the new command body: + +````markdown +1. If the request contains `--resume`, use `--resume-last`; if `--fresh`, use a fresh task. Otherwise run `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task-resume-candidate --json`: when it reports a resumable thread ask ONCE with `AskUserQuestion` — `Continue current Codex thread` (→ `--resume-last`) / `Start a new Codex thread` (→ fresh) — otherwise fresh. This decision is made here for BOTH the synchronous and the `--background` path. Pass `--model`, `--effort`, `--config key=value` through; never add `--write` unless the user explicitly asked Codex to modify files. +2. Synchronous path (default): ONE Bash call (`timeout: 600000`); flags on the command line, the request prose in a quoted heredoc whose delimiter is `CODEX_PROMPT_` + 8 fresh random hex chars that do not appear as an exact line in the request: +```bash +node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --await --prompt-stdin <<'CODEX_PROMPT_' + +CODEX_PROMPT_ +``` +Exit 0 → show the output verbatim, then your assessment. Exit 3 → the output ends with `Re-run: node "…" result --wait --timeout-ms 540000` — run exactly that line (again `timeout: 600000`) until it exits 0 or 1; never report "no result". Exit 1 → the job failed or was cancelled: show the output verbatim and stop. +3. `--background`: invoke the `Agent` tool with `codex:codex-rescue`, passing the request minus `--background` PLUS the explicit `--resume-last` or `--fresh` decided in step 1; tell the user the result arrives as a completion notification and via `/codex:status` / `/codex:result `. +```` +Agent body: the same single `task --await --prompt-stdin` call (never `task-resume-candidate`, never a resume heuristic — it must receive `--resume-last`/`--fresh` from the command), then `result --wait` for its own job on exit 3; failures returned verbatim. +- [ ] **Step 1: Failing tests** in `tests/commands.test.mjs`: rescue and agent bodies match `/task --await --prompt-stdin/`; the payload fenced block contains exactly one `codex-companion.mjs` invocation and no `mktemp`/`cat >`/`while`/`sleep`/`$JOB=`; `allowed-tools: Bash(node:*), AskUserQuestion, Agent` for rescue.md; the AskUserQuestion step precedes the `--background` branch text (assert index ordering); the agent body has no `task-resume-candidate` and no `--resume-last` heuristic; the delimiter guidance sentence is present; `SKILL.md` execution rules updated to the single-call flow. Existing two-step assertions updated (edit, don't delete). +- [ ] **Step 2: RED → rewrite the bodies + SKILL.md → GREEN; gate; commit** `feat(rescue): single node call via task --await; resume decision before the background split; Bash(node:*) grant restored`. +- [ ] **Step 3: Manual permission smoke (record in the report as not automatable):** in a fresh Claude Code 2.1.247 session run `/codex:status` and `/codex:rescue --model sol --effort low Strictly read-only: reply PONG` — both must run without a permission prompt; also confirm a non-`node` command inside the same block would NOT be covered (documented expectation, no test). + +--- + +### Task 3: Lifecycle — #355 (semantic merge), #425 (adapted reaper), bounded turns (own implementation), regression tests + +Codex-review rulings (ledger F7–F10): #355 and #425 are merged as branches but each needs a semantic resolution against the fork; #376 is NOT merged — its behaviour is implemented here directly. The cached diffs are in the SDD scratchpad (`prs/pr-355.diff`, `pr-425.diff`, `pr-376.diff`); `git fetch upstream pull/N/head:pr/N` for the merges. + +**Files:** `plugins/codex/scripts/session-lifecycle-hook.mjs`, `plugins/codex/scripts/codex-companion.mjs` (`enqueueBackgroundTask`, `handleTaskWorker`, `handleStatus` reaper hook, `--turn-timeout-ms`), `plugins/codex/scripts/lib/{state,tracked-jobs,job-control,codex,broker-lifecycle}.mjs`, `tests/{runtime,tracked-jobs,broker-stale-pid}.test.mjs`, `tests/fake-codex-fixture.mjs`. + +**Interfaces / rulings:** +- **#355 (own-jobs-only SessionEnd):** after the merge, `handleSessionEnd` order must be: (1) `cleanupSessionJobs` — terminate this session's *foreground* jobs, keep *background* jobs (queued records gain `background: true`; the fork's `requestFile` + redacted `request` fields must survive); (2) if any owned background job is still `queued`/`running` → skip broker shutdown (early return); (3) otherwise `sendBrokerShutdown` → `teardownBrokerSession` (v1.1.1 `ownsBrokerProcess` check stays as the fallback-signal guard); (4) hook-side `clearBrokerSession(cwd)` only when the record's `endpoint` equals the one captured at step start (a replacement broker's record must survive). Document that with `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0` a broker kept alive for a background job never exits on its own. +- **#425 (PID liveness reaper), adapted:** `enqueueBackgroundTask` records the worker pid after spawn via a narrow pid-only update (`updateJobPid(workspaceRoot, jobId, pid)` that rewrites ONLY `pid` and never `status` — no load-mutate-save of the whole record; if the worker already wrote `running`, keep it). The reaper (run by `status`, `result`, `task-resume-candidate`): a `queued`/`running` job whose `pid` is dead (`process.kill(pid, 0)` throws) → `failed` with `errorMessage: "worker exited before completing"`; after the terminal reap delete `jobs/.request.json` and set `requestFile: null`; never touch a live worker or its payload; never copy an unredacted payload into `state.json`. Tests: dead-queued-before-consume (payload removed after reap), dead-running, live job with payload untouched, cancelled job, secret sentinel absent from state after reap. +- **Bounded turns (instead of #376):** `captureTurn` gets a client-side timeout: `--turn-timeout-ms ` on `task`/`review`/`adversarial-review` (persisted into the worker request for background/await jobs), env `CODEX_TURN_TIMEOUT_MS` as default, `0`/unset = unbounded (preserves current behaviour). On timeout: `turn/interrupt` the thread, then return a structured failed result (`status: failed`, `errorMessage: "turn timed out after ms"`, partial `rawOutput` if any) — never an exception that only reaches stderr. `TurnStartParams` has no timeout field: do not add one to the RPC. The stop-review gate's own 13-minute limit is unaffected. +- **Regression tests (not RED-first; baseline is green):** in `tests/broker-stale-pid.test.mjs` / `tests/runtime.test.mjs`: (a) live owned broker → SessionEnd → pid gone within 3 s, record + endpoint cleared; (b) wrong/recycled pid never signalled (exists); (c) owned background job running → SessionEnd keeps the broker; after the job completes the broker idle-exits (use `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=2000` via `buildEnv` override) and only its own record is cleared; (d) a replacement broker started during the old one's shutdown keeps its record. + +- [ ] **Step 1:** `git fetch upstream pull/355/head:pr/355 && git merge --no-ff --no-edit pr/355`; resolve per the #355 ruling (read the merged `handleSessionEnd` end-to-end; reorder if needed); gate. +- [ ] **Step 2:** `git fetch upstream pull/425/head:pr/425 && git merge --no-ff --no-edit pr/425`; adapt per the #425 ruling (`updateJobPid`, reaper rules, payload deletion); write the five reaper tests (RED where the behaviour is new, e.g. dead-queued-before-consume); gate. +- [ ] **Step 3:** bounded turns: failing test (fake app-server with `FAKE_CODEX_TURN_DELAY_MS=5000`, `task --turn-timeout-ms 500` → exit 1, `status` shows `failed` with the timeout message, `turn/interrupt` recorded by the fixture) → implement → GREEN; gate. +- [ ] **Step 4:** the four broker regression tests; gate; commit `fix(lifecycle): own-jobs-only SessionEnd (#355), pid-liveness reaper (#425, adapted), bounded turns; broker teardown regression tests`. + +--- + +### Task 4: Release v1.2.0 + +- [ ] `npm run bump-version -- 1.2.0 && npm run check-version`; CHANGELOG 1.2.0 (Tasks 1–3, plus "rescue bodies are single node calls; `allowed-tools` narrowed back to `Bash(node:*)`; random-delimiter guidance now only protects the prompt heredoc; exit-code contract 0/1/3 for `task --await` and `result`; worker-survival caveat; `CODEX_TURN_TIMEOUT_MS`; `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0` keeps a background-job broker alive indefinitely"); `plugins/codex/commands/result.md` argument-hint `[job-id] [--wait] [--timeout-ms ]`; README task/rescue/result sections (exit codes, JSON shapes, `--prompt-stdin`, `--turn-timeout-ms`); `tests/commands.test.mjs` assertions for the hints; gate/build/validate; commit `chore(release): v1.2.0`. + +## Verification +1. Fresh session: `/codex:status` (no prompt), `/codex:rescue --model sol --effort max Strictly read-only: reply PONG` → answer in the same turn via the single call; `--background` variant → agent returns the result. +2. Force exit-3: `/codex:rescue --await-timeout-ms 5000 …` on a slow prompt → hint line → rerun `result --wait` → result. +3. SessionEnd on a session with a live broker → broker gone (test + manual `pgrep`). +4. `status` on a job whose worker was `kill -9`'d → shows `failed` (PID liveness), not `running` forever. diff --git a/package-lock.json b/package-lock.json index 1ee96d7dc..677b8830c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@cbepx/codex-plugin-cc", - "version": "1.1.1", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@cbepx/codex-plugin-cc", - "version": "1.1.1", + "version": "1.2.0", "license": "Apache-2.0", "devDependencies": { "@types/node": "^25.5.0", diff --git a/package.json b/package.json index 671051558..3c0694d26 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@cbepx/codex-plugin-cc", - "version": "1.1.1", + "version": "1.2.0", "private": true, "type": "module", "description": "Use Codex from Claude Code to review code or delegate tasks.", diff --git a/plugins/codex/.claude-plugin/plugin.json b/plugins/codex/.claude-plugin/plugin.json index bb6034608..5146a42c1 100644 --- a/plugins/codex/.claude-plugin/plugin.json +++ b/plugins/codex/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex", - "version": "1.1.1", + "version": "1.2.0", "description": "Use Codex from Claude Code to review code or delegate tasks.", "author": { "name": "OpenAI" diff --git a/plugins/codex/agents/codex-rescue.md b/plugins/codex/agents/codex-rescue.md index 9bab35bb6..0077fdee6 100644 --- a/plugins/codex/agents/codex-rescue.md +++ b/plugins/codex/agents/codex-rescue.md @@ -18,55 +18,28 @@ Selection guidance: Forwarding rules: -- Launch the job, then wait for it — two separate Bash calls, in ≤9-minute wait slices so the Bash tool's 10-minute cap never kills a long run. Bash calls share no variables — always set `JOB=` literally at the top of every later call; never rely on a `$JOB` left over from a previous call. - - The request prose and the runtime flags travel in two separate channels of the same Bash call: the prose is written byte-exact to `$PROMPT` by its own quoted heredoc and passed as `--prompt-file`, while the `--args-stdin` heredoc carries only the runtime flags (`--model`, `--effort`, `--config key=value`, `--resume-last`, `--write` as applicable). Never put the request text in the flags heredoc: it is tokenized, so quotes, backslashes and newlines in a stack trace, a regex or a code block would be mangled. Give both heredoc delimiters a fresh random suffix on every call — `CODEX_PROMPT_` / `CODEX_ARGS_`, e.g. 8 hex characters — and never reuse a suffix that appears in the request text: a payload line equal to the delimiter would end the heredoc early and run the rest on the host shell. - - Launch (one Bash call): +- You receive the resume decision already made: an explicit `--resume-last` or `--fresh` in the prompt. Pass it straight through in ``. Never call `task-resume-candidate`, never infer resume from prose (no "continue"/"keep going"/"apply the top fix" heuristics), and never guess — if neither flag is present, run fresh. You have no `AskUserQuestion` tool to ask with. +- ONE Bash call (tool `timeout: 600000`); flags on the command line, the request prose in a quoted heredoc whose delimiter is `CODEX_PROMPT_` + 8 fresh random hex characters that do not appear as an exact line in the request. Never reuse a delimiter suffix that appears as an exact line in the request: a payload line equal to it would end the heredoc early and run the rest on the host shell. +- `` may contain only bare tokens — `--model `, `--effort `, `--turn-timeout-ms `, `--config key=value` with a literal value, `--resume-last`/`--fresh`, `--write`. If any flag value contains `$`, a backtick, a quote, `;`, `&`, `|`, or a newline, drop it and mention it in the prose instead; never place it on the command line. ```bash -trap 'command rm -f -- "$ERR" "$OUT" "$PROMPT"' EXIT -ERR=$(mktemp); PROMPT=$(mktemp) -cat > "$PROMPT" <<'CODEX_PROMPT_' +node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --await --prompt-stdin <<'CODEX_PROMPT_' CODEX_PROMPT_ -JOB=$(node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --background --json --prompt-file "$PROMPT" --args-stdin <<'CODEX_ARGS_' 2>"$ERR" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{process.stdout.write(JSON.parse(d).jobId||"")}catch(e){}})' - -CODEX_ARGS_ -) -[ -n "$JOB" ] || { cat "$ERR"; exit 1; } -[[ "$JOB" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "invalid job id"; exit 1; } -echo "JOB=$JOB" ``` - If this call exits non-zero, its output is the launch failure — return it verbatim and stop; never return an empty result. Otherwise its last line is `JOB=`; read `` from it. - - Wait and fetch the result (one Bash call, tool `timeout: 600000`). Set `JOB=` literally as the first line, using the id you just read: -```bash -trap 'command rm -f -- "$ERR" "$OUT" "$PROMPT"' EXIT -JOB= -[[ "$JOB" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "invalid job id"; exit 1; } -OUT=$(mktemp); ERR=$(mktemp) -while node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" status "$JOB" --wait --timeout-ms 540000 --json >"$OUT" 2>"$ERR"; node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{const s=(JSON.parse(d).job||{}).status;process.exit(s==="queued"||s==="running"?3:(s?0:2))}catch(e){process.exit(2)}})' < "$OUT"; rc=$?; [ "$rc" -eq 3 ]; do sleep 1; done -[ "$rc" -eq 0 ] || { cat "$OUT" "$ERR"; exit "$rc"; } -node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" result "$JOB" -``` - Exits 3 while the job is `queued`/`running` (loop, with `sleep 1` so it can't spin hot), 0 on a terminal status, 2 if the status output is empty or unparseable — either non-3 outcome ends the loop. If this call is cut off by the tool's own timeout, the job keeps running server-side — run it again with the same literal `JOB=` line. If it exits non-zero, return its output verbatim and stop; never return an empty result. Otherwise return the `result` stdout as-is. -- You may check this job's own `status` and fetch its `result` to carry out the launch/wait above; do not inspect the repository, read files, grep, cancel jobs, summarize output, or do any other follow-up work of your own. + Exit 0 → return the output verbatim. Exit 3 → the output ends with a `Re-run:` line — run exactly that line (again `timeout: 600000`) until it exits 0; its output is the final job record whether the job completed, failed, or was cancelled — return it verbatim either way. Exit 1 from the first call → the job failed or was cancelled: return the output verbatim and stop. +- Each of those calls — the launch and any `result --wait` re-run — uses `timeout: 600000` to match the Bash tool's 10-minute cap; if one is cut off by it, the job keeps running server-side, re-run the printed `Re-run:` line with its literal job id. +- Re-running the exact printed `Re-run:` line for this job is the only permitted follow-up; do not inspect the repository, read files, grep, cancel jobs, summarize output, or do any other follow-up work of your own. - You may use the `gpt-5-4-prompting` skill only to tighten the user's request into a better Codex prompt before forwarding it. - Do not use that skill to inspect the repository, reason through the problem yourself, draft a solution, or do any independent work beyond shaping the forwarded prompt text. -- Do not call `review`, `adversarial-review`, or `cancel`. This subagent only forwards to `task` and checks its own job's `status`/`result`. +- Do not call `review`, `adversarial-review`, or `cancel`. This subagent only forwards to `task` and, on exit 3, re-runs its own job's printed `result --wait` hint. - Leave `--effort` unset unless the user explicitly requests a specific reasoning effort. - Leave model unset by default. Only add `--model` when the user explicitly asks for a specific model. - If the user asks for `spark`, map that to `--model gpt-5.3-codex-spark`. - If the user asks for a concrete model name such as `gpt-5.4-mini`, pass it through with `--model`. - Treat `--effort `, `--model `, and `--config key=value` as runtime controls and do not include them in the task text you pass through. - Never add `--write` unless the user explicitly asked Codex to modify files. -- Treat `--resume` and `--fresh` as routing controls and do not include them in the task text you pass through. -- `--resume` means add `--resume-last`. -- `--fresh` means do not add `--resume-last`. -- If the user is clearly asking to continue prior Codex work in this repository, such as "continue", "keep going", "resume", "apply the top fix", or "dig deeper", add `--resume-last` unless `--fresh` is present. -- Otherwise forward the task as a fresh `task` run. - Preserve the user's task text as-is apart from stripping routing flags. - Return the `result` stdout exactly as-is. - If the Bash call fails or Codex cannot be invoked, return the command's exit status and stderr verbatim so the failure is visible; never return an empty result. diff --git a/plugins/codex/commands/adversarial-review.md b/plugins/codex/commands/adversarial-review.md index 71c8d7493..0482349ed 100644 --- a/plugins/codex/commands/adversarial-review.md +++ b/plugins/codex/commands/adversarial-review.md @@ -1,6 +1,6 @@ --- description: Run a Codex review that challenges the implementation approach and design choices -argument-hint: '[--wait|--background] [--base ] [--scope auto|working-tree|branch] [--model ] [--effort ] [--config key=value] [focus ...]' +argument-hint: '[--wait|--background] [--base ] [--scope auto|working-tree|branch] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value] [focus ...]' disable-model-invocation: true allowed-tools: Read, Glob, Grep, Bash(node:*), Bash(git:*), AskUserQuestion --- diff --git a/plugins/codex/commands/rescue.md b/plugins/codex/commands/rescue.md index e6eea5aef..176e22cd2 100644 --- a/plugins/codex/commands/rescue.md +++ b/plugins/codex/commands/rescue.md @@ -1,7 +1,7 @@ --- description: Delegate investigation, an explicit fix request, or follow-up rescue work to the Codex rescue subagent -argument-hint: "[--background|--wait] [--resume|--fresh] [--model ] [--effort ] [--config key=value]... [what Codex should investigate, solve, or continue]" -allowed-tools: Bash, AskUserQuestion, Agent +argument-hint: "[--background] [--resume|--fresh] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value]... [what Codex should investigate, solve, or continue]" +allowed-tools: Bash(node:*), AskUserQuestion, Agent --- Delegate the request to Codex through the shared companion runtime. Default is synchronous: the user gets Codex's answer in this turn. @@ -9,45 +9,20 @@ Delegate the request to Codex through the shared companion runtime. Default is s Raw slash-command arguments: `$ARGUMENTS` -If the request contains `--background`, skip directly to step 3 — steps 1 and 2 are the default synchronous path and do not run for a `--background` request. - -1. Strip `--wait` if present (it is the default). If the request contains `--resume`, use `task --resume-last`; if `--fresh`, use a fresh `task`. Otherwise run `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task-resume-candidate --json`: when it reports no resumable thread, start a fresh `task`; when it reports one, ask with `AskUserQuestion` exactly once before choosing — options `Continue current Codex thread` (use `task --resume-last`) and `Start a new Codex thread` (use a fresh `task`). Resuming silently would append this request to an unrelated earlier thread, so never pick `--resume-last` on your own. Pass `--model`, `--effort` and every `--config key=value` through unchanged. Never add `--write` unless the user explicitly asked Codex to modify files. - -2. Launch the job, then wait for it — two separate Bash calls, in ≤9-minute wait slices so the Bash tool's 10-minute cap never kills a long run. Bash calls share no variables — always set `JOB=` literally at the top of every later call; never rely on a `$JOB` left over from a previous call. - -The request prose and the runtime flags travel in two separate channels of the same Bash call: the prose is written byte-exact to `$PROMPT` by its own quoted heredoc and passed as `--prompt-file`, while the `--args-stdin` heredoc carries only the runtime flags (`--model`, `--effort`, `--config key=value`, `--resume-last`, `--write` as applicable). Never put the request text in the flags heredoc: it is tokenized, so quotes, backslashes and newlines in a stack trace, a regex or a code block would be mangled. Give both heredoc delimiters a fresh random suffix on every call — `CODEX_PROMPT_` / `CODEX_ARGS_`, e.g. 8 hex characters — and never reuse a suffix that appears in the request text: a payload line equal to the delimiter would end the heredoc early and run the rest on the host shell. - -2a. Launch (one Bash call): +Strip `--background`, `--wait`, `--resume`, and `--fresh` out of `` before the Bash call below — they are routing controls Claude Code consumes here, not `task` flags forwarded to the script. +1. If the request contains `--resume`, use `--resume-last`; if `--fresh`, use a fresh task. Otherwise run `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task-resume-candidate --json`: when it reports a resumable thread ask ONCE with `AskUserQuestion` — `Continue current Codex thread` (→ `--resume-last`) / `Start a new Codex thread` (→ fresh) — otherwise fresh. This decision is made here for BOTH the synchronous and the `--background` path. Pass `--model`, `--effort`, `--config key=value` through; never add `--write` unless the user explicitly asked Codex to modify files. +2. Synchronous path (default): ONE Bash call (`timeout: 600000`); flags on the command line, the request prose in a quoted heredoc whose delimiter is `CODEX_PROMPT_` + 8 fresh random hex chars that do not appear as an exact line in the request: ```bash -trap 'command rm -f -- "$ERR" "$OUT" "$PROMPT"' EXIT -ERR=$(mktemp); PROMPT=$(mktemp) -cat > "$PROMPT" <<'CODEX_PROMPT_' +node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --await --prompt-stdin <<'CODEX_PROMPT_' CODEX_PROMPT_ -JOB=$(node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --background --json --prompt-file "$PROMPT" --args-stdin <<'CODEX_ARGS_' 2>"$ERR" | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{process.stdout.write(JSON.parse(d).jobId||"")}catch(e){}})' - -CODEX_ARGS_ -) -[ -n "$JOB" ] || { cat "$ERR"; exit 1; } -[[ "$JOB" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "invalid job id"; exit 1; } -echo "JOB=$JOB" ``` -If this call exits non-zero, its output is the launch failure (Codex missing, unauthenticated, a bad flag, etc.) — show it to the user verbatim and stop; never report "no result". Otherwise its last line is `JOB=`; read `` from it. +Exit 0 → show the output verbatim, then your assessment. Exit 3 → the output ends with a `Re-run:` line — run exactly that line (again `timeout: 600000`) until it exits 0; its output is the final job record: if it shows the job completed, show the Codex result verbatim and add your assessment; if it shows failed or cancelled, show the output verbatim and stop. Exit 1 from the first call → the job failed or was cancelled: show the output verbatim and stop. +3. `--background`: invoke the `Agent` tool with `codex:codex-rescue`, passing the request minus `--background` PLUS the explicit `--resume-last` or `--fresh` decided in step 1; tell the user the result arrives as a completion notification and via `/codex:status` / `/codex:result `. -2b. Wait and fetch the result (one Bash call, tool `timeout: 600000`). Set `JOB=` literally as the first line, using the id you just read from 2a: - -```bash -trap 'command rm -f -- "$ERR" "$OUT" "$PROMPT"' EXIT -JOB= -[[ "$JOB" =~ ^[A-Za-z0-9_-]+$ ]] || { echo "invalid job id"; exit 1; } -OUT=$(mktemp); ERR=$(mktemp) -while node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" status "$JOB" --wait --timeout-ms 540000 --json >"$OUT" 2>"$ERR"; node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{const s=(JSON.parse(d).job||{}).status;process.exit(s==="queued"||s==="running"?3:(s?0:2))}catch(e){process.exit(2)}})' < "$OUT"; rc=$?; [ "$rc" -eq 3 ]; do sleep 1; done -[ "$rc" -eq 0 ] || { cat "$OUT" "$ERR"; exit "$rc"; } -node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" result "$JOB" -``` -The status check exits 3 while the job is still `queued`/`running` (loop — `sleep 1` keeps a fast exit-3 from spinning hot), 0 once the job reaches a terminal status, or 2 if the status output was empty or unparseable; either non-3 outcome ends the loop. If this Bash call is itself cut off by the tool's own timeout before the loop finishes, the job keeps running server-side — run 2b again with the same literal `JOB=` line. If it exits non-zero, show its output verbatim and stop; never report "no result". Otherwise show the `result` output to the user verbatim, then add your own assessment. +Never reuse a delimiter suffix that appears as an exact line in the request: a payload line equal to it would end the heredoc early and run the rest on the host shell. -3. Only when the request contains `--background`: invoke the `codex:codex-rescue` subagent via the `Agent` tool (`subagent_type: "codex:codex-rescue"`, prompt = the raw request minus `--background`) and tell the user the job id will arrive as a completion notification; they can also run `/codex:status` and `/codex:result `. +`` may contain only bare tokens — `--model `, `--effort `, `--turn-timeout-ms `, `--config key=value` with a literal value, `--resume-last`/`--fresh`, `--write`. If any flag value contains `$`, a backtick, a quote, `;`, `&`, `|`, or a newline, drop it and mention it in the prose instead; never place it on the command line. Do not call `Skill(codex:rescue)` from here (it re-enters this command). If any Bash step exits non-zero, show its stderr to the user — never report "no result". diff --git a/plugins/codex/commands/result.md b/plugins/codex/commands/result.md index 56d3858c3..f9fea37e6 100644 --- a/plugins/codex/commands/result.md +++ b/plugins/codex/commands/result.md @@ -1,6 +1,6 @@ --- description: Show the stored final output for a finished Codex job in this repository -argument-hint: '[job-id]' +argument-hint: '[job-id] [--wait] [--timeout-ms ]' disable-model-invocation: true allowed-tools: Bash(node:*) --- diff --git a/plugins/codex/commands/review.md b/plugins/codex/commands/review.md index b3ddddaf2..93f1af661 100644 --- a/plugins/codex/commands/review.md +++ b/plugins/codex/commands/review.md @@ -1,6 +1,6 @@ --- description: Run a Codex code review against local git state -argument-hint: '[--wait|--background] [--base ] [--scope auto|working-tree|branch] [--model ] [--effort ] [--config key=value]' +argument-hint: '[--wait|--background] [--base ] [--scope auto|working-tree|branch] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value]' disable-model-invocation: true allowed-tools: Read, Glob, Grep, Bash(node:*), Bash(git:*), AskUserQuestion --- diff --git a/plugins/codex/hooks/hooks.json b/plugins/codex/hooks/hooks.json index bd54ad05a..0c0e1a5f8 100644 --- a/plugins/codex/hooks/hooks.json +++ b/plugins/codex/hooks/hooks.json @@ -18,7 +18,7 @@ { "type": "command", "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/session-lifecycle-hook.mjs\" SessionEnd", - "timeout": 5 + "timeout": 15 } ] } diff --git a/plugins/codex/scripts/app-server-broker.mjs b/plugins/codex/scripts/app-server-broker.mjs index b4d0a661d..862ad68fc 100644 --- a/plugins/codex/scripts/app-server-broker.mjs +++ b/plugins/codex/scripts/app-server-broker.mjs @@ -21,6 +21,13 @@ const STREAMING_METHODS = new Set(["turn/start", "review/start", "thread/compact // and needs no PID/liveness signal, so it covers the abnormal-exit orphan, the // dead-co-owner orphan, and the lock-contention skip in one mechanism. See #108, // #380, and #450. +// How long a client is given to close its side once the broker has said goodbye. +// `socket.end()` is a graceful half-close, so a peer that never answers the FIN — +// one that is wedged, or one whose process is gone but whose close event has not +// been processed yet — leaves the connection open. Anything still open after this +// is closed outright. +const SHUTDOWN_SOCKET_GRACE_MS = 1000; + const IDLE_TIMEOUT_ENV = "CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; @@ -102,7 +109,11 @@ async function main() { let activeStreamThreadIds = null; const sockets = new Set(); let idleTimer = null; - let shuttingDown = false; + // One shutdown, one exit. Both are memoized: every trigger — either signal, the + // `broker/shutdown` RPC, the idle timeout — joins the same teardown and the + // process leaves only when that teardown is done. + let shutdownPromise = null; + let exitPromise = null; function disarmIdleTimer() { if (idleTimer) { @@ -122,9 +133,7 @@ async function main() { } idleTimer = setTimeout(() => { idleTimer = null; - shutdown(server) - .catch(() => {}) - .finally(() => process.exit(0)); + void shutdownAndExit(server); }, idleTimeoutMs); } @@ -177,19 +186,52 @@ async function main() { // first await instead of after it: a client accepted in that window would be // served the broker-local `initialize` and then fail its first real RPC with // "codex app-server client is closed", which callers do not retry. - async function shutdown(server) { - if (shuttingDown) { - return; + function shutdown(server) { + if (!shutdownPromise) { + shutdownPromise = runShutdown(server); } - shuttingDown = true; + return shutdownPromise; + } + + // The one place the process is allowed to leave from. A trigger that arrives + // while the teardown is running joins it instead of exiting out from under it — + // which used to orphan the app-server child and leave the endpoint, the pid file + // and the ownership record behind. + function shutdownAndExit(server) { + if (!exitPromise) { + exitPromise = shutdown(server) + .catch((error) => { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + }) + .then(() => process.exit(0)); + } + return exitPromise; + } + + async function runShutdown(server) { disarmIdleTimer(); - clearOwnSessionRecord(); const serverClosed = new Promise((resolve) => server.close(resolve)); for (const socket of sockets) { socket.end(); } await appClient.close().catch(() => {}); - await serverClosed; + // Never wait on a client indefinitely. SIGTERM is handled here, so a shutdown + // that does not return is a broker that ignores SIGTERM — a SessionEnd could + // signal it, get on with its teardown, and leave the process running forever. + let graceTimer = null; + await Promise.race([ + serverClosed, + new Promise((resolve) => { + graceTimer = setTimeout(resolve, SHUTDOWN_SOCKET_GRACE_MS); + }) + ]); + clearTimeout(graceTimer); + for (const socket of sockets) { + socket.destroy(); + } + // Last: stop advertising this broker. Doing it first would leave the record + // gone while the process is still serving its own teardown. + clearOwnSessionRecord(); if (listenTarget.kind === "unix" && fs.existsSync(listenTarget.path)) { fs.unlinkSync(listenTarget.path); } @@ -201,7 +243,7 @@ async function main() { appClient.setNotificationHandler(routeNotification); const server = net.createServer((socket) => { - if (shuttingDown) { + if (shutdownPromise) { // Already accepted before the listener finished closing: reset it so the // client retries or reports a connection error instead of half-working. socket.destroy(); @@ -250,9 +292,26 @@ async function main() { } if (message.id !== undefined && message.method === "broker/shutdown") { + // The caller (a SessionEnd hook) decides to shut the broker down from a + // snapshot of the job index, and another session can enqueue a job and + // connect in the gap between that snapshot and this request. Only the + // broker knows whether it is actually idle, so it refuses while anyone + // else is connected at all: a client is a client from the moment it is + // accepted, and `CodexAppServerClient` connects before it writes its + // first line. The requester's own connection is the one that does not + // count. Refusing is fail-safe and costs nothing that the idle timer + // does not already cost — that timer is likewise held off by any open + // socket, so a client that hangs up lets both mechanisms proceed. + const busy = + (activeRequestSocket && activeRequestSocket !== socket) || + (activeStreamSocket && activeStreamSocket !== socket) || + [...sockets].some((other) => other !== socket); + if (busy) { + send(socket, { id: message.id, result: { busy: true } }); + continue; + } send(socket, { id: message.id, result: {} }); - await shutdown(server); - process.exit(0); + await shutdownAndExit(server); } if (message.id === undefined) { @@ -327,14 +386,12 @@ async function main() { }); }); - process.on("SIGTERM", async () => { - await shutdown(server); - process.exit(0); + process.on("SIGTERM", () => { + void shutdownAndExit(server); }); - process.on("SIGINT", async () => { - await shutdown(server); - process.exit(0); + process.on("SIGINT", () => { + void shutdownAndExit(server); }); server.listen(listenTarget.path, () => { diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 2c00f5637..d3641f6d1 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -31,9 +31,14 @@ import { generateJobId, getConfig, listJobs, + removeJobPidFile, + redactConfigValues, removeJobRequestFile, + resolveJobPid, setConfig, + updateJobPid, upsertJob, + withStateLock, writeJobFile, writeJobRequestFile } from "./lib/state.mjs"; @@ -52,6 +57,8 @@ import { createJobRecord, createProgressReporter, nowIso, + reapDeadJobs, + registerWorkerCrashGuard, runTrackedJob, SESSION_ID_ENV } from "./lib/tracked-jobs.mjs"; @@ -68,8 +75,13 @@ import { } from "./lib/render.mjs"; const ROOT_DIR = path.resolve(fileURLToPath(new URL("..", import.meta.url))); +const COMPANION_SCRIPT = path.join(ROOT_DIR, "scripts", "codex-companion.mjs"); const REVIEW_SCHEMA = path.join(ROOT_DIR, "schemas", "review-output.schema.json"); const DEFAULT_STATUS_WAIT_TIMEOUT_MS = 240000; +// Claude Code kills a Bash tool call at 600000ms, so an awaited task has to +// hand control back before that with a resumable hint. +const DEFAULT_AWAIT_TIMEOUT_MS = 540000; +const DEFAULT_AWAIT_POLL_INTERVAL_MS = 1000; const DEFAULT_STATUS_POLL_INTERVAL_MS = 2000; const VALID_REASONING_EFFORTS = new Set([ "none", @@ -95,16 +107,22 @@ function printUsage() { [ "Usage:", " node scripts/codex-companion.mjs setup [--enable-review-gate|--disable-review-gate] [--json]", - " node scripts/codex-companion.mjs review [--wait|--background] [--base ] [--scope ] [--model ] [--effort ] [--config key=value]...", - " node scripts/codex-companion.mjs adversarial-review [--wait|--background] [--base ] [--scope ] [--model ] [--effort ] [--config key=value]... [focus text]", - " node scripts/codex-companion.mjs task [--background] [--write] [--resume-last|--resume|--fresh] [--model ] [--effort ] [--config key=value]... [prompt]", + " node scripts/codex-companion.mjs review [--wait|--background] [--base ] [--scope ] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value]...", + " node scripts/codex-companion.mjs adversarial-review [--wait|--background] [--base ] [--scope ] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value]... [focus text]", + " node scripts/codex-companion.mjs task [--background|--await [--await-timeout-ms ]] [--prompt-stdin] [--write] [--resume-last|--resume|--fresh] [--model ] [--effort ] [--turn-timeout-ms ] [--config key=value]... [prompt]", " node scripts/codex-companion.mjs transfer [--source ] [--json]", " node scripts/codex-companion.mjs status [job-id] [--all] [--json]", - " node scripts/codex-companion.mjs result [job-id] [--json]", + " node scripts/codex-companion.mjs result [job-id] [--wait [--timeout-ms ]] [--json]", " node scripts/codex-companion.mjs cancel [job-id] [--json]", "", "Any subcommand also accepts --args-stdin: the whole argument string is read", - "from stdin and tokenized here, so no shell ever sees the caller's text." + "from stdin and tokenized here, so no shell ever sees the caller's text.", + "`task --prompt-stdin` instead takes stdin verbatim as the prompt, so flags", + "must be on the command line and --args-stdin cannot be combined with it.", + "`task --await` and `result --wait` exit 3 with a re-run hint on timeout.", + "--turn-timeout-ms (or CODEX_TURN_TIMEOUT_MS) fails a single Codex turn after", + "that many ms — it interrupts the turn and returns a structured failed result;", + "unset means unbounded." ].join("\n") ); } @@ -167,10 +185,24 @@ function parseConfigOverrides(list = []) { // `--args-stdin` tokenizes it here with the same shell-like splitter // `normalizeArgv` already uses — never through a shell. const ARGS_STDIN_FLAG = "--args-stdin"; +const PROMPT_STDIN_FLAG = "--prompt-stdin"; let argvTokenizedFromStdin = false; function applyArgsStdin(argv) { const flagIndex = argv.indexOf(ARGS_STDIN_FLAG); + + // Decided before anything reads stdin: both flags consume it and it can only + // be read once. `--prompt-stdin` therefore has to be on the command line, and + // is never visible inside the `--args-stdin` heredoc. + if (argv.includes(PROMPT_STDIN_FLAG)) { + if (flagIndex !== -1) { + throw new Error( + `${PROMPT_STDIN_FLAG} cannot be combined with ${ARGS_STDIN_FLAG}; put flags on the command line.` + ); + } + return argv; + } + if (flagIndex === -1) { return argv; } @@ -402,10 +434,64 @@ async function waitForSingleJobSnapshot(cwd, reference, options = {}) { }; } +function buildResumeWaitCommand(jobId) { + return `node "${COMPANION_SCRIPT}" result ${jobId} --wait --timeout-ms ${DEFAULT_AWAIT_TIMEOUT_MS}`; +} + +// Every "the job outlived this command" exit looks the same: the lead-in, the +// exact command that resumes the wait, and exit code 3. +function outputActiveJobHint(snapshot, leadIn, asJson) { + const resumeCommand = buildResumeWaitCommand(snapshot.job.id); + outputCommandResult({ ...snapshot, resumeCommand }, `${leadIn} Re-run: ${resumeCommand}\n`, asJson); + process.exitCode = 3; +} + +function parseTimeoutOption(value, flag) { + if (value == null) { + return null; + } + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`${flag} expects a positive integer number of milliseconds, got "${value}".`); + } + return parsed; +} + +// Waits for a job to reach a terminal status and returns its id. On timeout it +// prints the re-run hint, sets exit code 3 and returns null, so the caller +// (`task --await`, `result --wait`) hands control back before Claude Code's +// Bash timeout kills it mid-run. +async function waitForTerminalJobOrHint(cwd, reference, options = {}) { + const snapshot = await waitForSingleJobSnapshot(cwd, reference, { + timeoutMs: Number(options.timeoutMs ?? DEFAULT_AWAIT_TIMEOUT_MS), + pollIntervalMs: DEFAULT_AWAIT_POLL_INTERVAL_MS + }); + if (!snapshot.waitTimedOut) { + return snapshot.job.id; + } + + outputActiveJobHint(snapshot, `Still running: job ${snapshot.job.id}.`, options.json); + return null; +} + +// Prints exactly what `result ` prints — the awaited task path reuses +// it so both commands stay on one rendering — and returns the resolved job. +function outputJobResult(cwd, reference, asJson) { + const { workspaceRoot, job } = resolveResultJob(cwd, reference); + if (isActiveJobStatus(job.status)) { + outputActiveJobHint(buildSingleJobSnapshot(cwd, job.id), `Job ${job.id} is still ${job.status}.`, asJson); + return job; + } + + const storedJob = readStoredJob(workspaceRoot, job.id); + outputCommandResult({ job, storedJob }, renderStoredJobResult(job, storedJob), asJson); + return job; +} + async function resolveLatestTrackedTaskThread(cwd, options = {}) { const workspaceRoot = resolveWorkspaceRoot(cwd); const sessionId = getCurrentClaudeSessionId(); - const jobs = sortJobsNewestFirst(listJobs(workspaceRoot)).filter((job) => job.id !== options.excludeJobId); + const jobs = sortJobsNewestFirst(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot))).filter((job) => job.id !== options.excludeJobId); const visibleJobs = filterJobsForCurrentClaudeSession(jobs); const activeTask = visibleJobs.find((job) => job.jobClass === "task" && (job.status === "queued" || job.status === "running")); if (activeTask) { @@ -441,6 +527,7 @@ async function executeReviewRun(request) { model: request.model, effort: request.effort, config: request.config, + turnTimeoutMs: request.turnTimeoutMs, onProgress: request.onProgress }); const payload = { @@ -471,6 +558,7 @@ async function executeReviewRun(request) { resolved: result.resolved, payload, rendered, + errorMessage: result.error?.message ?? null, summary: firstMeaningfulLine(result.reviewText, `${reviewName} completed.`), jobTitle: `Codex ${reviewName}`, jobClass: "review", @@ -487,6 +575,7 @@ async function executeReviewRun(request) { config: request.config, sandbox: "read-only", outputSchema: readOutputSchema(REVIEW_SCHEMA), + turnTimeoutMs: request.turnTimeoutMs, onProgress: request.onProgress }); const parsed = parseStructuredOutput(result.finalMessage, { @@ -525,6 +614,7 @@ async function executeReviewRun(request) { targetLabel: context.target.label, reasoningSummary: result.reasoningSummary }), + errorMessage: result.error?.message ?? null, summary: parsed.parsed?.summary ?? parsed.parseError ?? firstMeaningfulLine(result.finalMessage, `${reviewName} finished.`), jobTitle: `Codex ${reviewName}`, jobClass: "review", @@ -561,12 +651,14 @@ async function executeTaskRun(request) { resumeThreadId, excludeJobId: request.jobId, prompt: request.prompt, + promptRaw: request.promptRaw, defaultPrompt: resumeThreadId ? DEFAULT_CONTINUE_PROMPT : "", model: request.model, effort: request.effort, config: request.config, approvalPolicy: request.write ? "on-request" : "never", sandbox: request.write ? "workspace-write" : "read-only", + turnTimeoutMs: request.turnTimeoutMs, onProgress: request.onProgress, persistThread: true, threadName: resumeThreadId ? null : buildPersistentTaskThreadName(request.prompt || DEFAULT_CONTINUE_PROMPT) @@ -601,6 +693,7 @@ async function executeTaskRun(request) { resolved: result.resolved, payload, rendered, + errorMessage: failureMessage || null, summary: firstMeaningfulLine(rawOutput, firstMeaningfulLine(failureMessage, `${taskMetadata.title} finished.`)), jobTitle: taskMetadata.title, jobClass: "task", @@ -643,7 +736,7 @@ function getJobKindLabel(kind, jobClass) { return jobClass === "review" ? "review" : "rescue"; } -function createCompanionJob({ prefix, kind, title, workspaceRoot, jobClass, summary, write = false }) { +function createCompanionJob({ prefix, kind, title, workspaceRoot, jobClass, summary, write = false, background = false }) { return createJobRecord({ id: generateJobId(prefix), kind, @@ -652,7 +745,10 @@ function createCompanionJob({ prefix, kind, title, workspaceRoot, jobClass, summ workspaceRoot, jobClass, summary, - write + write, + // Marks a job SessionEnd must leave alone — and whose record it must keep, + // so `result` still works after the dispatching session is gone. + ...(background ? { background: true } : {}) }); } @@ -680,15 +776,19 @@ function buildTaskJob(workspaceRoot, taskMetadata, write) { }); } -function buildTaskRequest({ cwd, model, effort, config, prompt, write, resumeLast, jobId }) { +function buildTaskRequest({ cwd, model, effort, config, prompt, promptRaw, write, resumeLast, turnTimeoutMs, jobId }) { return { cwd, model, effort, config, prompt, + promptRaw, write, resumeLast, + // Persisted so the detached worker runs under the same budget: it is a + // separate process and never sees this command's flags. + turnTimeoutMs, jobId }; } @@ -721,6 +821,19 @@ async function executeTransfer(cwd, options = {}) { } function readTaskPrompt(cwd, options, positionals) { + if (options["prompt-stdin"]) { + if (options["prompt-file"] || positionals.length > 0) { + throw new Error(`${PROMPT_STDIN_FLAG} cannot be combined with --prompt-file or prompt text.`); + } + // Raw bytes, no tokenization: only the one trailing newline the caller's + // heredoc adds is removed, so indentation and blank lines survive. + const prompt = readStdinIfPiped().replace(/\r?\n$/, ""); + if (!prompt.trim()) { + throw new Error(`${PROMPT_STDIN_FLAG} was set but stdin was empty.`); + } + return prompt; + } + if (options["prompt-file"]) { return fs.readFileSync(path.resolve(cwd, options["prompt-file"]), "utf8"); } @@ -749,8 +862,7 @@ async function runForegroundCommand(job, runner, options = {}) { } function spawnDetachedTaskWorker(cwd, jobId) { - const scriptPath = path.join(ROOT_DIR, "scripts", "codex-companion.mjs"); - const child = spawn(process.execPath, [scriptPath, "task-worker", "--cwd", cwd, "--job-id", jobId], { + const child = spawn(process.execPath, [COMPANION_SCRIPT, "task-worker", "--cwd", cwd, "--job-id", jobId], { cwd, env: process.env, detached: true, @@ -761,21 +873,6 @@ function spawnDetachedTaskWorker(cwd, jobId) { return child; } -const PRIVATE_CONFIG_KEY_PATTERN = /key|token|secret|auth|password/i; - -// `status --json` / `result --json` echo the stored job record back to the user -// and to Claude, so a `--config model_providers.x.http_headers.Authorization=...` -// would end up in the transcript. The worker reads the real values from the -// private one-shot payload file; the record keeps only a redacted copy. -function redactPrivateConfigValues(config) { - if (!config || typeof config !== "object") { - return config; - } - return Object.fromEntries( - Object.entries(config).map(([key, value]) => [key, PRIVATE_CONFIG_KEY_PATTERN.test(key) ? "[redacted]" : value]) - ); -} - function enqueueBackgroundTask(cwd, job, request) { const { logFile } = createTrackedProgress(job); appendLogLine(logFile, "Queued for background execution."); @@ -787,10 +884,14 @@ function enqueueBackgroundTask(cwd, job, request) { ...job, status: "queued", phase: "queued", + // Marks the job as one SessionEnd must leave running (#355). The pid is null + // here because this record is written BEFORE the spawn; `updateJobPid` + // patches the real one in as soon as the worker exists. + background: true, pid: null, logFile, requestFile, - request: { ...request, config: redactPrivateConfigValues(request.config) } + request: { ...request, config: redactConfigValues(request.config) } }; writeJobFile(job.workspaceRoot, job.id, queuedRecord); upsertJob(job.workspaceRoot, queuedRecord); @@ -812,10 +913,12 @@ function enqueueBackgroundTask(cwd, job, request) { throw error; } - // Nothing writes this record from here on: the worker owns it from the moment - // it starts, and `runTrackedJob` stores its own pid (the same `child.pid`) as - // its first act. A post-spawn patch from the parent would race the worker's - // own `upsertJob` and could rewind `running` back to `queued`. + // The record was written before the spawn, so this is the first moment the + // worker's pid exists. `updateJobPid` never touches the job file — the worker + // owns it — it writes an atomic `jobs/.pid` sidecar plus a pid-only index + // patch. Without it a `cancel` inside the queued window signals nothing and + // the reaper cannot tell a dead queued worker from a live one. + updateJobPid(job.workspaceRoot, job.id, child.pid); return { payload: { @@ -831,7 +934,7 @@ function enqueueBackgroundTask(cwd, job, request) { async function handleReviewCommand(argv, config) { const { options, positionals } = parseCommandInput(argv, { - valueOptions: ["base", "scope", "model", "effort", "cwd"], + valueOptions: ["base", "scope", "model", "effort", "cwd", "turn-timeout-ms"], booleanOptions: ["json", "background", "wait"], repeatableOptions: ["config"], // Only the adversarial variant takes free-form focus text; stop option @@ -850,6 +953,7 @@ async function handleReviewCommand(argv, config) { const model = normalizeRequestedModel(options.model); const effort = normalizeReasoningEffort(options.effort); const configOverrides = parseConfigOverrides(options.config); + const turnTimeoutMs = parseTimeoutOption(options["turn-timeout-ms"], "--turn-timeout-ms"); const focusText = positionals.join(" ").trim(); const target = resolveReviewTarget(cwd, { base: options.base, @@ -864,7 +968,10 @@ async function handleReviewCommand(argv, config) { title: metadata.title, workspaceRoot, jobClass: "review", - summary: metadata.summary + summary: metadata.summary, + // A `--background` review is dispatched (under `nohup`/`&`) to outlive the + // session that started it, so its record has to outlive it too. + background: Boolean(options.background) }); await runForegroundCommand( job, @@ -878,6 +985,7 @@ async function handleReviewCommand(argv, config) { config: configOverrides, focusText, reviewName: config.reviewName, + turnTimeoutMs, onProgress: progress }), { json: options.json } @@ -893,8 +1001,8 @@ async function handleReview(argv) { async function handleTask(argv) { const { options, positionals } = parseCommandInput(argv, { - valueOptions: ["model", "effort", "cwd", "prompt-file"], - booleanOptions: ["json", "write", "resume-last", "resume", "fresh", "background"], + valueOptions: ["model", "effort", "cwd", "prompt-file", "await-timeout-ms", "turn-timeout-ms"], + booleanOptions: ["json", "write", "resume-last", "resume", "fresh", "background", "await", "prompt-stdin"], repeatableOptions: ["config"], stopAtFirstPositional: true, aliasMap: { @@ -910,20 +1018,36 @@ async function handleTask(argv) { const model = normalizeRequestedModel(options.model); const effort = normalizeReasoningEffort(options.effort); const configOverrides = parseConfigOverrides(options.config); - const prompt = readTaskPrompt(cwd, options, positionals); - + // Every flag conflict is decided before the prompt is read: `--prompt-stdin` + // blocks on an open stdin, so a usage error must never wait for EOF. const resumeLast = Boolean(options["resume-last"] || options.resume); const fresh = Boolean(options.fresh); if (resumeLast && fresh) { throw new Error("Choose either --resume/--resume-last or --fresh."); } + if (options.await && options.background) { + throw new Error("Choose either --await or --background."); + } + if (options["await-timeout-ms"] != null && !options.await) { + throw new Error("--await-timeout-ms requires --await."); + } + const awaitTimeoutMs = parseTimeoutOption(options["await-timeout-ms"], "--await-timeout-ms"); + const turnTimeoutMs = parseTimeoutOption(options["turn-timeout-ms"], "--turn-timeout-ms"); + + const prompt = readTaskPrompt(cwd, options, positionals); + // A `--prompt-stdin` prompt is already exactly what the caller typed; nothing + // downstream may trim it further. + const promptRaw = Boolean(options["prompt-stdin"]); const write = Boolean(options.write); const taskMetadata = buildTaskRunMetadata({ prompt, resumeLast }); - if (options.background) { + // `--await` runs the same detached worker as `--background` — same job + // record, so status/result/cancel work on it — and only differs in waiting for + // it here instead of returning the queued line. + if (options.background || options.await) { ensureCodexAvailable(cwd); requireTaskRequest(prompt, resumeLast); @@ -934,12 +1058,25 @@ async function handleTask(argv) { effort, config: configOverrides, prompt, + promptRaw, write, resumeLast, + turnTimeoutMs, jobId: job.id }); const { payload } = enqueueBackgroundTask(cwd, job, request); - outputCommandResult(payload, renderQueuedTaskLaunch(payload), options.json); + if (!options.await) { + outputCommandResult(payload, renderQueuedTaskLaunch(payload), options.json); + return; + } + + const jobId = await waitForTerminalJobOrHint(cwd, job.id, { + timeoutMs: awaitTimeoutMs, + json: options.json + }); + if (jobId && outputJobResult(cwd, jobId, options.json).status !== "completed") { + process.exitCode = 1; + } return; } @@ -953,8 +1090,10 @@ async function handleTask(argv) { effort, config: configOverrides, prompt, + promptRaw, write, resumeLast, + turnTimeoutMs, jobId: job.id, onProgress: progress }), @@ -1010,6 +1149,7 @@ async function handleTaskWorker(argv) { logFile: storedJob.logFile ?? null } ); + registerWorkerCrashGuard(workspaceRoot, options["job-id"], logFile); await runTrackedJob( { ...storedJob, @@ -1055,25 +1195,35 @@ async function handleStatus(argv) { outputResult(renderStatusPayload(report, options.json), options.json); } -function handleResult(argv) { +async function handleResult(argv) { const { options, positionals } = parseCommandInput(argv, { - valueOptions: ["cwd"], - booleanOptions: ["json"] + valueOptions: ["cwd", "timeout-ms"], + booleanOptions: ["json", "wait"] }); if (maybePrintCommandHelp(options)) { return; } const cwd = resolveCommandCwd(options); - const reference = positionals[0] ?? ""; - const { workspaceRoot, job } = resolveResultJob(cwd, reference); - const storedJob = readStoredJob(workspaceRoot, job.id); - const payload = { - job, - storedJob - }; + if (options["timeout-ms"] != null && !options.wait) { + throw new Error("--timeout-ms requires --wait."); + } + let reference = positionals[0] ?? ""; + if (options.wait) { + if (!reference) { + throw new Error("`result --wait` requires a job id."); + } + const jobId = await waitForTerminalJobOrHint(cwd, reference, { + timeoutMs: parseTimeoutOption(options["timeout-ms"], "--timeout-ms"), + json: options.json + }); + if (!jobId) { + return; + } + reference = jobId; + } - outputCommandResult(payload, renderStoredJobResult(job, storedJob), options.json); + outputJobResult(cwd, reference, options.json); } function handleTaskResumeCandidate(argv) { @@ -1088,7 +1238,7 @@ function handleTaskResumeCandidate(argv) { const cwd = resolveCommandCwd(options); const workspaceRoot = resolveCommandWorkspace(options); const sessionId = getCurrentClaudeSessionId(); - const jobs = filterJobsForCurrentClaudeSession(sortJobsNewestFirst(listJobs(workspaceRoot))); + const jobs = filterJobsForCurrentClaudeSession(sortJobsNewestFirst(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot)))); const candidate = findLatestResumableTaskJob(jobs); const payload = { @@ -1140,7 +1290,7 @@ async function handleCancel(argv) { ); } - terminateProcessTree(job.pid ?? Number.NaN); + terminateProcessTree(resolveJobPid(workspaceRoot, job) ?? Number.NaN); appendLogLine(job.logFile, "Cancelled by user."); const completedAt = nowIso(); @@ -1149,22 +1299,36 @@ async function handleCancel(argv) { status: "cancelled", phase: "cancelled", pid: null, + requestFile: null, completedAt, errorMessage: "Cancelled by user." }; - writeJobFile(workspaceRoot, job.id, { - ...existing, - ...nextJob, - cancelledAt: completedAt - }); - upsertJob(workspaceRoot, { - id: job.id, - status: "cancelled", - phase: "cancelled", - pid: null, - errorMessage: "Cancelled by user.", - completedAt + // Deleting the artifacts and writing the terminal record is one step: another + // process's `saveState` prune works off a diff of the index, so a cancel split + // across that write can have the record it just wrote pruned away — or the + // payload it just deleted counted as still owned. + withStateLock(workspaceRoot, () => { + // A worker cancelled inside the queued window may never have consumed its + // private payload, and a cancelled job is terminal — the reaper will never + // look at it again — so the 0600 file (possibly holding `--config` secrets) + // has to be released here. + removeJobRequestFile(workspaceRoot, job.id); + removeJobPidFile(workspaceRoot, job.id); + writeJobFile(workspaceRoot, job.id, { + ...(readStoredJob(workspaceRoot, job.id) ?? existing), + ...nextJob, + cancelledAt: completedAt + }); + upsertJob(workspaceRoot, { + id: job.id, + status: "cancelled", + phase: "cancelled", + pid: null, + requestFile: null, + errorMessage: "Cancelled by user.", + completedAt + }); }); const payload = { @@ -1213,7 +1377,7 @@ async function main() { await handleStatus(argv); break; case "result": - handleResult(argv); + await handleResult(argv); break; case "task-resume-candidate": handleTaskResumeCandidate(argv); diff --git a/plugins/codex/scripts/lib/app-server.mjs b/plugins/codex/scripts/lib/app-server.mjs index c5c65de46..025bdf969 100644 --- a/plugins/codex/scripts/lib/app-server.mjs +++ b/plugins/codex/scripts/lib/app-server.mjs @@ -222,6 +222,15 @@ export class AppServerClientBase { } } +// Closing a direct app-server is a teardown step, not a negotiation: the turn +// timeout calls it precisely because the child is misbehaving. Ask (stdin EOF), +// then tell (SIGTERM), then insist (SIGKILL), and in the worst case return to the +// caller anyway — a leaked child process is a smaller problem than a companion +// that never finishes writing the job record. +const CLOSE_TERM_MS = 50; +const CLOSE_KILL_MS = 2000; +const CLOSE_DEADLINE_MS = 5000; + class SpawnedCodexAppServerClient extends AppServerClientBase { constructor(cwd, options = {}) { super(cwd, options); @@ -271,40 +280,63 @@ class SpawnedCodexAppServerClient extends AppServerClientBase { this.notify("initialized", {}); } - async close() { - if (this.closed) { - await this.exitPromise; + // On Windows with shell: true the direct child is cmd.exe, so the whole tree + // has to go — `taskkill /T /F` is the only escalation available there. + terminateChild(signal) { + if (!this.proc || this.proc.exitCode !== null || this.proc.signalCode !== null) { return; } + try { + if (process.platform === "win32") { + terminateProcessTree(this.proc.pid); + } else { + this.proc.kill(signal); + } + } catch { + // Best-effort teardown: never throw on the way out. + } + } + // One bounded close per client, memoized: a second call must return the first + // call's outcome, never fall through to an unbounded wait on a process that may + // have outlived the deadline. The timeout path closes twice by design. + close() { + if (!this.closePromise) { + this.closePromise = this.closeOnce(); + } + return this.closePromise; + } + + async closeOnce() { this.closed = true; if (this.readline) { this.readline.close(); } - if (this.proc && !this.proc.killed) { + const timers = []; + if (this.proc && this.proc.exitCode === null && this.proc.signalCode === null) { this.proc.stdin.end(); - setTimeout(() => { - if (this.proc && !this.proc.killed && this.proc.exitCode === null) { - // On Windows with shell: true, the direct child is cmd.exe. - // Use terminateProcessTree to kill the entire tree including - // the grandchild node process. - if (process.platform === "win32") { - try { - terminateProcessTree(this.proc.pid); - } catch { - // Best-effort cleanup inside an unref'd timer — swallow errors - // to avoid crashing the host process during shutdown. - } - } else { - this.proc.kill("SIGTERM"); - } - } - }, 50).unref?.(); + timers.push(setTimeout(() => this.terminateChild("SIGTERM"), CLOSE_TERM_MS)); + timers.push(setTimeout(() => this.terminateChild("SIGKILL"), CLOSE_KILL_MS)); } - await this.exitPromise; + let deadlineTimer = null; + try { + await Promise.race([ + this.exitPromise, + new Promise((resolve) => { + deadlineTimer = setTimeout(resolve, CLOSE_DEADLINE_MS); + }) + ]); + } finally { + for (const timer of timers) { + clearTimeout(timer); + } + if (deadlineTimer) { + clearTimeout(deadlineTimer); + } + } } sendMessage(message) { @@ -374,6 +406,15 @@ class BrokerCodexAppServerClient extends AppServerClientBase { } } +// The connection went away underneath us: reset by the peer, a write to a closed +// pipe, or our own "connection closed" error, which carries no errno at all. +function isConnectionDropped(error) { + if (error?.code) { + return error.code === "ECONNRESET" || error.code === "EPIPE"; + } + return /connection closed/i.test(error?.message ?? ""); +} + export class CodexAppServerClient { static async connect(cwd, options = {}) { let brokerEndpoint = null; @@ -387,10 +428,27 @@ export class CodexAppServerClient { brokerEndpoint = brokerSession?.endpoint ?? null; } } - const client = brokerEndpoint - ? new BrokerCodexAppServerClient(cwd, { ...options, brokerEndpoint }) - : new SpawnedCodexAppServerClient(cwd, options); - await client.initialize(); - return client; + if (!brokerEndpoint) { + const direct = new SpawnedCodexAppServerClient(cwd, options); + await direct.initialize(); + return direct; + } + + const client = new BrokerCodexAppServerClient(cwd, { ...options, brokerEndpoint }); + try { + await client.initialize(); + return client; + } catch (error) { + // A broker that is shutting down accepts the connection and drops it before + // answering `initialize`. That is a race with someone else's SessionEnd, not + // a reason to fail a background job — run on our own app-server instead. + if (!isConnectionDropped(error)) { + throw error; + } + await client.close().catch(() => {}); + const direct = new SpawnedCodexAppServerClient(cwd, options); + await direct.initialize(); + return direct; + } } } diff --git a/plugins/codex/scripts/lib/broker-lifecycle.mjs b/plugins/codex/scripts/lib/broker-lifecycle.mjs index bb51f5d91..11092dcd1 100644 --- a/plugins/codex/scripts/lib/broker-lifecycle.mjs +++ b/plugins/codex/scripts/lib/broker-lifecycle.mjs @@ -27,10 +27,15 @@ export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { while (Date.now() - start < timeoutMs) { const ready = await new Promise((resolve) => { const socket = connectToEndpoint(endpoint); + let connected = false; socket.on("connect", () => { + connected = true; socket.end(); - resolve(true); }); + // Report ready only once the probe connection is fully closed. A probe the + // broker still sees as open is a phantom client: it holds off the idle + // timer and makes the broker refuse a shutdown. + socket.on("close", () => resolve(connected)); socket.on("error", () => resolve(false)); }); if (ready) { @@ -41,19 +46,63 @@ export async function waitForBrokerEndpoint(endpoint, timeoutMs = 2000) { return false; } -export async function sendBrokerShutdown(endpoint) { - await new Promise((resolve) => { +const SHUTDOWN_REQUEST_ID = 1; +// A broker that has not answered by now is not going to: the handshake is one +// round trip to a local socket. Without this bound a peer that connects and +// stays silent blocks the SessionEnd hook forever. +const SHUTDOWN_HANDSHAKE_MS = 5000; + +// Three answers, not two: +// `false` — the broker said it is idle, or it is gone (connection error, or the +// socket closed without answering): the caller must clean up after it. +// `true` — the broker refused because another client is still using it. +// `null` — nothing usable came back before the deadline, or the reply was not +// parsable JSONL. The caller cannot prove the broker is idle, so it +// must leave it alone. +// The reply is framed by newline and matched by request id: a socket is a byte +// stream, and parsing whatever a single `data` event happened to carry turned a +// split `{"busy":true}` into "not busy" — a broker destroyed under a live turn. +export async function sendBrokerShutdown(endpoint, { timeoutMs = SHUTDOWN_HANDSHAKE_MS } = {}) { + return await new Promise((resolve) => { const socket = connectToEndpoint(endpoint); socket.setEncoding("utf8"); + let buffer = ""; + const finish = (busy) => { + clearTimeout(deadline); + socket.destroy(); + resolve({ busy }); + }; + const deadline = setTimeout(() => finish(null), timeoutMs); + socket.on("connect", () => { - socket.write(`${JSON.stringify({ id: 1, method: "broker/shutdown", params: {} })}\n`); + socket.write(`${JSON.stringify({ id: SHUTDOWN_REQUEST_ID, method: "broker/shutdown", params: {} })}\n`); }); - socket.on("data", () => { - socket.end(); - resolve(); + socket.on("data", (chunk) => { + buffer += chunk; + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + newlineIndex = buffer.indexOf("\n"); + if (!line.trim()) { + continue; + } + let message; + try { + message = JSON.parse(line); + } catch { + finish(null); + return; + } + if (message.id !== SHUTDOWN_REQUEST_ID) { + continue; + } + finish(message.result?.busy === true); + return; + } }); - socket.on("error", resolve); - socket.on("close", resolve); + socket.on("error", () => finish(false)); + socket.on("close", () => finish(false)); }); } @@ -176,21 +225,26 @@ export async function ensureBrokerSession(cwd, options = {}) { // record behind long enough for the OS to hand the PID — and with it the process // group `terminateProcessTree` kills — to something unrelated. Windows has no // cheap equivalent probe, so it keeps the previous unconditional behavior. -function ownsBrokerProcess(pid, endpoint) { +function ownsBrokerProcess(pid, endpoint, timeoutMs) { if (process.platform === "win32") { return true; } - const commandLine = processCommandLine(pid); + const commandLine = processCommandLine(pid, { timeoutMs }); if (!commandLine || !commandLine.includes("app-server-broker.mjs")) { return false; } return !endpoint || commandLine.includes(endpoint); } -export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null }) { - if (Number.isFinite(pid) && killProcess && ownsBrokerProcess(pid, endpoint)) { +// Reports whether the recorded process was actually signalled: a PID that no +// longer looks like this broker is deliberately left alone, and a caller that +// wonders why a broker outlived its teardown needs to know which it was. +export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessionDir = null, pid = null, killProcess = null, timeoutMs = undefined }) { + let signalled = false; + if (Number.isFinite(pid) && killProcess && ownsBrokerProcess(pid, endpoint, timeoutMs)) { try { killProcess(pid); + signalled = true; } catch { // Ignore missing or already-exited broker processes. } @@ -223,4 +277,6 @@ export function teardownBrokerSession({ endpoint = null, pidFile, logFile, sessi // Ignore non-empty or missing directories. } } + + return { signalled }; } diff --git a/plugins/codex/scripts/lib/codex.mjs b/plugins/codex/scripts/lib/codex.mjs index 17750584b..f2a4a3775 100644 --- a/plugins/codex/scripts/lib/codex.mjs +++ b/plugins/codex/scripts/lib/codex.mjs @@ -45,6 +45,7 @@ import { BROKER_BUSY_RPC_CODE, BROKER_ENDPOINT_ENV, CodexAppServerClient } from import { loadBrokerSession } from "./broker-lifecycle.mjs"; import { binaryAvailable } from "./process.mjs"; import { listJobs } from "./state.mjs"; +import { reapDeadJobs } from "./tracked-jobs.mjs"; const SERVICE_NAME = "claude_code_codex_plugin"; const TASK_THREAD_PREFIX = "Codex Companion Task"; @@ -609,9 +610,112 @@ function applyTurnNotification(state, message) { } } +export const TURN_TIMEOUT_ENV = "CODEX_TURN_TIMEOUT_MS"; + +// Client-side per-turn budget: `--turn-timeout-ms` > `CODEX_TURN_TIMEOUT_MS` > +// unbounded. `TurnStartParams` has no timeout field, so this never rides on the +// app-server RPC. Read at call time, not import time, so a value the companion +// sets after this module was loaded still takes effect. +export function resolveTurnTimeoutMs(value = null) { + const fromOption = Number(value); + if (Number.isFinite(fromOption) && fromOption > 0) { + return fromOption; + } + const fromEnv = Number(process.env[TURN_TIMEOUT_ENV]); + return Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : 0; +} + +// How long the interrupt may take before the turn is failed anyway: the app +// server is already misbehaving, so this must never become a second hang. +const TURN_INTERRUPT_GRACE_MS = 2000; + +// How long the turn's own terminal notification (`turn/completed`, whatever +// status it carries) may take after the interrupt. Answering `turn/interrupt` +// proves nothing: only that notification proves the runtime stopped the turn. +const TURN_INTERRUPT_ACK_MS = 10000; + +// Resolves true once the turn reached a terminal notification (which is what +// runs `completeTurn`), false if the runtime stayed silent for the whole window +// or died without sending one. +function waitForTurnAcknowledgement(client, state, timeoutMs) { + if (state.completed) { + return Promise.resolve(true); + } + let timer = null; + return Promise.race([ + state.completion.then(() => true), + // A transport that is gone is an answer too: no notification can follow it, + // so waiting out the rest of the window would only delay the terminal record. + client.exitPromise.then(() => state.completed), + new Promise((resolve) => { + // Deliberately referenced: this timer is the only thing keeping the process + // alive once a dead transport has released its handles, and the job still + // has to be written terminal before we exit. + timer = setTimeout(() => resolve(false), timeoutMs); + }) + ]).finally(() => { + if (timer) { + clearTimeout(timer); + } + }); +} + +// A timed-out turn is resolved as a normal failed turn — never thrown — so the +// caller still gets its partial output, the job record still gets a result, and +// the message reaches stdout instead of only stderr. +async function failTurnOnTimeout(client, state, timeoutMs) { + if (state.completed) { + return; + } + const timeoutMessage = `turn timed out after ${timeoutMs} ms`; + state.error = { message: timeoutMessage }; + emitProgress(state.onProgress, `Turn timed out after ${timeoutMs} ms; interrupting.`, "failed"); + + if (state.turnId) { + let graceTimer = null; + try { + await Promise.race([ + client.request("turn/interrupt", { threadId: state.threadId, turnId: state.turnId }), + new Promise((resolve) => { + graceTimer = setTimeout(resolve, TURN_INTERRUPT_GRACE_MS); + graceTimer.unref?.(); + }) + ]); + } catch { + // The turn is being abandoned either way. + } finally { + if (graceTimer) { + clearTimeout(graceTimer); + } + } + } + + // Wait for the turn to actually end. With no turnId there was nothing to + // interrupt (and notifications are still buffered), so this window only ever + // expires — the report then says the turn may still be running, which is the + // truth. `completeTurn` already ran if the notification arrived. + if (await waitForTurnAcknowledgement(client, state, TURN_INTERRUPT_ACK_MS)) { + return; + } + + state.error = { + message: `${timeoutMessage}; interrupt not acknowledged — the turn may still be running in the shared runtime, check status or cancel` + }; + // A client that owns its app-server can still stop the turn: killing the + // process takes the turn with it. A broker's app-server is shared, so closing + // this connection would only hide a turn that keeps writing. + if (client.transport !== "broker") { + await client.close().catch(() => {}); + } + + completeTurn(state, { id: state.turnId ?? "timed-out-turn", status: "failed" }); +} + async function captureTurn(client, threadId, startRequest, options = {}) { const state = createTurnCaptureState(threadId, options); const previousHandler = client.notificationHandler; + const timeoutMs = resolveTurnTimeoutMs(options.turnTimeoutMs); + let timeoutTimer = null; client.setNotificationHandler((message) => { if (!state.turnId) { @@ -654,10 +758,20 @@ async function captureTurn(client, threadId, startRequest, options = {}) { if (response.turn?.status && response.turn.status !== "inProgress") { completeTurn(state, response.turn); + } else if (timeoutMs > 0) { + // Armed only once the turn is actually running: before that there is no + // turnId to interrupt, and `startRequest` has its own failure paths. + timeoutTimer = setTimeout(() => { + void failTurnOnTimeout(client, state, timeoutMs); + }, timeoutMs); + timeoutTimer.unref?.(); } return await state.completion; } finally { + if (timeoutTimer) { + clearTimeout(timeoutTimer); + } clearCompletionTimer(state); client.setNotificationHandler(previousHandler ?? null); } @@ -1093,6 +1207,7 @@ export async function runAppServerReview(cwd, options = {}) { }), { onProgress: options.onProgress, + turnTimeoutMs: options.turnTimeoutMs, onResponse(response, state) { if (response.reviewThreadId) { state.threadIds.add(response.reviewThreadId); @@ -1159,8 +1274,11 @@ export async function importExternalAgentSession(cwd, options = {}) { // Two concurrent turns on one thread interleave their history. The // session-scoped resume-candidate lookup cannot see jobs from other Claude // sessions, so the thread itself is checked here, where every resume passes. +// Reaped, not raw: a job whose worker is gone — or whose terminal record only +// reached its job file — is not holding the thread, and its stale index entry +// would block the resume forever. function assertThreadIsFree(cwd, threadId, excludeJobId = null) { - const busy = listJobs(cwd).find( + const busy = reapDeadJobs(cwd, listJobs(cwd)).find( (job) => job.id !== excludeJobId && job.threadId === threadId && @@ -1217,7 +1335,10 @@ export async function runAppServerTurn(cwd, options = {}) { resolved }); - const prompt = options.prompt?.trim() || options.defaultPrompt || ""; + // `promptRaw` marks a prompt the caller already normalised byte for byte + // (`task --prompt-stdin`): trimming it here would eat indentation the user + // typed on purpose. + const prompt = (options.promptRaw ? options.prompt : options.prompt?.trim()) || options.defaultPrompt || ""; if (!prompt) { throw new Error("A prompt is required for this Codex run."); } @@ -1235,6 +1356,7 @@ export async function runAppServerTurn(cwd, options = {}) { }), { onProgress: options.onProgress, + turnTimeoutMs: options.turnTimeoutMs, onResponse() { if (!options.effort) { return; diff --git a/plugins/codex/scripts/lib/job-control.mjs b/plugins/codex/scripts/lib/job-control.mjs index ad152c157..d6a6ceb59 100644 --- a/plugins/codex/scripts/lib/job-control.mjs +++ b/plugins/codex/scripts/lib/job-control.mjs @@ -2,7 +2,7 @@ import fs from "node:fs"; import { getSessionRuntimeStatus } from "./codex.mjs"; import { getConfig, listJobs, readJobFile, resolveJobFile } from "./state.mjs"; -import { SESSION_ID_ENV } from "./tracked-jobs.mjs"; +import { reapDeadJobs, SESSION_ID_ENV } from "./tracked-jobs.mjs"; import { resolveWorkspaceRoot } from "./workspace.mjs"; export const DEFAULT_MAX_STATUS_JOBS = 8; @@ -188,7 +188,7 @@ export function readStoredJob(workspaceRoot, jobId) { return readJobFile(jobFile); } -function matchJobReference(jobs, reference, predicate = () => true) { +function matchJobReference(jobs, reference, predicate = () => true, options = {}) { const filtered = jobs.filter(predicate); if (!reference) { return filtered[0] ?? null; @@ -207,13 +207,17 @@ function matchJobReference(jobs, reference, predicate = () => true) { throw new Error(`Job reference "${reference}" is ambiguous. Use a longer job id.`); } + if (options.optional) { + return null; + } + throw new Error(`No job found for "${reference}". Run /codex:status to list known jobs.`); } export function buildStatusSnapshot(cwd, options = {}) { const workspaceRoot = resolveWorkspaceRoot(cwd); const config = getConfig(workspaceRoot); - const jobs = sortJobsNewestFirst(filterJobsForCurrentSession(listJobs(workspaceRoot), options)); + const jobs = sortJobsNewestFirst(filterJobsForCurrentSession(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot)), options)); const maxJobs = options.maxJobs ?? DEFAULT_MAX_STATUS_JOBS; const maxProgressLines = options.maxProgressLines ?? DEFAULT_MAX_PROGRESS_LINES; @@ -241,7 +245,7 @@ export function buildStatusSnapshot(cwd, options = {}) { export function buildSingleJobSnapshot(cwd, reference, options = {}) { const workspaceRoot = resolveWorkspaceRoot(cwd); - const jobs = sortJobsNewestFirst(listJobs(workspaceRoot)); + const jobs = sortJobsNewestFirst(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot))); const selected = matchJobReference(jobs, reference); if (!selected) { throw new Error(`No job found for "${reference}". Run /codex:status to inspect known jobs.`); @@ -253,26 +257,36 @@ export function buildSingleJobSnapshot(cwd, reference, options = {}) { }; } +// Resolves the job `result` should report on: a finished one when there is one, +// otherwise the still-active job the reference points at. Filtering by terminal +// status *before* matching used to make `result ` fail with +// "No job found" (#498/#524); the caller decides how to report an active job. export function resolveResultJob(cwd, reference) { const workspaceRoot = resolveWorkspaceRoot(cwd); - const jobs = sortJobsNewestFirst(reference ? listJobs(workspaceRoot) : filterJobsForCurrentSession(listJobs(workspaceRoot))); + const jobs = sortJobsNewestFirst(reference ? reapDeadJobs(workspaceRoot, listJobs(workspaceRoot)) : filterJobsForCurrentSession(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot)))); const selected = matchJobReference( jobs, reference, - (job) => job.status === "completed" || job.status === "failed" || job.status === "cancelled" + (job) => job.status === "completed" || job.status === "failed" || job.status === "cancelled", + { optional: true } ); if (selected) { return { workspaceRoot, job: selected }; } - const active = matchJobReference(jobs, reference, (job) => job.status === "queued" || job.status === "running"); + const active = matchJobReference( + jobs, + reference, + (job) => job.status === "queued" || job.status === "running", + { optional: true } + ); if (active) { - throw new Error(`Job ${active.id} is still ${active.status}. Check /codex:status and try again once it finishes.`); + return { workspaceRoot, job: active }; } if (reference) { - throw new Error(`No finished job found for "${reference}". Run /codex:status to inspect active jobs.`); + throw new Error(`No job found for "${reference}". Run /codex:status to list known jobs.`); } throw new Error("No finished Codex jobs found for this repository yet."); @@ -280,7 +294,7 @@ export function resolveResultJob(cwd, reference) { export function resolveCancelableJob(cwd, reference, options = {}) { const workspaceRoot = resolveWorkspaceRoot(cwd); - const jobs = sortJobsNewestFirst(listJobs(workspaceRoot)); + const jobs = sortJobsNewestFirst(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot))); const activeJobs = jobs.filter((job) => job.status === "queued" || job.status === "running"); if (reference) { diff --git a/plugins/codex/scripts/lib/process.mjs b/plugins/codex/scripts/lib/process.mjs index 05109a7ff..e2060491d 100644 --- a/plugins/codex/scripts/lib/process.mjs +++ b/plugins/codex/scripts/lib/process.mjs @@ -9,6 +9,7 @@ export function runCommand(command, args = [], options = {}) { input: options.input, maxBuffer: options.maxBuffer, stdio: options.stdio ?? "pipe", + timeout: options.timeoutMs, shell: options.shell ?? (process.platform === "win32" ? (process.env.SHELL || true) : false), windowsHide: true }); @@ -68,13 +69,29 @@ export function processCommandLine(pid, options = {}) { } const runCommandImpl = options.runCommandImpl ?? runCommand; - const result = runCommandImpl("ps", ["-o", "command=", "-p", String(pid)]); + const result = runCommandImpl("ps", ["-o", "command=", "-p", String(pid)], { timeoutMs: options.timeoutMs }); if (result.error || result.status !== 0) { return null; } return result.stdout.trim() || null; } +// True when the PID is running, false when it is provably gone (ESRCH), null +// when the question does not apply (no PID) — EPERM means it exists but belongs +// to someone else. Known limitation: a zombie reads as alive, and a recycled PID +// reads as the process that inherited it. +export function isPidAlive(pid) { + if (!Number.isInteger(pid) || pid <= 0) { + return null; + } + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "ESRCH" ? false : true; + } +} + export function terminateProcessTree(pid, options = {}) { if (!Number.isFinite(pid)) { return { attempted: false, delivered: false, method: null }; diff --git a/plugins/codex/scripts/lib/state.mjs b/plugins/codex/scripts/lib/state.mjs index a9a54246b..509c7e0fe 100644 --- a/plugins/codex/scripts/lib/state.mjs +++ b/plugins/codex/scripts/lib/state.mjs @@ -1,8 +1,9 @@ -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { isPidAlive } from "./process.mjs"; import { resolveWorkspaceRoot } from "./workspace.mjs"; const STATE_VERSION = 1; @@ -55,6 +56,71 @@ export function ensureStateDir(cwd) { fs.mkdirSync(resolveJobsDir(cwd), { recursive: true }); } +export const REDACTED_CONFIG_VALUE = "[redacted]"; + +// `status --json` / `result --json` echo the stored job record back to the user +// and to Claude, so a `--config model_providers.x.http_headers.Cookie=…` would end +// up in the transcript and in long-lived state. No key-name heuristic can tell a +// credential from a harmless override — `Cookie` matches no denylist — so every +// value is dropped and only the keys are recorded. The real values reach Codex +// from the private 0600 one-shot payload file instead. +export function redactConfigValues(config) { + if (!config || typeof config !== "object") { + return config; + } + return Object.fromEntries(Object.keys(config).map((key) => [key, REDACTED_CONFIG_VALUE])); +} + +function hasStoredConfigValues(record) { + const config = record?.request?.config; + return ( + Boolean(config) && + typeof config === "object" && + Object.values(config).some((value) => value !== REDACTED_CONFIG_VALUE) + ); +} + +function withRedactedRequest(record) { + return hasStoredConfigValues(record) + ? { ...record, request: { ...record.request, config: redactConfigValues(record.request.config) } } + : record; +} + +function readJsonOrNull(filePath) { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return null; + } +} + +// Records written before values were redacted (1.1.1 and earlier) carry the raw +// ones under the same STATE_VERSION, so an upgrade alone would keep serving them. +// Redaction happens on every read — that is the boundary every `status`/`result` +// output crosses — and the file is rewritten once so the values stop living in +// long-lived state. The rewrite re-reads inside the lock and never goes through +// `saveState`, whose prune is a diff against a snapshot this does not have. +function migrateStoredConfigValues(cwd, state) { + if (!state.jobs.some(hasStoredConfigValues)) { + return state; + } + + try { + withStateLock(cwd, () => { + const current = readJsonOrNull(resolveStateFile(cwd)); + if (!Array.isArray(current?.jobs)) { + return; + } + current.jobs = current.jobs.map(withRedactedRequest); + writeFileAtomic(resolveStateFile(cwd), `${JSON.stringify(current, null, 2)}\n`); + }); + } catch { + // Best effort: what this read returns is redacted either way. + } + + return { ...state, jobs: state.jobs.map(withRedactedRequest) }; +} + export function loadState(cwd) { const stateFile = resolveStateFile(cwd); if (!fs.existsSync(stateFile)) { @@ -63,7 +129,7 @@ export function loadState(cwd) { try { const parsed = JSON.parse(fs.readFileSync(stateFile, "utf8")); - return { + return migrateStoredConfigValues(cwd, { ...defaultState(), ...parsed, config: { @@ -71,7 +137,7 @@ export function loadState(cwd) { ...(parsed.config ?? {}) }, jobs: Array.isArray(parsed.jobs) ? parsed.jobs : [] - }; + }); } catch { return defaultState(); } @@ -83,13 +149,392 @@ function pruneJobs(jobs) { .slice(0, MAX_JOBS); } +// Every reader of these files tolerates a corrupt one by falling back to a +// default (`loadState` returns an empty job list), so a reader that catches a +// plain `writeFileSync` mid-flight cannot tell a truncated file from an idle +// workspace — that is how a SessionEnd with a live job shut the shared broker. +// Writing a sibling temp file and renaming it swaps the content in one step: +// a reader sees either the old file or the new one, never half of either. +/** @param {import("node:fs").WriteFileOptions} options */ +function writeFileAtomic(filePath, contents, options = "utf8") { + const tempFile = `${filePath}.${process.pid}.tmp`; + fs.writeFileSync(tempFile, contents, options); + fs.renameSync(tempFile, filePath); + return filePath; +} + function removeFileIfExists(filePath) { if (filePath && fs.existsSync(filePath)) { fs.unlinkSync(filePath); } } +// Every workspace command reads `state.json`, changes it and writes it back, and +// `saveState` deletes the artifacts of every job that was in the file it read but +// is not in the snapshot it writes. Unserialized, a process whose snapshot went +// stale therefore does not merely lose another process's job from the index — it +// deletes that job's file, private payload, PID sidecar and log. +// +// The lock is a Lamport bakery on files. Every scheme that takes a lock away from +// its owner is a check-then-act on a shared path — judge it, then remove or +// replace it — and POSIX gives no conditional replace, so whatever was judged can +// change before the act. Here nothing shared is ever replaced or removed: an +// acquirer only ever creates files whose names are unique to it +// (`choosing.`, `..ticket`), and the only deletions are `unlink`s +// of one exact name whose content never changes — so a verdict about that file +// cannot go stale between the verdict and the unlink. The directory itself is +// created once and never removed. +// +// A pre-1.2.0 `state.lock` directory, if one is left over, is not read, removed or +// otherwise touched: that format never shipped. +// ponytail: one lock per workspace, no reader/writer split — every mutation is a +// few file writes, so a shared-read lock would only add ways to get it wrong. +const LOCK_DIR_NAME = "state.lock.d"; +const LOCK_CHOOSING_PREFIX = "choosing."; +const LOCK_TICKET_SUFFIX = ".ticket"; +const LOCK_WAIT_MS = 5000; +const LOCK_POLL_MS = 25; +// How long an entry nothing can be learned from — unreadable, or written by a +// process that died between the two syscalls — may sit before it counts as debris. +const LOCK_ORPHAN_MS = 2000; +// ... and how long one whose PID cannot be checked at all may hold up the queue. +const LOCK_STALE_MS = 30000; + +// Locks this process already holds, with the ticket each was taken under: a locked +// section may call another one (`updateState` → `saveState`) and must not deadlock +// against itself. +const heldLocks = new Map(); + +function sleepSync(ms) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +// Three answers about a foreign entry, and every other outcome is an error worth +// failing on. `GONE` — it left the queue between the listing and the look, so it +// blocks nobody and there is nothing to unlink. `HELD` — someone is using it. +// `ABANDONED` — its owner is provably gone, or nothing can be learned from it and +// it is old enough to be debris. +const LOCK_ENTRY_GONE = "gone"; +const LOCK_ENTRY_HELD = "held"; +const LOCK_ENTRY_ABANDONED = "abandoned"; + +// Only the entry having disappeared is an answer. Every other stat failure — +// EACCES, EIO, ELOOP — says nothing about the owner, and guessing there is how a +// live holder's ticket gets unlinked: swallowing the error left the age comparison +// with NaN, which reads as infinitely old. +function statLockEntry(entryPath) { + try { + return fs.statSync(entryPath); + } catch (error) { + if (error.code === "ENOENT") { + return null; + } + throw error; + } +} + +// Only the entry having disappeared is an answer here too. A read that fails for +// any other reason (EACCES, EIO, EISDIR) says nothing about the owner, so it fails +// the acquisition rather than letting the entry be aged out. +// +// Content that reads but does not parse is different: entries are published by +// rename, so a live holder's entry is never half-written — junk can only be debris, +// and debris is judged by its age. +function readLockEntryOwner(entryPath) { + let contents; + try { + contents = fs.readFileSync(entryPath, "utf8"); + } catch (error) { + if (error.code === "ENOENT") { + return { present: false, owner: null }; + } + throw error; + } + + try { + return { present: true, owner: JSON.parse(contents) }; + } catch { + return { present: true, owner: null }; + } +} + +// A live owner is never evicted, whatever the clock says: from out here a slow +// process and a stuck one look identical, and evicting either puts two writers in +// the critical section. A dead one releases its place at once; an entry whose +// content is junk is debris after a short grace; one that carries no usable PID to +// check after a long one. (A PID that exists but belongs to another user reads +// as alive, which is the safe answer.) A stuck live owner is the operator's call — +// the timeout error names it. +function judgeLockEntry(entryPath) { + const { present, owner } = readLockEntryOwner(entryPath); + if (!present) { + return LOCK_ENTRY_GONE; + } + + if (owner) { + const alive = isPidAlive(owner.pid); + if (alive === true) { + return LOCK_ENTRY_HELD; + } + if (alive === false) { + return LOCK_ENTRY_ABANDONED; + } + } + + // Age is the only signal left, and it has to be a real one — a timestamp we + // could not read must never decide that somebody else's entry is stale. + const stats = statLockEntry(entryPath); + if (!stats) { + return LOCK_ENTRY_GONE; + } + const startedAt = owner ? Date.parse(owner.startedAt ?? "") : Number.NaN; + const since = Number.isFinite(startedAt) ? startedAt : stats.mtimeMs; + if (!Number.isFinite(since)) { + throw new Error(`Cannot judge the Codex state lock entry ${entryPath}: it has no usable timestamp.`); + } + // An entry that says nothing about its owner gets the short grace; one that + // named a PID nothing can check gets the long one. + const grace = owner ? LOCK_STALE_MS : LOCK_ORPHAN_MS; + return Date.now() - since > grace ? LOCK_ENTRY_ABANDONED : LOCK_ENTRY_HELD; +} + +// A queue that cannot be read is not an empty queue: answering a listing failure +// with "nobody is ahead of me" would put two processes in the critical section. +// Every error propagates and fails the acquisition. The one exception is a missing +// directory on the numbering read: something removed the whole directory — and our +// own choosing file with it — from underneath us, so it is recreated and read once +// more rather than failing a command over someone else's `rm -rf`. +function readLockEntries(lockDir, { createMissing = false } = {}) { + const choosing = []; + const tickets = []; + let names; + try { + names = fs.readdirSync(lockDir); + } catch (error) { + if (error.code !== "ENOENT" || !createMissing) { + throw error; + } + fs.mkdirSync(lockDir, { recursive: true }); + names = fs.readdirSync(lockDir); + } + + for (const name of names) { + if (name.startsWith(LOCK_CHOOSING_PREFIX)) { + const token = name.slice(LOCK_CHOOSING_PREFIX.length); + if (token) { + choosing.push({ name, token }); + } + continue; + } + if (!name.endsWith(LOCK_TICKET_SUFFIX)) { + continue; + } + const [rawNumber, token] = name.slice(0, -LOCK_TICKET_SUFFIX.length).split("."); + const number = Number.parseInt(rawNumber, 10); + if (Number.isInteger(number) && token) { + tickets.push({ name, number, token }); + } + } + return { choosing, tickets }; +} + +// The destination name is unique to this acquisition, so the rename creates it and +// can never replace anyone else's entry; the staging name is unique too, and the +// rename is what makes the entry appear complete or not at all. +function writeLockEntry(lockDir, name, token, contents) { + const staged = path.join(lockDir, `${token}.tmp`); + fs.writeFileSync(staged, contents, "utf8"); + fs.renameSync(staged, path.join(lockDir, name)); +} + +// True only when this call actually removed the entry. Failure is never raised: +// the name may already be retired (someone else judged the same entry abandoned, +// or its owner released it — tokens are one-off, so a name is never reused), and a +// foreign entry this process may not remove must not fail a release. Callers that +// need to know whether the queue really moved read the answer instead. +function unlinkLockEntry(lockDir, name) { + try { + fs.unlinkSync(path.join(lockDir, name)); + return true; + } catch { + return false; + } +} + +// Numbers can tie (two acquirers reading the ticket list at the same moment), so +// the token settles the order. Both parts are fixed at acquisition, so every +// process derives the same queue from the same directory listing. +function ticketIsBefore(left, right) { + return left.number === right.number ? left.token < right.token : left.number < right.number; +} + +// A stable discriminator for the one lock failure a caller may reasonably absorb. +// Matching on the message cannot tell this apart from an integrity error that names +// the same lock — or from a filesystem error whose path happens to contain the +// phrase — and swallowing either of those would hide a real fault. +export const STATE_LOCK_TIMEOUT_CODE = "CODEX_STATE_LOCK_TIMEOUT"; + +function lockTimeoutError(lockDir, blockers, waitMs) { + // The remembered blockers are from the last scan, and some may have been cleared + // since — including by this waiter. Name one that is still there, or say plainly + // that there is nothing left to name. + let visible = blockers; + try { + const names = new Set(fs.readdirSync(lockDir)); + visible = blockers.filter((entry) => names.has(entry.name)); + } catch { + // Re-listing failed (quite possibly the reason we are here): fall back to what + // the last scan saw. + } + const lowest = + visible + .filter((entry) => entry.number !== undefined) + .sort((left, right) => (ticketIsBefore(left, right) ? -1 : 1))[0] ?? visible[0]; + if (!lowest) { + return Object.assign( + new Error(`Timed out after ${waitMs} ms waiting for the Codex state lock (${lockDir}); no blocker visible now.`), + { code: STATE_LOCK_TIMEOUT_CODE } + ); + } + // The blocker may be a ticket or a client still choosing its number. + const entryPath = path.join(lockDir, lowest.name); + const pid = readJsonOrNull(entryPath)?.pid ?? "unknown"; + return Object.assign( + new Error( + `Timed out after ${waitMs} ms waiting for the Codex state lock. It is held by pid ${pid} (entry ${entryPath}). ` + + `Stop that process if it is stuck; if pid ${pid} is not a Codex process (pid reuse), remove that entry file.` + ), + { code: STATE_LOCK_TIMEOUT_CODE } + ); +} + +// The bakery's doorway: announce the choice, take a number one higher than any on +// display, then stop announcing. A later acquirer is therefore always visible as +// `choosing` to anyone still deciding, which is what stops it slipping in with a +// lower number behind a holder's back. +function acquireTicket(lockDir, waitMs) { + fs.mkdirSync(lockDir, { recursive: true }); + const token = `${process.pid}-${randomBytes(8).toString("hex")}`; + const owner = `${JSON.stringify({ pid: process.pid, startedAt: nowIso() })}\n`; + const choosingName = `${LOCK_CHOOSING_PREFIX}${token}`; + + writeLockEntry(lockDir, choosingName, token, owner); + let ticket; + try { + const highest = readLockEntries(lockDir, { createMissing: true }).tickets.reduce( + (max, entry) => Math.max(max, entry.number), + 0 + ); + ticket = { number: highest + 1, token, name: `${highest + 1}.${token}${LOCK_TICKET_SUFFIX}` }; + writeLockEntry(lockDir, ticket.name, token, owner); + } finally { + // The choosing file must go before the wait, not after it: two acquirers that + // waited for each other's choosing files would deadlock. + unlinkLockEntry(lockDir, choosingName); + } + + try { + waitForTurn(lockDir, ticket, waitMs); + } catch (error) { + // We are not holding the lock, so our ticket must leave the queue — this + // process is alive, so nothing would ever judge it abandoned and everyone + // behind it would wait on an acquisition that was given up. Only our own + // entries are touched, whatever went wrong. + unlinkLockEntry(lockDir, ticket.name); + throw error; + } + return ticket; +} + +function waitForTurn(lockDir, ticket, waitMs) { + const deadline = Date.now() + waitMs; + let blockers = []; + let scanned = false; + for (;;) { + // Checked before every scan but the first, so no path can skip the bound: an + // entry judged abandoned that cannot actually be unlinked (a permission problem + // on a foreign entry) would otherwise be re-judged forever. The first scan + // always happens, so a lock nobody else wants is taken whatever the budget was. + if (scanned && Date.now() >= deadline) { + throw lockTimeoutError(lockDir, blockers, waitMs); + } + scanned = true; + + const { choosing, tickets } = readLockEntries(lockDir); + blockers = [ + // Everyone still choosing may yet take a number below ours. + ...choosing.filter((entry) => entry.token !== ticket.token), + ...tickets.filter((entry) => entry.token !== ticket.token && ticketIsBefore(entry, ticket)) + ]; + if (blockers.length === 0) { + return; + } + + // Judge every blocker before touching any of them. Done in one pass, a verdict + // that throws halfway through would leave the entries it had already evicted + // gone — breaking the one promise a failed acquisition makes, that it removed + // nothing but its own files. + const verdicts = blockers.map((blocker) => ({ + blocker, + verdict: judgeLockEntry(path.join(lockDir, blocker.name)) + })); + + let evicted = false; + for (const { blocker, verdict } of verdicts) { + if (verdict === LOCK_ENTRY_ABANDONED && unlinkLockEntry(lockDir, blocker.name)) { + evicted = true; + } + } + // Only a removal that actually happened earns an immediate re-scan. An entry + // this process may not remove would otherwise be re-judged and re-unlinked with + // no pause at all, hammering the filesystem until the deadline; a queue that + // merely keeps changing under us must not become a spin either. + if (!evicted) { + sleepSync(LOCK_POLL_MS); + } + } +} + +// Runs `fn` — which must be synchronous — as the only writer of this workspace's +// state, across processes. Re-read whatever you are about to change inside it: +// anything read before the call may already be stale. +export function withStateLock(cwd, fn, options = {}) { + ensureStateDir(cwd); + return withLockDir(path.join(resolveStateDir(cwd), LOCK_DIR_NAME), fn, options); +} + +function withLockDir(lockDir, fn, options = {}) { + const held = heldLocks.get(lockDir); + if (held) { + held.depth += 1; + try { + return fn(); + } finally { + held.depth -= 1; + } + } + + const ticket = acquireTicket(lockDir, options.waitMs ?? LOCK_WAIT_MS); + heldLocks.set(lockDir, { depth: 1, ticket }); + try { + return fn(); + } finally { + heldLocks.delete(lockDir); + // Leaving the queue is one unlink of one name only this acquisition ever had. + unlinkLockEntry(lockDir, ticket.name); + } +} + +// The prune below is a diff against what is on disk right now, so both halves — +// the read and the write — belong inside the lock. `state` is the caller's +// snapshot: anything it did not carry over is treated as deleted, which is only +// safe because no other process can have added to the file since the caller +// re-read it under this same lock. export function saveState(cwd, state) { + return withStateLock(cwd, () => saveStateLocked(cwd, state)); +} + +function saveStateLocked(cwd, state) { const previousJobs = loadState(cwd).jobs; ensureStateDir(cwd); const nextJobs = pruneJobs(state.jobs ?? []); @@ -109,17 +554,23 @@ export function saveState(cwd, state) { } removeJobFile(resolveJobFile(cwd, job.id)); removeFileIfExists(resolveJobRequestFile(cwd, job.id)); + removeFileIfExists(resolveJobPidFile(cwd, job.id)); removeFileIfExists(job.logFile); } - fs.writeFileSync(resolveStateFile(cwd), `${JSON.stringify(nextState, null, 2)}\n`, "utf8"); + writeFileAtomic(resolveStateFile(cwd), `${JSON.stringify(nextState, null, 2)}\n`); return nextState; } +// Read, change and write as one indivisible step: the read has to happen inside +// the lock, or the snapshot handed to `mutate` can already be missing a job +// another process just added — which `saveState` would then delete. export function updateState(cwd, mutate) { - const state = loadState(cwd); - mutate(state); - return saveState(cwd, state); + return withStateLock(cwd, () => { + const state = loadState(cwd); + mutate(state); + return saveStateLocked(cwd, state); + }); } export function generateJobId(prefix = "job") { @@ -147,6 +598,29 @@ export function upsertJob(cwd, jobPatch) { }); } +// The queued record is written before the worker is spawned, so its pid can only +// be filled in afterwards — and the worker owns the job file from its first line +// (`runTrackedJob` writes `running` with the same pid). So the parent never +// writes that file back: a read-modify-write from the parent could put a queued +// snapshot over a record the worker had already completed, losing its result, +// threadId and turnId. The pid goes into an atomic sidecar plus the pid-only +// index patch, and readers fall back to the sidecar (`resolveJobPid`) only while +// the job is still active. +export function updateJobPid(cwd, jobId, pid) { + writeJobPidFile(cwd, jobId, pid); + // The index is patch-based, so it cannot lose a field — but a worker that + // already reported `running` wrote its own pid there, and that record is the + // newer one. A job that is gone from the index needs no pid at all. The read + // and the patch share one lock: between them the worker could otherwise report + // `running`, and the patch would put this stale pid over its own. + withStateLock(cwd, () => { + const indexed = listJobs(cwd).find((job) => job.id === jobId); + if (indexed?.status === "queued") { + upsertJob(cwd, { id: jobId, pid }); + } + }); +} + export function listJobs(cwd) { return loadState(cwd).jobs; } @@ -167,12 +641,49 @@ export function getConfig(cwd) { export function writeJobFile(cwd, jobId, payload) { ensureStateDir(cwd); const jobFile = resolveJobFile(cwd, jobId); - fs.writeFileSync(jobFile, `${JSON.stringify(payload, null, 2)}\n`, "utf8"); + writeFileAtomic(jobFile, `${JSON.stringify(payload, null, 2)}\n`); return jobFile; } export function readJobFile(jobFile) { - return JSON.parse(fs.readFileSync(jobFile, "utf8")); + const record = JSON.parse(fs.readFileSync(jobFile, "utf8")); + if (!hasStoredConfigValues(record)) { + return record; + } + + // Same one-shot migration as the index, plus the move that makes it safe for an + // active job: 1.1.1 wrote no private payload, so its record is the only copy of + // the request — and that record is exactly what `handleTaskWorker` falls back to. + // Redacting it in place would hand the worker "[redacted]" as its Codex config, + // so the raw request goes to the 0600 payload file the worker consumes first, + // and only then leaves the record. + const requestFile = jobFile.replace(/\.json$/, ".request.json"); + try { + withLockDir(path.join(path.dirname(path.dirname(jobFile)), LOCK_DIR_NAME), () => { + const current = readJsonOrNull(jobFile); + if (!current || !hasStoredConfigValues(current)) { + return; + } + // Only a `queued` record still has a worker coming for its request: the + // worker consumes the payload (or falls back to the record) *before* + // `runTrackedJob` flips the status to `running`, so staging one for a + // running job would write plaintext `--config` values that nothing reads + // and nothing deletes. + const active = current.status === "queued"; + const migrated = withRedactedRequest(current); + if (active && !fs.existsSync(requestFile)) { + // Same shape and mode as `writeJobRequestFile`: the temp file carries the + // 0600 through the rename, so the payload is never world-readable. + writeFileAtomic(requestFile, `${JSON.stringify(current.request, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); + migrated.requestFile = requestFile; + } + writeFileAtomic(jobFile, `${JSON.stringify(migrated, null, 2)}\n`); + }); + } catch { + // Best effort: the record this returns is redacted either way. + } + + return withRedactedRequest(record); } function removeJobFile(jobFile) { @@ -191,6 +702,42 @@ export function resolveJobFile(cwd, jobId) { return path.join(resolveJobsDir(cwd), `${jobId}.json`); } +export function resolveJobPidFile(cwd, jobId) { + ensureStateDir(cwd); + return path.join(resolveJobsDir(cwd), `${jobId}.pid`); +} + +export function writeJobPidFile(cwd, jobId, pid) { + return writeFileAtomic(resolveJobPidFile(cwd, jobId), `${pid}\n`); +} + +export function removeJobPidFile(cwd, jobId) { + removeFileIfExists(resolveJobPidFile(cwd, jobId)); +} + +function readJobPidSidecar(cwd, jobId) { + try { + const pid = Number.parseInt(fs.readFileSync(resolveJobPidFile(cwd, jobId), "utf8").trim(), 10); + return Number.isInteger(pid) && pid > 0 ? pid : null; + } catch { + return null; + } +} + +// The pid every reader (cancel, reaper, SessionEnd cleanup) should use: the +// record's own pid once the worker has taken the record over, the sidecar during +// the queued window before that. A terminal job never reports one — its worker +// is gone and the sidecar may name a pid the OS has recycled. +export function resolveJobPid(cwd, job) { + if (job?.pid != null) { + return job.pid; + } + if (job?.status !== "queued" && job?.status !== "running") { + return null; + } + return readJobPidSidecar(cwd, job.id); +} + // The full task request can carry secrets (`--config` values such as auth // headers), so background workers read it from a private one-shot file instead // of the job record that `status`/`result` echo back to the user. @@ -200,9 +747,12 @@ export function resolveJobRequestFile(cwd, jobId) { } export function writeJobRequestFile(cwd, jobId, payload) { - const requestFile = resolveJobRequestFile(cwd, jobId); - fs.writeFileSync(requestFile, `${JSON.stringify(payload, null, 2)}\n`, { encoding: "utf8", mode: 0o600 }); - return requestFile; + // The temp file carries the 0600 mode through the rename, so the payload is + // never briefly world-readable. + return writeFileAtomic(resolveJobRequestFile(cwd, jobId), `${JSON.stringify(payload, null, 2)}\n`, { + encoding: "utf8", + mode: 0o600 + }); } export function removeJobRequestFile(cwd, jobId) { diff --git a/plugins/codex/scripts/lib/tracked-jobs.mjs b/plugins/codex/scripts/lib/tracked-jobs.mjs index 0c7d56d34..ac24856d5 100644 --- a/plugins/codex/scripts/lib/tracked-jobs.mjs +++ b/plugins/codex/scripts/lib/tracked-jobs.mjs @@ -1,7 +1,19 @@ import fs from "node:fs"; import process from "node:process"; -import { readJobFile, resolveJobFile, resolveJobLogFile, upsertJob, writeJobFile } from "./state.mjs"; +import { isPidAlive } from "./process.mjs"; + +import { + readJobFile, + removeJobPidFile, + removeJobRequestFile, + resolveJobFile, + resolveJobLogFile, + resolveJobPid, + upsertJob, + withStateLock, + writeJobFile +} from "./state.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; @@ -164,9 +176,13 @@ export async function runTrackedJob(job, runner, options = {}) { const execution = await runner(); const completionStatus = execution.exitStatus === 0 ? "completed" : "failed"; const completedAt = nowIso(); + // A run that fails without throwing (a timed-out or interrupted turn) still + // has to say why: `status`/`result` read the reason off the record. + const errorMessage = completionStatus === "failed" ? execution.errorMessage ?? null : null; writeJobFile(job.workspaceRoot, job.id, { ...runningRecord, status: completionStatus, + errorMessage, threadId: execution.threadId ?? null, turnId: execution.turnId ?? null, resolved: execution.resolved ?? null, @@ -179,6 +195,7 @@ export async function runTrackedJob(job, runner, options = {}) { upsertJob(job.workspaceRoot, { id: job.id, status: completionStatus, + errorMessage, threadId: execution.threadId ?? null, turnId: execution.turnId ?? null, resolved: execution.resolved ?? null, @@ -187,6 +204,11 @@ export async function runTrackedJob(job, runner, options = {}) { pid: null, completedAt }); + removeJobPidFile(job.workspaceRoot, job.id); + // Nothing revisits a terminal job, so this is the last chance to release a + // payload the worker never consumed (a crash before the read, or one staged + // by the legacy-record migration). + removeJobRequestFile(job.workspaceRoot, job.id); appendLogBlock(options.logFile ?? job.logFile ?? null, "Final output", execution.rendered); return execution; } catch (error) { @@ -210,6 +232,183 @@ export async function runTrackedJob(job, runner, options = {}) { errorMessage, completedAt }); + removeJobPidFile(job.workspaceRoot, job.id); + // Nothing revisits a terminal job, so this is the last chance to release a + // payload the worker never consumed (a crash before the read, or one staged + // by the legacy-record migration). + removeJobRequestFile(job.workspaceRoot, job.id); throw error; } } + +// Reconciles a job the caller believes is dead. Everything here — the job file +// it trusts, the index entry it rewrites, the artifacts it deletes — has to move +// as one step, or a concurrent writer's prune can delete the files this just +// wrote (or resurrect the ones it deleted). +function markJobDead(workspaceRoot, jobSummary, errorMessage, lockWaitMs = undefined) { + return withStateLock(workspaceRoot, () => markJobDeadLocked(workspaceRoot, jobSummary, errorMessage), { + waitMs: lockWaitMs + }); +} + +function markJobDeadLocked(workspaceRoot, jobSummary, errorMessage) { + const jobFile = resolveJobFile(workspaceRoot, jobSummary.id); + const stored = fs.existsSync(jobFile) ? readJobFile(jobFile) : null; + const base = stored ?? jobSummary; + if (base.status !== "running" && base.status !== "queued") { + // The job finished between the caller's read and now — keep the real result, + // and put it in the index too: a worker that died between its terminal + // `writeJobFile` and its `upsertJob` leaves an active index entry that + // `assertThreadIsFree` reads as a phantom running job, blocking every later + // resume of that thread. + upsertJob(workspaceRoot, { + id: jobSummary.id, + status: base.status, + phase: base.phase ?? null, + errorMessage: base.errorMessage ?? null, + threadId: base.threadId ?? null, + turnId: base.turnId ?? null, + resolved: base.resolved ?? null, + requestFile: base.requestFile ?? null, + pid: null, + completedAt: base.completedAt ?? null + }); + return base; + } + const completedAt = nowIso(); + // Nothing will ever read the private payload now, so it must not stay on disk + // (0600, possibly holding `--config` secrets) until the job is pruned. Only + // its path is cleared on the record: the values themselves are never lifted + // into the record or the state index, which `status`/`result` echo back. + removeJobRequestFile(workspaceRoot, jobSummary.id); + removeJobPidFile(workspaceRoot, jobSummary.id); + const record = { + ...base, + status: "failed", + phase: "failed", + errorMessage, + pid: null, + requestFile: null, + completedAt, + // Keep updatedAt current so the reaped job sorts newest-first in the same + // read that recorded it — otherwise a stale updatedAt can page it out of + // the first /codex:status report. + updatedAt: completedAt + }; + writeJobFile(workspaceRoot, jobSummary.id, record); + upsertJob(workspaceRoot, { + id: jobSummary.id, + status: "failed", + phase: "failed", + pid: null, + requestFile: null, + errorMessage, + completedAt + }); + appendLogLine(base.logFile ?? null, `Marked failed: ${errorMessage}`); + return record; +} + +const DEAD_WORKER_MESSAGE = "worker exited before completing"; + +// How long a queued job may sit without a recorded pid before it counts as dead. +// `enqueueBackgroundTask` patches the pid in immediately after the spawn, so the +// window is milliseconds wide in practice; the grace period only has to outlast +// a heavily loaded machine. +const QUEUED_WITHOUT_PID_GRACE_MS = 30000; + +// A worker killed between the spawn and `updateJobPid` leaves a queued record +// with no pid at all — not in the record and not in the sidecar. `isPidAlive(null)` +// cannot tell that apart from a record that was written microseconds ago, so age +// decides it. +function isQueuedWithoutWorker(job, pid) { + if (job.status !== "queued" || pid != null) { + return false; + } + const createdAt = Date.parse(job.createdAt ?? ""); + return Number.isFinite(createdAt) && Date.now() - createdAt > QUEUED_WITHOUT_PID_GRACE_MS; +} + +// A worker that dies without throwing (SIGKILL, OOM, native crash) never +// reaches runTrackedJob's catch, so its job stays "running" — or, if it died +// before taking the record over, "queued" — forever. Rewrite any active job +// whose worker is gone as failed. +// +// The job file is read first, for every active entry: it is the authoritative +// record, and a worker that died between writing it and updating the index +// leaves an index entry PID liveness cannot correct — `kill(pid, 0)` reads a +// zombie as alive and cannot see a recycled pid, so that phantom `running` entry +// would block resume on its thread for as long as anything held that pid. Pid +// liveness is only consulted for jobs whose own file still says they are active. +// Below this there is no point starting another lock wait. +const REAP_MIN_STEP_MS = 100; + +/** + * @param {{ lockWaitMs?: number, remainingMs?: () => number }} [options] Bounds the + * reaper's own state-lock waits. Each dead job costs one acquisition, so a caller + * working to a deadline passes `remainingMs` and every wait is clamped to what is + * left of it; once that is spent the remaining jobs are left for the next run + * rather than reaped past the caller's budget. + */ +export function reapDeadJobs(workspaceRoot, jobs, options = {}) { + const { lockWaitMs, remainingMs } = options; + const waitFor = () => { + if (!remainingMs) { + return lockWaitMs; + } + const left = Math.max(0, remainingMs()); + return lockWaitMs === undefined ? left : Math.min(lockWaitMs, left); + }; + const deferred = []; + const reaped = jobs.map((job) => { + if (remainingMs && remainingMs() < REAP_MIN_STEP_MS) { + // Left as-is for the next run — say so, or a job that is dead but still + // listed as running looks like a live one to whoever reads the decision. + deferred.push(job.id); + return job; + } + if (job.status !== "running" && job.status !== "queued") { + return job; + } + const stored = readStoredJobOrNull(workspaceRoot, job.id); + if (stored && stored.status !== "running" && stored.status !== "queued") { + // Terminal on disk: markJobDead keeps the real result and reconciles it + // into the index rather than failing the job. + return markJobDead(workspaceRoot, job, DEAD_WORKER_MESSAGE, waitFor()); + } + // The queued record carries no pid of its own — the parent records it in an + // atomic sidecar instead of rewriting the worker's job file. + const pid = resolveJobPid(workspaceRoot, job); + if (isPidAlive(pid) === false || isQueuedWithoutWorker(job, pid)) { + return markJobDead(workspaceRoot, job, DEAD_WORKER_MESSAGE, waitFor()); + } + return job; + }); + if (deferred.length > 0) { + process.stderr.write(`[codex] Reaper ran out of budget; not judged this run: ${deferred.join(", ")}.\n`); + } + return reaped; +} + +// Guards only against in-process crashes (uncaughtException / unhandledRejection) +// where a precise error is available and no other command is writing the job. +// Signal-based deaths (SIGTERM/SIGINT/SIGHUP/SIGKILL) are intentionally NOT +// caught here: SIGKILL is uncatchable so the reader-side reapDeadJobs must cover +// it regardless, and /codex:cancel delivers SIGTERM as its teardown signal after +// writing the job "cancelled" — catching it here would race that terminal state +// back to "failed". reapDeadJobs handles every signal death and never rewrites a +// job that already reached a terminal status. +export function registerWorkerCrashGuard(workspaceRoot, jobId, logFile = null) { + const mark = (label) => (reason) => { + try { + const detail = reason instanceof Error ? reason.stack ?? reason.message : String(reason ?? ""); + appendLogLine(logFile, `Worker ${label}: ${detail}`); + markJobDead(workspaceRoot, { id: jobId, status: "running", logFile }, `worker ${label}: ${detail.split("\n")[0]}`); + } catch { + // Never let the guard itself throw during teardown. + } + process.exit(1); + }; + process.on("uncaughtException", mark("uncaughtException")); + process.on("unhandledRejection", mark("unhandledRejection")); +} diff --git a/plugins/codex/scripts/session-lifecycle-hook.mjs b/plugins/codex/scripts/session-lifecycle-hook.mjs index 34a22646b..6f8ff0b44 100644 --- a/plugins/codex/scripts/session-lifecycle-hook.mjs +++ b/plugins/codex/scripts/session-lifecycle-hook.mjs @@ -13,11 +13,51 @@ import { sendBrokerShutdown, teardownBrokerSession } from "./lib/broker-lifecycle.mjs"; -import { loadState, resolveStateFile, saveState } from "./lib/state.mjs"; +import { loadState, resolveJobPid, resolveStateFile, saveState, STATE_LOCK_TIMEOUT_CODE, withStateLock } from "./lib/state.mjs"; +import { reapDeadJobs } from "./lib/tracked-jobs.mjs"; import { TRANSCRIPT_PATH_ENV } from "./lib/claude-session-transfer.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; export const SESSION_ID_ENV = "CODEX_COMPANION_SESSION_ID"; +// How long a `busy` broker is given to shed a client this hook has just reaped, +// and how often to ask. Bounded: a broker that is really in use stays busy for the +// whole window and keeps everything it owns. +const BROKER_BUSY_RETRY_MS = 1000; +const BROKER_BUSY_POLL_MS = 100; + +// One absolute budget for the whole SessionEnd hook. Claude Code kills the hook at +// the timeout in hooks.json, and the steps below have bounds of their own (state +// lock 5 s, each broker handshake 5 s, the busy retries 1 s, the teardown probe): +// added up they can exceed any single bound, so each step is clamped to what is +// left of this budget and the hook reports what it decided instead of being killed +// mid-decision. KEEP hooks.json's SessionEnd timeout ABOVE this — the pair is +// asserted by `tests/commands.test.mjs` and documented in the README. The env +// override can only shorten this ceiling, never raise it, so the pair holds +// whatever the environment says. +const SESSION_END_BUDGET_MS = 12000; +const SESSION_END_BUDGET_ENV = "CODEX_COMPANION_SESSION_END_BUDGET_MS"; +const STATE_LOCK_STEP_MS = 5000; +const BROKER_HANDSHAKE_STEP_MS = 5000; +// Below this there is no point starting another bounded step. +const MIN_STEP_MS = 100; + +// The override may only ever SHORTEN the budget. `hooks.json`'s timeout is a fixed +// number that cannot be raised from the environment, so an override above the +// ceiling would put the deadline past the point where Claude Code kills the hook — +// the very failure the budget exists to prevent. +function resolveSessionEndBudgetMs(env = process.env) { + const configured = Number(env[SESSION_END_BUDGET_ENV]); + if (!Number.isFinite(configured) || configured <= 0) { + return SESSION_END_BUDGET_MS; + } + if (configured > SESSION_END_BUDGET_MS) { + process.stderr.write( + `[codex] SessionEnd budget override ${configured} ignored: above the ${SESSION_END_BUDGET_MS} ms ceiling.\n` + ); + return SESSION_END_BUDGET_MS; + } + return configured; +} const PLUGIN_DATA_ENV = "CLAUDE_PLUGIN_DATA"; function readHookInput() { @@ -60,7 +100,7 @@ function appendEnvVar(name, value) { ); } -function cleanupSessionJobs(cwd, sessionId) { +function cleanupSessionJobs(cwd, sessionId, lockWaitMs) { if (!cwd || !sessionId) { return; } @@ -71,28 +111,62 @@ function cleanupSessionJobs(cwd, sessionId) { return; } - const state = loadState(workspaceRoot); - const removedJobs = state.jobs.filter((job) => job.sessionId === sessionId); - if (removedJobs.length === 0) { - return; - } - - for (const job of removedJobs) { - const stillRunning = job.status === "queued" || job.status === "running"; - if (!stillRunning) { - continue; + // One locked read-modify-write: the jobs this decides to stop and the list it + // writes back have to come from the same snapshot, or another session's job — + // created between the read and the write — is dropped from the index and its + // files are pruned with it. + withStateLock(workspaceRoot, () => { + const state = loadState(workspaceRoot); + const sessionJobs = state.jobs.filter((job) => job.sessionId === sessionId); + if (sessionJobs.length === 0) { + return; } - try { - terminateProcessTree(job.pid ?? Number.NaN); - } catch { - // Ignore teardown failures during session shutdown. + + for (const job of sessionJobs) { + // Background jobs are explicitly dispatched to outlive the session that + // started them. Leave them running and leave their state entry intact so + // any session in the workspace can still poll for status/results. + if (job.background) { + continue; + } + const stillRunning = job.status === "queued" || job.status === "running"; + if (!stillRunning) { + continue; + } + try { + terminateProcessTree(resolveJobPid(workspaceRoot, job) ?? Number.NaN); + } catch { + // Ignore teardown failures during session shutdown. + } } - } - saveState(workspaceRoot, { - ...state, - jobs: state.jobs.filter((job) => job.sessionId !== sessionId) - }); + saveState(workspaceRoot, { + ...state, + jobs: state.jobs.filter((job) => job.sessionId !== sessionId || job.background) + }); + }, { waitMs: lockWaitMs }); +} + +// Read AFTER cleanupSessionJobs: the state it saved is the one that decides. +// Every kind of job counts, from every session — a foreground job of another +// Claude session survives this session's cleanup and is talking to the same +// shared broker, so tearing the broker down here would break its live turn. +function activeWorkspaceJobs(cwd, lockWaitMs, remainingMs) { + if (!cwd) { + return []; + } + const workspaceRoot = resolveWorkspaceRoot(cwd); + const stateFile = resolveStateFile(workspaceRoot); + if (!fs.existsSync(stateFile)) { + return []; + } + const state = loadState(workspaceRoot); + // Reap first: a worker killed outright (SIGKILL, OOM) leaves `running` behind, + // and trusting that record would keep this broker — and every later session's — + // alive forever, with the dead job's private payload still on disk. + return reapDeadJobs(workspaceRoot, state.jobs, { lockWaitMs, remainingMs }) + .filter((job) => job.status === "queued" || job.status === "running") + .map((job) => `${job.id}:${job.status}`); } function handleSessionStart(input) { @@ -103,6 +177,9 @@ function handleSessionStart(input) { async function handleSessionEnd(input) { const cwd = input.cwd || process.cwd(); + const budgetEndsAt = Date.now() + resolveSessionEndBudgetMs(); + const remainingMs = () => budgetEndsAt - Date.now(); + const stepBudget = (bound) => Math.min(bound, Math.max(0, remainingMs())); const brokerSession = loadBrokerSession(cwd) ?? (process.env[BROKER_ENDPOINT_ENV] @@ -118,20 +195,110 @@ async function handleSessionEnd(input) { const sessionDir = brokerSession?.sessionDir ?? null; const pid = brokerSession?.pid ?? null; + let activeJobs; + try { + cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV], stepBudget(STATE_LOCK_STEP_MS)); + activeJobs = activeWorkspaceJobs(cwd, stepBudget(STATE_LOCK_STEP_MS), remainingMs); + } catch (error) { + // A lock this hook could not take says nothing about the broker, and a + // SessionEnd that dies here would take its decision with it: report and leave + // everything alone. Only that one typed failure — matching the message would + // also swallow an integrity error naming the same lock, or a filesystem error + // whose path contains the phrase. Anything else is a real fault and still fails + // the hook. + if (error?.code !== STATE_LOCK_TIMEOUT_CODE) { + throw error; + } + process.stderr.write( + `[codex] SessionEnd could not take the state lock (budgetExhausted=true lock=${String(error?.message ?? error)}); leaving the broker running.\n` + ); + return; + } + + // Every job in this workspace — background worker or another session's + // foreground run — reaches Codex through the same broker. If any of them is + // still active, leave the broker running; a later SessionEnd (or the broker + // itself) tears it down once nothing depends on it anymore. + // + // What keeps this bounded is the broker's own idle self-terminate (#457): + // once the last client disconnects it exits and clears its own record. With + // `CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS=0` that safety net is off, and a + // broker kept alive here for an active job never exits on its own. + if (activeJobs.length > 0) { + process.stderr.write( + `[codex] Workspace jobs still active (${activeJobs.join(", ")} budgetExhausted=${remainingMs() < MIN_STEP_MS}); leaving the broker running.\n` + ); + return; + } + + // The check above is a snapshot; the broker itself has the live answer. If a + // job started in that gap the broker refuses, and refusing means every teardown + // step below would be wrong: the endpoint, pid file and record all still belong + // to a broker somebody is talking to. Leave it to the next SessionEnd or to its + // own idle timeout. + // Only a broker that answered "not busy" (or is provably gone) may be torn + // down. An unanswered or unreadable handshake is not evidence of an idle + // broker, and every step below assumes there is nothing left to talk to. + let busyRetries = 0; if (brokerEndpoint) { - await sendBrokerShutdown(brokerEndpoint); + if (remainingMs() < MIN_STEP_MS) { + process.stderr.write(`[codex] SessionEnd budget spent before the broker handshake (budgetExhausted=true); leaving the broker running.\n`); + return; + } + let shutdown = await sendBrokerShutdown(brokerEndpoint, { timeoutMs: stepBudget(BROKER_HANDSHAKE_STEP_MS) }); + // A `busy` answer straight after this hook reaped the workspace's jobs is + // usually a phantom: the broker counts every connected socket, and a worker + // that was just killed still has one until its close event is processed. That + // clears in milliseconds, so ask again for a moment before believing it. A + // broker that is genuinely serving someone stays busy for the whole window and + // is left alone, exactly as before. + const retriesEndAt = Date.now() + BROKER_BUSY_RETRY_MS; + while (shutdown.busy === true && Date.now() < retriesEndAt && remainingMs() >= MIN_STEP_MS) { + await new Promise((resolve) => setTimeout(resolve, stepBudget(BROKER_BUSY_POLL_MS))); + shutdown = await sendBrokerShutdown(brokerEndpoint, { timeoutMs: stepBudget(BROKER_HANDSHAKE_STEP_MS) }); + busyRetries += 1; + } + if (shutdown.busy !== false) { + process.stderr.write( + shutdown.busy === true + ? `[codex] Shared broker is still serving another session (busyRetries=${busyRetries} budgetExhausted=${remainingMs() < MIN_STEP_MS}); leaving it running.\n` + : `[codex] Shared broker did not confirm it is idle (busyRetries=${busyRetries} budgetExhausted=${remainingMs() < MIN_STEP_MS}); leaving it running.\n` + ); + return; + } } - cleanupSessionJobs(cwd, input.session_id || process.env[SESSION_ID_ENV]); - teardownBrokerSession({ + // Nothing below is worth starting on a spent budget: a teardown interrupted by + // the host mid-way is worse than one that has not begun. + if (remainingMs() < MIN_STEP_MS) { + process.stderr.write( + `[codex] SessionEnd budget spent before teardown (busyRetries=${busyRetries} budgetExhausted=true); leaving the broker running.\n` + ); + return; + } + + const teardown = teardownBrokerSession({ endpoint: brokerEndpoint, pidFile, logFile, sessionDir, pid, - killProcess: terminateProcessTree + killProcess: terminateProcessTree, + timeoutMs: stepBudget(STATE_LOCK_STEP_MS) }); - clearBrokerSession(cwd); + // Every branch of this hook says what it decided: when a broker outlives a + // SessionEnd the only question worth asking is which of these four paths ran. + process.stderr.write( + `[codex] Broker teardown: endpoint=${brokerEndpoint ?? "none"} pid=${pid ?? "none"} signalled=${teardown.signalled} busyRetries=${busyRetries} budgetExhausted=false\n` + ); + + // A replacement broker can have started — and recorded itself — while this one + // was shutting down. Clearing unconditionally would delete the live broker's + // ownership record, which is exactly what the broker's own endpoint-guarded + // `clearOwnSessionRecord` avoids on its side. + if (loadBrokerSession(cwd)?.endpoint === brokerEndpoint) { + clearBrokerSession(cwd); + } } async function main() { diff --git a/plugins/codex/scripts/stop-review-gate-hook.mjs b/plugins/codex/scripts/stop-review-gate-hook.mjs index 4d35a2aa5..ebe185a4f 100644 --- a/plugins/codex/scripts/stop-review-gate-hook.mjs +++ b/plugins/codex/scripts/stop-review-gate-hook.mjs @@ -10,7 +10,7 @@ import { getCodexAvailability } from "./lib/codex.mjs"; import { loadPromptTemplate, interpolateTemplate } from "./lib/prompts.mjs"; import { getConfig, setConfig, listJobs } from "./lib/state.mjs"; import { sortJobsNewestFirst } from "./lib/job-control.mjs"; -import { SESSION_ID_ENV } from "./lib/tracked-jobs.mjs"; +import { reapDeadJobs, SESSION_ID_ENV } from "./lib/tracked-jobs.mjs"; import { resolveWorkspaceRoot } from "./lib/workspace.mjs"; const STOP_REVIEW_TIMEOUT_MINUTES = 13; @@ -192,7 +192,7 @@ function main() { const workspaceRoot = resolveWorkspaceRoot(cwd); const config = getConfig(workspaceRoot); - const jobs = sortJobsNewestFirst(filterJobsForCurrentSession(listJobs(workspaceRoot), input)); + const jobs = sortJobsNewestFirst(filterJobsForCurrentSession(reapDeadJobs(workspaceRoot, listJobs(workspaceRoot)), input)); const runningJob = jobs.find((job) => job.status === "queued" || job.status === "running"); const runningTaskNote = runningJob ? `Codex task ${runningJob.id} is still running. Check /codex:status and use /codex:cancel ${runningJob.id} if you want to stop it before ending the session.` diff --git a/plugins/codex/skills/codex-cli-runtime/SKILL.md b/plugins/codex/skills/codex-cli-runtime/SKILL.md index 15a8151ae..f2b242f49 100644 --- a/plugins/codex/skills/codex-cli-runtime/SKILL.md +++ b/plugins/codex/skills/codex-cli-runtime/SKILL.md @@ -9,12 +9,16 @@ user-invocable: false Use this skill only inside the `codex:codex-rescue` subagent. Primary helper: -- `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task ""` +```bash +node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" task --await --prompt-stdin <<'CODEX_PROMPT_' + +CODEX_PROMPT_ +``` Execution rules: -- The rescue subagent is a forwarder, not an orchestrator. It launches once with `task --background --json`, polls only that job's own `status`, then returns the `result` stdout unchanged. +- The rescue subagent is a forwarder, not an orchestrator. It launches once with `task --await --prompt-stdin`, and on exit 3 re-runs only its own job's printed `result --wait` hint until it reaches a terminal status, then returns the output unchanged. - Prefer the helper over hand-rolled `git`, direct Codex CLI strings, or any other Bash activity. -- Do not call `setup`, `review`, `adversarial-review`, or `cancel` from `codex:codex-rescue`. `status` and `result` are allowed, but only for the job you just launched — never another job. +- Do not call `setup`, `review`, `adversarial-review`, or `cancel` from `codex:codex-rescue`. Re-running the printed `result --wait` hint for the job you just launched is the only follow-up call allowed — never a bare `status` call, never another job. - Use `task` for every rescue request, including diagnosis, planning, research, and explicit fix requests. - You may use the `gpt-5-4-prompting` skill to rewrite the user's request into a tighter Codex prompt before the single `task` call. - That prompt drafting is the only Claude-side work allowed. Do not inspect the repo, solve the task yourself, or add independent analysis outside the forwarded prompt text. @@ -26,19 +30,18 @@ Execution rules: Command selection: - If the request names an uncommon domain (private infra runbooks, vendor-specific tooling), prepend to the task text: "If a matching skill is not already loaded, run `$agent-compat:skill-router` to find a reviewed playbook before starting." (Codex has the agent-compat plugin installed; it routes to reviewed route-only skills offline.) -- Launch exactly one job per rescue handoff with `task --background --json`, then poll only that job with `status --wait --timeout-ms 540000 --json` until it reaches a terminal status, then fetch it with `result `. -- Bash calls share no shell state — carry the job id as literal text between calls, never as a leftover `$JOB` shell variable. If a wait call is cut off by the Bash tool's own 10-minute timeout, re-issue it with the same literal id; the job keeps running server-side. +- Launch exactly one job per rescue handoff with `task --await --prompt-stdin`; on exit 3 the job is still running — re-run exactly the printed `Re-run: node "…" result --wait --timeout-ms 540000` line for that same job until it exits 0 (that call's exit code only reports whether a terminal record was retrieved, not whether the job succeeded). +- The detached worker outlives the companion only when the companion returns on its own (exit 3); a host process-tree kill — e.g. Claude Code's Bash timeout — also kills the worker, so keep `--await-timeout-ms` below the host limit (default 540000 < 600000). +- There is no shell state between calls — the retry is the literal `Re-run:` hint text printed by the previous call, not a `$JOB` shell variable. If the retry itself is cut off by the Bash tool's own 10-minute timeout, re-issue the same literal id again; the job keeps running server-side. - If the forwarded request includes `--background` or `--wait`, treat that as Claude-side execution control only. Strip it before calling `task`, and do not treat it as part of the natural-language task text. - If the forwarded request includes `--model`, normalize `spark` to `gpt-5.3-codex-spark` and pass it through to `task`. - If the forwarded request includes `--effort`, pass it through to `task`. - If the forwarded request includes `--config key=value`, pass every occurrence through to `task` unchanged. -- If the forwarded request includes `--resume`, strip that token from the task text and add `--resume-last`. -- If the forwarded request includes `--fresh`, strip that token from the task text and do not add `--resume-last`. -- `--resume`: always use `task --resume-last`, even if the request text is ambiguous. -- `--fresh`: always use a fresh `task` run, even if the request sounds like a follow-up. +- If the forwarded request includes `--turn-timeout-ms `, pass it through to `task` unchanged; it bounds a single Codex turn (also settable via `CODEX_TURN_TIMEOUT_MS`) and is carried into the background worker with the job, so it applies whether the request resolves synchronously or through the exit-3 retry. +- The invoking command always resolves resume before delegating and hands you a literal `--resume-last` or `--fresh` flag already decided — pass it straight through in ``; never infer it yourself, never call `task-resume-candidate`. - `--config key=value` (repeatable) forwards a `config.toml` override to the Codex thread (`thread/start.config`), e.g. `--config model_provider=ollama`. On `--resume-last` the plugin opens a fresh app-server session (cold resume) so `--config` overrides, sandbox and approval policy take effect; model and effort for the resumed turn are sent on the turn, never on the resume request. - `--effort`: accepted values are `none`, `minimal`, `low`, `medium`, `high`, `xhigh`, `max`, `ultra`. Not every model supports every value; Codex validates the value against the reasoning levels the selected model advertises. -- `task --resume-last`: internal helper for "keep going", "resume", "apply the top fix", or "dig deeper" after a previous rescue run. +- `task --resume-last`: passed through verbatim when the invoking command decided to continue the previous Codex thread — the agent never decides this itself. Safety rules: - Never add `--write` unless the user explicitly asked Codex to modify files. diff --git a/tests/app-server.test.mjs b/tests/app-server.test.mjs index 6f70230fb..e9177d733 100644 --- a/tests/app-server.test.mjs +++ b/tests/app-server.test.mjs @@ -1,7 +1,11 @@ +import net from "node:net"; import { test } from "node:test"; import assert from "node:assert/strict"; -import { AppServerClientBase } from "../plugins/codex/scripts/lib/app-server.mjs"; +import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; +import { makeTempDir } from "./helpers.mjs"; +import { AppServerClientBase, CodexAppServerClient } from "../plugins/codex/scripts/lib/app-server.mjs"; +import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; /** Minimal client that records the JSON-RPC messages it would send. */ class CapturingClient extends AppServerClientBase { @@ -89,3 +93,86 @@ test("permission approval requests grant nothing for the turn", () => { }); assert.deepEqual(client.sent, [{ id: 25, result: { permissions: {}, scope: "turn" } }]); }); + +// `close()` is what the turn timeout uses to kill a runaway turn on a transport +// it owns, so it must never become the second hang: an app-server that ignores +// SIGTERM (or is wedged in a tool call) used to leave it awaiting process exit +// forever. TERM, then KILL, then give up on the process rather than the caller. +test("close() bounds an app-server that ignores SIGTERM", { timeout: 8000 }, async (t) => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const client = await CodexAppServerClient.connect(binDir, { + disableBroker: true, + env: buildEnv(binDir, { FAKE_CODEX_IGNORE_SIGTERM: "1" }) + }); + t.after(() => { + try { + client.proc.kill("SIGKILL"); + } catch { + // Already gone, which is the point of the test. + } + }); + + const started = Date.now(); + await client.close(); + const elapsed = Date.now() - started; + + assert.ok(elapsed < 6000, `close() must be bounded, took ${elapsed} ms`); + assert.equal(client.proc.signalCode, "SIGKILL", "a SIGTERM-immune app-server must be killed outright"); +}); + +// The bound only ever applied to the first call: a second one took the "already +// closed" branch and awaited the raw process-exit promise with no deadline at +// all. The turn timeout always closes twice — `failTurnOnTimeout` closes the +// runaway app-server, then `withAppServer` closes it again on the way out — so +// the one case the deadline exists for is exactly the case that hung. +test("close() stays bounded when it is called twice", { timeout: 15000 }, async (t) => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const client = await CodexAppServerClient.connect(binDir, { + disableBroker: true, + env: buildEnv(binDir, { FAKE_CODEX_IGNORE_SIGTERM: "1" }) + }); + const childPid = client.proc.pid; + t.after(() => { + try { + process.kill(childPid, "SIGKILL"); + } catch { + // Already gone. + } + }); + // A child nothing can kill: the fixture ignores SIGTERM and stdin EOF, and + // swallowing the signals means even the SIGKILL escalation never lands. + client.proc.kill = () => true; + + await client.close(); + const started = Date.now(); + await client.close(); + + assert.ok(Date.now() - started < 1000, `a repeated close must not wait again, took ${Date.now() - started} ms`); + assert.equal(client.proc.exitCode, null, "the test needs a child that never exits"); +}); + +// The broker can be tearing down at the exact moment a background job dials it: +// the connection is accepted and then dropped before `initialize` is answered. +// That is a race, not a failure — the job must go on with its own app-server. +test("a broker that drops the connection during initialize falls back to a direct app-server", async (t) => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const stub = net.createServer((socket) => socket.destroy()); + await new Promise((resolve, reject) => { + stub.once("error", reject); + stub.listen(parseBrokerEndpoint(endpoint).path, resolve); + }); + + let client = null; + t.after(async () => { + await client?.close(); + stub.close(); + }); + + client = await CodexAppServerClient.connect(binDir, { brokerEndpoint: endpoint, env: buildEnv(binDir) }); + assert.equal(client.transport, "direct", "a broker that hangs up must not fail the run"); +}); diff --git a/tests/broker-idle-timeout.test.mjs b/tests/broker-idle-timeout.test.mjs index 8d17d1255..51f4993ff 100644 --- a/tests/broker-idle-timeout.test.mjs +++ b/tests/broker-idle-timeout.test.mjs @@ -7,9 +7,9 @@ import { spawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; -import { makeTempDir } from "./helpers.mjs"; +import { makeTempDir, run } from "./helpers.mjs"; import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; -import { waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { clearBrokerSession, loadBrokerSession, saveBrokerSession, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const BROKER_SCRIPT = path.join(ROOT, "plugins", "codex", "scripts", "app-server-broker.mjs"); @@ -53,6 +53,17 @@ function connectClient(endpoint) { }); } +async function waitFor(predicate, timeoutMs, message) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) { + return; + } + await delay(25); + } + throw new Error(message); +} + function delay(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -196,3 +207,258 @@ test("broker refuses clients that connect after the idle shutdown starts", async } } }); + +// A JSON-RPC client that keeps every message the broker sent, so a test can wait +// for one instead of guessing at timings. +async function openClient(endpoint) { + const socket = await connectClient(endpoint); + socket.setEncoding("utf8"); + const messages = []; + let buffer = ""; + socket.on("data", (chunk) => { + buffer += chunk; + let newlineIndex = buffer.indexOf("\n"); + while (newlineIndex !== -1) { + const line = buffer.slice(0, newlineIndex); + buffer = buffer.slice(newlineIndex + 1); + newlineIndex = buffer.indexOf("\n"); + if (line.trim()) { + messages.push(JSON.parse(line)); + } + } + }); + + const client = { + socket, + messages, + send(message) { + socket.write(`${JSON.stringify(message)}\n`); + }, + async waitFor(predicate, { timeoutMs = 5000 } = {}) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const found = messages.find(predicate); + if (found) { + return found; + } + await delay(25); + } + throw new Error(`Timed out waiting for a broker message. Got: ${JSON.stringify(messages)}`); + }, + async request(id, method, params = {}) { + client.send({ id, method, params }); + return client.waitFor((message) => message.id === id); + }, + close() { + socket.end(); + return new Promise((resolve) => socket.once("close", resolve)); + } + }; + return client; +} + +// The SessionEnd hook's `activeWorkspaceJobs()` check is only a snapshot: another +// session can enqueue a job and connect between that check and the shutdown RPC. +// The broker is the only place that knows whether it is actually idle, so it +// refuses a shutdown while any other client is connected — the requester's own +// connection is the one that does not count. +test("broker refuses shutdown while another client is using it", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const child = spawnBroker({ + cwd: sessionDir, + endpoint, + env: buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "1500" }), + idleTimeoutMs: 20000 + }); + + let holder = null; + let requester = null; + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + holder = await openClient(endpoint); + await holder.request(1, "initialize", {}); + const started = await holder.request(2, "thread/start", { cwd: sessionDir }); + const threadId = started.result.thread.id; + await holder.request(3, "turn/start", { threadId, input: [{ type: "text", text: "hold the runtime" }] }); + + requester = await openClient(endpoint); + const refused = await requester.request(1, "broker/shutdown", {}); + assert.equal(refused.result?.busy, true, `shutdown must be refused: ${JSON.stringify(refused)}`); + assert.equal(child.exitCode, null, "the broker must keep serving after refusing a shutdown"); + + // The refused shutdown must not have cost the holder its turn. + const completed = await holder.waitFor((message) => message.method === "turn/completed", { timeoutMs: 8000 }); + assert.equal(completed.params.threadId, threadId); + + await holder.close(); + holder = null; + await requester.close(); + requester = null; + + const closer = await openClient(endpoint); + const accepted = await closer.request(1, "broker/shutdown", {}); + assert.notEqual(accepted.result?.busy, true, "an idle broker must accept the shutdown"); + const exited = await waitForExit(child, { timeoutMs: 5000 }); + assert.equal(exited.code, 0, "the broker must exit once nothing else is connected"); + } finally { + holder?.socket.destroy(); + requester?.socket.destroy(); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + } +}); + +// A client is a client from the moment it is accepted: `CodexAppServerClient` +// connects and only then writes `initialize`, and a shutdown that lands in that +// gap kills the broker under it. Refusing costs nothing — the requester's own +// connection never counts, and a probe that hangs up only postpones teardown. +test("broker refuses shutdown while a client is connected but has not spoken yet", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const child = spawnBroker({ cwd: sessionDir, endpoint, env: buildEnv(binDir), idleTimeoutMs: 20000 }); + + let silent = null; + let requester = null; + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + silent = await connectClient(endpoint); + requester = await openClient(endpoint); + + const refused = await requester.request(1, "broker/shutdown", {}); + assert.equal(refused.result?.busy, true, `shutdown must be refused: ${JSON.stringify(refused)}`); + assert.equal(child.exitCode, null, "the broker must survive a refused shutdown"); + + const closed = new Promise((resolve) => silent.once("close", resolve)); + silent.end(); + await closed; + silent = null; + await delay(100); + + const accepted = await requester.request(2, "broker/shutdown", {}); + assert.notEqual(accepted.result?.busy, true, "the last client leaving makes the broker shuttable"); + const exited = await waitForExit(child, { timeoutMs: 5000 }); + assert.equal(exited.code, 0); + } finally { + silent?.destroy(); + requester?.socket.destroy(); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + } +}); + +// A client that stops answering must not make the broker immortal. `shutdown()` +// awaited `server.close()`, which fires only once every connection has closed, and +// `socket.end()` is a *graceful* half-close: a peer that never answers the FIN +// leaves the connection open forever. Because SIGTERM is handled (shut down, then +// exit), that made the broker ignore SIGTERM outright — the process a SessionEnd +// had just signalled stayed alive, which is what CI caught on a worker whose +// socket had not been cleaned up yet. +test("broker exits on SIGTERM even when a client never answers the FIN", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const broker = spawnBroker({ cwd: sessionDir, endpoint, env: buildEnv(binDir), idleTimeoutMs: 60000 }); + + let halfOpen = null; + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + // `allowHalfOpen` keeps this client's side open when the broker sends its FIN, + // exactly like a peer that is gone, wedged, or simply slow to notice. + halfOpen = net.createConnection({ path: parseBrokerEndpoint(endpoint).path, allowHalfOpen: true }); + await new Promise((resolve, reject) => { + halfOpen.once("connect", resolve); + halfOpen.once("error", reject); + }); + + broker.kill("SIGTERM"); + const exited = await waitForExit(broker, { timeoutMs: 8000 }); + assert.equal(exited.code, 0, "the broker must still exit cleanly"); + } finally { + halfOpen?.destroy(); + if (broker.exitCode === null && broker.signalCode === null) { + broker.kill("SIGKILL"); + } + } +}); + +function processesMatching(pattern) { + const found = run("pgrep", ["-f", pattern]); + return found.stdout.split("\n").map((line) => line.trim()).filter(Boolean); +} + +// Shutting down takes time — the app-server child has to go, then the client +// sockets. A second trigger arriving in that window (another signal, or a +// `broker/shutdown` already in the queue) used to be answered with an immediate +// `return`, and *its* caller then exited the process: out from under the child +// still being killed, the sockets still being closed, and the endpoint, pid file +// and ownership record still on disk. +test("a second shutdown trigger does not exit before the first has cleaned up", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const endpointPath = parseBrokerEndpoint(endpoint).path; + const pidFile = path.join(sessionDir, "broker.pid"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + const broker = spawn( + process.execPath, + [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", workspace, "--pid-file", pidFile, "--idle-timeout", "60000"], + { + cwd: workspace, + // The app-server lingers after stdin closes and ignores SIGTERM, so closing + // it is slow enough for a second trigger to land mid-shutdown. + env: buildEnv(binDir, { FAKE_CODEX_CLOSE_DELAY_MS: "1500" }), + stdio: ["ignore", "pipe", "pipe"] + } + ); + + let halfOpen = null; + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + await waitFor(() => processesMatching(binDir).length > 0, 3000, "the fake app-server never started"); + + halfOpen = net.createConnection({ path: endpointPath, allowHalfOpen: true }); + await new Promise((resolve, reject) => { + halfOpen.once("connect", resolve); + halfOpen.once("error", reject); + }); + + broker.kill("SIGTERM"); + await delay(150); + broker.kill("SIGTERM"); + + const exited = await waitForExit(broker, { timeoutMs: 10000 }); + assert.equal(exited.code, 0, "the broker must exit cleanly, once"); + + assert.deepEqual(processesMatching(binDir), [], "the app-server child must be gone before the broker exits"); + assert.equal(fs.existsSync(endpointPath), false, "the endpoint socket must be removed"); + assert.equal(fs.existsSync(pidFile), false, "the pid file must be removed"); + assert.equal(loadBrokerSession(workspace), null, "the ownership record must be cleared"); + } finally { + halfOpen?.destroy(); + if (broker.exitCode === null && broker.signalCode === null) { + broker.kill("SIGKILL"); + } + for (const pid of processesMatching(binDir)) { + try { + process.kill(Number(pid), "SIGKILL"); + } catch { + // Already gone. + } + } + clearBrokerSession(workspace); + } +}); diff --git a/tests/broker-stale-pid.test.mjs b/tests/broker-stale-pid.test.mjs index 332b17bb0..150073d6c 100644 --- a/tests/broker-stale-pid.test.mjs +++ b/tests/broker-stale-pid.test.mjs @@ -1,3 +1,5 @@ +import fs from "node:fs"; +import net from "node:net"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; @@ -6,17 +8,20 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { makeTempDir, run } from "./helpers.mjs"; -import { createBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; +import { createBrokerEndpoint, parseBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-endpoint.mjs"; import { clearBrokerSession, loadBrokerSession, saveBrokerSession, + sendBrokerShutdown, waitForBrokerEndpoint } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; +import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const BROKER_SCRIPT = path.join(ROOT, "plugins", "codex", "scripts", "app-server-broker.mjs"); const SESSION_HOOK = path.join(ROOT, "plugins", "codex", "scripts", "session-lifecycle-hook.mjs"); +const SCRIPT = path.join(ROOT, "plugins", "codex", "scripts", "codex-companion.mjs"); function waitForExit(child, { timeoutMs = 10000 } = {}) { return new Promise((resolve, reject) => { @@ -128,3 +133,775 @@ test("session end teardown does not signal a recycled pid that is not this broke clearBrokerSession(workspace); } }); + +function spawnOwnedBroker(workspace, { binDir, sessionDir, endpoint, env }) { + const child = spawn( + process.execPath, + [BROKER_SCRIPT, "serve", "--endpoint", endpoint, "--cwd", workspace, "--pid-file", path.join(sessionDir, "broker.pid")], + { cwd: workspace, env: env ?? buildEnv(binDir), stdio: ["ignore", "pipe", "pipe"] } + ); + saveBrokerSession(workspace, { + endpoint, + pidFile: path.join(sessionDir, "broker.pid"), + logFile: path.join(sessionDir, "broker.log"), + sessionDir, + pid: child.pid + }); + return child; +} + +function runSessionEndHook(workspace, { env = process.env, sessionId = null } = {}) { + return run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: workspace, + env: sessionId ? { ...env, CODEX_COMPANION_SESSION_ID: sessionId } : env, + input: JSON.stringify({ + hook_event_name: "SessionEnd", + cwd: workspace, + ...(sessionId ? { session_id: sessionId } : {}) + }) + }); +} + +// `run` is spawnSync, which blocks this process's event loop — an in-process stub +// server could never accept the hook's connection. Anything that answers the hook +// from within the test has to run it asynchronously. +function runSessionEndHookAsync(workspace, { env = process.env, sessionId = null } = {}) { + const child = spawn("node", [SESSION_HOOK, "SessionEnd"], { + cwd: workspace, + env: sessionId ? { ...env, CODEX_COMPANION_SESSION_ID: sessionId } : env, + stdio: ["pipe", "pipe", "pipe"] + }); + child.stdin.end( + JSON.stringify({ hook_event_name: "SessionEnd", cwd: workspace, ...(sessionId ? { session_id: sessionId } : {}) }) + ); + let stdout = ""; + let stderr = ""; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + // `close`, not `exit`: the process can exit before its stdio is drained, and + // callers assert on what it logged. + return new Promise((resolve) => child.on("close", (status) => resolve({ status, stdout, stderr }))); +} + +async function waitUntil(predicate, { timeoutMs = 8000, intervalMs = 100 } = {}) { + const start = Date.now(); + while (Date.now() - start < timeoutMs) { + const value = await predicate(); + if (value) { + return value; + } + await new Promise((resolve) => setTimeout(resolve, intervalMs)); + } + return null; +} + +// Regression cover for the graceful path: the session that owns the broker ends, +// nothing else depends on it, so the process must be gone and the record must +// not survive to be signalled by a later hook. +test("session end shuts down the live broker it owns and clears its record", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const child = spawnOwnedBroker(workspace, { binDir, sessionDir, endpoint }); + + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + const cleanup = runSessionEndHook(workspace); + assert.equal(cleanup.status, 0, cleanup.stderr); + + const exited = await waitForExit(child, { timeoutMs: 3000 }); + assert.equal(exited.code, 0, "the owned broker must exit on SessionEnd"); + assert.equal(loadBrokerSession(workspace), null, "the broker record must be cleared"); + assert.equal(fs.existsSync(parseBrokerEndpoint(endpoint).path), false, "the endpoint socket must be removed"); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + clearBrokerSession(workspace); + } +}); + +// The broker is per-workspace, so a foreground job of ANOTHER Claude session is +// talking to it too. That job survives this session's cleanup (own jobs only) +// but used to be invisible to the active-job check, which only counted +// `background: true` — so this hook tore the shared runtime out from under a +// live foreign turn. +test("session end keeps the broker while another session's foreground job is running", async (t) => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const child = spawnOwnedBroker(workspace, { binDir, sessionDir, endpoint }); + + // Stands in for the other session's live foreground worker. + const foreignWorker = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + cwd: workspace, + detached: true, + stdio: "ignore" + }); + foreignWorker.unref(); + + t.after(() => { + for (const pid of [foreignWorker.pid, child.pid]) { + try { + process.kill(-pid, "SIGKILL"); + } catch { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Already gone. + } + } + } + clearBrokerSession(workspace); + }); + + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(path.join(stateDir, "jobs"), { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify( + { + version: 1, + config: { stopReviewGate: false }, + jobs: [ + { + id: "task-foreign-foreground", + status: "running", + phase: "running", + title: "Codex Task", + jobClass: "task", + sessionId: "sess-other", + pid: foreignWorker.pid, + logFile: null, + createdAt: "2026-03-18T15:30:00.000Z", + updatedAt: "2026-03-18T15:31:00.000Z" + }, + // The ending session's own finished job: makes the cleanup rewrite run. + { + id: "task-own-done", + status: "completed", + phase: "done", + title: "Codex Task", + jobClass: "task", + sessionId: "sess-current", + pid: null, + logFile: null, + createdAt: "2026-03-18T15:20:00.000Z", + updatedAt: "2026-03-18T15:21:00.000Z" + } + ] + }, + null, + 2 + )}\n`, + "utf8" + ); + + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + const cleanup = runSessionEndHook(workspace, { env: buildEnv(binDir), sessionId: "sess-current" }); + assert.equal(cleanup.status, 0, cleanup.stderr); + + assert.equal( + loadBrokerSession(workspace)?.endpoint, + endpoint, + "SessionEnd must not tear down a broker another session's job is using" + ); + assert.equal(isAlive(child.pid), true, "the shared broker process must survive a foreign active job"); + assert.equal(isAlive(foreignWorker.pid), true, "the foreign session's worker must not be signalled"); + + const jobs = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")).jobs; + assert.deepEqual( + jobs.map((job) => job.id), + ["task-foreign-foreground"], + "the foreign job record must survive while the ending session's own job is pruned" + ); + assert.equal(jobs[0].status, "running"); + assert.equal(jobs[0].pid, foreignWorker.pid); +}); + +// A background job is dispatched to outlive its session, and it talks to Codex +// through the broker: SessionEnd must leave both alone, and the broker's own +// idle timer — not the hook — is what finally reclaims it. +test("session end keeps the broker while an owned background job runs, and the broker idle-exits after it finishes", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const env = buildEnv(binDir, { + CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS: "2000", + CODEX_COMPANION_SESSION_ID: "sess-current", + FAKE_CODEX_TURN_DELAY_MS: "3000" + }); + + const launched = run("node", [SCRIPT, "task", "--background", "--json", "keep me running"], { cwd: workspace, env }); + assert.equal(launched.status, 0, launched.stderr); + const { jobId } = JSON.parse(launched.stdout); + + const stateFile = path.join(resolveStateDir(workspace), "state.json"); + const broker = await waitUntil(() => loadBrokerSession(workspace)); + assert.ok(broker, "the background worker must have started a broker"); + + const cleanup = runSessionEndHook(workspace, { env, sessionId: "sess-current" }); + assert.equal(cleanup.status, 0, cleanup.stderr); + + // The worker's broker is still there, and so is the job it belongs to. + assert.equal(loadBrokerSession(workspace)?.endpoint, broker.endpoint, "SessionEnd must not tear down a broker a background job needs"); + assert.equal(isAlive(broker.pid), true, "the broker process must survive SessionEnd"); + const jobs = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs; + assert.ok(jobs.some((job) => job.id === jobId), "the background job record must survive SessionEnd"); + + const finished = await waitUntil(() => { + const job = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs.find((entry) => entry.id === jobId); + return job && job.status !== "queued" && job.status !== "running" ? job : null; + }, { timeoutMs: 20000 }); + assert.equal(finished?.status, "completed", `background job did not complete: ${JSON.stringify(finished)}`); + + // Nothing is connected any more, so the broker reclaims itself and takes its + // own record with it. + const cleared = await waitUntil(() => (loadBrokerSession(workspace) === null ? "cleared" : null), { timeoutMs: 10000 }); + assert.equal(cleared, "cleared", "the idle broker must clear its own record once the job is done"); + // The record is dropped first and the app-server child is closed after, so the + // process disappears a moment later. + const exited = await waitUntil(() => (isAlive(broker.pid) ? null : "exited"), { timeoutMs: 10000 }); + assert.equal(exited, "exited", "the idle broker must exit once the job is done"); +}); + +// The broker's own `clearOwnSessionRecord` compares endpoints before deleting +// the record; the hook has to be just as careful, or a replacement broker that +// started while the old one was shutting down loses its ownership record and +// becomes unreachable. +test("session end does not clear the record of a replacement broker started during shutdown", async () => { + const workspace = makeTempDir(); + const oldSessionDir = makeTempDir("cxc-"); + const newSessionDir = makeTempDir("cxc-"); + const oldEndpoint = createBrokerEndpoint(oldSessionDir); + const newEndpoint = createBrokerEndpoint(newSessionDir); + const replacement = { + endpoint: newEndpoint, + pidFile: path.join(newSessionDir, "broker.pid"), + logFile: path.join(newSessionDir, "broker.log"), + sessionDir: newSessionDir, + pid: null + }; + + saveBrokerSession(workspace, { + endpoint: oldEndpoint, + pidFile: path.join(oldSessionDir, "broker.pid"), + logFile: path.join(oldSessionDir, "broker.log"), + sessionDir: oldSessionDir, + pid: null + }); + + // Stands in for the broker that is shutting down: it accepts the graceful + // `broker/shutdown`, and the replacement records itself inside that window. + let replaced = false; + const stub = net.createServer((socket) => { + socket.once("data", () => { + saveBrokerSession(workspace, replacement); + replaced = true; + setTimeout(() => socket.end(`${JSON.stringify({ id: 1, result: {} })}\n`), 100); + }); + }); + await new Promise((resolve, reject) => { + stub.once("error", reject); + stub.listen(parseBrokerEndpoint(oldEndpoint).path, resolve); + }); + + try { + const hook = spawn("node", [SESSION_HOOK, "SessionEnd"], { cwd: workspace, env: process.env, stdio: ["pipe", "pipe", "pipe"] }); + hook.stdin.end(JSON.stringify({ hook_event_name: "SessionEnd", cwd: workspace })); + const code = await new Promise((resolve) => hook.on("exit", resolve)); + + assert.equal(code, 0); + assert.equal(replaced, true, "the replacement must have been recorded during the shutdown window"); + assert.equal(loadBrokerSession(workspace)?.endpoint, newEndpoint, "the replacement broker must keep its record"); + } finally { + stub.close(); + clearBrokerSession(workspace); + } +}); + +// A worker killed outright (SIGKILL/OOM) never writes a terminal status. If the +// active-background check trusts that stale `running` record, every later +// SessionEnd in the workspace takes the early return and the broker — plus its +// app-server child — lingers forever. +test("session end reaps a SIGKILLed background worker instead of keeping its broker alive", async (t) => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const env = buildEnv(binDir, { + // Long enough that only the hook can be the reason the broker goes away. + CODEX_COMPANION_BROKER_IDLE_TIMEOUT_MS: "20000", + CODEX_COMPANION_SESSION_ID: "sess-current", + FAKE_CODEX_TURN_DELAY_MS: "20000" + }); + + const launched = run("node", [SCRIPT, "task", "--background", "--json", "die mid-turn"], { cwd: workspace, env }); + assert.equal(launched.status, 0, launched.stderr); + const { jobId } = JSON.parse(launched.stdout); + + const stateFile = path.join(resolveStateDir(workspace), "state.json"); + const running = await waitUntil(() => { + const job = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs.find((entry) => entry.id === jobId); + return job && job.status === "running" && job.pid ? job : null; + }, { timeoutMs: 20000 }); + assert.ok(running, "the background worker must have taken over its record"); + const broker = await waitUntil(() => loadBrokerSession(workspace)); + assert.ok(broker, "the background worker must have started a broker"); + + t.after(() => { + for (const pid of [running.pid, broker.pid]) { + try { + process.kill(-pid, "SIGKILL"); + } catch { + try { + process.kill(pid, "SIGKILL"); + } catch { + // Already gone. + } + } + } + clearBrokerSession(workspace); + }); + + process.kill(-running.pid, "SIGKILL"); + await waitUntil(() => (isAlive(running.pid) ? null : "dead")); + + const cleanup = runSessionEndHook(workspace, { env, sessionId: "sess-current" }); + assert.equal(cleanup.status, 0, cleanup.stderr); + + const exited = await waitUntil(() => (isAlive(broker.pid) ? null : "exited"), { timeoutMs: 5000 }); + assert.equal(exited, "exited", `a dead worker must not keep the broker alive; hook said: ${cleanup.stderr.trim()}`); + assert.equal(loadBrokerSession(workspace), null, "the broker record must be cleared"); + + const job = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs.find((entry) => entry.id === jobId); + assert.equal(job.status, "failed", "the dead worker's job must be reaped"); + assert.equal(job.requestFile, null, "the reaped job must not keep its private payload path"); +}); + +// The failure this reproduces: a background worker SIGKILLed while it was taking +// the workspace state lock left the lock directory behind with nothing inside to +// identify its holder. The dead-PID takeover had no PID to check, so the bounded +// wait expired and the SessionEnd hook died with "Timed out … waiting for the +// Codex state lock" — leaving the broker and its app-server child running. +test("session end recovers the lock a killed worker left behind", async () => { + const binDir = makeTempDir(); + installFakeCodex(binDir); + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const child = spawnOwnedBroker(workspace, { binDir, sessionDir, endpoint }); + + try { + assert.equal(await waitForBrokerEndpoint(endpoint, 3000), true); + + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(path.join(stateDir, "jobs"), { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify( + { + version: 1, + config: { stopReviewGate: false }, + jobs: [ + { + id: "task-finished", + status: "completed", + phase: "done", + sessionId: "sess-current", + updatedAt: "2026-03-24T20:05:00.000Z" + } + ] + }, + null, + 2 + )}\n`, + "utf8" + ); + // A worker SIGKILLed while it held the lock leaves its ticket behind; the + // pre-1.2.0 lock directory alongside it must simply be ignored. + const deadWorker = run(process.execPath, ["-e", "process.exit(0)"], { env: process.env }); + const lockDir = path.join(stateDir, "state.lock.d"); + fs.mkdirSync(lockDir, { recursive: true }); + fs.writeFileSync( + path.join(lockDir, `1.${deadWorker.pid}-killed.ticket`), + `${JSON.stringify({ pid: deadWorker.pid, startedAt: new Date().toISOString() })}\n`, + "utf8" + ); + fs.mkdirSync(path.join(stateDir, "state.lock"), { recursive: true }); + + const cleanup = runSessionEndHook(workspace, { env: buildEnv(binDir), sessionId: "sess-current" }); + assert.equal(cleanup.status, 0, cleanup.stderr); + + const exited = await waitForExit(child, { timeoutMs: 5000 }); + assert.equal(exited.code, 0, "an abandoned lock must not keep the broker alive"); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + clearBrokerSession(workspace); + } +}); + +function listenStub(endpoint, onConnection) { + const stub = net.createServer(onConnection); + return new Promise((resolve, reject) => { + stub.once("error", reject); + stub.listen(parseBrokerEndpoint(endpoint).path, () => resolve(stub)); + }); +} + +// A socket is a byte stream, not a message stream: the reply can arrive in as +// many chunks as the kernel feels like. Parsing each chunk on its own turned a +// split `{"busy":true}` into a parse error — read as "not busy", which is how a +// SessionEnd would tear down a broker in the middle of another session's turn. +test("sendBrokerShutdown reads a busy reply that arrives in fragments", async () => { + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const sockets = []; + const stub = await listenStub(endpoint, (socket) => { + sockets.push(socket); + socket.once("data", () => { + socket.write('{"id":1,"result":{"bu'); + setTimeout(() => socket.write('sy":true}}\n'), 50); + }); + }); + + try { + assert.deepEqual(await sendBrokerShutdown(endpoint), { busy: true }); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + } +}); + +// A peer that accepts the connection and never answers used to block the hook +// forever. It is also not proof of anything: an unanswered handshake must not be +// read as "idle, safe to destroy". +test("a broker that never answers is bounded and never assumed idle", { timeout: 20000 }, async () => { + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const sockets = []; + const stub = await listenStub(endpoint, (socket) => sockets.push(socket)); + + try { + const started = Date.now(); + const outcome = await sendBrokerShutdown(endpoint); + const elapsed = Date.now() - started; + + assert.equal(outcome.busy, null, "an unanswered handshake must report unknown, not idle"); + assert.ok(elapsed < 9000, `the handshake must be bounded, took ${elapsed} ms`); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + } +}); + +test("session end leaves everything alone when the broker never answers", { timeout: 20000 }, async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999\n", "utf8"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + const sockets = []; + const stub = await listenStub(endpoint, (socket) => sockets.push(socket)); + + try { + const cleanup = await runSessionEndHookAsync(workspace); + assert.equal(cleanup.status, 0, cleanup.stderr); + assert.equal(loadBrokerSession(workspace)?.endpoint, endpoint, "an unconfirmed broker must keep its record"); + assert.equal(fs.existsSync(pidFile), true, "an unconfirmed broker must not be torn down"); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + clearBrokerSession(workspace); + } +}); + +// The broker counts every connected socket as a client, and a worker that was +// just SIGKILLed still has one until its close event is processed. A SessionEnd +// that reaped that very worker and then believed the resulting `busy` answer left +// the broker running for nothing — the phantom clears milliseconds later. +test("session end retries a busy answer that is about to clear", async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999\n", "utf8"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + // Count the answers rather than the clock: under load the first handshake can + // land after any wall-clock window, and then the retry path is never exercised. + let answered = 0; + const sockets = []; + const stub = await listenStub(endpoint, (socket) => { + sockets.push(socket); + socket.once("data", () => { + const stillBusy = answered++ < 2; + socket.write(`${JSON.stringify({ id: 1, result: stillBusy ? { busy: true } : {} })}\n`); + }); + }); + + try { + const cleanup = await runSessionEndHookAsync(workspace); + assert.equal(cleanup.status, 0, cleanup.stderr); + assert.equal(loadBrokerSession(workspace), null, `a busy answer that clears must not stop the teardown: ${cleanup.stderr.trim()}`); + assert.equal(fs.existsSync(pidFile), false, "the pid file must be removed"); + assert.match(cleanup.stderr, /busyRetries=[1-9]/, "the decision line must report the retries"); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + clearBrokerSession(workspace); + } +}); + +// The other side of the same rule: a broker that is genuinely busy stays busy, and +// the retry window changes nothing about leaving it alone. +test("session end still leaves a persistently busy broker alone", async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999\n", "utf8"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + const sockets = []; + const stub = await listenStub(endpoint, (socket) => { + sockets.push(socket); + socket.once("data", () => socket.write(`${JSON.stringify({ id: 1, result: { busy: true } })}\n`)); + }); + + try { + const cleanup = await runSessionEndHookAsync(workspace); + assert.equal(cleanup.status, 0, cleanup.stderr); + assert.equal(loadBrokerSession(workspace)?.endpoint, endpoint, "a busy broker keeps its record"); + assert.equal(fs.existsSync(pidFile), true, "a busy broker keeps its pid file"); + assert.match(cleanup.stderr, /still serving another session/); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + clearBrokerSession(workspace); + } +}); + +// Every step of this hook has a bound of its own — the state lock, each broker +// handshake, the busy retries — and they add up past any single one of them. Claude +// Code kills the hook at the timeout in hooks.json, so a `busy` answer followed by +// a broker that stops answering used to get the hook killed before it could decide +// anything. One absolute budget, clamped into every step, keeps the decision inside +// the host's timeout. +test("session end stays inside its budget when a busy broker then goes silent", async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999\n", "utf8"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + const sockets = []; + let answered = 0; + const stub = await listenStub(endpoint, (socket) => { + sockets.push(socket); + socket.once("data", () => { + answered += 1; + // Busy once, then nothing at all. + if (answered === 1) { + socket.write(`${JSON.stringify({ id: 1, result: { busy: true } })}\n`); + } + }); + }); + + const budgetMs = 3000; + try { + const started = Date.now(); + const cleanup = await runSessionEndHookAsync(workspace, { + env: { ...process.env, CODEX_COMPANION_SESSION_END_BUDGET_MS: String(budgetMs) } + }); + const elapsed = Date.now() - started; + + assert.equal(cleanup.status, 0, cleanup.stderr); + assert.ok(elapsed < budgetMs + 2000, `the hook must stay inside its budget, took ${elapsed} ms: ${cleanup.stderr.trim()}`); + assert.match(cleanup.stderr, /busyRetries=\d+/, "the decision line must report the retries"); + assert.match(cleanup.stderr, /leaving it running/, "the hook must log the decision it made"); + assert.doesNotMatch(cleanup.stderr, /ignored/, "an override below the ceiling must be honoured, not clamped"); + assert.equal(loadBrokerSession(workspace)?.endpoint, endpoint, "an unconfirmed broker keeps its record"); + assert.equal(fs.existsSync(pidFile), true, "an unconfirmed broker keeps its pid file"); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + clearBrokerSession(workspace); + } +}); + +// The budget's ceiling is not negotiable from the environment: `hooks.json`'s +// timeout is a fixed number, so an override above the ceiling would push the +// deadline past the point where Claude Code kills the hook — reintroducing exactly +// the failure the budget prevents. +test("a SessionEnd budget override above the ceiling is ignored", async () => { + const workspace = makeTempDir(); + const sessionDir = makeTempDir("cxc-"); + const endpoint = createBrokerEndpoint(sessionDir); + const pidFile = path.join(sessionDir, "broker.pid"); + fs.writeFileSync(pidFile, "999999\n", "utf8"); + saveBrokerSession(workspace, { endpoint, pidFile, logFile: path.join(sessionDir, "broker.log"), sessionDir, pid: null }); + + const sockets = []; + const stub = await listenStub(endpoint, (socket) => sockets.push(socket)); + + try { + const started = Date.now(); + const cleanup = await runSessionEndHookAsync(workspace, { + env: { ...process.env, CODEX_COMPANION_SESSION_END_BUDGET_MS: "20000" } + }); + const elapsed = Date.now() - started; + + assert.equal(cleanup.status, 0, cleanup.stderr); + assert.match( + cleanup.stderr, + /budget override 20000 ignored: above the 12000 ms ceiling/, + "an override above the ceiling must be refused, with a reason" + ); + assert.ok(elapsed < 14000, `the effective budget must stay at the ceiling, took ${elapsed} ms`); + assert.match(cleanup.stderr, /leaving it running/, "the hook must still log its decision"); + assert.equal(loadBrokerSession(workspace)?.endpoint, endpoint, "an unconfirmed broker keeps its record"); + } finally { + for (const socket of sockets) { + socket.destroy(); + } + stub.close(); + clearBrokerSession(workspace); + } +}); + +// Reaping costs one lock acquisition per dead job, so a wedged lock holder used to +// cost the hook that bound N times over — and a wait that expires throws, which +// escaped as a crash with no decision at all. The waits are clamped to what is left +// of the budget, and a lock this hook cannot take is reported, not fatal. +test("session end reports a wedged state lock instead of dying on it", async () => { + const workspace = makeTempDir(); + const stateDir = resolveStateDir(workspace); + fs.mkdirSync(path.join(stateDir, "jobs"), { recursive: true }); + + const deadJobs = [1, 2, 3].map((index) => { + const finished = run(process.execPath, ["-e", "process.exit(0)"], { env: process.env }); + return { + id: `task-dead-${index}`, + status: "running", + phase: "running", + pid: finished.pid, + background: true, + updatedAt: "2026-03-24T20:05:00.000Z" + }; + }); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, jobs: deadJobs }, null, 2)}\n`, + "utf8" + ); + + // A live holder: never evictable, so every acquisition can only time out. + const lockDir = path.join(stateDir, "state.lock.d"); + fs.mkdirSync(lockDir, { recursive: true }); + fs.writeFileSync( + path.join(lockDir, `1.${process.pid}-wedged.ticket`), + `${JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() })}\n`, + "utf8" + ); + + const budgetMs = 3000; + const started = Date.now(); + const cleanup = await runSessionEndHookAsync(workspace, { + env: { ...process.env, CODEX_COMPANION_SESSION_END_BUDGET_MS: String(budgetMs) } + }); + const elapsed = Date.now() - started; + + assert.equal(cleanup.status, 0, `a lock this hook cannot take must not fail it: ${cleanup.stderr.trim()}`); + assert.ok(elapsed < budgetMs + 2000, `the hook must stay inside its budget, took ${elapsed} ms`); + assert.match(cleanup.stderr, /budgetExhausted=true/, "the decision line must say the budget decided it"); + assert.match(cleanup.stderr, /state lock/i, "the decision line must name the lock"); +}); + +// Matching the message would swallow more than the timeout: the lock's own +// integrity error names the same lock, and any filesystem error whose path happens +// to contain the phrase reads the same way. Only the typed timeout may be absorbed; +// everything else has to fail the hook loudly. +test( + "session end fails on a lock error that is not the timeout", + { skip: process.platform === "win32" || process.getuid?.() === 0 }, + async () => { + const workspace = makeTempDir(); + const pluginData = path.join(makeTempDir(), "state lock data"); + fs.mkdirSync(pluginData, { recursive: true }); + + const previous = process.env.CLAUDE_PLUGIN_DATA; + process.env.CLAUDE_PLUGIN_DATA = pluginData; + let stateDir; + try { + stateDir = resolveStateDir(workspace); + } finally { + if (previous === undefined) { + delete process.env.CLAUDE_PLUGIN_DATA; + } else { + process.env.CLAUDE_PLUGIN_DATA = previous; + } + } + // The lock's path now contains the phrase a message match would key on. + assert.match(stateDir, /state lock/); + + fs.mkdirSync(path.join(stateDir, "jobs"), { recursive: true }); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify( + { + version: 1, + config: { stopReviewGate: false }, + jobs: [{ id: "task-done", status: "completed", sessionId: "sess-current", updatedAt: "2026-03-24T20:05:00.000Z" }] + }, + null, + 2 + )}\n`, + "utf8" + ); + const lockDir = path.join(stateDir, "state.lock.d"); + fs.mkdirSync(lockDir, { recursive: true }); + fs.chmodSync(lockDir, 0o000); + + try { + const cleanup = await runSessionEndHookAsync(workspace, { + env: { ...process.env, CLAUDE_PLUGIN_DATA: pluginData }, + sessionId: "sess-current" + }); + + assert.equal(cleanup.status, 1, `a real lock failure must fail the hook: ${cleanup.stderr.trim()}`); + assert.match(cleanup.stderr, /EACCES/, "the real error must reach the operator"); + assert.doesNotMatch(cleanup.stderr, /budgetExhausted/, "a real failure must not be reported as a spent budget"); + } finally { + fs.chmodSync(lockDir, 0o700); + } + } +); diff --git a/tests/commands.test.mjs b/tests/commands.test.mjs index 8629e7b5e..0a1205ae8 100644 --- a/tests/commands.test.mjs +++ b/tests/commands.test.mjs @@ -22,6 +22,7 @@ test("review command uses AskUserQuestion and background Bash while staying revi assert.match(source, /```typescript/); assert.match(source, /review --args-stdin <<'CODEX_ARGS'/); assert.match(source, /\[--scope auto\|working-tree\|branch\]/); + assert.match(source, /\[--turn-timeout-ms \]/); assert.match(source, /run_in_background:\s*true/); assert.match(source, /command:\s*`node "\$\{CLAUDE_PLUGIN_ROOT\}\/scripts\/codex-companion\.mjs" review --args-stdin <<'CODEX_ARGS'\n\$ARGUMENTS\nCODEX_ARGS`/); assert.match(source, /description:\s*"Codex review"/); @@ -50,6 +51,7 @@ test("adversarial review command uses AskUserQuestion and background Bash while assert.match(source, /```typescript/); assert.match(source, /adversarial-review --args-stdin <<'CODEX_ARGS'/); assert.match(source, /\[--scope auto\|working-tree\|branch\].*\[focus \.\.\.\]/); + assert.match(source, /\[--turn-timeout-ms \]/); assert.match(source, /run_in_background:\s*true/); assert.match(source, /command:\s*`node "\$\{CLAUDE_PLUGIN_ROOT\}\/scripts\/codex-companion\.mjs" adversarial-review --args-stdin <<'CODEX_ARGS'\n\$ARGUMENTS\nCODEX_ARGS`/); assert.match(source, /description:\s*"Codex adversarial review"/); @@ -90,40 +92,39 @@ test("rescue command absorbs continue semantics", () => { const readme = fs.readFileSync(path.join(ROOT, "README.md"), "utf8"); const runtimeSkill = read("skills/codex-cli-runtime/SKILL.md"); - assert.match(rescue, /Show the `result` output to the user verbatim/i); - assert.match(rescue, /allowed-tools:\s*Bash,\s*AskUserQuestion,\s*Agent/); + assert.match(rescue, /show the output verbatim, then your assessment/i); + assert.match(rescue, /allowed-tools:\s*Bash\(node:\*\),\s*AskUserQuestion,\s*Agent/); // Regression for #234: `Skill(codex:rescue)` from the main agent recursed // because rescue.md named the routing with ambiguous prose ("Route this // request to the `codex:codex-rescue` subagent") while running under // `context: fork` — forked general-purpose subagents do not expose the // `Agent` tool, so the fork fell back to `Skill` and re-entered this - // command. Pin the explicit transport and the inline (no-fork) execution. - assert.match(rescue, /subagent_type: "codex:codex-rescue"/); + // command. Pin the explicit `Agent` tool invocation naming `codex:codex-rescue` + // and the inline (no-fork) execution. + assert.match(rescue, /invoke the `Agent` tool with `codex:codex-rescue`/); assert.match(rescue, /do not call `Skill\(codex:rescue\)`/i); + // Covers the separate `task-resume-candidate --json` Bash call too: it isn't + // part of the payload block, so it needs its own non-zero-exit fallback. + assert.match(rescue, /If any Bash step exits non-zero, show its stderr to the user — never report "no result"/i); assert.doesNotMatch(rescue, /^context:\s*fork\b/m); - assert.match(rescue, /--background\|--wait/); + assert.match(rescue, /\[--background\]/); assert.match(rescue, /--resume\|--fresh/); assert.match(rescue, /--model /); assert.match(rescue, /--effort /); + assert.match(rescue, /\[--turn-timeout-ms \]/); assert.match(rescue, /task-resume-candidate --json/); assert.match(rescue, /AskUserQuestion.*Continue current Codex thread/s); assert.match(rescue, /Start a new Codex thread/); assert.match(rescue, /Default is synchronous/i); - assert.match(rescue, /Strip `--wait` if present/i); - assert.match(rescue, /Pass `--model`, `--effort` and every `--config key=value` through unchanged/i); + assert.match(rescue, /Strip `--background`, `--wait`, `--resume`, and `--fresh` out of ``/i); + assert.match(rescue, /Pass `--model`, `--effort`, `--config key=value` through/i); assert.match(rescue, /Delegate the request to Codex through the shared companion runtime/i); assert.match(agent, /--resume/); assert.match(agent, /--fresh/); assert.match(agent, /thin forwarding wrapper/i); - assert.match(agent, /result "\$JOB"/); + assert.match(agent, /the output ends with a `Re-run:` line/i); assert.doesNotMatch(agent, /prefer background execution/i); - // The rescue shell traps must use `command rm -f --` so a shadowing/aliased - // `rm` (or a filename starting with `-`) can't hijack cleanup on EXIT. - assert.match(rescue, /trap 'command rm -f -- /); - assert.match(agent, /trap 'command rm -f -- /); - assert.doesNotMatch(rescue, /trap 'rm -f/); - assert.doesNotMatch(agent, /trap 'rm -f/); - assert.match(runtimeSkill, /Launch exactly one job per rescue handoff with `task --background --json`/i); + assert.match(runtimeSkill, /Launch exactly one job per rescue handoff with `task --await --prompt-stdin`/i); assert.match(agent, /Bash tool's 10-minute cap/i); assert.match(agent, /do not inspect the repository, read files, grep, cancel jobs, summarize output, or do any other follow-up work of your own/i); assert.match(agent, /Do not call `review`, `adversarial-review`, or `cancel`/i); @@ -136,9 +137,9 @@ test("rescue command absorbs continue semantics", () => { assert.match(agent, /gpt-5-4-prompting/); assert.match(agent, /only to tighten the user's request into a better Codex prompt/i); assert.match(agent, /Do not use that skill to inspect the repository, reason through the problem yourself, draft a solution, or do any independent work/i); - assert.match(runtimeSkill, /launches once with `task --background --json`, polls only that job's own `status`, then returns the `result` stdout unchanged/i); + assert.match(runtimeSkill, /launches once with `task --await --prompt-stdin`, and on exit 3 re-runs only its own job's printed `result --wait` hint/i); assert.match(runtimeSkill, /Do not call `setup`, `review`, `adversarial-review`, or `cancel` from `codex:codex-rescue`/i); - assert.match(runtimeSkill, /`status` and `result` are allowed, but only for the job you just launched/i); + assert.match(runtimeSkill, /Re-running the printed `result --wait` hint for the job you just launched is the only follow-up call allowed/i); assert.match(runtimeSkill, /use the `gpt-5-4-prompting` skill to rewrite the user's request into a tighter Codex prompt/i); assert.match(runtimeSkill, /That prompt drafting is the only Claude-side work allowed/i); assert.match(runtimeSkill, /Leave `--effort` unset unless the user explicitly requests a specific effort/i); @@ -170,17 +171,20 @@ test("rescue runs synchronously through the companion and uses Agent only for -- const rescue = fs.readFileSync(path.join(PLUGIN_ROOT, "commands", "rescue.md"), "utf8"); const agent = fs.readFileSync(path.join(PLUGIN_ROOT, "agents", "codex-rescue.md"), "utf8"); const runtimeSkill = fs.readFileSync(path.join(PLUGIN_ROOT, "skills", "codex-cli-runtime", "SKILL.md"), "utf8"); - assert.match(rescue, /task --background --json/); - assert.match(rescue, /status "\$JOB" --wait --timeout-ms 540000/); - assert.match(rescue, /result "\$JOB"/); - assert.match(rescue, /Only when the request contains `--background`.*Agent/s); + assert.match(rescue, /task --await --prompt-stdin/); + assert.match(rescue, /the output ends with a `Re-run:` line/i); + assert.match(rescue, /`--background`: invoke the `Agent` tool with `codex:codex-rescue`/); assert.match(rescue, /--config/); assert.doesNotMatch(agent, /^model:/m); assert.doesNotMatch(agent, /return nothing/i); assert.match(agent, /exit status and stderr/i); - assert.match(agent, /task --background --json/); - assert.match(agent, /status "\$JOB" --wait --timeout-ms 540000/); - assert.doesNotMatch(agent, /Do not .*poll status/i); + assert.match(agent, /task --await --prompt-stdin/); + assert.match(agent, /the output ends with a `Re-run:` line/i); + assert.doesNotMatch(agent, /task-resume-candidate --json/); + // Fix round 1: the agent must defer to SKILL.md's "only permitted follow-up + // is the printed Re-run line" rule instead of granting itself a separate + // bare-`status` permission. + assert.doesNotMatch(agent, /own `status`/); assert.match(agent, /--config/); assert.doesNotMatch(runtimeSkill, /return nothing/i); assert.match(runtimeSkill, /Map `sol` to `--model gpt-5\.6-sol`/i); @@ -199,6 +203,7 @@ test("transfer, result, and cancel commands are exposed as deterministic runtime assert.match(transfer, /codex-companion\.mjs" transfer --args-stdin <<'CODEX_ARGS'/); assert.match(transfer, /codex resume /); assert.match(result, /disable-model-invocation:\s*true/); + assert.match(result, /argument-hint:\s*'\[job-id\] \[--wait\] \[--timeout-ms \]'/); assert.match(result, /codex-companion\.mjs" result --args-stdin <<'CODEX_ARGS'/); assert.match(cancel, /disable-model-invocation:\s*true/); assert.match(cancel, /codex-companion\.mjs" cancel --args-stdin <<'CODEX_ARGS'/); @@ -211,7 +216,7 @@ test("internal docs use task terminology for rescue runs", () => { const promptingSkill = read("skills/gpt-5-4-prompting/SKILL.md"); const promptRecipes = read("skills/gpt-5-4-prompting/references/codex-prompt-recipes.md"); - assert.match(runtimeSkill, /codex-companion\.mjs" task ""/); + assert.match(runtimeSkill, /codex-companion\.mjs" task --await --prompt-stdin/); assert.match(runtimeSkill, /Use `task` for every rescue request/i); assert.match(runtimeSkill, /task --resume-last/i); assert.match(promptingSkill, /Use `task` when the task is diagnosis/i); @@ -296,8 +301,10 @@ test("command bodies hand arguments to the companion via a quoted heredoc, never // whole body line of a quoted heredoc. Anywhere else the shell expands what // Claude Code substituted before bash ever ran. assertArgumentsNeverReachTheShell(file, body); - // rescue.md randomizes its delimiter suffix per call; the flag-only bodies keep the fixed one. - const expectedDelimiter = file === "rescue.md" ? /--args-stdin <<'CODEX_ARGS_/ : /--args-stdin <<'CODEX_ARGS'/; + // rescue.md sends only the request prose through a randomized --prompt-stdin + // heredoc (flags travel on the command line); the other seven command bodies + // keep the fixed `--args-stdin <<'CODEX_ARGS'` delimiter for their flag-only payload. + const expectedDelimiter = file === "rescue.md" ? /--prompt-stdin <<'CODEX_PROMPT_/ : /--args-stdin <<'CODEX_ARGS'/; assert.match(body, expectedDelimiter, `${file} must pass arguments through a quoted heredoc`); } @@ -307,49 +314,118 @@ test("command bodies hand arguments to the companion via a quoted heredoc, never assert.doesNotMatch(body, /""/, `${label} still interpolates the request text inside a shell string`); assert.match( body, - /task --background --json --prompt-file "\$PROMPT" --args-stdin <<'CODEX_ARGS/, + /task --await --prompt-stdin <<'CODEX_PROMPT_/, `${label} must launch through a quoted heredoc` ); + } +}); + +test("rescue sends the request prose through --prompt-stdin, never through the argument tokenizer", () => { + for (const [label, body] of [ + ["rescue.md", read("commands/rescue.md")], + ["codex-rescue.md", read("agents/codex-rescue.md")] + ]) { + assert.match(body, /--prompt-stdin <<'CODEX_PROMPT_/, `${label} must pass the request prose via --prompt-stdin`); + assert.doesNotMatch(body, /--prompt-file/, `${label} must not use the old --prompt-file channel`); + assert.doesNotMatch(body, /--args-stdin/, `${label} must not use the old --args-stdin channel`); + assert.doesNotMatch(body, /\$PROMPT\b/, `${label} must not carry the prompt through a shell variable`); + + // A payload line equal to a fixed delimiter would close the heredoc early and + // run the rest on the host shell. + assert.match(body, /8 fresh random hex/i, `${label} must require a fresh random heredoc delimiter`); + assert.match(body, /CODEX_PROMPT_/, `${label} must name the randomized delimiter placeholder`); assert.match( body, - /\[\[ "\$JOB" =~ \^\[A-Za-z0-9_-\]\+\$ \]\] \|\| \{ echo "invalid job id"; exit 1; \}/, - `${label} must validate the job id before using it` + /a payload line equal to it would end the heredoc early and run the rest on the host shell/i, + `${label} must explain why the delimiter must not collide with request text` ); } }); -// The request prose and the runtime flags travel in separate channels: the prose -// via --prompt-file (byte-exact) and only the flags through the tokenizer. -function readArgsHeredocBody(body) { - const lines = body.split("\n"); - const start = lines.findIndex((line) => line.includes("--args-stdin <<'CODEX_ARGS")); - assert.notEqual(start, -1, "no --args-stdin heredoc found"); - const end = lines.findIndex((line, index) => index > start && line.trim().startsWith("CODEX_ARGS")); - assert.notEqual(end, -1, "unterminated --args-stdin heredoc"); - return lines.slice(start + 1, end).join("\n"); +function extractFirstBashBlock(body, label) { + const match = body.match(/```bash\n([\s\S]*?)```/); + assert.ok(match, `${label} must contain a fenced bash block`); + return match[1]; } -test("rescue sends the request prose through --prompt-file, never through the argument tokenizer", () => { +test("rescue and agent payload blocks are a single node call with no leftover shell scaffolding", () => { for (const [label, body] of [ ["rescue.md", read("commands/rescue.md")], ["codex-rescue.md", read("agents/codex-rescue.md")] ]) { - assert.match(body, /cat > "\$PROMPT" <<'CODEX_PROMPT_/, `${label} must write the request prose with its own quoted heredoc`); - assert.match(body, /--prompt-file "\$PROMPT"/, `${label} must pass the prose file to the companion`); - assert.doesNotMatch( - readArgsHeredocBody(body), - //, - `${label} still routes the request text through the argument tokenizer` + const block = extractFirstBashBlock(body, label); + const invocations = block.match(/codex-companion\.mjs/g) || []; + assert.equal(invocations.length, 1, `${label} payload block must invoke codex-companion.mjs exactly once`); + assert.match(block, /task --await --prompt-stdin/, `${label} payload block must call task --await --prompt-stdin`); + for (const banned of [/mktemp/, /cat >/, /\bwhile\b/, /sleep/, /\$JOB=/]) { + assert.doesNotMatch(block, banned, `${label} payload block must not contain ${banned}`); + } + // sit on the host command line unconstrained; the payload block + // itself must never carry command substitution or a literal backtick. + assert.doesNotMatch(block, /\$\(|`/, `${label} payload block must not contain $() or backticks`); + assert.match( + body, + /may contain only bare tokens/i, + `${label} must document the hygiene rule` + ); + assert.match( + body, + /never place it on the command line/i, + `${label} hygiene rule must tell the caller to drop unsafe flag values instead of using them` ); - assert.match(body, /command rm -f -- "\$ERR" "\$OUT" "\$PROMPT"/, `${label} must clean up the prose file`); - - // A payload line equal to a fixed delimiter would close the heredoc early and - // run the rest on the host shell. - assert.match(body, /fresh random suffix on every call/, `${label} must require per-call heredoc delimiters`); - assert.match(body, /`CODEX_PROMPT_` \/ `CODEX_ARGS_`/, `${label} must name both randomized delimiters`); } }); +test("rescue resolves the resume decision before the --background branch", () => { + const rescue = read("commands/rescue.md"); + const resumeDecisionIndex = rescue.indexOf("ask ONCE with `AskUserQuestion`"); + const backgroundBranchIndex = rescue.indexOf("`--background`: invoke the `Agent` tool with `codex:codex-rescue`"); + assert.notEqual(resumeDecisionIndex, -1, "resume-decision AskUserQuestion step must be present"); + assert.notEqual(backgroundBranchIndex, -1, "--background branch must be present"); + assert.ok(resumeDecisionIndex < backgroundBranchIndex, "resume decision must precede the --background branch"); +}); + +test("agent never decides the resume choice itself", () => { + const agent = read("agents/codex-rescue.md"); + assert.doesNotMatch(agent, /task-resume-candidate --json/); + assert.doesNotMatch(agent, /clearly asking to continue/i); + assert.match(agent, /You have no `AskUserQuestion` tool to ask with/i); + assert.match(agent, /never call `task-resume-candidate`/i); + assert.match(agent, /if neither flag is present, run fresh/i); +}); + +test("SKILL.md execution rules describe the single-call flow, not the old two-step poll loop", () => { + const runtimeSkill = read("skills/codex-cli-runtime/SKILL.md"); + assert.doesNotMatch(runtimeSkill, /task --background --json/); + assert.doesNotMatch(runtimeSkill, /polls only that job's own `status`/); + assert.match(runtimeSkill, /task --await --prompt-stdin/); + assert.match(runtimeSkill, /result --wait --timeout-ms 540000/); + assert.doesNotMatch(runtimeSkill, /task ""/); +}); + +// `task --await` reports the job's outcome; `result` reports whether a record +// could be retrieved. Automation cannot act on a published contract that claims +// both return 0 for a failed job one line after saying `--await` returns 1. +test("README keeps the task --await and result exit-code contracts apart", () => { + const readme = fs.readFileSync(path.join(ROOT, "README.md"), "utf8"); + + assert.match( + readme, + /Exit code is 0 when the job completed, 1 when it failed or was cancelled, and 3 when the wait times out/, + "the task --await contract (0/1/3) must stay stated" + ); + assert.match( + readme, + /`result` exits 0 for any terminal record \(completed, failed or cancelled\) and 3 while the job is still active/, + "the result contract must be stated separately" + ); + assert.doesNotMatch( + readme, + /`result` and `task --await` exit 0 for \*\*any\*\* terminal record/, + "the two contracts must not be merged back into one claim" + ); +}); + test("README documents the fork's own install commands", () => { const readme = fs.readFileSync(path.join(ROOT, "README.md"), "utf8"); @@ -373,3 +449,21 @@ test("bump-version --check pins the lockfile identity to package.json", () => { assert.equal(lock.name, pkg.name); assert.equal(lock.packages[""].name, pkg.name); }); + +// The hook bounds its own work with `SESSION_END_BUDGET_MS`; Claude Code kills it at +// the timeout in hooks.json. If the second ever drops below the first the hook is +// killed mid-decision instead of reporting one, so the two numbers are asserted +// together — that is the only thing keeping them from drifting apart. +test("the SessionEnd hook timeout stays above the hook's own budget", () => { + const hooks = JSON.parse(read("hooks/hooks.json")); + const timeoutSeconds = hooks.hooks.SessionEnd[0].hooks[0].timeout; + const source = read("scripts/session-lifecycle-hook.mjs"); + const budgetMs = Number(/const SESSION_END_BUDGET_MS = (\d+);/.exec(source)?.[1]); + + assert.ok(Number.isFinite(budgetMs), "the hook must declare SESSION_END_BUDGET_MS"); + assert.ok(Number.isFinite(timeoutSeconds), "hooks.json must give SessionEnd a timeout"); + assert.ok( + timeoutSeconds * 1000 > budgetMs, + `hooks.json SessionEnd timeout (${timeoutSeconds}s) must exceed the hook budget (${budgetMs}ms)` + ); +}); diff --git a/tests/fake-codex-fixture.mjs b/tests/fake-codex-fixture.mjs index 40a351061..507bfb401 100644 --- a/tests/fake-codex-fixture.mjs +++ b/tests/fake-codex-fixture.mjs @@ -288,6 +288,22 @@ const rl = readline.createInterface({ input: process.stdin }); // escalation) so a test can observe the window while a broker is shutting its // app-server child down. const CLOSE_DELAY_MS = Number(process.env.FAKE_CODEX_CLOSE_DELAY_MS || 0); + +// Test knob: hold a plain turn open for this long before completing it, so a +// test can observe a job that is still running. +const TURN_DELAY_MS = Number(process.env.FAKE_CODEX_TURN_DELAY_MS || 0); + +// Test knob: answer turn/interrupt but keep running the turn, the way a real +// app-server that has wedged on a tool call does. Also records that the client +// closed the connection, which is the only thing that stops such a turn. +const IGNORE_INTERRUPT = process.env.FAKE_CODEX_IGNORE_INTERRUPT === "1"; +if (IGNORE_INTERRUPT) { + rl.on("close", () => { + const closingState = loadState(); + closingState.clientClosed = true; + saveState(closingState); + }); +} if (CLOSE_DELAY_MS > 0) { process.on("SIGTERM", () => {}); rl.on("close", () => { @@ -295,6 +311,19 @@ if (CLOSE_DELAY_MS > 0) { }); } +// Test knob: exit the moment the interrupt is answered and never send the +// turn's terminal notification, the way an app-server that dies (or is killed) +// mid-turn does. +const EXIT_AFTER_INTERRUPT = process.env.FAKE_CODEX_EXIT_AFTER_INTERRUPT === "1"; + +// Test knob: an app-server that survives stdin EOF and ignores SIGTERM, so only +// SIGKILL can end it. The interval is what keeps the process alive once stdin +// is gone. +if (process.env.FAKE_CODEX_IGNORE_SIGTERM === "1") { + process.on("SIGTERM", () => {}); + setInterval(() => {}, 1000); +} + rl.on("line", (line) => { if (!line.trim()) { return; @@ -611,7 +640,12 @@ rl.on("line", (line) => { } ]; - if (BEHAVIOR === "interruptible-slow-task") { + // Any held-open turn is interruptible: an unregistered timer would fire + // after a turn/interrupt and complete a turn the client already cancelled. + const heldTurnDelayMs = BEHAVIOR === "interruptible-slow-task" ? 5000 : TURN_DELAY_MS; + if (BEHAVIOR === "slow-task") { + emitTurnCompletedLater(thread.id, turnId, items, 400); + } else if (heldTurnDelayMs > 0) { send({ method: "turn/started", params: { threadId: thread.id, turn: buildTurn(turnId) } }); const timer = setTimeout(() => { if (!interruptibleTurns.has(turnId)) { @@ -624,10 +658,8 @@ rl.on("line", (line) => { } } send({ method: "turn/completed", params: { threadId: thread.id, turn: buildTurn(turnId, "completed") } }); - }, 5000); + }, heldTurnDelayMs); interruptibleTurns.set(turnId, { threadId: thread.id, timer }); - } else if (BEHAVIOR === "slow-task") { - emitTurnCompletedLater(thread.id, turnId, items, 400); } else { emitTurnCompleted(thread.id, turnId, items); } @@ -640,7 +672,12 @@ rl.on("line", (line) => { turnId: message.params.turnId }; saveState(state); - const pending = interruptibleTurns.get(message.params.turnId); + if (EXIT_AFTER_INTERRUPT) { + send({ id: message.id, result: {} }); + setTimeout(() => process.exit(0), 10); + break; + } + const pending = IGNORE_INTERRUPT ? null : interruptibleTurns.get(message.params.turnId); if (pending) { clearTimeout(pending.timer); interruptibleTurns.delete(message.params.turnId); diff --git a/tests/runtime.test.mjs b/tests/runtime.test.mjs index 07c06f1f9..791ea3182 100644 --- a/tests/runtime.test.mjs +++ b/tests/runtime.test.mjs @@ -8,7 +8,13 @@ import { fileURLToPath } from "node:url"; import { buildEnv, installFakeCodex } from "./fake-codex-fixture.mjs"; import { initGitRepo, makeTempDir, run } from "./helpers.mjs"; import { loadBrokerSession, saveBrokerSession } from "../plugins/codex/scripts/lib/broker-lifecycle.mjs"; -import { resolveStateDir } from "../plugins/codex/scripts/lib/state.mjs"; +import { + consumeJobRequestFile, + readJobFile, + resolveJobFile, + resolveJobRequestFile, + resolveStateDir +} from "../plugins/codex/scripts/lib/state.mjs"; const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const PLUGIN_ROOT = path.join(ROOT, "plugins", "codex"); @@ -590,6 +596,48 @@ test("task-resume-candidate returns the latest rescue thread from the current se assert.equal(payload.candidate.threadId, "thr_current"); }); +test("task-resume-candidate reaps a crashed running task so it becomes resumable", () => { + const workspace = makeTempDir(); + const stateDir = resolveStateDir(workspace); + const jobsDir = path.join(stateDir, "jobs"); + fs.mkdirSync(jobsDir, { recursive: true }); + + // A pid that has already exited: process.kill(pid, 0) will throw ESRCH. + const deadPid = run(process.execPath, ["-e", ""]).pid; + const crashedJob = { + id: "task-crashed", + status: "running", + phase: "delegating", + title: "Codex Task", + jobClass: "task", + sessionId: "sess-current", + threadId: "thr_crashed", + summary: "Investigate the crash", + pid: deadPid, + updatedAt: "2026-03-24T20:00:00.000Z" + }; + fs.writeFileSync(path.join(jobsDir, "task-crashed.json"), `${JSON.stringify(crashedJob, null, 2)}\n`, "utf8"); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, jobs: [crashedJob] }, null, 2)}\n`, + "utf8" + ); + + const result = run("node", [SCRIPT, "task-resume-candidate", "--json"], { + cwd: workspace, + env: { ...process.env, CODEX_COMPANION_SESSION_ID: "sess-current" } + }); + + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout); + // Without the reaper the job would still read as "running" and be skipped, + // leaving the resume probe with no candidate. + assert.equal(payload.available, true); + assert.equal(payload.candidate.id, "task-crashed"); + assert.equal(payload.candidate.status, "failed"); + assert.equal(payload.candidate.threadId, "thr_crashed"); +}); + test("task --resume-last does not resume a task from another Claude session", () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -2125,6 +2173,170 @@ test("session end fully cleans up jobs for the ending session", async (t) => { assert.equal(otherJob.logFile, otherSessionLog); }); +test("session end preserves background jobs and their broker so workers survive their dispatching session", async (t) => { + const repo = makeTempDir(); + initGitRepo(repo); + fs.writeFileSync(path.join(repo, "README.md"), "hello\n"); + run("git", ["add", "README.md"], { cwd: repo }); + run("git", ["commit", "-m", "init"], { cwd: repo }); + + const stateDir = resolveStateDir(repo); + const jobsDir = path.join(stateDir, "jobs"); + fs.mkdirSync(jobsDir, { recursive: true }); + + const backgroundLog = path.join(jobsDir, "background.log"); + const foregroundLog = path.join(jobsDir, "foreground.log"); + const backgroundJobFile = path.join(jobsDir, "task-background.json"); + const foregroundJobFile = path.join(jobsDir, "review-foreground.json"); + fs.writeFileSync(backgroundLog, "background\n", "utf8"); + fs.writeFileSync(foregroundLog, "foreground\n", "utf8"); + fs.writeFileSync(backgroundJobFile, JSON.stringify({ id: "task-background" }, null, 2), "utf8"); + fs.writeFileSync(foregroundJobFile, JSON.stringify({ id: "review-foreground" }, null, 2), "utf8"); + + const backgroundSleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + cwd: repo, + detached: true, + stdio: "ignore" + }); + backgroundSleeper.unref(); + const foregroundSleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + cwd: repo, + detached: true, + stdio: "ignore" + }); + foregroundSleeper.unref(); + + t.after(() => { + for (const proc of [backgroundSleeper, foregroundSleeper]) { + try { + process.kill(-proc.pid, "SIGTERM"); + } catch { + try { + process.kill(proc.pid, "SIGTERM"); + } catch { + // Ignore missing process. + } + } + } + }); + + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify( + { + version: 1, + config: { stopReviewGate: false }, + jobs: [ + { + id: "task-background", + status: "running", + title: "Codex Task", + sessionId: "sess-current", + background: true, + pid: backgroundSleeper.pid, + logFile: backgroundLog, + createdAt: "2026-03-18T15:30:00.000Z", + updatedAt: "2026-03-18T15:31:00.000Z" + }, + { + id: "review-foreground", + status: "running", + title: "Codex Review", + sessionId: "sess-current", + pid: foregroundSleeper.pid, + logFile: foregroundLog, + createdAt: "2026-03-18T15:32:00.000Z", + updatedAt: "2026-03-18T15:33:00.000Z" + } + ] + }, + null, + 2 + )}\n`, + "utf8" + ); + + const result = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env: { + ...process.env, + CODEX_COMPANION_SESSION_ID: "sess-current" + }, + input: JSON.stringify({ + hook_event_name: "SessionEnd", + session_id: "sess-current", + cwd: repo + }) + }); + + assert.equal(result.status, 0, result.stderr); + + // Foreground job killed + pruned from state. + await waitFor(() => { + try { + process.kill(foregroundSleeper.pid, 0); + return false; + } catch (error) { + return error?.code === "ESRCH"; + } + }); + + // Background job still alive — its worker outlives the session that started it. + assert.equal( + (() => { + try { + process.kill(backgroundSleeper.pid, 0); + return true; + } catch { + return false; + } + })(), + true, + "background job worker should not be terminated by SessionEnd" + ); + + const state = JSON.parse(fs.readFileSync(path.join(stateDir, "state.json"), "utf8")); + assert.deepEqual( + state.jobs.map((job) => job.id), + ["task-background"], + "background job stays in state so later sessions can poll it" + ); + assert.equal(fs.existsSync(backgroundJobFile), true, "background job file preserved"); + assert.equal(fs.existsSync(backgroundLog), true, "background log preserved"); +}); + +// `--background` on a review means what it means on a task: the job is +// dispatched to outlive the session that started it. The flag was parsed and +// then dropped, so SessionEnd pruned the review's record — and with it the +// only way to read the review back with `/codex:result`. +test("an adversarial review dispatched with --background survives its own session's SessionEnd", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + fs.writeFileSync(path.join(repo, "README.md"), "hello world\n"); + const env = { ...buildEnv(binDir), CODEX_COMPANION_SESSION_ID: "sess-current" }; + + const review = run("node", [SCRIPT, "adversarial-review", "--background"], { cwd: repo, env }); + assert.equal(review.status, 0, review.stderr); + + const stateFile = path.join(resolveStateDir(repo), "state.json"); + const recorded = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs[0]; + assert.equal(recorded.background, true, "a --background review must be recorded as a background job"); + + const cleanup = run("node", [SESSION_HOOK, "SessionEnd"], { + cwd: repo, + env, + input: JSON.stringify({ hook_event_name: "SessionEnd", session_id: "sess-current", cwd: repo }) + }); + assert.equal(cleanup.status, 0, cleanup.stderr); + + assert.deepEqual( + JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs.map((job) => job.id), + [recorded.id], + "the background review record must survive the dispatching session's SessionEnd" + ); +}); + test("stop hook runs a stop-time review task and blocks on findings when the review gate is enabled", () => { const repo = makeTempDir(); const binDir = makeTempDir(); @@ -2695,6 +2907,9 @@ test("task --background persists the job record before spawning the worker", () assert.equal(JSON.parse(waited.stdout).job.status, "completed"); }); +// Classifying secrets by key name always misses one: a session cookie header is +// every bit a credential and matches no denylist. Every `--config` value stays +// out of the public record; only the keys are kept. test("task --background keeps secret --config values out of every job record", () => { const repo = seededRepo(); const binDir = makeTempDir(); @@ -2709,7 +2924,7 @@ test("task --background keeps secret --config values out of every job record", ( "--background", "--json", "--config", - "model_providers.x.http_headers.Authorization=SECRET_SENTINEL_42", + "model_providers.x.http_headers.Cookie=SECRET_SENTINEL_42", "--config", "model_provider=ollama", "x" @@ -2738,7 +2953,8 @@ test("task --background keeps secret --config values out of every job record", ( for (const [label, text] of Object.entries(exposures)) { assert.equal(text.includes("SECRET_SENTINEL_42"), false, `${label} leaked the secret --config value`); assert.equal(text.includes("[redacted]"), true, `${label} should keep the redacted placeholder`); - assert.equal(text.includes("ollama"), true, `${label} should keep non-secret config values readable`); + assert.equal(text.includes("model_provider"), true, `${label} should still record which config keys were set`); + assert.equal(text.includes("ollama"), false, `${label} stored a --config value; keys are recorded, values never are`); } // The one-shot payload file is deleted by the worker once it has read it. @@ -2746,10 +2962,134 @@ test("task --background keeps secret --config values out of every job record", ( // The worker still forwarded the real value to Codex. const fakeState = JSON.parse(fs.readFileSync(statePath, "utf8")); - assert.equal(fakeState.lastThreadStart.config["model_providers.x.http_headers.Authorization"], "SECRET_SENTINEL_42"); + assert.equal(fakeState.lastThreadStart.config["model_providers.x.http_headers.Cookie"], "SECRET_SENTINEL_42"); assert.equal(fakeState.lastThreadStart.config.model_provider, "ollama"); }); +// Redaction was added when the job is created, which does nothing for records +// v1.1.1 already wrote: same `STATE_VERSION`, raw `--config` values, and every +// `status`/`result --json` still echoing them back after the upgrade. +test("a v1.1.1 record's --config values never reach status/result and are redacted on disk", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir); + + const seeded = run("node", [SCRIPT, "task", "seed the state dir"], { cwd: repo, env }); + assert.equal(seeded.status, 0, seeded.stderr); + + const stateDir = resolveStateDir(repo); + const statePath = path.join(stateDir, "state.json"); + const legacyRequest = { + cwd: repo, + prompt: "legacy prompt", + config: { "model_providers.x.http_headers.Cookie": "SESSION_SECRET_FROM_1_1_1" } + }; + const legacyJob = { + id: "task-legacy", + status: "completed", + phase: "done", + jobClass: "task", + kind: "task", + title: "Codex Task", + summary: "Legacy job written by 1.1.1", + threadId: "thr_legacy", + updatedAt: "2026-03-24T20:05:00.000Z", + completedAt: "2026-03-24T20:06:00.000Z", + request: legacyRequest + }; + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.jobs.push(legacyJob); + fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + const legacyJobFile = path.join(stateDir, "jobs", "task-legacy.json"); + fs.writeFileSync( + legacyJobFile, + `${JSON.stringify({ ...legacyJob, result: { status: 0, finalMessage: "legacy output" }, rendered: "legacy output\n" }, null, 2)}\n`, + "utf8" + ); + + const status = run("node", [SCRIPT, "status", "task-legacy", "--json"], { cwd: repo, env }); + assert.equal(status.status, 0, status.stderr); + const result = run("node", [SCRIPT, "result", "task-legacy", "--json"], { cwd: repo, env }); + assert.equal(result.status, 0, result.stderr); + + const exposures = { + "status --json stdout": status.stdout, + "result --json stdout": result.stdout, + "state index": fs.readFileSync(statePath, "utf8"), + "job file": fs.readFileSync(legacyJobFile, "utf8") + }; + for (const [label, text] of Object.entries(exposures)) { + assert.equal(text.includes("SESSION_SECRET_FROM_1_1_1"), false, `${label} leaked a legacy --config value`); + assert.equal(text.includes("[redacted]"), true, `${label} should carry the redacted placeholder`); + assert.equal( + text.includes("model_providers.x.http_headers.Cookie"), + true, + `${label} should still record which config keys were set` + ); + } +}); + +// The record of an ACTIVE 1.1.1 job is the only copy of its request — 1.1.1 wrote +// no private payload file — and `handleTaskWorker` falls back to exactly that +// record when there is none. Redacting it in place would hand the worker +// "[redacted]" as its Codex config (auth headers included), so the raw request is +// moved into a fresh 0600 payload file first and only then dropped from the record. +test("an active v1.1.1 record keeps its real --config for the worker while output stays redacted", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir); + + const seeded = run("node", [SCRIPT, "task", "seed the state dir"], { cwd: repo, env }); + assert.equal(seeded.status, 0, seeded.stderr); + + const stateDir = resolveStateDir(repo); + const statePath = path.join(stateDir, "state.json"); + const now = new Date().toISOString(); + const legacyRequest = { + cwd: repo, + prompt: "legacy queued prompt", + config: { "model_providers.x.http_headers.Cookie": "SESSION_SECRET_FROM_1_1_1" } + }; + const legacyJob = { + id: "task-legacy-queued", + status: "queued", + phase: "queued", + jobClass: "task", + kind: "task", + title: "Codex Task", + summary: "Legacy queued job written by 1.1.1", + createdAt: now, + updatedAt: now, + request: legacyRequest + }; + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.jobs.push(legacyJob); + fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + const legacyJobFile = resolveJobFile(repo, "task-legacy-queued"); + fs.writeFileSync(legacyJobFile, `${JSON.stringify(legacyJob, null, 2)}\n`, "utf8"); + + const status = run("node", [SCRIPT, "status", "task-legacy-queued", "--json"], { cwd: repo, env }); + assert.equal(status.status, 0, status.stderr); + assert.equal(status.stdout.includes("SESSION_SECRET_FROM_1_1_1"), false, "status --json leaked a legacy --config value"); + assert.equal(status.stdout.includes("[redacted]"), true); + + // The worker's own read path (`readStoredJob` → `readJobFile`). + const workerView = readJobFile(legacyJobFile); + assert.equal(workerView.request.config["model_providers.x.http_headers.Cookie"], "[redacted]"); + + const requestFile = resolveJobRequestFile(repo, "task-legacy-queued"); + assert.equal(fs.existsSync(requestFile), true, "the raw request must be moved into the private payload file"); + assert.equal(fs.statSync(requestFile).mode & 0o777, 0o600, "the payload file must be owner-only"); + assert.equal(fs.readFileSync(legacyJobFile, "utf8").includes("SESSION_SECRET_FROM_1_1_1"), false, "the record must be redacted on disk"); + + // What the worker actually runs with. + const consumed = consumeJobRequestFile(repo, "task-legacy-queued"); + assert.equal(consumed.config["model_providers.x.http_headers.Cookie"], "SESSION_SECRET_FROM_1_1_1"); + assert.equal(consumed.prompt, "legacy queued prompt"); +}); + test("a resume refuses to start a second turn on a thread another job is still using", () => { const repo = seededRepo(); const binDir = makeTempDir(); @@ -2797,6 +3137,70 @@ test("a resume refuses to start a second turn on a thread another job is still u assert.equal(fakeState.lastTurnStart.prompt, "follow up"); }); +// The crash window between a worker's terminal `writeJobFile` and its +// `upsertJob`: the job file is terminal, the index still says running. The +// reaper handed the terminal file back to its caller but left the index alone, +// so the raw `listJobs()` behind `assertThreadIsFree` kept seeing a phantom +// running job and blocked every later resume of that thread. The recorded pid +// here is alive and unrelated (this test runner) — exactly what a zombie or a +// recycled pid looks like — so nothing but the job file itself can settle it. +test("a terminal job file reconciles the state index and unblocks resume", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = { ...buildEnv(binDir), CODEX_COMPANION_SESSION_ID: "sess-current" }; + + const first = run("node", [SCRIPT, "task", "first"], { cwd: repo, env }); + assert.equal(first.status, 0, first.stderr); + + const stateDir = resolveStateDir(repo); + const statePath = path.join(stateDir, "state.json"); + const state = JSON.parse(fs.readFileSync(statePath, "utf8")); + state.jobs.push({ + id: "task-crash-window", + status: "running", + phase: "running", + title: "Codex Task", + jobClass: "task", + sessionId: "sess-other", + threadId: "thr_1", + pid: process.pid, + summary: "Other session task that died after writing its result", + updatedAt: "2026-03-24T20:05:00.000Z" + }); + fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + fs.writeFileSync( + path.join(stateDir, "jobs", "task-crash-window.json"), + `${JSON.stringify( + { + id: "task-crash-window", + status: "completed", + phase: "done", + pid: null, + threadId: "thr_1", + turnId: "turn_9", + result: { status: 0, finalMessage: "done" }, + rendered: "done\n", + completedAt: "2026-03-24T20:06:00.000Z" + }, + null, + 2 + )}\n`, + "utf8" + ); + + // Resume first: only a reaped/reconciled job list can tell this thread is free. + const resumed = run("node", [SCRIPT, "task", "--resume-last", "follow up"], { cwd: repo, env }); + assert.equal(resumed.status, 0, resumed.stderr); + + const reconciled = JSON.parse(fs.readFileSync(statePath, "utf8")).jobs.find((job) => job.id === "task-crash-window"); + assert.equal(reconciled.status, "completed", "the terminal job file must be reconciled into the state index"); + assert.equal(reconciled.pid, null); + + const status = run("node", [SCRIPT, "status", "--json"], { cwd: repo, env }); + assert.equal(status.status, 0, status.stderr); +}); + test("task --prompt-file wins over --args-stdin and keeps the prompt byte-exact", () => { const repo = seededRepo(); const binDir = makeTempDir(); @@ -2820,3 +3224,438 @@ test("task --prompt-file wins over --args-stdin and keeps the prompt byte-exact" assert.equal(fakeState.lastTurnStart.prompt, promptText); assert.equal(fakeState.lastTurnStart.effort, "max"); }); + +test("task --await launches a tracked job, waits, and prints the result", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + const statePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir); + const result = run("node", [SCRIPT, "task", "--await", "--json", "--model", "sol", "--effort", "low", "--prompt-stdin"], { + cwd: repo, env: buildEnv(binDir), input: "line one \\d+ \"quoted\" 'single'\nline two\n" + }); + assert.equal(result.status, 0, result.stderr); + const out = JSON.parse(result.stdout); + assert.match(out.job.id, /^task-/); + assert.equal(out.job.status, "completed"); + assert.ok(typeof out.storedJob.result.rawOutput === "string" && out.storedJob.result.rawOutput.length > 0); + const fakeState = JSON.parse(fs.readFileSync(statePath, "utf8")); + assert.equal(fakeState.lastTurnStart.prompt, "line one \\d+ \"quoted\" 'single'\nline two"); + assert.equal(fakeState.lastTurnStart.effort, "low"); + assert.equal(fakeState.lastTurnStart.model, "gpt-5.6-sol"); + const status = run("node", [SCRIPT, "status", out.job.id, "--json"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(JSON.parse(status.stdout).job.status, "completed"); +}); + +test("task --await exits 3 with a resumable hint when the await timeout elapses", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "3000" }); + const result = run("node", [SCRIPT, "task", "--await", "--await-timeout-ms", "500", "--prompt-stdin"], { cwd: repo, env, input: "slow task\n" }); + assert.equal(result.status, 3); + assert.match(result.stdout, /Still running: job task-[A-Za-z0-9_-]+\. Re-run: node .*result task-[A-Za-z0-9_-]+ --wait --timeout-ms 540000/); + const jobId = result.stdout.match(/job (task-[A-Za-z0-9_-]+)/)[1]; + + const timedOutJson = run("node", [SCRIPT, "result", jobId, "--wait", "--timeout-ms", "100", "--json"], { cwd: repo, env }); + assert.equal(timedOutJson.status, 3, timedOutJson.stderr); + const snapshot = JSON.parse(timedOutJson.stdout); + assert.ok(["queued", "running"].includes(snapshot.job.status), snapshot.job.status); + assert.match( + snapshot.resumeCommand, + new RegExp(`^node ".*codex-companion\\.mjs" result ${jobId} --wait --timeout-ms 540000$`) + ); + + const done = run("node", [SCRIPT, "result", jobId, "--wait", "--timeout-ms", "20000"], { cwd: repo, env }); + assert.equal(done.status, 0, done.stderr); +}); + +test("task rejects --prompt-stdin combined with --args-stdin or --prompt-file", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const r = run("node", [SCRIPT, "task", "--prompt-stdin", "--args-stdin"], { cwd: repo, env: buildEnv(binDir), input: "x" }); + assert.notEqual(r.status, 0); + assert.match(r.stderr, /--prompt-stdin/); +}); + +test("result on a still-running job exits 3 with the wait hint instead of \"No job found\"", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "3000" }); + const launch = run("node", [SCRIPT, "task", "--background", "--json", "--prompt-stdin"], { + cwd: repo, env, input: "slow background task\n" + }); + assert.equal(launch.status, 0, launch.stderr); + const { jobId } = JSON.parse(launch.stdout); + + const active = run("node", [SCRIPT, "result", jobId], { cwd: repo, env }); + assert.equal(active.status, 3, active.stderr); + assert.match( + active.stdout, + new RegExp(`Job ${jobId} is still (queued|running)\\. Re-run: node .*result ${jobId} --wait --timeout-ms 540000`) + ); + + const done = run("node", [SCRIPT, "result", jobId, "--wait", "--timeout-ms", "20000"], { cwd: repo, env }); + assert.equal(done.status, 0, done.stderr); + assert.ok(done.stdout.trim().length > 0); +}); + +test("task usage errors around --prompt-stdin arrive without waiting for stdin", async () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const cases = [ + [["--prompt-stdin", "--args-stdin"], /--prompt-stdin cannot be combined with --args-stdin/], + [["--prompt-stdin", "--await", "--background"], /Choose either --await or --background/] + ]; + + for (const [args, pattern] of cases) { + const startedAt = Date.now(); + const child = spawn("node", [SCRIPT, "task", ...args], { + cwd: repo, + env: buildEnv(binDir), + stdio: ["pipe", "pipe", "pipe"] + }); + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + // stdin stays open and empty: the error must not wait for EOF. + const code = await new Promise((resolve, reject) => { + child.on("error", reject); + child.on("exit", resolve); + }); + child.stdin.destroy(); + + assert.notEqual(code, 0, args.join(" ")); + assert.match(stderr, pattern, args.join(" ")); + assert.ok(Date.now() - startedAt < 2000, `${args.join(" ")} took ${Date.now() - startedAt}ms`); + } +}); + +test("task --prompt-stdin sends the prompt verbatim minus one trailing newline", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + const statePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir); + const promptText = "\n indented first line \r\nsecond\tline\n\nlast line without a newline"; + + const withNewline = run("node", [SCRIPT, "task", "--prompt-stdin"], { + cwd: repo, env: buildEnv(binDir), input: `${promptText}\r\n` + }); + assert.equal(withNewline.status, 0, withNewline.stderr); + assert.equal(JSON.parse(fs.readFileSync(statePath, "utf8")).lastTurnStart.prompt, promptText); + + const withoutNewline = run("node", [SCRIPT, "task", "--prompt-stdin"], { + cwd: repo, env: buildEnv(binDir), input: promptText + }); + assert.equal(withoutNewline.status, 0, withoutNewline.stderr); + assert.equal(JSON.parse(fs.readFileSync(statePath, "utf8")).lastTurnStart.prompt, promptText); +}); + +test("task rejects contradictory await and prompt flag combinations", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const cases = [ + [["--prompt-stdin", "inline prompt text"], /--prompt-stdin cannot be combined with --prompt-file or prompt text/], + [["--prompt-stdin", "--prompt-file", "prompt.txt"], /--prompt-stdin cannot be combined with --prompt-file or prompt text/], + [["--await", "--background", "do it"], /Choose either --await or --background/], + [["--await-timeout-ms", "1000", "do it"], /--await-timeout-ms requires --await/], + [["--await", "--await-timeout-ms", "0", "do it"], /--await-timeout-ms expects a positive integer/], + [["--await", "--await-timeout-ms", "-5", "do it"], /--await-timeout-ms expects a positive integer/], + [["--await", "--await-timeout-ms", "1.5", "do it"], /--await-timeout-ms expects a positive integer/], + [["--await", "--await-timeout-ms", "nope", "do it"], /--await-timeout-ms expects a positive integer/], + [["--await", "--await-timeout-ms", "1e400", "do it"], /--await-timeout-ms expects a positive integer/] + ]; + + for (const [args, pattern] of cases) { + const result = run("node", [SCRIPT, "task", ...args], { cwd: repo, env: buildEnv(binDir), input: "" }); + assert.notEqual(result.status, 0, args.join(" ")); + assert.match(result.stderr, pattern, args.join(" ")); + } + + const badResultTimeout = run("node", [SCRIPT, "result", "task-x", "--wait", "--timeout-ms", "0"], { + cwd: repo, env: buildEnv(binDir) + }); + assert.notEqual(badResultTimeout.status, 0); + assert.match(badResultTimeout.stderr, /--timeout-ms expects a positive integer/); + + const missingWaitTimeout = run("node", [SCRIPT, "result", "task-x", "--timeout-ms", "1000"], { + cwd: repo, env: buildEnv(binDir) + }); + assert.notEqual(missingWaitTimeout.status, 0); + assert.match(missingWaitTimeout.stderr, /--timeout-ms requires --wait/); +}); + +test("task --await reports a failed job with exit 1 while result stays exit 0", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir, "turn-start-fails"); + + const awaited = run("node", [SCRIPT, "task", "--await", "--json", "--prompt-stdin"], { + cwd: repo, env: buildEnv(binDir), input: "break on purpose\n" + }); + assert.equal(awaited.status, 1, awaited.stderr); + const out = JSON.parse(awaited.stdout); + assert.equal(out.job.status, "failed"); + assert.match(out.storedJob.errorMessage, /turn\/start failed after thread resolution/); + + const stored = run("node", [SCRIPT, "result", out.job.id], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(stored.status, 0, stored.stderr); + assert.match(stored.stdout, /turn\/start failed after thread resolution/); +}); + +test("cancelling an awaited job ends the await with exit 1 and leaves a readable result", async () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "6000" }); + + const child = spawn("node", [SCRIPT, "task", "--await", "--await-timeout-ms", "30000", "--prompt-stdin"], { + cwd: repo, + env, + stdio: ["pipe", "pipe", "pipe"] + }); + child.stdin.end("cancel me\n"); + const exited = new Promise((resolve, reject) => { + child.on("error", reject); + child.on("exit", resolve); + }); + + const stateFile = path.join(resolveStateDir(repo), "state.json"); + const jobId = await waitFor(() => { + if (!fs.existsSync(stateFile)) { + return null; + } + const job = JSON.parse(fs.readFileSync(stateFile, "utf8")).jobs?.[0]; + // Wait for the worker to own the record: the turn has to be under way for + // the cancel to have a running turn to interrupt. + return job && job.status === "running" && job.pid ? job.id : null; + }, { timeoutMs: 15000 }); + + const cancelled = run("node", [SCRIPT, "cancel", jobId], { cwd: repo, env }); + assert.equal(cancelled.status, 0, cancelled.stderr); + assert.equal(await exited, 1); + + const stored = run("node", [SCRIPT, "result", jobId, "--json"], { cwd: repo, env }); + assert.equal(stored.status, 0, stored.stderr); + assert.equal(JSON.parse(stored.stdout).job.status, "cancelled"); +}); + +// A turn that never completes used to hang the companion until Claude Code's +// Bash tool SIGKILLed it, leaving the job "running" and no output at all. +test("task --turn-timeout-ms interrupts a stalled turn and fails the job with the timeout message", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + const statePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "5000" }); + + const started = Date.now(); + const result = run("node", [SCRIPT, "task", "--turn-timeout-ms", "500", "--json", "stall please"], { + cwd: repo, + env + }); + + assert.equal(result.status, 1, result.stderr); + assert.ok(Date.now() - started < 5000, "the turn budget must fire long before the fake turn completes"); + assert.equal(JSON.parse(result.stdout).status, 1); + + const fakeState = JSON.parse(fs.readFileSync(statePath, "utf8")); + assert.ok(fakeState.lastInterrupt, "a timed-out turn must be interrupted, not abandoned"); + + const status = run("node", [SCRIPT, "status", "--json"], { cwd: repo, env }); + assert.equal(status.status, 0, status.stderr); + const latest = JSON.parse(status.stdout).latestFinished; + assert.equal(latest.status, "failed"); + assert.match(latest.errorMessage, /turn timed out after 500 ms/); +}); + +// `turn/interrupt` returning is not proof the turn stopped: a wedged app-server +// answers the RPC and keeps going. Writing `failed` right there claims a turn +// (possibly a `--write` one) is over while it is still editing files, so the +// timeout waits for the terminal turn notification and says so when it never +// arrives. `--resume-last` is the path that owns a direct app-server, so +// closing the connection is what actually kills the runaway turn. +test("an unacknowledged interrupt is reported and closes a direct app-server", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + const statePath = path.join(binDir, "fake-codex-state.json"); + installFakeCodex(binDir); + + const seeded = run("node", [SCRIPT, "task", "initial task"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(seeded.status, 0, seeded.stderr); + + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "20000", FAKE_CODEX_IGNORE_INTERRUPT: "1" }); + const result = run("node", [SCRIPT, "task", "--resume-last", "--turn-timeout-ms", "500", "--json", "stall please"], { + cwd: repo, + env + }); + + assert.equal(result.status, 1, result.stderr); + + const fakeState = JSON.parse(fs.readFileSync(statePath, "utf8")); + assert.ok(fakeState.lastInterrupt, "the timed-out turn must still be interrupted"); + assert.equal(fakeState.clientClosed, true, "a direct app-server must be closed so the runaway turn dies with it"); + + const status = run("node", [SCRIPT, "status", "--json"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(status.status, 0, status.stderr); + const latest = JSON.parse(status.stdout).latestFinished; + assert.equal(latest.status, "failed"); + assert.match(latest.errorMessage, /turn timed out after 500 ms; interrupt not acknowledged/); + assert.match(latest.errorMessage, /may still be running in the shared runtime/); +}); + +// The other degraded shape: the app-server answers the interrupt and dies before +// any terminal notification. Nothing can arrive after that, and the acknowledgement +// window used to be an unref'd timer with no exit awareness — so the companion +// waited on a promise nothing would settle and could exit before writing the job's +// terminal record, leaving it `running` forever. +test("a transport that exits after the interrupt still writes a terminal record", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const seeded = run("node", [SCRIPT, "task", "initial task"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(seeded.status, 0, seeded.stderr); + + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "20000", FAKE_CODEX_EXIT_AFTER_INTERRUPT: "1" }); + const started = Date.now(); + const result = run("node", [SCRIPT, "task", "--resume-last", "--turn-timeout-ms", "500", "--json", "stall please"], { + cwd: repo, + env + }); + const elapsed = Date.now() - started; + + assert.equal(result.status, 1, result.stderr); + assert.ok(elapsed < 9000, `a dead transport must end the acknowledgement wait early, took ${elapsed} ms`); + + const status = run("node", [SCRIPT, "status", "--json"], { cwd: repo, env: buildEnv(binDir) }); + assert.equal(status.status, 0, status.stderr); + const latest = JSON.parse(status.stdout).latestFinished; + assert.equal(latest.status, "failed", "the job must not be left running"); + assert.match(latest.errorMessage, /turn timed out after 500 ms; interrupt not acknowledged/); +}); + +test("task --turn-timeout-ms survives into the detached background worker", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + const env = buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "5000" }); + + const launched = run("node", [SCRIPT, "task", "--background", "--turn-timeout-ms", "500", "--json", "stall please"], { + cwd: repo, + env + }); + assert.equal(launched.status, 0, launched.stderr); + const { jobId } = JSON.parse(launched.stdout); + + const waited = run("node", [SCRIPT, "status", jobId, "--wait", "--timeout-ms", "20000", "--json"], { cwd: repo, env }); + assert.equal(waited.status, 0, waited.stderr); + const job = JSON.parse(waited.stdout).job; + assert.equal(job.status, "failed"); + assert.match(job.errorMessage, /turn timed out after 500 ms/); +}); + +test("task without a turn budget is unbounded", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const result = run("node", [SCRIPT, "task", "--json", "take your time"], { + cwd: repo, + env: buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "700" }) + }); + + assert.equal(result.status, 0, result.stderr); + assert.equal(JSON.parse(result.stdout).status, 0); +}); + +test("CODEX_TURN_TIMEOUT_MS bounds a turn when no flag is passed", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const result = run("node", [SCRIPT, "task", "--json", "stall please"], { + cwd: repo, + env: buildEnv(binDir, { FAKE_CODEX_TURN_DELAY_MS: "5000", CODEX_TURN_TIMEOUT_MS: "500" }) + }); + + assert.equal(result.status, 1, result.stderr); +}); + +test("task rejects a non-positive turn budget", () => { + const repo = seededRepo(); + const binDir = makeTempDir(); + installFakeCodex(binDir); + + const result = run("node", [SCRIPT, "task", "--turn-timeout-ms", "0", "hi"], { cwd: repo, env: buildEnv(binDir) }); + + assert.equal(result.status, 1); + assert.match(result.stderr, /--turn-timeout-ms expects a positive integer/); +}); + +// Recording the worker pid on the queued record (so a queued job can be +// cancelled at all) means cancel can now kill a worker *before* it consumed its +// private one-shot payload. A cancelled job is terminal, so the reaper will +// never look at it again — cancel has to release the file itself. +test("cancel removes the private request payload of a job killed in the queued window", async (t) => { + const repo = seededRepo(); + const stateDir = resolveStateDir(repo); + const jobsDir = path.join(stateDir, "jobs"); + fs.mkdirSync(jobsDir, { recursive: true }); + + const secret = "sk-cancel-secret-value"; + const requestFile = path.join(jobsDir, "task-queued.request.json"); + fs.writeFileSync(requestFile, JSON.stringify({ prompt: "hi", config: { auth_header: secret } }), { + encoding: "utf8", + mode: 0o600 + }); + + const sleeper = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + cwd: repo, + detached: true, + stdio: "ignore" + }); + sleeper.unref(); + t.after(() => { + try { + process.kill(-sleeper.pid, "SIGKILL"); + } catch { + // Already gone. + } + }); + + const job = { + id: "task-queued", + status: "queued", + phase: "queued", + jobClass: "task", + title: "Codex Task", + background: true, + pid: sleeper.pid, + logFile: null, + requestFile, + request: { prompt: "hi", config: { auth_header: "[redacted]" } }, + createdAt: "2026-03-18T15:30:00.000Z", + updatedAt: "2026-03-18T15:30:00.000Z" + }; + fs.writeFileSync(path.join(jobsDir, "task-queued.json"), `${JSON.stringify(job, null, 2)}\n`, "utf8"); + fs.writeFileSync( + path.join(stateDir, "state.json"), + `${JSON.stringify({ version: 1, config: { stopReviewGate: false }, jobs: [job] }, null, 2)}\n`, + "utf8" + ); + + const cancelled = run("node", [SCRIPT, "cancel", "task-queued", "--json"], { cwd: repo, env: process.env }); + assert.equal(cancelled.status, 0, cancelled.stderr); + assert.equal(JSON.parse(cancelled.stdout).status, "cancelled"); + + assert.equal(fs.existsSync(requestFile), false, "the private payload must not outlive the cancelled job"); + const stored = readPersistedJob(repo, "task-queued"); + assert.equal(stored.status, "cancelled"); + assert.equal(stored.requestFile, null); + assert.equal(fs.readFileSync(path.join(stateDir, "state.json"), "utf8").includes(secret), false); +}); diff --git a/tests/state.test.mjs b/tests/state.test.mjs index b1148f06e..ed1e740d4 100644 --- a/tests/state.test.mjs +++ b/tests/state.test.mjs @@ -3,19 +3,35 @@ import os from "node:os"; import path from "node:path"; import test from "node:test"; import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath, pathToFileURL } from "node:url"; -import { makeTempDir } from "./helpers.mjs"; +import { makeTempDir, run } from "./helpers.mjs"; import { consumeJobRequestFile, + listJobs, + readJobFile, resolveJobFile, resolveJobLogFile, resolveJobRequestFile, resolveStateDir, resolveStateFile, saveState, + upsertJob, + withStateLock, writeJobRequestFile } from "../plugins/codex/scripts/lib/state.mjs"; +const STATE_MODULE = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "..", + "plugins", + "codex", + "scripts", + "lib", + "state.mjs" +); + test("resolveStateDir uses a temp-backed per-workspace directory", () => { const workspace = makeTempDir(); const stateDir = resolveStateDir(workspace); @@ -136,3 +152,639 @@ test("saveState drops the private request payload of pruned jobs", () => { saveState(workspace, { jobs: [] }); assert.equal(fs.existsSync(requestFile), false); }); + +// A reader that catches `state.json` mid-write parses a truncated file, and +// `loadState` turns that into "no jobs" — which is how a SessionEnd with a live +// job decided the workspace was idle and shut the shared broker down. Writers +// must swap the file in atomically so a reader sees the old or the new one. +test("concurrent writers never leave a torn state.json for a reader", async () => { + const workspace = makeTempDir(); + const jobs = Array.from({ length: 50 }, (_, index) => ({ + id: `job-${index}`, + status: "running", + updatedAt: `2026-03-18T15:${String(index % 60).padStart(2, "0")}:00.000Z`, + summary: "x".repeat(2048) + })); + saveState(workspace, { jobs }); + const stateFile = resolveStateFile(workspace); + + const writer = spawn( + process.execPath, + [ + "--input-type=module", + "-e", + `import { saveState } from ${JSON.stringify(pathToFileURL(STATE_MODULE).href)}; + const jobs = ${JSON.stringify(jobs)}; + const deadline = Date.now() + 2000; + while (Date.now() < deadline) { + saveState(${JSON.stringify(workspace)}, { jobs }); + }` + ], + { env: process.env, stdio: ["ignore", "ignore", "pipe"] } + ); + let writerStderr = ""; + writer.stderr.on("data", (chunk) => { + writerStderr += chunk; + }); + + let reads = 0; + const deadline = Date.now() + 2000; + while (Date.now() < deadline) { + const raw = fs.readFileSync(stateFile, "utf8"); + let parsed; + try { + parsed = JSON.parse(raw); + } catch (error) { + assert.fail(`torn read after ${reads} reads (${raw.length} bytes): ${error.message}`); + } + assert.equal(parsed.jobs.length, 50, `torn read after ${reads} reads: lost jobs`); + reads += 1; + } + + await new Promise((resolve) => writer.on("exit", resolve)); + assert.equal(writerStderr, ""); + assert.ok(reads > 100, `expected the reader to race the writer, got ${reads} reads`); +}); + +const STATE_MODULE_URL = JSON.stringify(pathToFileURL(STATE_MODULE).href); + +function runModule(source) { + return spawn(process.execPath, ["--input-type=module", "-e", source], { + env: process.env, + stdio: ["ignore", "ignore", "pipe"] + }); +} + +function collectExit(child) { + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + return new Promise((resolve) => child.on("exit", (code) => resolve({ code, stderr }))); +} + +// `updateState` reads, mutates and writes without serialization, and `saveState` +// deletes the artifacts of every job that is in the file it read but not in the +// snapshot it is about to write. A second process that creates a job inside that +// window is therefore not just lost from the index: its job file, private request +// payload, PID sidecar and log are deleted under it. +test("a job created during another process's read-modify-write survives it", async () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [{ id: "job-a", status: "completed", updatedAt: "2026-03-18T15:00:00.000Z" }] }); + + const holder = runModule(` + import { updateState } from ${STATE_MODULE_URL}; + updateState(${JSON.stringify(workspace)}, () => { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1500); + }); + `); + const holderExit = collectExit(holder); + + // Let the holder get past its read and into the slow part of its mutation. + await new Promise((resolve) => setTimeout(resolve, 400)); + + const enqueue = run(process.execPath, [ + "--input-type=module", + "-e", + ` + import { upsertJob, writeJobFile, writeJobRequestFile } from ${STATE_MODULE_URL}; + const workspace = ${JSON.stringify(workspace)}; + writeJobFile(workspace, "job-b", { id: "job-b", status: "queued" }); + writeJobRequestFile(workspace, "job-b", { prompt: "hello" }); + upsertJob(workspace, { id: "job-b", status: "queued" }); + ` + ], { env: process.env }); + assert.equal(enqueue.status, 0, enqueue.stderr); + + const finished = await holderExit; + assert.equal(finished.code, 0, finished.stderr); + + const ids = listJobs(workspace).map((job) => job.id).sort(); + assert.deepEqual(ids, ["job-a", "job-b"], "neither writer may lose the other's job"); + assert.equal(fs.existsSync(resolveJobFile(workspace, "job-b")), true, "the new job's file must not be pruned"); + assert.equal(fs.existsSync(resolveJobRequestFile(workspace, "job-b")), true, "the new job's payload must not be pruned"); +}); + +// A `running` legacy job has already consumed its request: the worker reads the +// payload (or the record) BEFORE `runTrackedJob` flips the record to `running`. +// Staging a payload for it would write plaintext `--config` values that nothing +// ever reads and nothing ever deletes — only a `queued` record still has a worker +// coming for them. +test("migrating a running legacy record does not stage a payload nobody consumes", () => { + const workspace = makeTempDir(); + const legacy = { + id: "job-running-legacy", + status: "running", + request: { prompt: "old", config: { "model_providers.x.http_headers.Cookie": "SESSION_SECRET" } } + }; + const jobFile = resolveJobFile(workspace, legacy.id); + fs.writeFileSync(jobFile, `${JSON.stringify(legacy, null, 2)}\n`, "utf8"); + + const view = readJobFile(jobFile); + + assert.equal(view.request.config["model_providers.x.http_headers.Cookie"], "[redacted]"); + assert.equal(fs.readFileSync(jobFile, "utf8").includes("SESSION_SECRET"), false, "the record must be redacted on disk"); + assert.equal( + fs.existsSync(resolveJobRequestFile(workspace, legacy.id)), + false, + "a running job's request has already been consumed; staging it again only leaks it" + ); +}); + +const LOCK_DIR = "state.lock.d"; + +function lockDirFor(workspace) { + const lockDir = path.join(resolveStateDir(workspace), LOCK_DIR); + fs.mkdirSync(lockDir, { recursive: true }); + return lockDir; +} + +function seedLockEntry(lockDir, name, pid, startedAt = new Date().toISOString()) { + const entry = path.join(lockDir, name); + fs.writeFileSync(entry, `${JSON.stringify({ pid, startedAt })}\n`, "utf8"); + return entry; +} + +function deadPid() { + const finished = run(process.execPath, ["-e", "process.exit(0)"], { env: process.env }); + assert.equal(finished.status, 0); + return finished.pid; +} + +// The property the whole lock exists for, checked the only way that means +// anything: a counter that is read, incremented and written back — non-atomic by +// construction, so any overlap loses increments. +test("two processes acquiring concurrently never overlap", async () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const counter = path.join(workspace, "counter.json"); + fs.writeFileSync(counter, "0", "utf8"); + + const rounds = 100; + const worker = ` + import fs from "node:fs"; + import { withStateLock } from ${STATE_MODULE_URL}; + const workspace = ${JSON.stringify(workspace)}; + const counter = ${JSON.stringify(counter)}; + for (let round = 0; round < ${rounds}; round += 1) { + withStateLock(workspace, () => { + const value = Number.parseInt(fs.readFileSync(counter, "utf8"), 10); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 1); + fs.writeFileSync(counter, String(value + 1), "utf8"); + }); + } + `; + + const [first, second] = await Promise.all([collectExit(runModule(worker)), collectExit(runModule(worker))]); + assert.equal(first.code, 0, first.stderr); + assert.equal(second.code, 0, second.stderr); + assert.equal(Number.parseInt(fs.readFileSync(counter, "utf8"), 10), rounds * 2, "an overlap lost increments"); +}); + +// A holder that died with its ticket in the directory releases it to the next +// acquirer immediately — its PID proves it is gone. +test("a ticket whose holder is gone is removed and the waiter acquires at once", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const gone = deadPid(); + const ticket = seedLockEntry(lockDir, `1.${gone}-gone.ticket`, gone); + + const started = Date.now(); + assert.equal(withStateLock(workspace, () => "ok"), "ok"); + + assert.ok(Date.now() - started < 1000, `a dead holder must not cost a grace period, took ${Date.now() - started} ms`); + assert.equal(fs.existsSync(ticket), false, "the dead holder's ticket must be cleared"); +}); + +// The opposite rule, and the one that keeps two writers apart: a holder that is +// still running keeps its ticket however long it has been working — a slow holder +// and a stuck one are indistinguishable from out here. The waiter gives up and +// says exactly which process to look at. +test("a ticket held by a live process is never removed and the waiter times out naming it", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const ticket = seedLockEntry(lockDir, `1.${process.pid}-live.ticket`, process.pid, new Date(Date.now() - 600000).toISOString()); + + assert.throws( + () => withStateLock(workspace, () => "stolen", { waitMs: 200 }), + (error) => + /state lock/i.test(error.message) && + error.message.includes(`pid ${process.pid}`) && + error.message.includes(ticket) && + /pid reuse/.test(error.message) + ); + assert.equal(fs.existsSync(ticket), true, "a live holder's ticket must survive"); +}); + +// A process killed between announcing its choice and taking a number leaves a +// `choosing` file behind. Every acquirer waits for those, so one that nobody will +// ever come back for would wedge the workspace. +test("a choosing file left by a dead process does not block the next acquirer", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const gone = deadPid(); + const chooser = seedLockEntry(lockDir, `choosing.${gone}-crashed`, gone); + + const started = Date.now(); + assert.equal(withStateLock(workspace, () => "ok"), "ok"); + + assert.ok(Date.now() - started < 1000, `a dead chooser must not cost a grace period, took ${Date.now() - started} ms`); + assert.equal(fs.existsSync(chooser), false, "the crashed chooser's file must be cleared"); +}); + +// Bakery order: a waiter that arrived while the lock was held is served before one +// that arrives later, and a latecomer can never take a lower number than a waiter +// already in line. +test("a latecomer queues behind the waiter that was already in line", async () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const trace = path.join(workspace, "trace.log"); + + const participant = (name, holdMs) => ` + import fs from "node:fs"; + import { withStateLock } from ${STATE_MODULE_URL}; + withStateLock(${JSON.stringify(workspace)}, () => { + fs.appendFileSync(${JSON.stringify(trace)}, "enter ${name}" + "\\n"); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ${holdMs}); + fs.appendFileSync(${JSON.stringify(trace)}, "leave ${name}" + "\\n"); + }); + `; + + const holder = collectExit(runModule(participant("holder", 900))); + await new Promise((resolve) => setTimeout(resolve, 250)); + const early = collectExit(runModule(participant("early", 50))); + await new Promise((resolve) => setTimeout(resolve, 250)); + const late = collectExit(runModule(participant("late", 50))); + + for (const finished of await Promise.all([holder, early, late])) { + assert.equal(finished.code, 0, finished.stderr); + } + + const steps = fs.readFileSync(trace, "utf8").trim().split("\n"); + assert.deepEqual( + steps, + ["enter holder", "leave holder", "enter early", "leave early", "enter late", "leave late"], + `the queue was not served in order: ${steps.join(" | ")}` + ); +}); + +// Two acquirers can take the same number — they read the tickets at the same +// moment — so the number alone cannot decide. The token breaks the tie, and both +// tickets are judged the same way: the dead one goes, the live one holds the line. +test("tickets that tie on a number are ordered and judged individually", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const gone = deadPid(); + const abandoned = seedLockEntry(lockDir, `3.${gone}-aaa.ticket`, gone); + const live = seedLockEntry(lockDir, `3.${process.pid}-zzz.ticket`, process.pid); + + assert.throws( + () => withStateLock(workspace, () => "stolen", { waitMs: 300 }), + (error) => error.message.includes(`pid ${process.pid}`) && error.message.includes(live) + ); + assert.equal(fs.existsSync(abandoned), false, "the dead ticket in the tie must be cleared"); + assert.equal(fs.existsSync(live), true, "the live ticket in the tie must survive"); +}); + +// A queue that cannot be read is not an empty queue. Answering a listing failure +// with "nobody is ahead of me" is fail-open: a directory that allows creating +// files but not listing them (or a transient I/O error) would put two processes in +// the critical section at once. +test( + "a lock directory that cannot be listed fails the acquisition, not the queue", + { skip: process.platform === "win32" || process.getuid?.() === 0 }, + () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const foreign = seedLockEntry(lockDir, `1.${process.pid}-live.ticket`, process.pid); + + let ran = false; + // Write and search allowed, read denied: files can still be created and + // unlinked, but the directory cannot be listed. + fs.chmodSync(lockDir, 0o300); + try { + assert.throws(() => withStateLock(workspace, () => { + ran = true; + }, { waitMs: 200 }), /EACCES/); + } finally { + fs.chmodSync(lockDir, 0o700); + } + + assert.equal(ran, false, "the callback must not run when the queue cannot be read"); + assert.equal(fs.existsSync(foreign), true, "a foreign ticket must not be touched"); + assert.deepEqual( + fs.readdirSync(lockDir), + [path.basename(foreign)], + "a failed acquisition must take its own entries back out of the queue" + ); + } +); + + +// `statSync` failing is not "the entry is infinitely old". When the entry's own +// content cannot be read either, a swallowed stat error left the age comparison +// with NaN, which read as "abandoned" — so the waiter unlinked a live holder's +// ticket and walked into the critical section behind it. +function withStatFailure(targetPath, error, run) { + const real = fs.statSync; + fs.statSync = (candidate, ...rest) => { + if (String(candidate) === targetPath) { + throw Object.assign(new Error(`${error}: injected, stat '${candidate}'`), { code: error }); + } + return real.call(fs, candidate, ...rest); + }; + try { + return run(); + } finally { + fs.statSync = real; + } +} + +test("a stat failure on a foreign entry fails the acquisition instead of evicting it", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + // Present but unreadable, so the decision falls to the entry's age. + const foreign = path.join(lockDir, `1.${process.pid}-live.ticket`); + fs.writeFileSync(foreign, "{ not json", "utf8"); + + let ran = false; + assert.throws( + () => + withStatFailure(foreign, "EIO", () => + withStateLock(workspace, () => { + ran = true; + }, { waitMs: 200 }) + ), + /EIO/ + ); + + assert.equal(ran, false, "the callback must not run when an entry cannot be judged"); + assert.equal(fs.existsSync(foreign), true, "a foreign entry must not be evicted on a stat failure"); + assert.deepEqual(fs.readdirSync(lockDir), [path.basename(foreign)], "the waiter must take its own entries back out"); +}); + +// The one stat error that is an answer: the entry was released between the listing +// and the look, so it is simply not in the queue any more. Modelled truthfully — +// the file really is removed as the waiter reaches for it. +test("an entry that vanishes between listing and stat is not a blocker", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const foreign = path.join(lockDir, `1.${process.pid}-live.ticket`); + fs.writeFileSync(foreign, "{ not json", "utf8"); + + const realStat = fs.statSync; + fs.statSync = (candidate, ...rest) => { + if (String(candidate) === foreign) { + fs.rmSync(foreign, { force: true }); + } + return realStat.call(fs, candidate, ...rest); + }; + + let ran = false; + try { + withStateLock(workspace, () => { + ran = true; + }, { waitMs: 1000 }); + } finally { + fs.statSync = realStat; + } + + assert.equal(ran, true, "a vanished entry must not hold up the queue"); + assert.deepEqual(fs.readdirSync(lockDir), [], "the lock must be released and nothing left behind"); +}); + +// A budget of zero is not a reason to fail an acquisition nobody is contending: +// the wait is a bound on queueing, not on trying. It still must not queue. +test("a zero budget takes a free lock but does not queue", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + + assert.equal(withStateLock(workspace, () => "ok", { waitMs: 0 }), "ok"); + + const lockDir = lockDirFor(workspace); + const held = seedLockEntry(lockDir, `1.${process.pid}-live.ticket`, process.pid); + assert.throws(() => withStateLock(workspace, () => "queued", { waitMs: 0 }), /state lock/i); + assert.equal(fs.existsSync(held), true, "the live holder keeps its ticket"); +}); + +// Judging and evicting have to be separate passes. Done in one, a verdict that +// throws halfway through leaves the entries it already evicted gone — breaking the +// only promise a failed acquisition makes: it touches nothing but its own files. +test("a blocker that cannot be judged leaves every foreign entry in place", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const abandoned = seedLockEntry(lockDir, `1.${deadPid()}-gone.ticket`, deadPid()); + // Unreadable content, so its age decides — and its age cannot be read. + const unjudgeable = path.join(lockDir, `2.${process.pid}-live.ticket`); + fs.writeFileSync(unjudgeable, "{ not json", "utf8"); + + let ran = false; + assert.throws( + () => + withStatFailure(unjudgeable, "EIO", () => + withStateLock(workspace, () => { + ran = true; + }, { waitMs: 200 }) + ), + /EIO/ + ); + + assert.equal(ran, false, "the callback must not run"); + assert.equal(fs.existsSync(abandoned), true, "an entry judged before the failure must not have been evicted"); + assert.equal(fs.existsSync(unjudgeable), true, "the unjudgeable entry must be left alone"); + assert.deepEqual( + fs.readdirSync(lockDir).sort(), + [path.basename(abandoned), path.basename(unjudgeable)].sort(), + "only our own entries may be removed on a failed acquisition" + ); +}); + +// An entry this process may not read says nothing about its owner, so it cannot be +// aged out either — that is the same fail-closed rule as a stat failure. +test( + "an unreadable foreign entry fails the acquisition instead of being aged out", + { skip: process.platform === "win32" || process.getuid?.() === 0 }, + () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const foreign = seedLockEntry(lockDir, `1.${process.pid}-live.ticket`, process.pid); + // Old enough that the rule this replaces would have aged it out, so the test + // fails against an implementation that keeps guessing rather than failing. + const longAgo = new Date(Date.now() - 600000); + fs.utimesSync(foreign, longAgo, longAgo); + fs.chmodSync(foreign, 0o000); + + let ran = false; + try { + assert.throws(() => withStateLock(workspace, () => { + ran = true; + }, { waitMs: 200 }), /EACCES/); + } finally { + fs.chmodSync(foreign, 0o600); + } + + assert.equal(ran, false, "the callback must not run when an entry cannot be read"); + assert.deepEqual(fs.readdirSync(lockDir), [path.basename(foreign)], "the entry must be intact and ours removed"); + } +); + +// A verdict that keeps saying "it left the queue" while the listing keeps showing +// it must still end: clearing something may skip the poll interval, never the +// deadline. +test("a blocker that never settles still ends at the deadline", { timeout: 10000 }, () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const foreign = seedLockEntry(lockDir, `1.${process.pid}-live.ticket`, process.pid); + + const realRead = fs.readFileSync; + fs.readFileSync = (candidate, ...rest) => { + if (String(candidate) === foreign) { + throw Object.assign(new Error(`ENOENT: injected, open '${candidate}'`), { code: "ENOENT" }); + } + return realRead.call(fs, candidate, ...rest); + }; + + const started = Date.now(); + try { + assert.throws(() => withStateLock(workspace, () => "never", { waitMs: 300 }), /state lock/i); + } finally { + fs.readFileSync = realRead; + } + + assert.ok(Date.now() - started < 5000, `the wait must end at its own deadline, took ${Date.now() - started} ms`); + assert.deepEqual(fs.readdirSync(lockDir), [path.basename(foreign)], "the waiter must take its own entries back out"); +}); + +// The blocker the last scan saw may be gone by the time the error is built — not +// least because this waiter cleared it. Naming a file that no longer exists helps +// nobody, so the message says plainly that there is nothing left to point at. +test("the timeout says so when no blocker is left to name", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const foreign = path.join(lockDir, `1.${process.pid}-live.ticket`); + const owner = JSON.stringify({ pid: process.pid, startedAt: new Date().toISOString() }); + fs.writeFileSync(foreign, owner, "utf8"); + + // Held at the moment it is judged, gone by the time the message is built. + const realRead = fs.readFileSync; + fs.readFileSync = (candidate, ...rest) => { + if (String(candidate) === foreign) { + fs.rmSync(foreign, { force: true }); + return owner; + } + return realRead.call(fs, candidate, ...rest); + }; + + try { + assert.throws( + () => withStateLock(workspace, () => "never", { waitMs: 0 }), + /waiting for the Codex state lock .*; no blocker visible now\./ + ); + } finally { + fs.readFileSync = realRead; + } +}); + +// An eviction that did not actually happen is not progress. Treating a failed +// unlink as one skipped the poll pause on every pass, so an abandoned entry this +// process may not remove turned the bounded wait into a busy loop hammering the +// filesystem. +test("an eviction that fails keeps the poll pause", () => { + const workspace = makeTempDir(); + saveState(workspace, { jobs: [] }); + const lockDir = lockDirFor(workspace); + const gone = deadPid(); + const stuck = seedLockEntry(lockDir, `1.${gone}-gone.ticket`, gone); + + let attempts = 0; + let listings = 0; + const realUnlink = fs.unlinkSync; + const realReaddir = fs.readdirSync; + fs.unlinkSync = (candidate, ...rest) => { + if (String(candidate) === stuck) { + attempts += 1; + throw Object.assign(new Error(`EPERM: injected, unlink '${candidate}'`), { code: "EPERM" }); + } + return realUnlink.call(fs, candidate, ...rest); + }; + fs.readdirSync = (candidate, ...rest) => { + if (String(candidate) === lockDir) { + listings += 1; + } + return realReaddir.call(fs, candidate, ...rest); + }; + + let ran = false; + const started = Date.now(); + try { + assert.throws(() => withStateLock(workspace, () => { + ran = true; + }, { waitMs: 300 }), /state lock/i); + } finally { + fs.unlinkSync = realUnlink; + fs.readdirSync = realReaddir; + } + const elapsed = Date.now() - started; + + assert.equal(ran, false, "the callback must not run while a blocker is still there"); + assert.ok(elapsed >= 250, `the wait must use its whole budget, took ${elapsed} ms`); + assert.ok(elapsed < 3000, `the wait must end at its own deadline, took ${elapsed} ms`); + assert.ok(attempts <= 300 / 25 + 5, `the waiter span instead of polling: ${attempts} unlink attempts`); + assert.ok(listings <= 300 / 25 + 5, `the waiter span instead of polling: ${listings} listings`); + assert.equal(fs.existsSync(stuck), true, "the entry that could not be removed must still be there"); + assert.deepEqual(fs.readdirSync(lockDir), [path.basename(stuck)], "our own entries must be cleaned up"); +}); + +// The two grace periods, stated directly: an entry whose content says nothing +// about an owner is debris after 2 s, and one that names a PID nothing can check +// gets 30 s. Both are judged against the entry's own mtime. +test("a junk entry holds its place for the orphan grace and no longer", () => { + const fresh = makeTempDir(); + saveState(fresh, { jobs: [] }); + const freshEntry = path.join(lockDirFor(fresh), `1.${process.pid}-junk.ticket`); + fs.writeFileSync(freshEntry, "{ not json", "utf8"); + assert.throws(() => withStateLock(fresh, () => "too soon", { waitMs: 150 }), /state lock/i); + assert.equal(fs.existsSync(freshEntry), true, "junk younger than the grace keeps its place"); + + const aged = makeTempDir(); + saveState(aged, { jobs: [] }); + const agedEntry = path.join(lockDirFor(aged), `1.${process.pid}-junk.ticket`); + fs.writeFileSync(agedEntry, "{ not json", "utf8"); + const pastGrace = new Date(Date.now() - 5000); + fs.utimesSync(agedEntry, pastGrace, pastGrace); + + assert.equal(withStateLock(aged, () => "ok", { waitMs: 500 }), "ok"); + assert.equal(fs.existsSync(agedEntry), false, "junk past the grace is debris"); +}); + +test("an entry with no checkable PID holds its place for the long grace", () => { + const fresh = makeTempDir(); + saveState(fresh, { jobs: [] }); + const freshEntry = path.join(lockDirFor(fresh), `1.${process.pid}-nopid.ticket`); + fs.writeFileSync(freshEntry, JSON.stringify({ pid: null }), "utf8"); + const withinGrace = new Date(Date.now() - 10000); + fs.utimesSync(freshEntry, withinGrace, withinGrace); + assert.throws(() => withStateLock(fresh, () => "too soon", { waitMs: 150 }), /state lock/i); + assert.equal(fs.existsSync(freshEntry), true, "an unusable PID is given the long grace, not the short one"); + + const aged = makeTempDir(); + saveState(aged, { jobs: [] }); + const agedEntry = path.join(lockDirFor(aged), `1.${process.pid}-nopid.ticket`); + fs.writeFileSync(agedEntry, JSON.stringify({ pid: null }), "utf8"); + const pastGrace = new Date(Date.now() - 60000); + fs.utimesSync(agedEntry, pastGrace, pastGrace); + + assert.equal(withStateLock(aged, () => "ok", { waitMs: 500 }), "ok"); + assert.equal(fs.existsSync(agedEntry), false, "past the long grace it is debris"); +}); diff --git a/tests/tracked-jobs.test.mjs b/tests/tracked-jobs.test.mjs new file mode 100644 index 000000000..ca39b3815 --- /dev/null +++ b/tests/tracked-jobs.test.mjs @@ -0,0 +1,398 @@ +import fs from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { makeTempDir, run } from "./helpers.mjs"; +import { reapDeadJobs, runTrackedJob } from "../plugins/codex/scripts/lib/tracked-jobs.mjs"; +import { + listJobs, + readJobFile, + resolveJobFile, + resolveJobPid, + resolveJobPidFile, + resolveJobRequestFile, + resolveStateFile, + updateJobPid, + upsertJob, + writeJobFile, + writeJobPidFile, + writeJobRequestFile +} from "../plugins/codex/scripts/lib/state.mjs"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const TRACKED_JOBS_URL = pathToFileURL(path.join(ROOT, "plugins", "codex", "scripts", "lib", "tracked-jobs.mjs")).href; + +function seedJob(workspace, job) { + writeJobFile(workspace, job.id, job); + upsertJob(workspace, job); +} + +function spawnDeadPid() { + const result = run(process.execPath, ["-e", ""]); + assert.equal(result.status, 0); + return result.pid; +} + +test("reapDeadJobs marks a running job with a dead pid as failed", () => { + const workspace = makeTempDir(); + seedJob(workspace, { id: "job-dead", status: "running", phase: "delegating", pid: spawnDeadPid(), logFile: null }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped.length, 1); + assert.equal(reaped[0].status, "failed"); + assert.equal(reaped[0].pid, null); + assert.match(reaped[0].errorMessage, /worker exited before completing/); + + const stored = readJobFile(resolveJobFile(workspace, "job-dead")); + assert.equal(stored.status, "failed"); + assert.equal(stored.pid, null); + assert.equal(listJobs(workspace).find((job) => job.id === "job-dead").status, "failed"); +}); + +test("reapDeadJobs refreshes updatedAt so the reaped job sorts newest-first", () => { + const workspace = makeTempDir(); + const stale = "2000-01-01T00:00:00.000Z"; + seedJob(workspace, { id: "job-stale", status: "running", phase: "delegating", pid: spawnDeadPid(), updatedAt: stale, logFile: null }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.notEqual(reaped[0].updatedAt, stale); + assert.equal(reaped[0].updatedAt, reaped[0].completedAt); + assert.equal(readJobFile(resolveJobFile(workspace, "job-stale")).updatedAt, reaped[0].updatedAt); +}); + +test("reapDeadJobs leaves a running job with a live pid untouched", () => { + const workspace = makeTempDir(); + seedJob(workspace, { id: "job-live", status: "running", phase: "delegating", pid: process.pid, logFile: null }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "running"); + assert.equal(readJobFile(resolveJobFile(workspace, "job-live")).status, "running"); +}); + +test("reapDeadJobs leaves a pid-less queued job untouched inside the grace window", () => { + const workspace = makeTempDir(); + seedJob(workspace, { id: "job-no-pid", status: "queued", phase: "queued", pid: null, logFile: null }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "queued"); +}); + +test("reapDeadJobs keeps the stored result when the job finished between read and probe", () => { + const workspace = makeTempDir(); + const deadPid = spawnDeadPid(); + writeJobFile(workspace, "job-raced", { id: "job-raced", status: "completed", phase: "completed", result: "done" }); + upsertJob(workspace, { id: "job-raced", status: "running", phase: "delegating", pid: deadPid, logFile: null }); + + const reaped = reapDeadJobs(workspace, [{ id: "job-raced", status: "running", pid: deadPid, logFile: null }]); + + assert.equal(reaped[0].status, "completed"); + assert.equal(reaped[0].result, "done"); + assert.equal(readJobFile(resolveJobFile(workspace, "job-raced")).status, "completed"); +}); + +test("registerWorkerCrashGuard marks the job failed when the worker dies on an unhandled rejection", () => { + const workspace = makeTempDir(); + seedJob(workspace, { id: "job-crash", status: "running", phase: "delegating", pid: null, logFile: null }); + + const workerFile = path.join(makeTempDir(), "crashing-worker.mjs"); + fs.writeFileSync( + workerFile, + [ + `import { registerWorkerCrashGuard } from ${JSON.stringify(TRACKED_JOBS_URL)};`, + "registerWorkerCrashGuard(process.argv[2], process.argv[3], null);", + 'Promise.reject(new Error("boom"));', + "setTimeout(() => {}, 5000);", + "" + ].join("\n"), + "utf8" + ); + + const result = run(process.execPath, [workerFile, workspace, "job-crash"]); + + assert.equal(result.status, 1); + const stored = readJobFile(resolveJobFile(workspace, "job-crash")); + assert.equal(stored.status, "failed"); + assert.match(stored.errorMessage, /unhandledRejection/); + assert.match(stored.errorMessage, /boom/); +}); + +test("registerWorkerCrashGuard does not rewrite a cancelled job when the worker is SIGTERMed", async () => { + const workspace = makeTempDir(); + // Simulate handleCancel having already written the terminal state before the + // worker processes the teardown SIGTERM it delivered. + seedJob(workspace, { id: "job-cancelled", status: "cancelled", phase: "cancelled", pid: null, errorMessage: "Cancelled by user." }); + + const workerFile = path.join(makeTempDir(), "long-worker.mjs"); + fs.writeFileSync( + workerFile, + [ + `import { registerWorkerCrashGuard } from ${JSON.stringify(TRACKED_JOBS_URL)};`, + "registerWorkerCrashGuard(process.argv[2], process.argv[3], null);", + 'process.stdout.write("ready\\n");', + "setInterval(() => {}, 1000);", + "" + ].join("\n"), + "utf8" + ); + + const child = spawn(process.execPath, [workerFile, workspace, "job-cancelled"], { stdio: ["ignore", "pipe", "ignore"] }); + await new Promise((resolve, reject) => { + child.stdout.on("data", (chunk) => { + if (chunk.toString().includes("ready")) { + resolve(); + } + }); + child.on("error", reject); + }); + + const exited = new Promise((resolve) => child.on("exit", (code, signal) => resolve({ code, signal }))); + child.kill("SIGTERM"); + const { signal } = await exited; + + assert.equal(signal, "SIGTERM"); + const stored = readJobFile(resolveJobFile(workspace, "job-cancelled")); + assert.equal(stored.status, "cancelled"); + assert.equal(stored.errorMessage, "Cancelled by user."); +}); + +// A `--config model_providers.x.http_headers.Authorization=...` value only ever +// lives in the private `jobs/.request.json`; the reaper must delete that file +// and must never lift it into the record it rewrites. +const REQUEST_SECRET = "sk-reaper-secret-value"; +const STALE_CREATED_AT = new Date(Date.now() - 120000).toISOString(); + +function seedQueuedJobWithPayload(workspace, id, overrides = {}) { + const requestFile = writeJobRequestFile(workspace, id, { + prompt: "investigate", + config: { auth_header: REQUEST_SECRET } + }); + const job = { + id, + status: "queued", + phase: "queued", + pid: null, + logFile: null, + createdAt: STALE_CREATED_AT, + requestFile, + request: { prompt: "investigate", config: { auth_header: "[redacted]" } }, + ...overrides + }; + seedJob(workspace, job); + return job; +} + +// The fork writes the queued record before the spawn, so a worker killed before +// it consumed the payload leaves a `queued` job with no usable pid: without the +// grace-period rule it would stay queued forever and keep its 0600 payload. +test("reapDeadJobs fails a queued job whose worker died before it consumed the request payload", () => { + const workspace = makeTempDir(); + seedQueuedJobWithPayload(workspace, "job-queued-dead"); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "failed"); + assert.match(reaped[0].errorMessage, /worker exited before completing/); + assert.equal(reaped[0].requestFile, null); + assert.equal(fs.existsSync(resolveJobRequestFile(workspace, "job-queued-dead")), false); + assert.equal(readJobFile(resolveJobFile(workspace, "job-queued-dead")).requestFile, null); +}); + +test("reapDeadJobs deletes the private request payload of a dead running job", () => { + const workspace = makeTempDir(); + seedQueuedJobWithPayload(workspace, "job-running-dead", { + status: "running", + phase: "delegating", + pid: spawnDeadPid() + }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "failed"); + assert.equal(reaped[0].requestFile, null); + assert.equal(fs.existsSync(resolveJobRequestFile(workspace, "job-running-dead")), false); +}); + +test("reapDeadJobs never touches a live worker or its request payload", () => { + const workspace = makeTempDir(); + const job = seedQueuedJobWithPayload(workspace, "job-live-payload", { pid: process.pid }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "queued"); + assert.equal(reaped[0].requestFile, job.requestFile); + assert.equal(fs.existsSync(job.requestFile), true); + assert.equal(JSON.parse(fs.readFileSync(job.requestFile, "utf8")).config.auth_header, REQUEST_SECRET); +}); + +test("reapDeadJobs leaves a cancelled job with a dead pid in its terminal state", () => { + const workspace = makeTempDir(); + seedJob(workspace, { + id: "job-cancelled-dead", + status: "cancelled", + phase: "cancelled", + pid: spawnDeadPid(), + errorMessage: "Cancelled by user.", + createdAt: STALE_CREATED_AT, + logFile: null + }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "cancelled"); + assert.equal(reaped[0].errorMessage, "Cancelled by user."); + assert.equal(readJobFile(resolveJobFile(workspace, "job-cancelled-dead")).status, "cancelled"); +}); + +test("a reaped job leaves no private --config secret behind in state.json", () => { + const workspace = makeTempDir(); + seedQueuedJobWithPayload(workspace, "job-secret"); + + reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(fs.readFileSync(resolveStateFile(workspace), "utf8").includes(REQUEST_SECRET), false); + assert.equal(fs.readFileSync(resolveJobFile(workspace, "job-secret"), "utf8").includes(REQUEST_SECRET), false); +}); + +// The worker owns its job file from its first line, so the parent must never +// write that file back — not even to add the pid. It goes to an atomic sidecar +// and to the pid-only index patch instead. +test("updateJobPid records the worker pid without rewriting the job file", () => { + const workspace = makeTempDir(); + const job = seedQueuedJobWithPayload(workspace, "job-pid"); + const jobFile = resolveJobFile(workspace, "job-pid"); + const before = fs.readFileSync(jobFile, "utf8"); + + updateJobPid(workspace, "job-pid", 424242); + + assert.equal(fs.readFileSync(jobFile, "utf8"), before, "the parent must not rewrite the worker's job file"); + const stored = readJobFile(jobFile); + assert.equal(stored.status, "queued"); + assert.equal(stored.requestFile, job.requestFile); + assert.equal(resolveJobPid(workspace, stored), 424242, "readers must find the pid in the sidecar"); + assert.equal(listJobs(workspace).find((entry) => entry.id === "job-pid").pid, 424242); +}); + +// The race the sidecar removes: the worker finishes between the parent's read +// and its write, and the parent puts the queued snapshot back — losing the +// result, threadId and turnId while the index already says completed. +test("updateJobPid leaves a record the worker already completed intact", () => { + const workspace = makeTempDir(); + seedQueuedJobWithPayload(workspace, "job-raced"); + + const completed = { + id: "job-raced", + status: "completed", + phase: "done", + pid: null, + threadId: "thr_1", + turnId: "turn_1", + result: { status: 0, finalMessage: "done" }, + completedAt: "2026-03-18T15:31:00.000Z" + }; + writeJobFile(workspace, "job-raced", completed); + upsertJob(workspace, completed); + + updateJobPid(workspace, "job-raced", 424242); + + const stored = readJobFile(resolveJobFile(workspace, "job-raced")); + assert.equal(stored.status, "completed"); + assert.equal(stored.threadId, "thr_1"); + assert.equal(stored.turnId, "turn_1"); + assert.deepEqual(stored.result, completed.result); + const indexed = listJobs(workspace).find((entry) => entry.id === "job-raced"); + assert.equal(indexed.status, "completed"); + assert.equal(indexed.pid, null, "a finished job must not get its pid back"); + assert.equal(resolveJobPid(workspace, stored), null, "a terminal record never reports a pid"); +}); + +// A worker that took the record over but has not written its own pid yet is +// still reapable through the sidecar, and the sidecar goes away with the job. +test("reapDeadJobs resolves a pid from the sidecar and releases it", () => { + const workspace = makeTempDir(); + seedQueuedJobWithPayload(workspace, "job-sidecar", { status: "running", phase: "starting" }); + writeJobPidFile(workspace, "job-sidecar", spawnDeadPid()); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "failed"); + assert.match(reaped[0].errorMessage, /worker exited before completing/); + assert.equal(fs.existsSync(resolveJobPidFile(workspace, "job-sidecar")), false, "a terminal job must not keep a stale pid file"); +}); + +// The parent patches the pid in after the spawn, but the worker owns the record +// from its first line: a load-mutate-save of the whole record would rewind +// `running` back to `queued`. +test("updateJobPid never rewinds a worker that already reported running", () => { + const workspace = makeTempDir(); + seedJob(workspace, { id: "job-started", status: "running", phase: "starting", pid: 777, logFile: null }); + + updateJobPid(workspace, "job-started", 424242); + + const stored = readJobFile(resolveJobFile(workspace, "job-started")); + assert.equal(stored.status, "running"); + assert.equal(stored.pid, 777); + const indexed = listJobs(workspace).find((entry) => entry.id === "job-started"); + assert.equal(indexed.status, "running"); + assert.equal(indexed.pid, 777); +}); + +// PID liveness cannot settle this: a zombie and a recycled pid both read as +// alive, so a worker that died right after its terminal `writeJobFile` kept a +// phantom `running` entry in the index — blocking resume on that thread — for as +// long as some process held its pid. The job file is the authoritative record, +// so it is read first, for every active entry, whatever the pid says. +test("reapDeadJobs reconciles a terminal job file even when the recorded pid is alive", () => { + const workspace = makeTempDir(); + upsertJob(workspace, { id: "job-zombie", status: "running", phase: "running", threadId: "thr_1", pid: process.pid }); + writeJobFile(workspace, "job-zombie", { + id: "job-zombie", + status: "completed", + phase: "done", + pid: null, + threadId: "thr_1", + turnId: "turn_9", + result: { status: 0, finalMessage: "done" }, + completedAt: "2026-03-24T20:06:00.000Z" + }); + + const reaped = reapDeadJobs(workspace, listJobs(workspace)); + + assert.equal(reaped[0].status, "completed", "the authoritative job file decides"); + assert.equal(reaped[0].turnId, "turn_9"); + const indexed = listJobs(workspace).find((entry) => entry.id === "job-zombie"); + assert.equal(indexed.status, "completed", "the terminal record must reach the index"); + assert.equal(indexed.pid, null); +}); + +// The payload is a 0600 file that can hold `--config` credentials. The worker +// consumes it on the way in, but a job that never got that far (or a migration +// that staged one) leaves it behind: nothing revisits a terminal job, so its +// terminal write is the last chance to release it. +test("a terminal write releases the job's private request payload", async () => { + const workspace = makeTempDir(); + + writeJobRequestFile(workspace, "job-done", { prompt: "x", config: { "http_headers.Cookie": "SECRET" } }); + await runTrackedJob({ id: "job-done", workspaceRoot: workspace, logFile: null }, async () => ({ + exitStatus: 0, + payload: { ok: true }, + rendered: "done\n", + summary: "done" + })); + assert.equal(fs.existsSync(resolveJobRequestFile(workspace, "job-done")), false, "a completed job must not keep its payload"); + + writeJobRequestFile(workspace, "job-thrown", { prompt: "x", config: { "http_headers.Cookie": "SECRET" } }); + await assert.rejects( + runTrackedJob({ id: "job-thrown", workspaceRoot: workspace, logFile: null }, async () => { + throw new Error("boom"); + }), + /boom/ + ); + assert.equal(fs.existsSync(resolveJobRequestFile(workspace, "job-thrown")), false, "a failed job must not keep its payload"); +});