diff --git a/.changeset/cli-contract-busy.md b/.changeset/cli-contract-busy.md new file mode 100644 index 0000000..9fafd66 --- /dev/null +++ b/.changeset/cli-contract-busy.md @@ -0,0 +1,44 @@ +--- +"@managers/server": minor +--- + +`managers busy`, `GET /api/busy`, a `service stop` that refuses when Managers is mid-turn, and a SIGTERM that finishes or records the work under way + +Wave F2 of the CLI-contract rollout (edspencer/cli-contract SPEC 0.2, §4), ported from Paddock 0.81. + +**New: `managers busy [--json]`.** Asks the running instance whether it is in the +middle of something a restart would lose. It exits **0** not busy, **75** busy, +**3** when nothing answers, and **1** when something answered but not as +Managers' busy check. 75 is `EX_TEMPFAIL`, which Watchtower's +`lifecycle.pre-update` hook reads as "skip this one". + +**`service stop` and `service restart` now run that check first and refuse with +exit 75 unless `--force` is given.** A script that stops Managers unconditionally +needs `--force`, or needs to handle 75: that is the behaviour change in this +release. `service uninstall` and `service start` don't check. A forced stop +prints the message it would have refused with, prefixed `Overriding:`. +**`--force` anywhere else is now a usage error (exit 2)** rather than being +silently ignored. When the busy work is chat turns, the refusal says to run the +command from a terminal rather than from inside a Managers chat. + +**New: `GET /api/busy`**, answering `{ busy, version, counts }` with four counts: +`chat` (chat turns running, triggers and schedules included), `background` +(background tasks a live session still holds after its turn), `job` (herdctl +scheduler jobs) and **`run`** (Managers trigger runs in flight: from the moment a +fire writes its `running` record until that record is finished and committed, +which is wider than the turn at both ends). No authentication, because it +carries counts and nothing else; in-memory state only. + +**New: `/runtime.json`**, written 0600 once the server is listening and +removed when it stops, holding `{ pid, port, url, version, startedAt }`. + +**SIGTERM now finishes or records the work under way, instead of closing at +once.** If chat turns or trigger runs are in flight, a stop waits up to 8 seconds +for them to end by themselves; whatever is left is recorded — each trigger run's +record is finished as `failed`, `error: interrupted by shutdown` (through its own +completion hook, so a consolidation still gets its `#reflection` episode and the +record is committed), and the log names what was cut short. An idle instance +stops at once. Signals are idempotent and the whole stop is bounded at 15 +seconds, inside launchd's 20-second kill budget. Under Docker, whose default +`docker stop` timeout is 10 seconds, give the container a `stop_grace_period` of +at least 20 seconds to let the wait run. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0235498..b463dba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,10 +40,11 @@ jobs: # The CLI contract, checked from the outside by @edspencer/cli-conformance: `--version` is # bare, `--help` exits 0, an unknown flag exits 2 with the usage after the message, # `service status` exits 4 with nothing installed and its --json matches the shared schema, - # `serve`'s first stdout line is `Managers : `, and /api/health answers with - # that same version. `cli-contract.json` at the root says which checks apply and why the - # two that don't (`busy`, wave C1) don't — a NEWER runner's new check fails until this repo - # says whether it applies, which is the point. + # `busy` exits 3 with nothing running and its --json matches the schema too, `serve`'s first + # stdout line is `Managers : `, and /api/health answers with that same version. + # `cli-contract.json` at the root says which checks apply; since wave C1 every one of the + # nine does, with no skips — and a NEWER runner's new check fails until this repo says + # whether IT applies, which is the point. # # `--dir dist-npm`, not the repo root: Managers' publishable package is SYNTHESIZED from two # private workspaces by scripts/make-npm-package.mjs (see RELEASING.md), so `npm pack` at the diff --git a/README.md b/README.md index 37a9d46..69b0799 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,11 @@ runs it in the background from login (a launchd LaunchAgent on macOS, a `systemd --user` unit on Linux), with `uninstall`, `start`, `stop`, `restart`, `status [--json]` and `logs [-f] [-n ]`. `service status` exits **0** when it is up and answering, **3** when installed but not, and **4** when there is no -unit; a bad command, flag or value exits **2**. The CLI follows the shared +unit; a bad command, flag or value exits **2**. `managers busy` exits **75** while +a chat turn or trigger run is in flight (0 when idle, 3 when nothing is running), +and `service stop`/`restart` refuse with 75 unless given `--force`. A SIGTERM +waits briefly for in-flight work, then records what it cut short; under Docker, +give the container a `stop_grace_period` of 20 s or more. The CLI follows the shared [CLI contract](https://github.com/edspencer/cli-contract) (`cli-contract.json` says which of its checks apply). diff --git a/cli-contract.json b/cli-contract.json index c30a49f..32e8861 100644 --- a/cli-contract.json +++ b/cli-contract.json @@ -38,12 +38,8 @@ "unknown-flag": true, "service-status": true, "service-status-json": true, - "busy": { - "skip": "no `managers busy` until wave F2 of the CLI-contract rollout" - }, - "busy-json": { - "skip": "no `managers busy` until wave F2 of the CLI-contract rollout" - }, + "busy": true, + "busy-json": true, "serve-banner": true, "serve-health": true } diff --git a/packages/server/src/app.ts b/packages/server/src/app.ts index 7ff503e..fc59b7f 100644 --- a/packages/server/src/app.ts +++ b/packages/server/src/app.ts @@ -48,6 +48,7 @@ import { installHerdctlBridgeLoopbackBind } from "./herdctl-bridge-bind.js"; import { renderIndexHtml } from "./brand.js"; import { makeChatHandler } from "./ws.js"; import type { SessionHub } from "./session-hub.js"; +import type { BackgroundRegistry } from "./background-live.js"; import { SweepService } from "./sweep.js"; import { ArchiveStore } from "./archive.js"; import { StarStore } from "./star.js"; @@ -107,6 +108,12 @@ export interface BuiltApp { * same way, via the route context). */ hub: SessionHub; + /** + * The live background-work registry (#604), exposed for the same reason `hub` + * is — it is WS-layer state a non-socket caller has to read. `GET /api/busy` + * reaches it through the route context; a test reaches it here. + */ + background: BackgroundRegistry; /** Tear down the fleet + close the server (no process.exit, for tests). */ close: () => Promise; } @@ -655,5 +662,5 @@ export async function buildApp(opts: BuildAppOptions = {}): Promise { // `hub` rides along like every other built piece: it is the WS layer's turn // bookkeeping, and the only place a caller can observe or annotate a turn // without a live socket (see SessionHub.noteCancel). - return { app, cfg, projects, herdctl, git, githubAuth, sweep, archive, star, readState, unread, parentDetach, runProvenance, queuedMessage, transcriber, events, triggers, managers, autocommit, dataSync, hub: chatHandler.managementOpsContext.hub, close }; + return { app, cfg, projects, herdctl, git, githubAuth, sweep, archive, star, readState, unread, parentDetach, runProvenance, queuedMessage, transcriber, events, triggers, managers, autocommit, dataSync, hub: chatHandler.managementOpsContext.hub, background: chatHandler.managementOpsContext.background, close }; } diff --git a/packages/server/src/auth.ts b/packages/server/src/auth.ts index e3b4e2b..c3e2b7c 100644 --- a/packages/server/src/auth.ts +++ b/packages/server/src/auth.ts @@ -18,9 +18,10 @@ * paddock holds no key material, only the JWKS URL, and * rejects missing/invalid/expired tokens with 401. * - * Paddock's health route (`/api/health`) is always exempt so the proxy and - * monitoring can poll a server that is otherwise locked down. Only registered - * routes may be exempted — see HEALTH_PATHS below for why (issue #569). + * Managers' health route (`/api/health`) is always exempt so the proxy and + * monitoring can poll a server that is otherwise locked down, as is the CLI + * contract's `/api/busy` — which carries counts and nothing else. Only + * registered routes may be exempted — see HEALTH_PATHS below for why (#569). * * Registered in app.ts BEFORE the routes so the `onRequest` hook guards every * REST + WS request. The decorator + hook are added directly to the root app @@ -78,7 +79,18 @@ declare module "fastify" { * to a real route returning JSON, so the set cannot drift from the router again. */ export const HEALTH_PATHS: ReadonlySet = new Set([ - "/api/health", // paddock's own health route (routes/meta.ts) + "/api/health", // Managers' own health route (routes/meta.ts) + // The CLI contract's busy check (SPEC §4, §8). Exempt for the same reason + // health is, and safe for a reason health does not need: the answer is four + // integers and the version, with no chat title, session id or project name in + // it — SPEC §10 keeps `reasons` out until there is a way for a caller to prove + // it is local. `managers service stop`, `managers update` and Watchtower's + // pre-update hook all ask it with no credential to offer, and a 401 would be + // read by `cli/busy.ts` as exit 1 rather than as "not busy" — so gating it + // would fail every stop on an authenticated instance. + // + // If this answer ever grows a label, this line has to be revisited with it. + "/api/busy", // routes/meta.ts ]); /** diff --git a/packages/server/src/background-live.ts b/packages/server/src/background-live.ts index 9c21b96..5441853 100644 --- a/packages/server/src/background-live.ts +++ b/packages/server/src/background-live.ts @@ -214,6 +214,30 @@ export class BackgroundRegistry { return !!e && e.tasks.size > 0; } + /** + * How many live background tasks there are, across the sessions `keep` + * accepts. + * + * The `/api/busy` check (CLI contract SPEC §4) passes herdctl's SDK liveness + * as `keep`, for the reason the per-process caveat in the module header gives: + * nothing resets this set when a one-shot `claude` exits, so on + * `driveMode: batch` a finished turn's tasks sit here indefinitely. Filtering + * on a live session is what stops a stale entry reporting a batch instance + * busy for good — and it costs nothing in `session` mode, where the reaper + * holds a session open precisely BECAUSE it has background work. + * + * Counted rather than listed so the busy check allocates nothing per task + * (§4: in-memory, well under a second). {@link snapshot} is the one that + * builds arrays, and it is for the connect-time frame. + */ + liveTaskCount(keep: (sessionId: string) => boolean = () => true): number { + let n = 0; + for (const [sessionId, e] of this.bySession) { + if (e.tasks.size > 0 && keep(sessionId)) n += e.tasks.size; + } + return n; + } + /** * Forget a session's set. Called when the session's process restarts or its * stream ends — see the per-process caveat in the module header. diff --git a/packages/server/src/busy.ts b/packages/server/src/busy.ts new file mode 100644 index 0000000..2b9399f --- /dev/null +++ b/packages/server/src/busy.ts @@ -0,0 +1,133 @@ +/** + * What `GET /api/busy` answers (CLI contract SPEC §4, ROADMAP "Busy, per tool"). + * + * ## What "busy" means here + * + * Only work **a restart would lose or cut short** — the kind someone would be + * annoyed to find killed. For Managers that is four things: Paddock's three, which + * the ROADMAP row names, and Managers' own trigger runs: + * + * - **`chat`** — a chat turn running. This is the one that matters: a turn is a + * live `claude` subprocess writing a transcript, and `herdctl.stop()` ends it + * with `waitForJobs: false`. A fired trigger or a cron schedule is NOT a + * fourth kind, because every one of them runs as a first-class chat on the + * hub (`ws-triggers.ts` — one `startAgentTurn` for every trigger type), so it + * is already counted here. + * - **`background`** — background work a live SDK session is still holding + * after its turn ended (#604): a `run_in_background` shell, a sub-agent, a + * `Monitor`. These outlive the `Turn` record, so the hub cannot see them. + * - **`job`** — herdctl's scheduler running a job. Managers routes a keeper's + * schedule fire back onto the hub, so in practice this double-counts with + * `chat` for the window where both are true; a count that is occasionally + * high by one is the right failure direction for a check whose answer is + * "wait a moment". + * - **`run`** — a Managers trigger run in flight (`ManagersState.liveRuns`): + * from the moment a fire writes its `status: running` record until its + * completion hook has finished that record and committed it. That window is + * wider than the chat turn at both ends — the briefing is built before the + * turn starts, and the record, the `#reflection` episode and the commit are + * written after it ends — and a restart inside it leaves a run failed with + * "interrupted by restart". The overlap with `chat` while the turn runs is + * the same deliberate high-by-one as `job`. + * + * ## What is deliberately NOT busy + * + * - **Idle live sessions.** A session the reaper is holding open with no turn + * and no background task resumes after a restart; that is the whole point of + * a resumable session. + * - **The post-turn sweeper.** It is re-runnable from its watermark, so losing + * one is a re-sweep rather than lost work. The ROADMAP row leaves it out too. + * - **Open browsers.** A page reconnects to `/ws` and replays. + * + * ## Cheap, and in-memory only + * + * §4 requires this to read in-memory state, never walk the disk, and answer in + * well under a second — it is called by Watchtower's pre-update hook and by + * every `service stop`. So each source here is a `Map.size` or a filtered + * iteration over one, and {@link collectBusy} is synchronous. + * + * That rules out the obvious source for `job`. `fleetStatus().counts.runningJobs` + * is what the ROADMAP names, and `FleetStatus` is built by `getAgentInfo()`, + * which **reads the fleet state file off disk** on every call (herdctl's + * `StatusQueries.readFleetStateSnapshot`) — the same re-parse that was 61% of + * this server's CPU once. `counts.runningJobs` reduces to + * `Σ scheduler.getRunningJobCount(agent)`, and the scheduler also publishes + * `getTotalRunningJobCount()` over the same in-memory map, so Managers asks the + * scheduler directly for an identical number with no file read. + * + * ## No labels, ids or titles + * + * Counts only, for every caller, until SPEC §10's first question (how a CLI + * proves it is local) is designed. That is why this route needs no auth: there + * is nothing in the answer to protect. A chat's TITLE would be a different + * matter, and is exactly what `reasons` will carry once there is a way to know + * who is asking — so nothing here may grow a string field by accident. + */ + +/** The non-zero kinds the CLI's message names: `Managers is busy: 2 chats are running.` */ +export interface BusyCounts { + /** Chat turns running — a human turn, a fired trigger, a cron fire. */ + chat: number; + /** Background tasks live sessions are still holding after their turn (#604). */ + background: number; + /** herdctl scheduler jobs in flight. */ + job: number; + /** Managers trigger runs in flight (a `running` run record this process owns). */ + run: number; +} + +/** The `/api/busy` body (SPEC §4). */ +export interface BusyBody { + busy: boolean; + version: string; + counts: BusyCounts; +} + +/** + * The in-memory readings, injected so the computation is unit-testable + * without a fleet, a hub or a socket. Each must be cheap enough to call on every + * request — see the module header. + */ +export interface BusySources { + /** Sessions with a running turn: `hub.runningSessions().length`. */ + runningChats: () => number; + /** Live background tasks, counted only in sessions whose SDK session is live. */ + backgroundTasks: () => number; + /** Scheduler jobs running right now. */ + runningJobs: () => number; + /** Trigger runs in flight in this process: `managers.liveRuns.size`. Absent → 0. */ + runningRuns?: () => number; +} + +/** + * Read the counts and decide busy. + * + * `busy` is `counts` having any non-zero member rather than a separately + * computed predicate, so the sentence the CLI prints can never disagree with the + * boolean it printed it for. A source that throws counts as zero: a busy check + * that 500s is worse than one that under-reports, because a 500 is read by + * `busy-client.ts` as "something answered, but not as Managers' busy check" — + * exit 1 — which fails a `service stop` outright instead of letting it proceed. + */ +export function collectBusy(sources: BusySources, version: string): BusyBody { + const counts: BusyCounts = { + chat: safeCount(sources.runningChats), + background: safeCount(sources.backgroundTasks), + job: safeCount(sources.runningJobs), + run: safeCount(sources.runningRuns ?? (() => 0)), + }; + return { + busy: counts.chat > 0 || counts.background > 0 || counts.job > 0 || counts.run > 0, + version, + counts, + }; +} + +function safeCount(read: () => number): number { + try { + const n = read(); + return Number.isInteger(n) && n > 0 ? n : 0; + } catch { + return 0; + } +} diff --git a/packages/server/src/cli/args.ts b/packages/server/src/cli/args.ts index f4c4df0..233f29e 100644 --- a/packages/server/src/cli/args.ts +++ b/packages/server/src/cli/args.ts @@ -74,6 +74,16 @@ export interface CliOptions { follow?: boolean; /** `service logs`: how many lines to print first (`-n`, default 50). */ lines?: number; + /** + * `service stop` / `service restart`: go ahead even though the instance is + * busy (SPEC §1, §4). + * + * The refusal it skips is the only thing it skips. It does not change what the + * busy check printed — a forced stop prints the same sentence, prefixed + * `Overriding:` — and in wave C2 it will not skip `update`'s health check or + * its rollback either. + */ + force?: boolean; } /** @@ -144,10 +154,10 @@ export type ConfigAction = (typeof CONFIG_ACTIONS)[number]; * * `serve` is the contract's name for running in the foreground (SPEC §1), and * `start` is kept as an alias **for good** — the spec says so explicitly for - * Managers and Managers, every existing service unit's `ExecStart` names it, and + * Paddock and Managers, every existing service unit's `ExecStart` names it, and * it is in two years of other people's scripts. They parse to the same command. */ -export const VERBS = ["serve", "start", "service", "config"] as const; +export const VERBS = ["serve", "start", "service", "config", "busy"] as const; /** * A parsed invocation: which verb, plus the flags that followed it. @@ -159,7 +169,9 @@ export const VERBS = ["serve", "start", "service", "config"] as const; export type Command = | { verb: "serve"; opts: CliOptions } | { verb: "service"; action: ServiceAction | undefined; opts: CliOptions } - | { verb: "config"; action: ConfigAction | undefined; opts: CliOptions }; + | { verb: "config"; action: ConfigAction | undefined; opts: CliOptions } + /** `busy` takes no action — it is one question with one answer (SPEC §1). */ + | { verb: "busy"; opts: CliOptions }; /** * Split an action off a verb's argv, then parse what follows as flags. @@ -223,6 +235,36 @@ function parseAction( * and load-bearing; it was only the example's error that was misdescribed. */ export function parseCommand(argv: string[]): Command { + const command = dispatch(argv); + // `--force` overrides the busy check (SPEC §1, §4) and nothing else, so it + // only means anything on the two commands that run one. Rejecting it + // everywhere else, rather than parsing and dropping it, is the difference + // between being told and guessing: the two readings of a quiet + // `service uninstall --force` are "it overrode something" and "it did + // nothing", and the exit code is 0 either way. + // + // Checked here, once, rather than in each command: a second copy of this rule + // is a second chance for the two to drift apart about what `--force` means. + if ( + command.opts.force === true && + // `--help` wins, the same way it wins over a missing action above: asking + // for the usage is not a usage error. + !command.opts.help && + !(command.verb === "service" && (command.action === "stop" || command.action === "restart")) + ) { + const what = + command.verb === "service" && command.action !== undefined + ? `\`service ${command.action}\`` + : `\`${command.verb}\``; + throw new UsageError( + `--force applies to \`service stop\` and \`service restart\` only, not ${what}.\n` + + "It overrides the busy check, and those are the two commands that run one.", + ); + } + return command; +} + +function dispatch(argv: string[]): Command { const [first, ...rest] = argv; if (first === "service") { @@ -233,6 +275,11 @@ export function parseCommand(argv: string[]): Command { return { verb: "config", ...parseAction("config", CONFIG_ACTIONS, rest) }; } + // `busy` (SPEC §1, §4). A top-level verb rather than a `service` action + // because its whole point is to be askable where there is no service to ask: + // a Docker container, a Watchtower pre-update hook, an Ansible task. + if (first === "busy") return { verb: "busy", opts: parseArgs(rest) }; + // `serve` and `start` are one command: see VERBS. if (first === "serve" || first === "start") return { verb: "serve", opts: parseArgs(rest) }; @@ -304,6 +351,10 @@ export function parseArgs(argv: string[]): CliOptions { case "--include-env": opts.includeEnv = true; break; + // `service stop` / `service restart`: override the busy check (SPEC §4). + case "--force": + opts.force = true; + break; // `service logs` flags, the names `tail`, `journalctl` and every other // log reader uses — this is not the place to be original. case "-f": @@ -385,6 +436,7 @@ export function explainListenError(err: unknown, host: string, port: string): st export const USAGE_SYNOPSIS = `Usage managers [serve] [options] run the server in this terminal managers service [options] install|uninstall|status|start|stop|restart|logs + managers busy [--json] is the running instance busy? managers config [options] what this instance's config resolved to managers -v | --version print the version and exit managers -h | --help the full help`; @@ -399,10 +451,13 @@ Usage managers service uninstall stop it and remove the unit managers service status [--json] installed? running? which version? managers service start start an installed service - managers service stop stop it, leave it installed - managers service restart stop it and start it again + managers service stop [--force] stop it, leave it installed + managers service restart [--force] stop it and start it again managers service logs [-f] [-n ] the service's log (\`managers service --help\`) + managers busy [--json] is it busy? exit 0 no, 75 yes, 3 + nothing answering + (\`managers busy --help\`) managers config show [--resolved] what this instance's config resolved to, and where each value came from managers config eject [--write] freeze that resolution into @@ -419,11 +474,12 @@ Options -h, --help Show this help Exit codes - 0 done; running and answering + 0 done; running and answering; not busy 1 failed 2 usage: an unknown command or flag, or a bad value - 3 not running: installed, but not running or not answering + 3 not running: installed, but not running or not answering; nothing answers 4 not installed: there is no service unit + 75 busy: refused, and why was printed. Try later, or use --force Opening your own directories Where you run this from does not matter — the instance is decided by its data @@ -536,14 +592,28 @@ Usage managers service status [--json] is it registered, is it running, which version, where are the logs managers service start start an installed service that is stopped - managers service stop stop it, and leave it installed - managers service restart stop it and start it again, re-reading config + managers service stop [--force] stop it, and leave it installed + managers service restart [--force] stop it and start it again, re-reading config managers service logs [-f] [-n ] the service's log (the last 50 lines) Options (logs) -f, --follow keep printing what gets appended, until Ctrl-C -n, --lines how many lines to print first (default 50) +Options (stop, restart) + --force stop it even though it is busy + +Busy + \`stop\` and \`restart\` ask the running instance whether it is busy first, and + refuse with exit 75 if it is — a chat turn in flight, a background task, a + scheduled job. \`managers busy\` asks the same question on its own. + + Being refused is the useful case: a restart kills a chat turn mid-sentence, + and the turn cannot be resumed from where it was. Wait, or --force. + + \`uninstall\` does not ask. Removing the service is an explicit act, and + someone uninstalling it has already decided. + Options (install only — recorded in the generated unit) -p, --port HTTP/WS port (default 7234) --host Bind address (default 127.0.0.1) @@ -588,6 +658,7 @@ Exit codes 4 not installed: there is no unit (status, logs, start, stop, restart) 1 something failed 2 usage: an unknown action or flag, or a bad value + 75 busy, and --force was not given (stop, restart) Installed from npx? \`service install\` refuses. An npx cache path is hash-keyed and npm may prune @@ -606,6 +677,67 @@ A note on access Docs: https://github.com/edspencer/managers`; +export const BUSY_USAGE = `managers busy — is the running instance in the middle of something? + +Usage + managers busy ask, and say so in a sentence + managers busy --json the same answer as one JSON document + managers busy -d ask a particular instance + +Exit codes — this command is FOR its exit code + 0 not busy. Nothing a restart would cut short. + 75 busy. What is running was printed. + 3 nothing answered. No instance is running, so there is nothing to wait for. + 1 something answered, but not as Managers' busy check. + +Options + -d, --data-dir Which instance to ask (default ~/.managers, or + \$MANAGERS_DATA_DIR) — the same rule \`managers serve\` + uses, so the two always mean the same instance + --json One JSON document on stdout: running, busy, url, + version, counts. Prose goes to stderr + -h, --help Show this help + +What counts as busy + Only work a restart would LOSE or cut short: + + chat a chat turn running — a human turn, a fired trigger, a cron + fire. A turn is a live claude subprocess writing a transcript, + and it cannot be resumed from where it was interrupted. + background a background task a live session is still holding after its + turn ended: a background shell, a sub-agent, a Monitor. + job a scheduled job herdctl is running. + + Not busy: an idle session (it resumes), an open browser tab (it reconnects), + a pending curation sweep (it re-runs from its watermark). + +Where to run it from + From a terminal. A Managers chat that runs this is itself one of the chat turns + it counts, so it will always see itself as busy — and \`service stop --force\` + from inside a chat stops the service that is running the chat, which does not + come back with an explanation. That is why the refusal says so. + +For scripts, containers and Watchtower + \`busy\` is there for callers with no service to ask. 75 is EX_TEMPFAIL, which + is the code Watchtower's pre-update hook reads as "skip this one": + + labels: + com.centurylinklabs.watchtower.lifecycle.pre-update: managers busy + + An Ansible task does the same thing with a retry. Nothing here contacts the + network or the npm registry, and the question is answered from memory — it is + cheap enough to ask in a loop. + +How it finds the instance + \`/runtime.json\`, written 0600 when the server starts listening and + removed when it stops. There is no fallback to a default port: guessing one + would ask whatever else happened to be listening there, and get an answer. + + A Managers from before this feature shipped writes no such file, so it reads as + not running (exit 3). Upgrade it and restart it once. + +Docs: https://github.com/edspencer/managers`; + export const CONFIG_USAGE = `managers config — what this instance's configuration actually resolved to Usage diff --git a/packages/server/src/cli/busy-command.ts b/packages/server/src/cli/busy-command.ts new file mode 100644 index 0000000..c149d57 --- /dev/null +++ b/packages/server/src/cli/busy-command.ts @@ -0,0 +1,129 @@ +/** + * Managers' bindings for the copied busy client (`cli/busy.ts`): the nouns its + * message uses, the keeper-chat trap, and the two entry points — `managers busy` + * and the check `service stop` / `service restart` run first (CLI contract + * SPEC §1, §4). + * + * Kept separate from `cli/busy.ts` so that file stays a straight copy of + * `reference/busy-client.ts`, which is what wave C4 re-syncs. Everything in + * here is Managers-specific by definition: the contract says each tool's kinds of + * work are its own. + */ +import { EXIT } from "./exit-codes.js"; +import { createOutput, type Output } from "./output.js"; +import { + type BusyResult, + type FindOptions, + type Nouns, + guardBusy, + runBusyCommand, +} from "./busy.js"; +import { MANAGERS_NAME } from "../version.js"; + +/** + * How each count in `/api/busy` is said in a sentence (`busy.ts` on the server + * decides the keys). + * + * `background` is spelled out rather than left to the default pluraliser, + * because "3 backgrounds are running" is not English — and because "background + * task" is the thing a user recognises from the running-work bar in the UI, + * where they watched it start. + */ +export const MANAGERS_NOUNS: Nouns = { + chat: ["chat turn", "chat turns"], + background: ["background task", "background tasks"], + job: ["scheduled job", "scheduled jobs"], + run: ["trigger run", "trigger runs"], +}; + +/** + * What to do about a busy answer — and for Managers, the first thing to say is + * **where to run the command from**. + * + * This is the keeper-chat trap, and it is specific to Managers and Paddock + * among the six tools: Managers' agents can drive Managers. A keeper chat that + * runs `managers service restart` or `managers update` is itself one of the chat + * turns the answer just counted, so it will always see itself as busy — and + * `--force` from there does not override a mistaken reading, it kills the chat + * that typed it, mid-sentence, along with whatever it had not finished. The + * agent then has no way to report what happened, because the thing that would + * report it is the thing that died. + * + * So when `chat` is non-zero the advice is not "wait or use --force": it is + * "run this from a terminal". When the busy work is background tasks or + * scheduled jobs only, nobody is holding the keyboard inside the service and + * the ordinary advice applies. + */ +export function managersBusyHint(result: BusyResult): string | undefined { + if (result.state !== "busy") return undefined; + const chats = result.answer.counts?.chat ?? 0; + if (chats > 0) { + // The lead sentence is short because `guardBusy` prints it on the same line + // as the message, after `managers: `. The reasoning goes underneath, indented + // like every other continuation this CLI prints. + return ( + "Run it from a terminal, not from inside a Managers chat.\n" + + " One of the turns counted above may be the chat asking, and --force from there\n" + + " stops the service running that chat — which then never gets to say so.\n" + + " Otherwise: wait for the turns to finish." + ); + } + return "Try again once it has finished, or use --force."; +} + +/** The options both entry points share: name, nouns and hint are fixed for Managers. */ +function managersBusyOptions(home: string, out: Output): FindOptions & { + name: string; + out: Output; + nouns: Nouns; + hint: typeof managersBusyHint; +} { + return { home, out, name: MANAGERS_NAME, nouns: MANAGERS_NOUNS, hint: managersBusyHint }; +} + +/** + * `managers busy [--json]` — exit 0 not busy, 75 busy, 3 nothing answering + * (SPEC §1, §2). + * + * The instance is found through `/runtime.json` and nowhere else, so + * this reports on the same instance `managers serve` and `managers config show` + * resolve — `resolveDataDir` in managers.ts is the one rule all three use. + */ +export async function runBusy(home: string, json: boolean): Promise { + const out = createOutput({ tool: "managers", json }); + return runBusyCommand(managersBusyOptions(home, out)); +} + +/** + * The check before `service stop` and `service restart` (SPEC §1): refuse with + * 75 unless `--force`, which prints the same message prefixed `Overriding:` and + * carries on. Returns null to go ahead. + * + * `home` is the data dir of the INSTALLED unit, not whatever `--data-dir` this + * invocation carries — the same rule `status` and `logs` follow (#824), because + * the instance being stopped is the one the unit describes. + * + * Two deliberate non-refusals. Nothing answering goes ahead: there is nothing + * to cut short, which is also the answer for a service that is already stopped. + * And an instance running a Managers from before this release has no + * `runtime.json`, so it reads as not running and the stop proceeds exactly as + * it did before — an upgrade cannot make a stop start failing. + */ +export async function guardServiceBusy( + action: "stop" | "restart", + home: string, + opts: { force?: boolean; json?: boolean }, +): Promise { + // No `result()` call on this path, so `--json` only decides which stream the + // prose goes to — the one JSON document a `--json` stop may print belongs to + // the stop, not to the check it ran first. + const out = createOutput({ tool: "managers", json: opts.json === true }); + const refusal = await guardBusy({ + ...managersBusyOptions(home, out), + ...(opts.force === true ? { force: true } : {}), + }); + if (refusal === null) return null; + // Spell out the way past it, after the hint about where to run it from. + out.warn(`\n managers service ${action} --force ${action} it anyway\n`); + return EXIT.busy; +} diff --git a/packages/server/src/cli/busy.ts b/packages/server/src/cli/busy.ts new file mode 100644 index 0000000..446f3d0 --- /dev/null +++ b/packages/server/src/cli/busy.ts @@ -0,0 +1,197 @@ +// From edspencer/cli-contract reference/busy-client.ts @ SPEC 0.1; local changes: `.ts` import specifiers → `.js` +// and `../runtime-file.js` (the writer lives server-side, next to the boot path that writes the file); `EXIT.OK` / +// `EXIT.BUSY` / `EXIT.NOT_RUNNING` / `EXIT.FAILED` → Managers' lowercase `EXIT.ok` / `busy` / `notRunning` / +// `failure` (see `exit-codes.ts`, which wave C4 re-syncs); and `hint` widened from a string to +// `string | ((result) => string | undefined)`, with {@link runBusyCommand} printing it too — that is what carries +// Managers' keeper-chat trap (ROADMAP "Busy, per tool"), where the line to print depends on WHICH counts are +// non-zero, so a fixed string could not say it. +// +// The CLI's side of the busy check (SPEC §4): find the running instance through `/runtime.json`, ask its +// `/api/busy`, and map the answer to exit 0 (idle), 75 (busy) or 3 (nothing answers), with §4's messages. +// Used by ` busy`, and before `service stop`, `service restart` and `update`. Node built-ins only. + +import { EXIT } from "./exit-codes.js"; +import type { Output } from "./output.js"; +import { readLiveRuntimeFile } from "../runtime-file.js"; + +/** What `/api/busy` answers (SPEC §4). Counts only, until §10's first question is settled. */ +export interface BusyAnswer { + busy: boolean; + version?: string; + counts?: Record; +} + +export type BusyResult = + | { state: "not-running" } + | { state: "idle" | "busy"; url: string; answer: BusyAnswer } + /** Something answered, but not with a busy answer: an older version without `/api/busy`, or another program. */ + | { state: "unknown"; url: string; detail: string }; + +export interface FindOptions { + /** The tool's data home, holding `runtime.json`. */ + home: string; + /** Ask this URL instead of the one in `runtime.json` (`--url`, or a port the user named). */ + url?: string; + /** How long to wait for an answer. §4 asks for well under a second; this allows for a slow machine. */ + timeoutMs?: number; + fetch?: typeof globalThis.fetch; +} + +/** + * Asks the running instance whether it's busy. There's no fallback to a default port: an instance that's running + * wrote `runtime.json`, and guessing a port would ask whatever else is listening there. + */ +export async function checkBusy(o: FindOptions): Promise { + const base = o.url ?? readLiveRuntimeFile(o.home)?.url; + if (!base) return { state: "not-running" }; + const url = new URL("/api/busy", base).toString(); + const doFetch = o.fetch ?? globalThis.fetch; + let res: Response; + try { + res = await doFetch(url, { + signal: AbortSignal.timeout(o.timeoutMs ?? 3000), + headers: { accept: "application/json" }, + }); + } catch { + // Refused, reset or timed out: nothing is answering, which §4 calls not running. + return { state: "not-running" }; + } + if (!res.ok) return { state: "unknown", url, detail: `HTTP ${res.status}` }; + let body: unknown; + try { + body = await res.json(); + } catch { + return { state: "unknown", url, detail: "the answer isn't JSON" }; + } + if (!isBusyAnswer(body)) return { state: "unknown", url, detail: "the answer has no boolean `busy`" }; + return { state: body.busy ? "busy" : "idle", url, answer: body }; +} + +function isBusyAnswer(v: unknown): v is BusyAnswer { + if (typeof v !== "object" || v === null || typeof (v as BusyAnswer).busy !== "boolean") return false; + const counts = (v as BusyAnswer).counts; + if (counts === undefined) return true; + return ( + typeof counts === "object" && + counts !== null && + Object.values(counts).every((n) => Number.isInteger(n) && n >= 0) + ); +} + +/** A kind's singular and plural, where adding an "s" is wrong: `{ batch: ['batch', 'batches'] }`. */ +export type Nouns = Record; + +/** `2 chats and 1 job are running`, from the non-zero counts; '' if there are none. */ +export function describeCounts(counts: Record = {}, nouns: Nouns = {}): string { + const parts = Object.entries(counts) + .filter(([, n]) => n > 0) + .map(([kind, n]) => { + const [one, many] = nouns[kind] ?? [kind, `${kind}s`]; + return `${n} ${n === 1 ? one : many}`; + }); + if (parts.length === 0) return ""; + const list = parts.length === 1 ? parts[0] : `${parts.slice(0, -1).join(", ")} and ${parts.at(-1)}`; + const total = Object.values(counts).reduce((a, n) => a + (n > 0 ? n : 0), 0); + return `${list} ${total === 1 ? "is" : "are"} running`; +} + +/** The sentence for a person (SPEC §4): `Managers is busy: 2 chats are running.` */ +export function busyMessage(name: string, result: BusyResult, nouns: Nouns = {}): string { + switch (result.state) { + case "not-running": + return `${name} isn't running.`; + case "idle": + return `${name} isn't busy.`; + case "busy": { + const what = describeCounts(result.answer.counts, nouns); + return what ? `${name} is busy: ${what}.` : `${name} is busy.`; + } + case "unknown": + return `Something answered at ${result.url}, but not as ${name}'s busy check (${result.detail}).`; + } +} + +/** 0 idle, 75 busy, 3 nothing answers, 1 an answer that isn't a busy answer. */ +export function busyExitCode(result: BusyResult): number { + switch (result.state) { + case "idle": + return EXIT.ok; + case "busy": + return EXIT.busy; + case "not-running": + return EXIT.notRunning; + case "unknown": + return EXIT.failure; + } +} + +/** What ` busy --json` prints: valid against `schemas/busy.schema.json`. */ +export function busyJson(result: BusyResult): Record { + if (result.state === "not-running") return { running: false, busy: false }; + if (result.state === "unknown") + return { running: true, busy: false, url: base(result.url), error: result.detail }; + return { + running: true, + busy: result.answer.busy, + url: base(result.url), + ...(result.answer.version !== undefined ? { version: result.answer.version } : {}), + counts: result.answer.counts ?? {}, + }; +} + +const base = (busyUrl: string) => busyUrl.replace(/\/api\/busy$/, ""); + +/** + * LOCAL CHANGE (see the header): what to suggest when the answer is busy, as a string or as a function of the + * result. {@link resolveHint} is what both the command and the guard call, so the two can never print different + * advice about the same answer. + */ +export type BusyHint = string | ((result: BusyResult) => string | undefined); + +function resolveHint(hint: BusyHint | undefined, result: BusyResult): string | undefined { + return typeof hint === "function" ? hint(result) : hint; +} + +export interface BusyCommandOptions extends FindOptions { + /** The product's name, for messages: `Local Review`. */ + name: string; + out: Output; + nouns?: Nouns; + /** LOCAL CHANGE: the advice printed after a busy message — Managers' keeper-chat trap. */ + hint?: BusyHint; +} + +/** ` busy [--json]`. Returns the exit code; the caller sets `process.exitCode`. */ +export async function runBusyCommand(o: BusyCommandOptions): Promise { + const result = await checkBusy(o); + const message = busyMessage(o.name, result, o.nouns); + o.out.result(busyJson(result)); + if (result.state === "unknown") o.out.error(message); + else o.out.say(message); + // LOCAL CHANGE: the hint rides along with a busy answer here too, not only on the guard's refusal — someone who + // ran `managers busy` from inside a chat needs the same sentence as someone whose `service stop` was refused. + const hint = result.state === "busy" ? resolveHint(o.hint, result) : undefined; + if (hint !== undefined) o.out.say(hint); + return busyExitCode(result); +} + +export interface GuardOptions extends BusyCommandOptions { + force?: boolean; +} + +/** + * The check before `service stop`, `service restart` and `update` (SPEC §4). Returns null to go ahead, or 75 after + * printing the refusal. With `--force` it prints the same message, prefixed `Overriding:`, and goes ahead. + * Nothing answering goes ahead: there's nothing to cut short. + */ +export async function guardBusy(o: GuardOptions): Promise { + const result = await checkBusy(o); + if (result.state !== "busy") return null; + const message = busyMessage(o.name, result, o.nouns); + if (o.force) { + o.out.warn(`Overriding: ${message}`); + return null; + } + const hint = resolveHint(o.hint, result) ?? "Try again once it has finished, or use --force."; + return o.out.error(`${message} ${hint}`, EXIT.busy); +} diff --git a/packages/server/src/cli/managers.ts b/packages/server/src/cli/managers.ts index 403ac94..23b6dc3 100644 --- a/packages/server/src/cli/managers.ts +++ b/packages/server/src/cli/managers.ts @@ -32,6 +32,7 @@ import { USAGE_SYNOPSIS, SERVICE_USAGE, CONFIG_USAGE, + BUSY_USAGE, parseCommand, nodeVersionProblem, explainListenError, @@ -285,7 +286,13 @@ async function main(): Promise { if (opts.help) { console.log( - command.verb === "service" ? SERVICE_USAGE : command.verb === "config" ? CONFIG_USAGE : USAGE, + command.verb === "service" + ? SERVICE_USAGE + : command.verb === "config" + ? CONFIG_USAGE + : command.verb === "busy" + ? BUSY_USAGE + : USAGE, ); return EXIT.ok; } @@ -302,6 +309,15 @@ async function main(): Promise { if (command.verb === "config") return config(command); + // `busy` (SPEC §1, §4). Nothing is imported from the server for this: the + // answer comes off the wire, and `cli/busy.ts` is Node built-ins only — which + // is what keeps a question meant for a Watchtower hook from loading Fastify, + // herdctl and the project layer to ask it. + if (command.verb === "busy") { + const { runBusy } = await import("./busy-command.js"); + return runBusy(resolveDataDir(opts), opts.json === true); + } + addBundledBinsToPath(); // NOTHING here reads `process.cwd()`, and that is the point (#798). `--here` diff --git a/packages/server/src/cli/output.ts b/packages/server/src/cli/output.ts new file mode 100644 index 0000000..3f714d2 --- /dev/null +++ b/packages/server/src/cli/output.ts @@ -0,0 +1,125 @@ +// From edspencer/cli-contract reference/output.ts @ SPEC 0.1; local changes: `./exit-codes.ts` → `./exit-codes.js` +// (Managers compiles to ESM-with-extensions, not a TS runtime); `EXIT.FAILED`/`EXIT.USAGE` → Managers' lowercase +// `EXIT.failure`/`EXIT.usage`, because Managers' own `exit-codes.ts` predates the reference module and wave C4 +// re-syncs that file; and the reference's `UsageError` / `reportUsageError` are NOT copied, because `cli/args.ts` +// already has a `UsageError` (a `CliError` subclass carrying its own code) that the entrypoint already reports. +// +// A CLI's output, the way SPEC §2 and §3 say: errors as `: ` on stderr, colour only on a TTY (honouring +// NO_COLOR and FORCE_COLOR), and with `--json`, one JSON document on stdout and every word of prose on stderr. +// Node built-ins only. + +import { EXIT } from "./exit-codes.js"; + +export interface OutputStream { + write(chunk: string): unknown; + isTTY?: boolean; +} + +export interface OutputOptions { + /** The bin, for `: `. */ + tool: string; + /** `--json` was given: stdout carries exactly one JSON document, and prose goes to stderr. */ + json?: boolean; + stdout?: OutputStream; + stderr?: OutputStream; + env?: NodeJS.ProcessEnv; +} + +/** + * Whether to colour what goes to this stream (SPEC §3). + * + * - `NO_COLOR` set to anything but the empty string: never (https://no-color.org). + * - `FORCE_COLOR` set: `0` or `false` means never, anything else always. + * - Otherwise only on a TTY, and not when `TERM=dumb`. + */ +export function useColor(stream: OutputStream, env: NodeJS.ProcessEnv = process.env): boolean { + if (env.NO_COLOR !== undefined && env.NO_COLOR !== "") return false; + if (env.FORCE_COLOR !== undefined) return env.FORCE_COLOR !== "0" && env.FORCE_COLOR !== "false"; + if (env.TERM === "dumb") return false; + return stream.isTTY === true; +} + +export interface Paint { + bold(s: string): string; + dim(s: string): string; + red(s: string): string; + green(s: string): string; + yellow(s: string): string; +} + +const sgr = (open: number, close: number) => (s: string) => `\u001b[${open}m${s}\u001b[${close}m`; +const plain = (s: string) => s; + +/** Colour functions that do nothing when `on` is false. */ +export function paint(on: boolean): Paint { + return on + ? { + bold: sgr(1, 22), + dim: sgr(2, 22), + red: sgr(31, 39), + green: sgr(32, 39), + yellow: sgr(33, 39), + } + : { bold: plain, dim: plain, red: plain, green: plain, yellow: plain }; +} + +export interface Output { + readonly json: boolean; + /** Colour for prose on stdout (never with `--json`, which has no prose on stdout). */ + readonly color: Paint; + /** Colour for stderr. */ + readonly errColor: Paint; + /** Prose for a person: stdout, or stderr under `--json`. */ + say(line: string): void; + /** A warning on stderr, always. */ + warn(line: string): void; + /** `: ` on stderr. Returns `code` (default 1), for `return out.error(…)`. Sets nothing itself. */ + error(message: string, code?: number): number; + /** The one JSON document `--json` promises. Prints nothing without `--json`. */ + result(doc: unknown): void; +} + +/** One place for a command's output. Make it once per command, after the flags are parsed. */ +export function createOutput(o: OutputOptions): Output { + const stdout = o.stdout ?? process.stdout; + const stderr = o.stderr ?? process.stderr; + const env = o.env ?? process.env; + const json = o.json === true; + let printed = false; + const line = (s: string) => (s.endsWith("\n") ? s : `${s}\n`); + return { + json, + color: paint(!json && useColor(stdout, env)), + errColor: paint(useColor(stderr, env)), + say(s) { + (json ? stderr : stdout).write(line(s)); + }, + warn(s) { + stderr.write(line(s)); + }, + error(message, code = EXIT.failure) { + stderr.write(line(`${o.tool}: ${message}`)); + return code; + }, + result(doc) { + if (!json) return; + if (printed) throw new Error("--json prints one document; result() was called twice"); + printed = true; + stdout.write(`${JSON.stringify(doc, null, 2)}\n`); + }, + }; +} + +/** + * A closed pipe (` … | head -1`) exits quietly (SPEC §2): without this, Node throws EPIPE from a later write + * and prints a stack trace. Call once, at startup. + */ +export function exitQuietlyOnEpipe( + stream: NodeJS.WriteStream = process.stdout, + exit: (code: number) => void = (c) => process.exit(c), +): void { + stream.on("error", (err: NodeJS.ErrnoException) => { + if (err.code === "EPIPE") exit(typeof process.exitCode === "number" ? process.exitCode : 0); + else throw err; + }); +} diff --git a/packages/server/src/cli/service/index.ts b/packages/server/src/cli/service/index.ts index bbe15c3..f8a817c 100644 --- a/packages/server/src/cli/service/index.ts +++ b/packages/server/src/cli/service/index.ts @@ -86,6 +86,19 @@ export interface ServiceContext { * is — `status` has to be assertable without a server to ask. */ probeHealth?: (url: string) => Promise; + /** + * The busy check `stop` and `restart` run first (SPEC §1, §4): null to go + * ahead, or 75 having printed the refusal. + * + * Injected for the third time for the third version of the same reason: the + * real one reads `/runtime.json` and fetches a URL, so a test of "does + * stop refuse" would otherwise need a server to be busy at it. + */ + guardBusy?: ( + action: "stop" | "restart", + home: string, + opts: { force?: boolean; json?: boolean }, + ) => Promise; } /** @@ -581,6 +594,7 @@ function notInstalled(backend: ServiceBackend): string { */ async function startOrRestart( action: "start" | "restart", + opts: CliOptions, ctx: ServiceContext, backend: ServiceBackend, ): Promise { @@ -596,6 +610,14 @@ async function startOrRestart( return EXIT.ok; } + // A restart stops the instance first, so it asks the same question `stop` + // asks (SPEC §1): refuse with 75 unless `--force`. `start` does not — there + // is nothing running for it to cut short, and asking would be theatre. + if (action === "restart") { + const refused = await busyCheck(action, opts, ctx, before); + if (refused !== null) return refused; + } + console.log(`\n ${action === "start" ? "Starting" : "Restarting"} Managers…`); if (action === "start") backend.start(); else backend.restart(); @@ -624,8 +646,36 @@ async function startOrRestart( return readiness === "ready" ? EXIT.ok : EXIT.failure; } +/** + * The busy check, asked of the instance the UNIT describes (SPEC §1, §4). + * + * `home` is `dataDirOf(state, ctx)` — the unit's own data dir, not whatever + * `--data-dir` this invocation carries. Same rule as `status` and `logs` + * (#824): the instance about to be stopped is the one the unit points at, and + * reading `runtime.json` from somewhere else would ask a different Managers + * whether this one is busy. + * + * Returns null to go ahead, or the code to return (75). + */ +async function busyCheck( + action: "stop" | "restart", + opts: CliOptions, + ctx: ServiceContext, + state: ServiceState, +): Promise { + const guard = ctx.guardBusy ?? (await import("../busy-command.js")).guardServiceBusy; + return guard(action, dataDirOf(state, ctx), { + ...(opts.force === true ? { force: true } : {}), + ...(opts.json === true ? { json: true } : {}), + }); +} + /** `stop` (#873): put it down, leave it installed. */ -function stop(backend: ServiceBackend): number { +async function stop( + opts: CliOptions, + ctx: ServiceContext, + backend: ServiceBackend, +): Promise { const state = backend.status(); if (!state.registered) { console.log(notInstalled(backend)); @@ -636,6 +686,12 @@ function stop(backend: ServiceBackend): number { return EXIT.ok; } + // Refuse with 75 if it is in the middle of something, unless --force (SPEC + // §4). Deliberately AFTER the two cases above: an unit that is not there, or + // a service already stopped, has no instance to ask and no work to cut short. + const refused = await busyCheck("stop", opts, ctx, state); + if (refused !== null) return refused; + backend.stop(); console.log( [ @@ -711,11 +767,13 @@ export async function runService( opts: CliOptions, ctx: ServiceContext, ): Promise { + // `--force` is rejected on every action but `stop` and `restart` — in + // `parseCommand`, once, for every verb. See the note there. const backend = backendFor(ctx); if (action === "install") return install(opts, ctx, backend); if (action === "uninstall") return uninstall(ctx, backend); - if (action === "start" || action === "restart") return startOrRestart(action, ctx, backend); - if (action === "stop") return stop(backend); + if (action === "start" || action === "restart") return startOrRestart(action, opts, ctx, backend); + if (action === "stop") return stop(opts, ctx, backend); if (action === "logs") return logs(opts, ctx, backend); return status(opts, ctx, backend); } diff --git a/packages/server/src/herdctl.ts b/packages/server/src/herdctl.ts index ba06ffb..2166103 100644 --- a/packages/server/src/herdctl.ts +++ b/packages/server/src/herdctl.ts @@ -758,6 +758,32 @@ export class HerdctlService { } } + /** + * How many scheduler jobs are running right now, read from memory. + * + * This exists for `GET /api/busy` (CLI contract SPEC §4), which must read + * in-memory state and never walk the disk. The number the contract's ROADMAP + * names is `fleetStatus().counts.runningJobs` — but {@link fleetStatus} is + * built from `getAgentInfo()`, which re-reads the fleet state FILE on every + * call, and that re-parse has been this server's single largest CPU cost. + * + * The two are the same number. `counts.runningJobs` is + * `Σ scheduler.getRunningJobCount(agent.qualifiedName)` over the configured + * agents, and `getTotalRunningJobCount()` is the size of the same scheduler + * map keyed `/` — one entry per running agent+schedule pair + * either way. So this reads the authority directly instead of a derived + * snapshot of it. + * + * Null-safe like {@link isSdkSessionLive}: no fleet or no scheduler means 0. + */ + runningJobCount(): number { + try { + return this.fleet?.getScheduler()?.getTotalRunningJobCount() ?? 0; + } catch { + return 0; + } + } + async fleetStatus(): Promise { return this.manager.getFleetStatus(); } diff --git a/packages/server/src/managers/state.ts b/packages/server/src/managers/state.ts index ca845d9..3349193 100644 --- a/packages/server/src/managers/state.ts +++ b/packages/server/src/managers/state.ts @@ -32,6 +32,16 @@ export class ManagersState { * unlocks `memory_op`) and the per-workspace early-fire claim. */ readonly consolidations = new ConsolidationTracker(); + /** + * The trigger runs in flight in THIS process, each with the way to finish it + * early: added once a fire's `running` record is written, removed when its + * completion hook has finished that record (or the turn never started). In + * memory only, so `GET /api/busy` can count them without reading `runs/` + * (CLI contract SPEC §4), a run left `running` on disk by a previous process + * never counts, and the SIGTERM path (start.ts) can record the ones a stop + * cuts short — `interrupt(reason)` finishes the record as failed, once. + */ + readonly liveRuns = new Map Promise>(); /** When each chat's previous human turn ran, for its next turn's delta (`chat-delta.ts`). */ readonly chatTurns = new ChatTurnStore(); diff --git a/packages/server/src/managers/trigger-runs.ts b/packages/server/src/managers/trigger-runs.ts index fecee3b..0e6da26 100644 --- a/packages/server/src/managers/trigger-runs.ts +++ b/packages/server/src/managers/trigger-runs.ts @@ -126,6 +126,13 @@ export async function beginTriggerRun(p: BeginTriggerRunParams): Promise ({ ok: true, version: MANAGERS_VERSION }), ); + // The contract's second server endpoint (SPEC §4, §8): is this instance busy? + // + // **Counts only, and no auth, and those two facts are the same fact.** There + // is nothing in the answer to protect — no chat title, no session id, no + // project name, just four integers and the version — which is why the route + // can sit beside `/api/health` in `HEALTH_PATHS` without widening what an + // unauthenticated caller can learn beyond "something is happening here". + // `reasons` (a line per piece of work, for a person) waits on SPEC §10's first + // question: how a CLI proves it is local, given that a reverse proxy on the + // same host also connects from loopback. Until that is designed, this answer + // is the whole answer for every caller — so nothing here may grow a string + // field without revisiting the auth decision above it. + // + // The callers that make the cheapness requirement real: `managers service + // stop`, `managers update`, and Watchtower's pre-update hook, which runs + // `managers busy` before every image pull. `collectBusy` is synchronous and + // reads four in-memory structures — see busy.ts for why `job` does NOT come + // from `fleetStatus()`. + app.get( + "/api/busy", + { + schema: { + tags: ["System"], + summary: "Is this instance busy?", + description: + "The CLI contract's busy check (SPEC §4). Returns `{ busy, version, counts }`, where `counts` has `chat` (chat turns running, including fired triggers and cron fires), `background` (live sessions holding background work after their turn ended) `job` (herdctl scheduler jobs) and `run` (Managers trigger runs in flight, from the run record being written until it is finished and committed). `busy` is true only for work a restart would lose or cut short. Counts only — never labels, ids or titles — so it needs no authentication. Reads in-memory state, never the disk.", + response: { + 200: { + description: "Whether work a restart would cut short is under way.", + type: "object", + additionalProperties: true, + }, + }, + }, + }, + async () => + collectBusy( + { + runningChats: () => hub?.runningSessions().length ?? 0, + // Tasks in a session count only while its SDK session is still live — + // see `BackgroundRegistry.liveTaskCount`, which explains why a stale + // batch-mode entry must not wedge this at busy. + backgroundTasks: () => background?.liveTaskCount((id) => herdctl.isSdkSessionLive(id)) ?? 0, + runningJobs: () => herdctl.runningJobCount(), + runningRuns: () => ctx.managers?.liveRuns.size ?? 0, + }, + MANAGERS_VERSION, + ), + ); + // --- instance-wide settings (issue #385) -------------------------------- // A top-level admin screen over the frozen instance config. GET reports every // surfaced field (value / default / editable / sensitive / env-shadow); PUT diff --git a/packages/server/src/runtime-file.ts b/packages/server/src/runtime-file.ts new file mode 100644 index 0000000..1166d0e --- /dev/null +++ b/packages/server/src/runtime-file.ts @@ -0,0 +1,108 @@ +// From edspencer/cli-contract reference/runtime-file.ts @ SPEC 0.1; local changes: `node:fs` / `node:path` imports +// unchanged, nothing else — the module is Node built-ins only and Managers' `` is its data dir, so it needed +// no adaptation. Kept verbatim on purpose: wave C4 re-syncs it, and a divergence here would be a divergence in the +// one file every tool's `busy`, `service status` and `update` agree on. +// +// `/runtime.json`: what a running instance tells its own CLI (SPEC §4, ROADMAP). `serve` writes it once it's +// listening and removes it on a clean exit; `busy`, `service status` and `update` read it to find the instance. +// +// It holds `{ pid, port, url, version, startedAt }`. A token for /api/busy will join it once SPEC §10's first question +// is settled; the file is written 0600 already, so adding one later changes nothing about who can read it. +// Node built-ins only. + +import fs from "node:fs"; +import path from "node:path"; + +export interface RuntimeInfo { + pid: number; + port: number; + /** Where the CLI reaches it, e.g. `http://127.0.0.1:3729`. */ + url: string; + version: string; + /** ISO 8601. */ + startedAt: string; +} + +export const RUNTIME_FILE = "runtime.json"; + +export function runtimeFilePath(home: string): string { + return path.join(home, RUNTIME_FILE); +} + +/** Writes the file atomically (a temporary file, then a rename), readable only by this user. */ +export function writeRuntimeFile( + home: string, + info: Omit & { startedAt?: string }, +): RuntimeInfo { + const full: RuntimeInfo = { + pid: info.pid, + port: info.port, + url: info.url, + version: info.version, + startedAt: info.startedAt ?? new Date().toISOString(), + }; + fs.mkdirSync(home, { recursive: true }); + const file = runtimeFilePath(home); + const tmp = `${file}.${process.pid}.tmp`; + fs.writeFileSync(tmp, `${JSON.stringify(full, null, 2)}\n`, { mode: 0o600 }); + try { + fs.renameSync(tmp, file); + } catch (err) { + fs.rmSync(tmp, { force: true }); + throw err; + } + return full; +} + +/** The file's contents, or null if it's missing or isn't a runtime file (a half-written or foreign file). */ +export function readRuntimeFile(home: string): RuntimeInfo | null { + let raw: string; + try { + raw = fs.readFileSync(runtimeFilePath(home), "utf8"); + } catch { + return null; + } + try { + const v = JSON.parse(raw) as Partial; + if ( + Number.isInteger(v.pid) && + Number.isInteger(v.port) && + typeof v.url === "string" && + typeof v.version === "string" && + typeof v.startedAt === "string" + ) { + return v as RuntimeInfo; + } + } catch { + // fall through + } + return null; +} + +/** + * Removes the file, but only if it's still ours: a second instance may have started since and written its own. + * Call it on a clean exit. Returns whether it removed anything. + */ +export function removeRuntimeFile(home: string, pid: number = process.pid): boolean { + const info = readRuntimeFile(home); + if (!info || info.pid !== pid) return false; + fs.rmSync(runtimeFilePath(home), { force: true }); + return true; +} + +/** Whether a process with this pid exists. EPERM means it exists but isn't ours, which still counts. */ +export function isPidAlive(pid: number): boolean { + if (!Number.isInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (err) { + return (err as NodeJS.ErrnoException).code === "EPERM"; + } +} + +/** The runtime file, if the process it names is still alive. A crashed instance leaves a stale file behind. */ +export function readLiveRuntimeFile(home: string): RuntimeInfo | null { + const info = readRuntimeFile(home); + return info && isPidAlive(info.pid) ? info : null; +} diff --git a/packages/server/src/shutdown-drain.ts b/packages/server/src/shutdown-drain.ts new file mode 100644 index 0000000..40c3226 --- /dev/null +++ b/packages/server/src/shutdown-drain.ts @@ -0,0 +1,136 @@ +/** + * The first half of a clean stop (CLI contract SPEC §4, §8): **finish or record + * the work under way**, inside the unit's stop timeout. + * + * Paddock's SIGTERM only records — it logs how many chat turns it is cutting + * short and closes at once. Managers does a little better, in two steps: + * + * 1. **Let it finish, briefly.** If a chat turn or a trigger run is in flight, + * wait up to {@link SHUTDOWN_DRAIN_MS} for it to end on its own. Many do: a + * scheduled wake that is writing its last episode, a report being composed, + * a run whose turn has ended and whose record is being committed. An idle + * instance does not wait at all, so a stop with nothing running is as quick + * as it ever was. + * 2. **Record what is left.** Every trigger run still in flight is finished as + * `failed`, `error: interrupted by shutdown` — through the run's own + * completion hook, so `expect` is evaluated, a consolidation still gets its + * `#reflection` episode, and the record is committed — rather than sitting + * `running` until the next boot calls it "interrupted by restart". A chat + * turn has no record of its own to write (its transcript belongs to `claude`, + * which may still append to it), so it is recorded by the count in the log. + * + * The drain is bounded by a count of polls rather than a wall-clock deadline, so + * an injected no-op sleep in a test cannot turn it into a spin. + */ + +/** + * How long a stop waits for in-flight work to end by itself. With the shutdown + * deadline (start.ts `SHUTDOWN_DEADLINE_MS`, 15 s) this leaves time to record + * what is left and close inside launchd's 20 s budget, the tightest of the unit + * stop timeouts (systemd's user default is 90 s). Docker's default `docker stop` + * timeout is 10 s, which is why the compose example and the home-lab role give + * the container a longer `stop_grace_period`. + */ +export const SHUTDOWN_DRAIN_MS = 8_000; + +/** How long recording the cut-short runs may take before the stop goes ahead anyway. */ +export const SHUTDOWN_RECORD_MS = 2_000; + +const POLL_MS = 250; + +export interface InFlight { + /** Chat turns running (`hub.runningSessions().length`). */ + turns: number; + /** Trigger runs in flight (`managers.liveRuns.size`). */ + runs: number; +} + +export interface DrainOptions { + drainMs?: number; + pollMs?: number; + sleep?: (ms: number) => Promise; +} + +export interface DrainResult { + /** What was in flight when the stop began. */ + before: InFlight; + /** What was still in flight when the wait ended. */ + after: InFlight; + /** How many polls the wait took (0 when nothing was in flight). */ + polls: number; +} + +const idle = (f: InFlight): boolean => f.turns === 0 && f.runs === 0; + +/** Wait, boundedly, for the in-flight work to end. Never throws; a reading that throws counts as zero. */ +export async function drainInFlight(read: () => InFlight, opts: DrainOptions = {}): Promise { + const pollMs = opts.pollMs ?? POLL_MS; + const maxPolls = Math.max(0, Math.ceil((opts.drainMs ?? SHUTDOWN_DRAIN_MS) / pollMs)); + const sleep = opts.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); + const safeRead = (): InFlight => { + try { + const f = read(); + return { turns: Math.max(0, f.turns | 0), runs: Math.max(0, f.runs | 0) }; + } catch { + return { turns: 0, runs: 0 }; + } + }; + const before = safeRead(); + let after = before; + let polls = 0; + while (!idle(after) && polls < maxPolls) { + await sleep(pollMs); + polls++; + after = safeRead(); + } + return { before, after, polls }; +} + +/** + * Finish every run still in flight as failed with `reason`, each through its own + * interrupt (which is once-only, so a turn that ends at the same moment cannot + * finish it twice). Bounded by `timeoutMs`; returns how many were recorded in time. + */ +export async function recordInterruptedRuns( + liveRuns: ReadonlyMap Promise> | undefined, + reason: string, + timeoutMs: number = SHUTDOWN_RECORD_MS, +): Promise { + if (!liveRuns || liveRuns.size === 0) return 0; + let recorded = 0; + const all = Promise.all( + [...liveRuns.values()].map((interrupt) => + interrupt(reason).then( + () => { + recorded++; + }, + () => undefined, + ), + ), + ); + let timer: NodeJS.Timeout | undefined; + await Promise.race([all, new Promise((r) => (timer = setTimeout(r, timeoutMs)))]); + if (timer) clearTimeout(timer); + return recorded; +} + +/** The log line for a stop, from what was in flight before and after the wait. */ +export function describeShutdown(signal: string, d: DrainResult, recorded: number): string { + if (idle(d.before)) return `${signal}: shutting down, nothing in flight.`; + const parts = (f: InFlight): string => + [ + f.turns > 0 ? `${f.turns} chat turn${f.turns === 1 ? "" : "s"}` : "", + f.runs > 0 ? `${f.runs} trigger run${f.runs === 1 ? "" : "s"}` : "", + ] + .filter(Boolean) + .join(" and "); + if (idle(d.after)) return `${signal}: shutting down; waited for ${parts(d.before)} to finish, and they did.`; + const tail = + d.after.runs > 0 + ? ` ${recorded} of ${d.after.runs} run record${d.after.runs === 1 ? "" : "s"} marked failed (interrupted by shutdown).` + : ""; + return ( + `${signal}: shutting down with ${parts(d.after)} still in flight — cut short.${tail} ` + + "Run `managers busy` before a stop or an update to see this before it happens." + ); +} diff --git a/packages/server/src/start.ts b/packages/server/src/start.ts index c7cc359..24d86c3 100644 --- a/packages/server/src/start.ts +++ b/packages/server/src/start.ts @@ -1,7 +1,8 @@ /** - * Process lifecycle for a Managers server: bind the port and wire signal-driven - * shutdown. Split out of `index.ts` so the `managers` CLI (src/cli/managers.ts) - * and the plain `node dist/index.js` entrypoint share one implementation. + * Process lifecycle for a Managers server: bind the port, publish + * `runtime.json`, and wire signal-driven shutdown. Split out of `index.ts` so + * the `managers` CLI (src/cli/managers.ts) and the plain `node dist/index.js` + * entrypoint share one implementation. * * Config is resolved INSIDE `buildApp()`, not at module load, so a caller may * still mutate `process.env` right up until this function is invoked. The CLI @@ -12,6 +13,53 @@ import { describeScrub, scrubInheritedEnv } from "./env-scrub.js"; import { applyRuntimeEnv } from "./managers/claude-overlay.js"; import { describeSequester, sequesterMcpSecrets } from "./managers/mcp-secret-env.js"; import { startServiceLogRotation } from "./service-log.js"; +import { removeRuntimeFile, writeRuntimeFile } from "./runtime-file.js"; +import { MANAGERS_VERSION } from "./version.js"; +import { describeShutdown, drainInFlight, recordInterruptedRuns } from "./shutdown-drain.js"; +import { INTERRUPTED_BY_SHUTDOWN } from "./managers/trigger-runs.js"; + +/** + * How long a clean shutdown gets before we exit anyway. + * + * The CLI contract requires the SIGTERM path to exit "within the unit's stop + * timeout" (SPEC §4, §8), and the tightest of those is **launchd's**: with no + * `ExitTimeOut` in the plist, a LaunchAgent gets 20 seconds before SIGKILL + * (systemd's user default is 90). So the budget is 20 seconds, and this sits + * inside it with room for the exit itself. + * + * The deadline matters because the two awaits below are not bounded by anything + * else. `app.close()` waits for in-flight requests and open WebSockets, and a + * browser holding `/ws` is the normal state of a running Managers; `fleet.stop()` + * talks to subprocesses. Exceeding the budget is strictly worse than exiting at + * it: SIGKILL skips the rest of this function, so the runtime file would be left + * behind for the next instance's `busy` to find and the log would have no record + * of what was cut short. + */ +export const SHUTDOWN_DEADLINE_MS = 15_000; + +/** + * The URL to record in `runtime.json` — the one a CLI on this machine can + * actually fetch, which is not always the one we bound. + * + * A wildcard bind is the normal case for a container or a dev box (`HOST=0.0.0.0` + * is what this repo's own `pm` wrapper injects), and `http://0.0.0.0:` is + * not an address to connect TO. Node happens to resolve it to loopback on Linux + * and fails on some other stacks, so the file records loopback explicitly rather + * than relying on that. A real interface address is left alone: someone who + * bound `192.168.1.10` meant it, and that address reaches the server from here + * as well as from anywhere else. + * + * IPv6 gets brackets, because `new URL('http://::1:7234')` is not a URL. + */ +export function runtimeUrl(host: string, port: number): string { + const loopback: Record = { + "0.0.0.0": "127.0.0.1", + "::": "::1", + "": "127.0.0.1", + }; + const h = loopback[host] ?? host; + return `http://${h.includes(":") ? `[${h}]` : h}:${String(port)}`; +} export interface StartOptions { /** @@ -36,7 +84,7 @@ export async function start(opts: StartOptions = {}): Promise { // keeper/trigger/sweeper child inherits it (belt-and-braces with the // `autoMemoryEnabled: false` in the generated settings.json). applyRuntimeEnv(); - const { app, cfg, close } = await buildApp({ loopbackNoAuth: opts.loopbackNoAuth === true }); + const { app, cfg, close, hub, managers } = await buildApp({ loopbackNoAuth: opts.loopbackNoAuth === true }); // warn, not info, when anything was removed: it means this process was started // from a shell configured for Paddock, which is worth seeing even at LOG_LEVEL=warn. if (scrubbed > 0) app.log.warn({ scrubbedEnvVars: scrubbed }, describeScrub(scrubbed)); @@ -53,14 +101,87 @@ export async function start(opts: StartOptions = {}): Promise { }, }); - const shutdown = async () => { + /** + * Exit on SIGTERM/SIGINT having finished or recorded the work under way + * (SPEC §4, §8). + * + * Three things are deliberate here. + * + * **It waits briefly, then records.** A chat turn is a `claude` subprocess + * that can run for minutes; no 20-second stop timeout can wait for one, and + * `herdctl.stop()` ends the fleet with `waitForJobs: false` precisely because + * waiting is not an option. So in-flight turns and trigger runs get a short, + * bounded window to end by themselves (shutdown-drain.ts), and whatever is + * left is recorded: each trigger run's record is finished as failed, + * "interrupted by shutdown", and the count goes in this `warn` line in the + * service log, the one artefact that survives the restart and which + * `managers service logs` reads. What is NOT done, and should not be, is + * writing a note into the cut-short chat's transcript: `claude` owns that file + * and may append to it after we are gone, so a server-side write cannot be + * ordered against the subprocess's own and would be the resurrection race + * `turn-interlock.ts` exists to avoid. + * + * The reason a count is enough: `/api/busy` reports the same numbers BEFORE + * the stop, and `managers service stop` refuses on them. The log line is for + * the race §4 admits — work that started between the check and the signal. + * + * **It is idempotent.** A supervisor that sends SIGTERM twice, or a Ctrl-C + * after a SIGTERM, used to re-enter `close()` concurrently: a second + * `app.close()` on a closing Fastify rejects, and a second `fleet.stop()` + * races the first. The guard makes the extra signals no-ops. + * + * **It exits 0.** The unit's `Restart=always` / `KeepAlive` means the + * supervisor brings Managers back whatever the code; 0 is what makes a logout + * or a `service stop` read as a clean stop rather than a crash in the logs. + */ + let shuttingDown = false; + const shutdown = async (signal: string) => { + if (shuttingDown) return; + shuttingDown = true; + const startedAt = Date.now(); stopRotation(); - await close(); + // Published once we were listening; removed here so a `busy` or a + // `service status` run after this exit does not find a live-looking + // instance. Before the slow awaits, because this is the part that must land + // even if they overrun the deadline below. + removeRuntimeFile(cfg.dataDir); + // Finish, then record (shutdown-drain.ts). An idle instance skips the wait. + const drained = await drainInFlight(() => ({ + turns: hub.runningSessions().length, + runs: managers.liveRuns.size, + })); + const recorded = await recordInterruptedRuns(managers.liveRuns, INTERRUPTED_BY_SHUTDOWN); + app.log.warn(describeShutdown(signal, drained, recorded)); + // Bounded: see SHUTDOWN_DEADLINE_MS, which the wait above counts against. + // `exit(0)` either way, so a hung `app.close()` cannot turn a stop into a + // SIGKILL. + const left = Math.max(1_000, SHUTDOWN_DEADLINE_MS - (Date.now() - startedAt)); + await Promise.race([close(), new Promise((r) => setTimeout(r, left))]).catch(() => undefined); process.exit(0); }; - process.on("SIGINT", shutdown); - process.on("SIGTERM", shutdown); + process.on("SIGINT", () => void shutdown("SIGINT")); + process.on("SIGTERM", () => void shutdown("SIGTERM")); await app.listen({ port: cfg.port, host: cfg.host }); + + // `/runtime.json`, 0600, written only once we are actually listening + // (SPEC §4) — it is how `managers busy`, and later `update`, find THIS instance + // without guessing a port. A failure to write it is logged and not fatal: the + // server is up and serving, and the one thing that stops working is a CLI + // finding it, which is better than refusing to run. + try { + writeRuntimeFile(cfg.dataDir, { + pid: process.pid, + port: cfg.port, + url: runtimeUrl(cfg.host, cfg.port), + version: MANAGERS_VERSION, + }); + } catch (err) { + app.log.warn( + `could not write ${cfg.dataDir}/runtime.json (${(err as Error).message}) — ` + + "`managers busy` will report this instance as not running", + ); + } + app.log.info(`managers-server listening on http://${cfg.host}:${cfg.port}`); } diff --git a/packages/server/src/ws-context.ts b/packages/server/src/ws-context.ts index 37cd604..1be7295 100644 --- a/packages/server/src/ws-context.ts +++ b/packages/server/src/ws-context.ts @@ -14,6 +14,7 @@ import type { SweepService } from "./sweep.js"; import type { PaddockConfig } from "./config.js"; import type { DriveMode } from "./models.js"; import type { SessionHub } from "./session-hub.js"; +import type { BackgroundRegistry } from "./background-live.js"; import type { AttachmentStore } from "./attachments.js"; import type { QueuedMessageStore } from "./queued-message.js"; import type { ArchiveStore } from "./archive.js"; @@ -215,6 +216,14 @@ export type FireTrigger = ( export interface ChatHandlerContext { deps: ChatHandlerDeps; hub: SessionHub; + /** + * The live background-work registry (#604). Here for the same reason `hub` is: + * it is per-server state owned by the WS layer that a NON-socket caller has to + * be able to read. `GET /api/busy` is that caller — a turn can end while its + * background tasks keep running, so the hub alone would report a session with + * a live sub-agent as idle (CLI contract SPEC §4). + */ + background: BackgroundRegistry; startAgentTurn: StartAgentTurn; composePreloadedPrompt: ComposePreloadedPrompt; fireTrigger: FireTrigger; diff --git a/packages/server/src/ws-triggers.ts b/packages/server/src/ws-triggers.ts index 28d772e..7e7a374 100644 --- a/packages/server/src/ws-triggers.ts +++ b/packages/server/src/ws-triggers.ts @@ -312,6 +312,31 @@ async function fireTriggerForProject( await finish(r); }; } + // CLI contract (SPEC §4): the run is in flight from here until its record is + // finished — counted by `/api/busy`, and recorded by a SIGTERM that cuts it + // short. Wrapped LAST, so an interrupt runs the consolidation's #reflection + // too, and made once-only, because a stop's interrupt and the dying turn's own + // completion can both arrive. + let runSessionId: string | null = null; + if (run && deps.managers) { + const state = deps.managers; + const runId = run.runId; + const complete = run.onComplete; + let finished = false; + run.onComplete = async (r) => { + if (finished) return; + finished = true; + try { + await complete(r); + } finally { + state.liveRuns.delete(runId); + } + }; + const onComplete = run.onComplete; + state.liveRuns.set(runId, (reason) => + onComplete({ success: false, sessionId: runSessionId, error: reason, mcpCalls: {} }), + ); + } if (run && opts.onRun) { try { opts.onRun(run.runId); @@ -387,6 +412,7 @@ async function fireTriggerForProject( project: slug, }, }); + runSessionId = sessionId; // First fire of an accreting trigger: remember the chat it created so the next // fire resumes THIS transcript (a resume already had an id). if (trigger.run.session === "resume" && !resume && deps.triggerSessions) { @@ -394,6 +420,9 @@ async function fireTriggerForProject( } return sessionId; } catch { + // The turn never started, so its completion hook will not run: the run is + // no longer in flight as far as `/api/busy` is concerned. + if (run) deps.managers?.liveRuns.delete(run.runId); return null; } } diff --git a/packages/server/src/ws-turn.ts b/packages/server/src/ws-turn.ts index f550f41..f860435 100644 --- a/packages/server/src/ws-turn.ts +++ b/packages/server/src/ws-turn.ts @@ -164,6 +164,7 @@ const triggers = makeTriggerCluster(deps, startAgentTurn); const selfMcpCtx: ChatHandlerContext = { deps, hub, + background, startAgentTurn, composePreloadedPrompt: triggers.composePreloadedPrompt, fireTrigger: triggers.fireTrigger, diff --git a/packages/server/test/integration/auth-health-paths.test.ts b/packages/server/test/integration/auth-health-paths.test.ts index 04f9808..1622340 100644 --- a/packages/server/test/integration/auth-health-paths.test.ts +++ b/packages/server/test/integration/auth-health-paths.test.ts @@ -14,6 +14,11 @@ * "every member of HEALTH_PATHS resolves to a registered route returning JSON, * with a real web dist mounted and auth ON". A dist is required: without one all * five aliases correctly 404 and the bug is invisible. + * + * The set now holds two members: `/api/health` and, from wave C1 of the + * CLI-contract rollout, `/api/busy`. The second one's exemption rests on its + * body being counts only (SPEC §4, §10), which is asserted here rather than + * only where the route is tested. */ import { describe, it, expect, afterEach } from "vitest"; import path from "node:path"; @@ -126,7 +131,33 @@ describe("integration: auth health-path exemptions match the router (issue #569) /application\/json/, ); expect(res.body, `${url} must not be the SPA shell`).not.toContain("paddock SPA"); - expect(res.json().ok).toBe(true); + // A JSON OBJECT rather than a particular key: the set now holds two + // routes with different bodies (`/api/health` answers `ok`, `/api/busy` + // answers `busy`), and asserting one route's field here is what would + // make the next addition look like a failure of the exemption rather than + // of the assertion. Each body's own shape is pinned where it belongs — + // cli-contract.test.ts §8 for health, busy-endpoint.test.ts for busy. + const body: unknown = res.json(); + expect(typeof body, `${url} must answer a JSON object`).toBe("object"); + expect(body).not.toBeNull(); + expect(Object.keys(body as object).length).toBeGreaterThan(0); + } + }); + + // `/api/busy` joined the set in wave C1 of the CLI-contract rollout, and the + // reason it is safe to exempt is a property of its BODY rather than of the + // route: counts only, never a label, an id or a title (SPEC §4, §10). Pinned + // here, where the exemption is, so growing the body a string field fails the + // test that justifies the exemption rather than only a shape test elsewhere. + it("the exempted busy answer carries no strings but the version", async () => { + h = await boot(); + const res = await h.built.app.inject({ method: "GET", url: "/api/busy" }); + expect(res.statusCode).toBe(200); + const body = res.json() as Record; + expect(body.busy).toBe(false); + expect(Object.keys(body).sort()).toEqual(["busy", "counts", "version"]); + for (const value of Object.values(body.counts as Record)) { + expect(typeof value).toBe("number"); } }); diff --git a/packages/server/test/integration/busy-endpoint.test.ts b/packages/server/test/integration/busy-endpoint.test.ts new file mode 100644 index 0000000..61872ac --- /dev/null +++ b/packages/server/test/integration/busy-endpoint.test.ts @@ -0,0 +1,278 @@ +/** + * `GET /api/busy` against a real app and a real turn (CLI contract SPEC §4, §8). + * + * ## What only this layer can prove + * + * `busy.ts` is pure and unit-tested with injected sources, so the arithmetic + * and the predicate are covered on their own. What a unit test cannot see is + * the WIRING: whether the route reads the hub that the turn engine actually + * registers turns on, or a different one. Three of these — a hub that is + * `undefined` on a server built without the WS layer, a registry hoisted into + * `ChatHandlerContext` for the first time in wave C1, and a herdctl scheduler + * reached through two optional accessors — are exactly the mistakes that type- + * check and then report an idle instance forever. + * + * So the load-bearing test here is: start a turn that will not finish, and ask + * the endpoint. That is the question `managers service stop` asks, through the + * same route, with nothing stubbed. + * + * ## The one thing it cannot prove + * + * `counts.background` is zero in this harness and should be. These tests run on + * `driveMode: batch` (the fake `claude` is a one-shot CLI), so there is no + * session-lifecycle reaper and `herdctl.isSdkSessionLive` is false for every + * session — which is the filter `/api/busy` applies. The unhappy path that + * MATTERS about that is tested here instead, and it is the one that bites: a + * batch turn's background tasks stay in the registry after the turn ends, and + * must not report the instance busy for the rest of its life. + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { startTestApp, type TestApp } from "../helpers/app.js"; +import { listen, connectWs } from "../helpers/ws.js"; +import { MANAGERS_VERSION } from "../../src/version.js"; + +interface BusyBody { + busy: boolean; + version: string; + counts: { chat: number; background: number; job: number; run: number }; +} + +describe("integration: GET /api/busy (CLI contract SPEC §4)", () => { + let t: TestApp; + let port: number; + let n = 0; + + beforeAll(async () => { + t = await startTestApp({ + script: { + "[[HANG]] hold": "Working on it.", + "[[BGTASK]] go": "Kicked off some background work.", + hello: "Hi.", + }, + // The post-turn sweeper fires its own `claude`, and a sweep in flight is + // deliberately NOT busy (see busy.ts) — but a sweep racing a count is + // still noise in an assertion, so it is parked for these tests. + sweepIntervalMs: 600_000, + env: { MANAGERS_FAKE_BGTASK_MS: "1500" }, + }); + ({ port } = await listen(t.app)); + }); + afterAll(async () => { + await t.teardown(); + }); + + async function busy(): Promise { + const res = await t.app.inject({ method: "GET", url: "/api/busy" }); + expect(res.statusCode).toBe(200); + return res.json() as BusyBody; + } + + async function freshProject(): Promise { + const name = `Busy ${++n}`; + await t.app.inject({ method: "POST", url: "/api/projects", payload: { name } }); + return name.toLowerCase().replace(/\s+/g, "-"); + } + + it("answers the shape the contract fixes, with this instance's version", async () => { + const body = await busy(); + expect(body).toEqual({ + busy: false, + version: MANAGERS_VERSION, + counts: { chat: 0, background: 0, job: 0, run: 0 }, + }); + // §4: counts only. No label, id or title, for any caller, until §10's first + // question is settled. Asserted on the live route as well as on the pure + // function, because this is the shape an unauthenticated caller receives. + expect(JSON.stringify(body)).not.toMatch(/sessionId|projectSlug|title|slug/i); + }); + + /** + * THE test. `[[HANG]]` streams its reply and then blocks without writing a + * terminal result, so the turn is genuinely in flight — the state a restart + * would cut short, and the state the busy check exists to refuse on. + */ + it("reports a running chat turn as busy, and goes quiet when it ends", async () => { + const slug = await freshProject(); + const ws = await connectWs(port); + try { + expect((await busy()).busy).toBe(false); + + ws.send({ + type: "chat:send", + payload: { projectSlug: slug, sessionId: null, message: "[[HANG]] hold" }, + }); + // Wait for the turn to be REGISTERED on the hub rather than for the text: + // `chat:active` with `running: true` is the hub's own announcement, so + // this cannot pass before the thing being counted exists. + await ws.waitFor( + (e) => e.type === "chat:active" && e.payload?.running === true, + { timeoutMs: 20_000 }, + ); + + const during = await busy(); + expect(during.busy).toBe(true); + expect(during.counts.chat).toBeGreaterThanOrEqual(1); + expect(during.version).toBe(MANAGERS_VERSION); + + // Stop it — the same thing the UI's Stop button does — and the instance + // must report itself free again. A count that only ever goes up is the + // failure that would make every later `service stop` refuse. + const active = ws.events.find( + (e) => e.type === "chat:active" && e.payload?.running === true, + ); + const jobId = active?.payload?.jobId as string | undefined; + expect(jobId).toBeTruthy(); + ws.send({ type: "chat:cancel", payload: { jobId } }); + await ws.waitFor( + (e) => + (e.type === "chat:active" && e.payload?.running === false) || + e.type === "chat:complete" || + e.type === "chat:error", + { timeoutMs: 20_000 }, + ); + + await expectEventually(async () => { + const after = await busy(); + expect(after.counts.chat).toBe(0); + expect(after.busy).toBe(false); + }); + } finally { + ws.close(); + } + }); + + /** + * Managers' own source: a trigger run is in flight from the moment its + * `running` record is written until its completion hook has finished it. And + * the SIGTERM path's half of the contract — record what a stop cuts short — + * is the run's interrupt, which finishes the record as failed ONCE: the + * dying turn's own completion arriving afterwards must not overwrite it. + */ + it("counts a trigger run in flight, and a shutdown's interrupt records it once", async () => { + const slug = await freshProject(); + const ws = await connectWs(port); + try { + await t.triggers.set(slug, "hold", { + trigger: { type: "schedule", cron: "0 0 1 1 *" }, + run: { prompt: "[[HANG]] hold" }, + enabled: false, + }); + const res = await t.app.inject({ method: "POST", url: `/api/projects/${slug}/triggers/hold/run` }); + expect(res.statusCode).toBe(202); + const active = await ws.waitFor( + (e) => e.type === "chat:active" && e.payload?.running === true && e.payload?.projectSlug === slug, + { timeoutMs: 20_000 }, + ); + + const during = await busy(); + expect(during.busy).toBe(true); + expect(during.counts.run).toBe(1); + expect(during.counts.chat).toBeGreaterThanOrEqual(1); + + // What start.ts's SIGTERM does with a run that is still in flight. + const [runId, interrupt] = [...t.managers.liveRuns.entries()][0]!; + await interrupt("interrupted by shutdown"); + expect(t.managers.liveRuns.size).toBe(0); + expect((await busy()).counts.run).toBe(0); + const runsUrl = `/api/projects/${slug}/managers/runs/${runId}`; + const recorded = ((await t.app.inject({ method: "GET", url: runsUrl })).json() as { + run: { status: string; error: string | null }; + }).run; + expect(recorded).toMatchObject({ status: "failed", error: "interrupted by shutdown" }); + + // Then the turn itself ends; its completion must not rewrite the record. + ws.send({ type: "chat:cancel", payload: { jobId: active.payload?.jobId } }); + await expectEventually(async () => expect((await busy()).counts.chat).toBe(0)); + await new Promise((r) => setTimeout(r, 300)); + const after = ((await t.app.inject({ method: "GET", url: runsUrl })).json() as { + run: { error: string | null }; + }).run; + expect(after.error).toBe("interrupted by shutdown"); + } finally { + ws.close(); + } + }); + + /** + * The unhappy path for the background count, and the reason it is filtered on + * SDK liveness rather than read straight off the registry. + * + * Nothing resets `BackgroundRegistry` when a one-shot `claude` exits — the + * SDK's level signal is per-process and emits nothing at startup — so on + * `driveMode: batch` a finished turn's tasks sit in the registry + * indefinitely. Counting them would report this instance busy for the rest of + * its life, and every `managers service stop` after that one turn would refuse + * with 75 and never recover. + * + * So: drive a turn that registers background tasks, let it END, confirm the + * registry still lists them, and confirm `/api/busy` says not busy. + */ + it("does not count background tasks left behind by a finished batch turn", async () => { + const slug = await freshProject(); + const ws = await connectWs(port); + try { + ws.send({ + type: "chat:send", + payload: { projectSlug: slug, sessionId: null, message: "[[BGTASK]] go" }, + }); + // The registry has the tasks: this is the server's own broadcast of them. + const frame = await ws.waitFor( + (e) => + e.type === "chat:background" && + e.payload?.projectSlug === slug && + ((e.payload?.tasks as unknown[]) ?? []).length > 0, + { timeoutMs: 20_000 }, + ); + const sessionId = frame.payload!.sessionId as string; + expect(t.hub).toBeDefined(); + + await ws.waitFor((e) => e.type === "chat:complete" || e.type === "chat:error", { + timeoutMs: 20_000, + }); + + // The turn is over, and its tasks are STILL in the registry — stated as + // an assertion so this test fails loudly if that leak is ever fixed, + // rather than silently becoming a test of nothing. + const registry = t.background; + expect(registry.list(sessionId).length).toBeGreaterThan(0); + expect(registry.liveTaskCount()).toBeGreaterThan(0); + + // And the endpoint does not report them, because the session is not live. + await expectEventually(async () => { + const body = await busy(); + expect(body.counts.background).toBe(0); + expect(body.counts.chat).toBe(0); + expect(body.busy).toBe(false); + }); + } finally { + ws.close(); + } + }); + + // §4: "it must be cheap… reads in-memory state, never walks the disk, and + // answers in well under a second". A loose bound, because a loaded CI runner + // is not a stopwatch — but 50 calls that each re-read the fleet state file + // (the shape the ROADMAP's `fleetStatus()` would have had) do not fit in it. + it("answers 50 times well inside a second", async () => { + const started = Date.now(); + for (let i = 0; i < 50; i++) await busy(); + expect(Date.now() - started).toBeLessThan(1000); + }); +}); + +/** Poll an assertion until it holds, for state that settles a tick after a frame. */ +async function expectEventually( + assertion: () => Promise, + opts: { timeoutMs?: number } = {}, +): Promise { + const deadline = Date.now() + (opts.timeoutMs ?? 10_000); + for (;;) { + try { + await assertion(); + return; + } catch (err) { + if (Date.now() >= deadline) throw err; + await new Promise((r) => setTimeout(r, 50)); + } + } +} diff --git a/packages/server/test/unit/cli-contract.test.ts b/packages/server/test/unit/cli-contract.test.ts index a199028..d179cec 100644 --- a/packages/server/test/unit/cli-contract.test.ts +++ b/packages/server/test/unit/cli-contract.test.ts @@ -28,9 +28,31 @@ import { USAGE, USAGE_SYNOPSIS, SERVICE_USAGE, + BUSY_USAGE, } from "../../src/cli/args.js"; import { EXIT } from "../../src/cli/exit-codes.js"; -import { buildSpec, serviceLogPath } from "../../src/cli/service/spec.js"; +import { + checkBusy, + busyExitCode, + busyJson, + busyMessage, + describeCounts, + guardBusy, + type BusyResult, +} from "../../src/cli/busy.js"; +import { MANAGERS_NOUNS, managersBusyHint } from "../../src/cli/busy-command.js"; +import { createOutput, type Output } from "../../src/cli/output.js"; +import { + readLiveRuntimeFile, + readRuntimeFile, + removeRuntimeFile, + runtimeFilePath, + writeRuntimeFile, +} from "../../src/runtime-file.js"; +import { runtimeUrl } from "../../src/start.js"; +import { collectBusy } from "../../src/busy.js"; +import { BackgroundRegistry } from "../../src/background-live.js"; +import { buildSpec, serviceLogPath, SYSTEMD_UNIT } from "../../src/cli/service/spec.js"; import { renderUnit, execWord, @@ -59,10 +81,20 @@ const BASE = { const OPTS = { open: false, verbose: false, help: false, version: false }; +/** + * With no argument, a machine with nothing loaded — so `launchctl print` + * answers 3. `install` and `restart` poll `print` until launchd says the label + * has gone (the bootout/bootstrap race, see launchd.ts), and a fake that kept + * saying "still loaded" would make those tests wait out the ten-second timeout + * and pass anyway. See the same note on cli-service.test.ts's copy. + */ function recorder(result: Partial = {}): { run: Runner; calls: string[][] } { const calls: string[][] = []; + const nothingLoaded = Object.keys(result).length === 0; const run: Runner = (cmd, args) => { calls.push([cmd, ...args]); + if (nothingLoaded && cmd === "launchctl" && args[0] === "print") + return { status: 3, stdout: "", stderr: "Could not find service" }; return { status: 0, stdout: "", stderr: "", ...result }; }; return { run, calls }; @@ -81,6 +113,10 @@ function answering(answers: Record>): { run: Runner; const run: Runner = (cmd, args) => { calls.push([cmd, ...args]); const key = Object.keys(answers).find((k) => [cmd, ...args].join(" ").includes(k)); + // Same reason as `recorder`: an unanswered `launchctl print` means "not + // loaded", or an install here waits out the bootout poll. + if (key === undefined && cmd === "launchctl" && args[0] === "print") + return { status: 3, stdout: "", stderr: "Could not find service" }; return { status: 0, stdout: "", stderr: "", ...(key !== undefined ? answers[key] : {}) }; }; return { run, calls }; @@ -781,3 +817,607 @@ describe("SPEC §8: /api/health carries the version", () => { expect(MANAGERS_VERSION).toMatch(/^\d+\.\d+\.\d+/); }); }); + +// --------------------------------------------------------------------------- +// §4 the busy check — the CLI's side (wave C1) +// --------------------------------------------------------------------------- + +describe("SPEC §4: runtime.json is how the CLI finds the instance", () => { + let home: string; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "managers-runtime-")); + }); + afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + }); + + it("writes { pid, port, url, version, startedAt } and reads it back", () => { + const written = writeRuntimeFile(home, { + pid: process.pid, + port: 7234, + url: "http://127.0.0.1:7234", + version: "0.80.0", + }); + expect(written.startedAt).toMatch(/^\d{4}-\d\d-\d\dT/); + expect(readRuntimeFile(home)).toEqual(written); + expect(runtimeFilePath(home)).toBe(path.join(home, "runtime.json")); + }); + + // 0600 now, so that adding the §10 token later changes nothing about who can + // read the file — which is the whole reason the reference writes it this way + // before there is a token to protect. + it("is readable only by this user", () => { + writeRuntimeFile(home, { pid: 1, port: 1, url: "http://x", version: "0" }); + expect(fs.statSync(runtimeFilePath(home)).mode & 0o777).toBe(0o600); + }); + + it("leaves no temporary file behind, so a reader never sees a half-written one", () => { + writeRuntimeFile(home, { pid: 1, port: 1, url: "http://x", version: "0" }); + expect(fs.readdirSync(home)).toEqual(["runtime.json"]); + }); + + it("reads a corrupt or foreign file as absent rather than throwing", () => { + fs.writeFileSync(runtimeFilePath(home), "{ not json"); + expect(readRuntimeFile(home)).toBeNull(); + fs.writeFileSync(runtimeFilePath(home), JSON.stringify({ hello: "world" })); + expect(readRuntimeFile(home)).toBeNull(); + expect(readRuntimeFile(path.join(home, "nope"))).toBeNull(); + }); + + /** + * THE case this file exists for: a crashed instance leaves the file behind, + * and `busy` must not report a dead pid's port as a running instance — it + * would fetch whatever took the port next and believe the answer. + */ + it("treats a file naming a dead pid as no instance", () => { + writeRuntimeFile(home, { pid: 2 ** 22, port: 1, url: "http://x", version: "0" }); + expect(readRuntimeFile(home)).not.toBeNull(); + expect(readLiveRuntimeFile(home)).toBeNull(); + writeRuntimeFile(home, { pid: process.pid, port: 1, url: "http://x", version: "0" }); + expect(readLiveRuntimeFile(home)).not.toBeNull(); + }); + + // A second instance may have started and written its own since: removing that + // one on our way out would blind the CLI to a server that IS running. + it("removes the file only while it is still ours", () => { + writeRuntimeFile(home, { pid: process.pid + 1, port: 1, url: "http://x", version: "0" }); + expect(removeRuntimeFile(home, process.pid)).toBe(false); + expect(fs.existsSync(runtimeFilePath(home))).toBe(true); + writeRuntimeFile(home, { pid: process.pid, port: 1, url: "http://x", version: "0" }); + expect(removeRuntimeFile(home, process.pid)).toBe(true); + expect(fs.existsSync(runtimeFilePath(home))).toBe(false); + // Idempotent: a SIGINT after a SIGTERM must not throw on the way out. + expect(removeRuntimeFile(home, process.pid)).toBe(false); + }); + + /** + * The URL recorded has to be one a CLI can fetch, which `cfg.host` is not + * when the bind is a wildcard — the normal case in a container, and what this + * repo's own `pm` wrapper injects (`HOST=0.0.0.0`). Found by reading the file + * a `pm`-run instance wrote. + */ + it("records a loopback URL for a wildcard bind, and leaves a real address alone", () => { + expect(runtimeUrl("0.0.0.0", 5101)).toBe("http://127.0.0.1:5101"); + expect(runtimeUrl("::", 5101)).toBe("http://[::1]:5101"); + expect(runtimeUrl("127.0.0.1", 7234)).toBe("http://127.0.0.1:7234"); + expect(runtimeUrl("192.168.1.10", 80)).toBe("http://192.168.1.10:80"); + // Bracketed, because `new URL('http://::1:7234')` is not a URL — and + // `cli/busy.ts` builds its request with `new URL('/api/busy', base)`. + expect(() => new URL("/api/busy", runtimeUrl("::", 5101))).not.toThrow(); + }); +}); + +describe("SPEC §4: `managers busy` and its exit codes", () => { + const answer = (body: unknown, ok = true): typeof fetch => + (async () => ({ ok, status: ok ? 200 : 503, json: async () => body }) as Response) as unknown as typeof fetch; + + it("is a top-level verb, taking --json and nothing else", () => { + expect(parseCommand(["busy"])).toEqual({ verb: "busy", opts: OPTS }); + expect(parseCommand(["busy", "--json"]).opts.json).toBe(true); + expect(parseCommand(["busy", "-d", "/tmp/x"]).opts.dataDir).toBe("/tmp/x"); + // Not a `service` action: the point of `busy` is to be askable where there + // is no service to ask (SPEC §1). + expect(() => parseCommand(["service", "busy"])).toThrow(UsageError); + }); + + it("is in the help, with its own page and its exit codes", () => { + expect(USAGE).toContain("managers busy"); + expect(USAGE_SYNOPSIS).toContain("managers busy"); + expect(USAGE.replace(/\s+/g, " ")).toContain("75 busy"); + expect(BUSY_USAGE).toContain("managers busy --json"); + for (const code of ["0 ", "75 ", "3 ", "1 "]) { + expect(BUSY_USAGE.replace(/\s+/g, " ")).toContain(code); + } + }); + + it("maps idle to 0, busy to 75, nothing answering to 3", async () => { + const idle = await checkBusy({ home: "/nope", url: "http://x", fetch: answer({ busy: false }) }); + expect(busyExitCode(idle)).toBe(EXIT.ok); + + const busy = await checkBusy({ + home: "/nope", + url: "http://x", + fetch: answer({ busy: true, version: "0.80.0", counts: { chat: 2, background: 0, job: 0 } }), + }); + expect(busyExitCode(busy)).toBe(EXIT.busy); + expect(busyExitCode({ state: "not-running" })).toBe(EXIT.notRunning); + }); + + // "Nothing answering" is not a failure: §4 says it means not running, and an + // update may go ahead. A refused connection has to land there and not on 1. + it("reads a refused connection as not running, not as a failure", async () => { + const refused = (async () => { + throw new Error("ECONNREFUSED"); + }) as unknown as typeof fetch; + const result = await checkBusy({ home: "/nope", url: "http://x", fetch: refused }); + expect(result).toEqual({ state: "not-running" }); + expect(busyExitCode(result)).toBe(EXIT.notRunning); + }); + + // With no runtime.json there is NO fallback to a default port, because + // guessing one asks whatever else is listening there and believes it. + it("never guesses a port: no runtime file means not running", async () => { + let asked = false; + const spy = (async () => { + asked = true; + return { ok: true, json: async () => ({ busy: true }) } as Response; + }) as unknown as typeof fetch; + const empty = fs.mkdtempSync(path.join(os.tmpdir(), "managers-nobusy-")); + try { + expect(await checkBusy({ home: empty, fetch: spy })).toEqual({ state: "not-running" }); + expect(asked).toBe(false); + } finally { + fs.rmSync(empty, { recursive: true, force: true }); + } + }); + + /** + * Something is listening, but it is not a Managers that knows this question — + * an older version with no `/api/busy`, or an unrelated program on the port. + * That is exit 1 and a sentence, NOT a silent "not busy": treating it as idle + * is how a stop proceeds against a server it could not actually ask. + */ + it("distinguishes an answer it cannot read from no answer at all", async () => { + const notFound = await checkBusy({ home: "/nope", url: "http://x", fetch: answer({}, false) }); + expect(notFound).toMatchObject({ state: "unknown", detail: "HTTP 503" }); + expect(busyExitCode(notFound)).toBe(EXIT.failure); + + const notBusyShaped = await checkBusy({ + home: "/nope", + url: "http://x", + fetch: answer({ hello: "world" }), + }); + expect(notBusyShaped).toMatchObject({ state: "unknown" }); + expect(busyExitCode(notBusyShaped)).toBe(EXIT.failure); + }); + + it("says what is running, in the sentence §4 gives", () => { + const busy = (counts: Record): BusyResult => ({ + state: "busy", + url: "http://127.0.0.1:7234/api/busy", + answer: { busy: true, counts }, + }); + expect(busyMessage("Managers", busy({ chat: 2 }), MANAGERS_NOUNS)).toBe( + "Managers is busy: 2 chat turns are running.", + ); + expect(busyMessage("Managers", busy({ chat: 1 }), MANAGERS_NOUNS)).toBe( + "Managers is busy: 1 chat turn is running.", + ); + expect(busyMessage("Managers", busy({ chat: 2, background: 1, job: 0 }), MANAGERS_NOUNS)).toBe( + "Managers is busy: 2 chat turns and 1 background task are running.", + ); + expect(busyMessage("Managers", { state: "idle", url: "u", answer: { busy: false } })).toBe( + "Managers isn't busy.", + ); + expect(busyMessage("Managers", { state: "not-running" })).toBe("Managers isn't running."); + // No noun ends up pluralised by adding an "s" to something that is not a + // noun: "3 backgrounds are running" was the version this guards against. + expect(describeCounts({ background: 3 }, MANAGERS_NOUNS)).toBe("3 background tasks are running"); + }); + + it("`--json` prints one document of the shape the schema fixes", () => { + const doc = busyJson({ + state: "busy", + url: "http://127.0.0.1:7234/api/busy", + answer: { busy: true, version: "0.80.0", counts: { chat: 1, background: 0, job: 0 } }, + }); + expect(doc).toEqual({ + running: true, + busy: true, + // The INSTANCE url, not the endpoint's — `/api/busy` is stripped. + url: "http://127.0.0.1:7234", + version: "0.80.0", + counts: { chat: 1, background: 0, job: 0 }, + }); + expect(busyJson({ state: "not-running" })).toEqual({ running: false, busy: false }); + expect( + busyJson({ state: "unknown", url: "http://127.0.0.1:7234/api/busy", detail: "HTTP 404" }), + ).toEqual({ running: true, busy: false, url: "http://127.0.0.1:7234", error: "HTTP 404" }); + }); +}); + +describe("SPEC §4: the guard `stop`, `restart` and `update` run first", () => { + const busyFetch = (counts: Record): typeof fetch => + (async () => + ({ ok: true, json: async () => ({ busy: true, version: "0.80.0", counts }) }) as Response) as unknown as typeof fetch; + + function recordingOutput(): { out: Output; lines: string[] } { + const lines: string[] = []; + const stream = { write: (s: string) => void lines.push(s.trimEnd()) }; + return { out: createOutput({ tool: "managers", stdout: stream, stderr: stream, env: {} }), lines }; + } + + it("refuses with 75, naming what is running and the way past it", async () => { + const { out, lines } = recordingOutput(); + const code = await guardBusy({ + home: "/nope", + url: "http://x", + fetch: busyFetch({ chat: 0, background: 2, job: 0 }), + name: "Managers", + nouns: MANAGERS_NOUNS, + hint: managersBusyHint, + out, + }); + expect(code).toBe(EXIT.busy); + expect(lines.join("\n")).toContain("managers: Managers is busy: 2 background tasks are running."); + expect(lines.join("\n")).toContain("--force"); + }); + + // `--force` prints the SAME message, prefixed `Overriding:` (SPEC §4) — the + // user who forced it should see exactly what they just overrode. + it("--force prints the same message prefixed `Overriding:` and carries on", async () => { + const { out, lines } = recordingOutput(); + const code = await guardBusy({ + home: "/nope", + url: "http://x", + fetch: busyFetch({ chat: 1, background: 0, job: 0 }), + name: "Managers", + nouns: MANAGERS_NOUNS, + out, + force: true, + }); + expect(code).toBeNull(); + expect(lines.join("\n")).toBe("Overriding: Managers is busy: 1 chat turn is running."); + }); + + it("goes ahead when nothing answers, and when it is idle", async () => { + const idle = (async () => ({ ok: true, json: async () => ({ busy: false }) }) as Response) as unknown as typeof fetch; + const { out } = recordingOutput(); + const base = { home: "/nope", url: "http://x", name: "Managers", out }; + expect(await guardBusy({ ...base, fetch: idle })).toBeNull(); + expect(await guardBusy({ ...base, home: "/definitely/not/here" })).toBeNull(); + }); + + /** + * THE Managers-specific trap (ROADMAP "Busy, per tool"). A keeper chat that + * runs `service restart` or `update` is itself one of the chat turns the + * answer counted, so it can only ever see itself as busy — and `--force` from + * there does not override a mistaken reading, it kills the chat that typed it. + */ + it("tells a chat to run it from a terminal, when chats are what is running", () => { + const withChats = managersBusyHint({ + state: "busy", + url: "http://x", + answer: { busy: true, counts: { chat: 1, background: 0, job: 0 } }, + }); + expect(withChats).toContain("from a terminal"); + expect(withChats).toContain("not from inside a Managers chat"); + + // Background work and jobs get the ordinary advice: nothing is holding a + // keyboard inside the service, so --force is a reasonable thing to suggest. + const withoutChats = managersBusyHint({ + state: "busy", + url: "http://x", + answer: { busy: true, counts: { chat: 0, background: 1, job: 2 } }, + }); + expect(withoutChats).toBe("Try again once it has finished, or use --force."); + expect(managersBusyHint({ state: "not-running" })).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// §4/§8 the server's side of the busy check +// --------------------------------------------------------------------------- + +describe("SPEC §4: what the server counts as busy", () => { + const sources = (chat: number, background: number, job: number, run = 0) => ({ + runningChats: () => chat, + backgroundTasks: () => background, + runningJobs: () => job, + runningRuns: () => run, + }); + + it("is not busy when nothing is running", () => { + expect(collectBusy(sources(0, 0, 0), "0.80.0")).toEqual({ + busy: false, + version: "0.80.0", + counts: { chat: 0, background: 0, job: 0, run: 0 }, + }); + }); + + it("is busy on any one of the four, and reports all four counts", () => { + expect(collectBusy(sources(1, 0, 0), "0.80.0").busy).toBe(true); + expect(collectBusy(sources(0, 1, 0), "0.80.0").busy).toBe(true); + expect(collectBusy(sources(0, 0, 1), "0.80.0").busy).toBe(true); + // Managers' own: a trigger run in flight, with no turn yet (its briefing is + // being built) or no turn any more (its record is being committed). + expect(collectBusy(sources(0, 0, 0, 1), "0.80.0").busy).toBe(true); + expect(collectBusy(sources(2, 3, 1, 4), "0.80.0").counts).toEqual({ + chat: 2, + background: 3, + job: 1, + run: 4, + }); + }); + + it("counts no runs when no run source is wired (a server built without Managers state)", () => { + const { runningRuns: _omit, ...three } = sources(0, 0, 0, 9); + expect(collectBusy(three, "0.80.0")).toEqual({ + busy: false, + version: "0.80.0", + counts: { chat: 0, background: 0, job: 0, run: 0 }, + }); + }); + + /** + * A busy check that 500s is worse than one that under-reports: `cli/busy.ts` + * reads a non-200 as "something answered, but not as Managers' busy check" — + * exit 1 — which FAILS a `service stop` outright rather than letting it + * through. So a source that throws counts as zero. + */ + it("survives a source that throws, rather than 500ing the endpoint", () => { + const body = collectBusy( + { + runningChats: () => { + throw new Error("no hub"); + }, + backgroundTasks: () => 2, + runningJobs: () => { + throw new Error("no fleet"); + }, + }, + "0.80.0", + ); + expect(body).toEqual({ busy: true, version: "0.80.0", counts: { chat: 0, background: 2, job: 0, run: 0 } }); + }); + + it("never reports a negative or fractional count", () => { + expect(collectBusy(sources(-1, 1.5, Number.NaN, -2), "0.80.0")).toEqual({ + busy: false, + version: "0.80.0", + counts: { chat: 0, background: 0, job: 0, run: 0 }, + }); + }); + + /** + * Background work is counted only in sessions whose SDK session is still + * live, and that filter is load-bearing rather than belt-and-braces. Nothing + * resets the registry when a one-shot `claude` exits, so on + * `driveMode: batch` a finished turn's tasks sit in it indefinitely — without + * the filter, one batch turn that launched a background task would report the + * instance busy for the rest of the process's life, and every `service stop` + * after it would refuse. + */ + it("ignores background tasks whose session is no longer live", () => { + const registry = new BackgroundRegistry(); + registry.observe("demo", { + type: "system", + subtype: "background_tasks_changed", + session_id: "live-one", + tasks: [{ task_id: "t1", task_type: "local_bash", description: "build" }], + }); + registry.observe("demo", { + type: "system", + subtype: "background_tasks_changed", + session_id: "dead-one", + tasks: [ + { task_id: "t2", task_type: "local_agent", description: "review" }, + { task_id: "t3", task_type: "local_bash", description: "test" }, + ], + }); + expect(registry.liveTaskCount()).toBe(3); + expect(registry.liveTaskCount((id) => id === "live-one")).toBe(1); + expect(registry.liveTaskCount(() => false)).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// §1/§4 `service stop` and `service restart` refuse when busy +// --------------------------------------------------------------------------- + +describe("SPEC §1: stop and restart run the busy check, uninstall does not", () => { + let home: string; + beforeEach(() => { + home = fs.mkdtempSync(path.join(os.tmpdir(), "managers-busy-stop-")); + }); + afterEach(() => { + fs.rmSync(home, { recursive: true, force: true }); + }); + + const ready = async () => "ready" as const; + + /** + * A context whose busy guard is injected, with a record of what it was asked. + * + * `refuse` is what `guardServiceBusy` returns when it has printed a refusal: + * 75, which `runService` must pass straight out rather than carrying on and + * returning its own 0. + */ + function withGuard( + refuse: number | null, + run: Runner, + ): { + ctx: Parameters[2]; + asked: { action: string; home: string; force: boolean }[]; + } { + const asked: { action: string; home: string; force: boolean }[] = []; + return { + asked, + ctx: { + ...contextFor("linux", home, run), + checkReady: ready, + guardBusy: async (action, askedHome, opts) => { + asked.push({ action, home: askedHome, force: opts.force === true }); + return opts.force === true ? null : refuse; + }, + }, + }; + } + + function installUnit(dataDir?: string): void { + const { run } = recorder(); + createSystemdBackend(run, home).install( + buildSpec({ + ...BASE, + dataDir: dataDir ?? path.join(home, ".managers"), + ...(dataDir !== undefined ? { dataDirArg: dataDir } : {}), + }), + ); + } + + it("stop refuses with 75 and does not touch the supervisor", async () => { + installUnit(); + const active = recorder({ stdout: "active" }); + const { ctx, asked } = withGuard(EXIT.busy, active.run); + + const { code } = await capture(() => runService("stop", OPTS, ctx)); + + expect(code).toBe(EXIT.busy); + expect(asked).toEqual([{ action: "stop", home: path.join(home, ".managers"), force: false }]); + // The point of the refusal: nothing was stopped. + expect(active.calls.flat()).not.toContain("stop"); + }); + + it("stop --force goes ahead", async () => { + installUnit(); + const active = recorder({ stdout: "active" }); + const { ctx, asked } = withGuard(EXIT.busy, active.run); + + const { code } = await capture(() => runService("stop", { ...OPTS, force: true }, ctx)); + + expect(code).toBe(EXIT.ok); + expect(asked[0].force).toBe(true); + expect(active.calls).toContainEqual(["systemctl", "--user", "stop", SYSTEMD_UNIT]); + }); + + it("restart refuses with 75, and --force goes ahead", async () => { + installUnit(); + const active = recorder({ stdout: "active" }); + const refusing = withGuard(EXIT.busy, active.run); + const { code } = await capture(() => runService("restart", OPTS, refusing.ctx)); + expect(code).toBe(EXIT.busy); + expect(refusing.asked).toEqual([ + { action: "restart", home: path.join(home, ".managers"), force: false }, + ]); + expect(active.calls.flat()).not.toContain("restart"); + + const forcing = withGuard(EXIT.busy, active.run); + const forced = await capture(() => runService("restart", { ...OPTS, force: true }, forcing.ctx)); + expect(forced.code).toBe(EXIT.ok); + expect(active.calls).toContainEqual(["systemctl", "--user", "restart", SYSTEMD_UNIT]); + }); + + /** + * `uninstall` does not ask (SPEC §1: "removing the service is an explicit + * act"), and neither does `start` — there is nothing running for a start to + * cut short, so asking would be theatre that could only ever refuse wrongly. + */ + it("start and uninstall never ask", async () => { + installUnit(); + const inactive = recorder({ stdout: "inactive" }); + const forStart = withGuard(EXIT.busy, inactive.run); + await capture(() => runService("start", OPTS, forStart.ctx)); + expect(forStart.asked).toEqual([]); + + const active = recorder({ stdout: "active" }); + const forUninstall = withGuard(EXIT.busy, active.run); + await capture(() => runService("uninstall", OPTS, forUninstall.ctx)); + expect(forUninstall.asked).toEqual([]); + }); + + /** + * `--force` where nothing runs a busy check is a usage error (SPEC §2), not a + * silent no-op. + * + * The reason it matters: the two readings of a quiet `service uninstall + * --force` are "it overrode something" and "it did nothing", and the exit + * code is 0 either way, so someone typing it at 2am cannot tell which they + * got. + */ + it("--force is exit 2 anywhere but stop and restart, and is not silently dropped", () => { + const rejected: string[][] = [ + ["service", "install", "--force"], + ["service", "uninstall", "--force"], + ["service", "start", "--force"], + ["service", "logs", "--force"], + ["service", "status", "--force"], + ["busy", "--force"], + ["serve", "--force"], + ["--force"], + ["config", "show", "--force"], + ]; + for (const argv of rejected) { + let thrown: unknown; + try { + parseCommand(argv); + } catch (e) { + thrown = e; + } + expect(thrown, `managers ${argv.join(" ")}`).toBeInstanceOf(UsageError); + expect((thrown as UsageError).exitCode).toBe(EXIT.usage); + expect((thrown as UsageError).message).toMatch( + /--force applies to `service stop` and `service restart` only/, + ); + } + + // The control, and the thing that makes the above a test of the RULE + // rather than of a flag nobody can pass: the two that do take it parse. + for (const action of ["stop", "restart"] as const) { + expect(parseCommand(["service", action, "--force"]).opts.force).toBe(true); + } + // And asking for the usage is never a usage error. + expect(parseCommand(["service", "status", "--force", "--help"]).opts.help).toBe(true); + }); + + // Nothing to ask and nothing to cut short: a missing unit is 4 and an + // already-stopped service is 0, both decided before the check runs. + it("does not ask when there is no unit, or the service is already stopped", async () => { + const noUnit = withGuard(EXIT.busy, recorder().run); + const missing = await capture(() => runService("stop", OPTS, noUnit.ctx)); + expect(missing.code).toBe(EXIT.notInstalled); + expect(noUnit.asked).toEqual([]); + + installUnit(); + const stopped = withGuard(EXIT.busy, recorder({ stdout: "inactive" }).run); + const already = await capture(() => runService("stop", OPTS, stopped.ctx)); + expect(already.code).toBe(EXIT.ok); + expect(stopped.asked).toEqual([]); + }); + + /** + * The instance asked about is the one the UNIT describes, not whatever + * `--data-dir` this invocation carries (#824). Getting this wrong would read + * `runtime.json` out of a different instance and answer "not busy" about the + * one being stopped — the quiet version of no busy check at all. + */ + it("asks about the unit's own data dir, ignoring today's --data-dir", async () => { + const unitDataDir = path.join(home, "service-instance"); + installUnit(unitDataDir); + const active = recorder({ stdout: "active" }); + const { ctx, asked } = withGuard(null, active.run); + + await capture(() => + runService("stop", { ...OPTS, dataDir: path.join(home, "somewhere-else") }, ctx), + ); + + expect(asked[0].home).toBe(unitDataDir); + }); + + it("`--force` parses, and nothing else turns it on", () => { + expect(parseArgs(["--force"]).force).toBe(true); + expect(parseArgs([]).force).toBeUndefined(); + expect(parseCommand(["service", "stop", "--force"]).opts.force).toBe(true); + expect(parseCommand(["service", "restart", "--force"]).opts.force).toBe(true); + expect(SERVICE_USAGE).toContain("--force"); + }); +}); diff --git a/packages/server/test/unit/shutdown-drain.test.ts b/packages/server/test/unit/shutdown-drain.test.ts new file mode 100644 index 0000000..5c87668 --- /dev/null +++ b/packages/server/test/unit/shutdown-drain.test.ts @@ -0,0 +1,123 @@ +/** + * The SIGTERM path's finish-or-record (CLI contract SPEC §4, §8): + * `shutdown-drain.ts`. start.ts wires it to the hub and `managers.liveRuns`. + */ +import { describe, expect, it } from "vitest"; +import { + describeShutdown, + drainInFlight, + recordInterruptedRuns, + type InFlight, +} from "../../src/shutdown-drain.js"; + +const noSleep = async () => undefined; + +describe("drainInFlight", () => { + it("does not wait at all when nothing is in flight", async () => { + let reads = 0; + const d = await drainInFlight( + () => { + reads++; + return { turns: 0, runs: 0 }; + }, + { sleep: async () => expect.fail("an idle stop must not sleep") }, + ); + expect(d).toEqual({ before: { turns: 0, runs: 0 }, after: { turns: 0, runs: 0 }, polls: 0 }); + expect(reads).toBe(1); + }); + + it("waits until the work ends by itself", async () => { + const readings: InFlight[] = [ + { turns: 1, runs: 1 }, + { turns: 1, runs: 1 }, + { turns: 0, runs: 1 }, // the turn ended; its run record is being committed + { turns: 0, runs: 0 }, + ]; + const d = await drainInFlight(() => readings.shift() ?? { turns: 0, runs: 0 }, { sleep: noSleep }); + expect(d.before).toEqual({ turns: 1, runs: 1 }); + expect(d.after).toEqual({ turns: 0, runs: 0 }); + expect(d.polls).toBe(3); + }); + + /** + * Bounded by a COUNT of polls, not a wall-clock deadline: with an injected + * no-op sleep a deadline would spin for the whole window instead. + */ + it("gives up after drainMs / pollMs polls, however fast the clock", async () => { + let reads = 0; + const d = await drainInFlight( + () => { + reads++; + return { turns: 2, runs: 0 }; + }, + { drainMs: 1_000, pollMs: 250, sleep: noSleep }, + ); + expect(d.polls).toBe(4); + expect(reads).toBe(5); + expect(d.after).toEqual({ turns: 2, runs: 0 }); + }); + + it("reads a throwing source as nothing in flight, and never a negative count", async () => { + const d = await drainInFlight( + () => { + throw new Error("no hub"); + }, + { sleep: noSleep }, + ); + expect(d.before).toEqual({ turns: 0, runs: 0 }); + const neg = await drainInFlight(() => ({ turns: -3, runs: -1 }), { sleep: noSleep }); + expect(neg.before).toEqual({ turns: 0, runs: 0 }); + }); +}); + +describe("recordInterruptedRuns", () => { + it("interrupts every live run with the reason, and counts the ones that landed", async () => { + const seen: string[] = []; + const runs = new Map Promise>([ + ["r-1", async (reason) => void seen.push(`r-1 ${reason}`)], + ["r-2", async (reason) => void seen.push(`r-2 ${reason}`)], + ["r-3", async () => Promise.reject(new Error("store is read-only"))], + ]); + expect(await recordInterruptedRuns(runs, "interrupted by shutdown")).toBe(2); + expect(seen.sort()).toEqual(["r-1 interrupted by shutdown", "r-2 interrupted by shutdown"]); + }); + + it("is bounded: a run that never finishes recording does not hold the stop", async () => { + const runs = new Map Promise>([ + ["r-1", () => new Promise(() => undefined)], + ["r-2", async () => undefined], + ]); + const t0 = Date.now(); + expect(await recordInterruptedRuns(runs, "x", 50)).toBe(1); + expect(Date.now() - t0).toBeLessThan(1_000); + }); + + it("is a no-op with no runs, or no Managers state", async () => { + expect(await recordInterruptedRuns(new Map(), "x")).toBe(0); + expect(await recordInterruptedRuns(undefined, "x")).toBe(0); + }); +}); + +describe("describeShutdown", () => { + const z = { turns: 0, runs: 0 }; + it("says so when nothing was in flight", () => { + expect(describeShutdown("SIGTERM", { before: z, after: z, polls: 0 }, 0)).toBe( + "SIGTERM: shutting down, nothing in flight.", + ); + }); + it("says what it waited for when everything finished", () => { + expect(describeShutdown("SIGTERM", { before: { turns: 1, runs: 1 }, after: z, polls: 3 }, 0)).toBe( + "SIGTERM: shutting down; waited for 1 chat turn and 1 trigger run to finish, and they did.", + ); + }); + it("names what was cut short, and how many run records it marked", () => { + const line = describeShutdown( + "SIGINT", + { before: { turns: 2, runs: 2 }, after: { turns: 2, runs: 2 }, polls: 32 }, + 2, + ); + expect(line).toContain("SIGINT: shutting down with 2 chat turns and 2 trigger runs still in flight"); + expect(line).toContain("2 of 2 run records marked failed (interrupted by shutdown)"); + expect(line).toContain("managers busy"); + }); +}); diff --git a/scripts/service-real.mjs b/scripts/service-real.mjs index 210d788..7739135 100644 --- a/scripts/service-real.mjs +++ b/scripts/service-real.mjs @@ -4,7 +4,8 @@ // // Ported from Local Review's scripts/service-real.mjs @ cli-contract spec 0.1; local changes: Managers' packaging // (scripts/make-npm-package.mjs synthesizes @edspencer/managers from two private workspaces), a throwaway -// MANAGERS_DATA_DIR, the contract's exit codes (0/2/3/4), `service logs`, and the file-logging check. +// MANAGERS_DATA_DIR, the contract's exit codes (0/2/3/4/75), `service logs`, the file-logging check, and wave C1's +// `runtime.json` + the busy check on `stop` / `restart`. // // CI ONLY. It writes the real ~/Library/LaunchAgents/net.edspencer.managers.plist or // ~/.config/systemd/user/managers.service and loads it, under the real label — so it must never run on a machine that @@ -19,11 +20,15 @@ // status / status --json exit 0, running + answering, installed version == running version // logs, logs -n exit 0, reads that same file // an unknown flag exit 2 +// runtime.json written 0600, naming the supervisor's pid and port +// busy / busy --json exit 0 against the idle service +// stop / restart while busy exit 75, and the service is STILL answering afterwards +// stop --force while busy exit 0, `Overriding: …`, and it really goes down // SIGKILL the supervisor brings it back (KeepAlive / Restart=always) -// stop exit 0; then status exit 3, logs still exit 0 +// stop an idle service NOT refused: exit 0; status 3, logs 0, busy 3, runtime.json gone // start / restart exit 0 and answering again // uninstall the unit is gone, status and logs exit 4, the data home is untouched -import { execFileSync } from "node:child_process"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; import assert from "node:assert/strict"; import fs from "node:fs"; import net from "node:net"; @@ -63,21 +68,34 @@ for (const k of [ const bin = path.join(prefix, "bin", "managers"); -/** Run the installed CLI. Returns { code, stdout, stderr } — never throws for a non-zero exit. */ +/** + * Run the installed CLI. Returns { code, stdout, stderr } — never throws for a non-zero exit. + * + * `spawnSync`, not `execFileSync`, and that is the whole point: `execFileSync` + * RETURNS stdout and only exposes stderr by throwing, so a command that + * succeeded handed back `stderr: ""` no matter what it had written. That is a + * lie for every command on this contract, because SPEC §3 puts prose on stderr + * and keeps stdout for the one JSON document — so `stop --force`'s whole + * observable output (`Overriding: …`, then exit 0) was being discarded, and the + * assertion on it failed against `''`. + */ function managers(...args) { - try { - const stdout = execFileSync(bin, args, { - cwd: work, - env, - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 120_000, - }); - return { code: 0, stdout, stderr: "" }; - } catch (e) { - if (e.status === undefined || e.status === null) throw e; - return { code: e.status, stdout: e.stdout ?? "", stderr: e.stderr ?? "" }; + const r = spawnSync(bin, args, { + cwd: work, + env, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + timeout: 120_000, + }); + if (r.error !== undefined) throw r.error; + // Killed by a signal (the timeout above) rather than exited: `status` is null, + // and reporting that as a code would turn a hang into a wrong-exit-code. + if (r.status === null) { + throw new Error( + `managers ${args.join(" ")} was killed by ${r.signal}\n${r.stdout ?? ""}\n${r.stderr ?? ""}`, + ); } + return { code: r.status, stdout: r.stdout ?? "", stderr: r.stderr ?? "" }; } /** Run it and insist on an exit code, so a wrong one fails here rather than three assertions later. */ @@ -216,6 +234,130 @@ try { if (process.platform === "linux") assert.equal(s.linger, true, "CI enables lingering"); else assert.equal(s.linger, null, "launchd has no linger"); + step("runtime.json: written 0600 once it is listening, naming this pid and port (SPEC §4)"); + const runtimeFile = path.join(data, "runtime.json"); + const runtime = JSON.parse(fs.readFileSync(runtimeFile, "utf8")); + assert.equal(runtime.pid, s.pid, "runtime.json names the pid the supervisor reports"); + assert.equal(runtime.port, port); + assert.equal(runtime.version, version); + assert.match(runtime.startedAt, /^\d{4}-\d\d-\d\dT/); + // Written 0600 before there is a token in it (SPEC §10) — adding one later + // must not have to change who may read the file. Asserted as "no group or + // other access" rather than an exact 0600, because `open()`'s mode is masked + // by the umask and a runner's umask is not ours to assume; the exact mode is + // pinned in cli-contract.test.ts, where it is. + assert.equal(fs.statSync(runtimeFile).mode & 0o077, 0, "runtime.json is not group/world readable"); + + step("busy against the real, idle service: exit 0"); + expectExit(0, "busy"); + const idleBusy = JSON.parse(expectExit(0, "busy", "--json").stdout); + assert.equal(idleBusy.running, true); + assert.equal(idleBusy.busy, false); + assert.equal(idleBusy.version, version); + assert.equal(idleBusy.counts.chat, 0); + + // --- the refusal, against a real service manager ------------------------ + // + // Making the REAL instance busy would mean driving a chat turn through it, + // which needs a project, a socket and a fake `claude` inside the unit. What is + // worth proving HERE is not the counting — unit and integration tests cover + // that against the real hub — but that a refusal reaches the real + // launchctl/systemctl path and stops it: `stop` returns 75 and the service is + // still answering afterwards, and `--force` gets past it. + // + // So the busy ANSWER is faked, at the seam the CLI actually uses. The CLI finds + // the instance through `/runtime.json` and nothing else, so pointing that + // file at a server answering `/api/busy` with `busy: true` is exactly what a + // busy Managers looks like from out here. The real service keeps running + // throughout, untouched, which is the whole assertion. + // + // ⚠️ The stand-in runs in a CHILD PROCESS, and it has to. An `http.createServer` + // in THIS process cannot answer anything while `execFileSync` is running the + // CLI, because that blocks the event loop the server listens on — the kernel + // accepts the connection into the backlog and Node never gets to it, so the + // CLI's fetch times out, `checkBusy` reads a timeout as "not running", and the + // stop goes ahead and PASSES its way to a green `exit 0`. That is how this test + // failed on both real runners the first time it ran. + step("stop and restart refuse with 75 while the instance reports itself busy"); + const fakePort = await freePort(); + const fake = spawn( + process.execPath, + [ + "-e", + "const b=JSON.parse(process.env.BODY);" + + "require('node:http').createServer((q,s)=>{" + + "s.writeHead(200,{'content-type':'application/json'});s.end(JSON.stringify(b));" + + "}).listen(Number(process.env.PORT),'127.0.0.1');", + ], + { + stdio: ["ignore", "ignore", "inherit"], + env: { + ...process.env, + PORT: String(fakePort), + BODY: JSON.stringify({ busy: true, version, counts: { chat: 2, background: 0, job: 0 } }), + }, + }, + ); + try { + // `pid` must name a LIVE process or the file reads as stale (a crashed + // instance leaves one behind), so it names the stand-in — which is also the + // honest answer: that process IS the thing answering on that port. + fs.writeFileSync( + runtimeFile, + JSON.stringify( + { ...runtime, pid: fake.pid, port: fakePort, url: `http://127.0.0.1:${fakePort}` }, + null, + 2, + ), + { mode: 0o600 }, + ); + // Prove the stand-in is answering BEFORE asserting on a refusal, so a child + // that failed to start reads as a broken rig rather than as a broken guard. + const standIn = await until(`the stand-in /api/busy on ${fakePort}`, async () => { + try { + const res = await fetch(`http://127.0.0.1:${fakePort}/api/busy`, { + signal: AbortSignal.timeout(2000), + }); + return res.ok ? await res.json() : undefined; + } catch { + return undefined; + } + }); + assert.equal(standIn.busy, true); + + const refusedStop = expectExit(75, "service", "stop"); + assert.match(refusedStop.stderr, /is busy: 2 chat turns are running/); + assert.match(refusedStop.stderr, /from a terminal/, "the keeper-chat trap (ROADMAP)"); + const refusedRestart = expectExit(75, "service", "restart"); + assert.match(refusedRestart.stderr, /is busy: 2 chat turns are running/); + // THE assertion: neither refusal touched the real service. + const stillUp = await health(port); + assert.equal(stillUp?.ok, true, "a refused stop leaves the service running"); + assert.equal( + JSON.parse(managers("service", "status", "--json").stdout).pid, + s.pid, + "a refused restart did not bounce it either", + ); + + step("stop --force prints the same message prefixed `Overriding:`, and stops it"); + const forced = expectExit(0, "service", "stop", "--force"); + assert.match(forced.stderr, /^Overriding: .*is busy: 2 chat turns are running\.$/m); + await until(`nothing on ${port}`, async () => (await health(port)) === undefined); + statusJson(3); + } finally { + fake.kill(); + // Ours, not the server's: `removeRuntimeFile` only removes a file naming its + // own pid, so the stop above correctly left this one alone. + fs.rmSync(runtimeFile, { force: true }); + } + + step("start again, and the real runtime.json comes back"); + expectExit(0, "service", "start"); + await until("health after start", () => health(port)); + await until("runtime.json", () => fs.existsSync(runtimeFile)); + expectExit(0, "busy"); + s = statusJson(0); + step("logs"); const logged = expectExit(0, "service", "logs", "-n", "5").stdout; assert.ok(logged.trim() !== "", "logs printed something"); @@ -233,13 +375,20 @@ try { return n.running && n.pid !== old && n.answering; }); - step("stop: exit 0, then status exit 3 and logs still exit 0"); + step("stop an IDLE service: not refused — exit 0, status 3, logs still 0, busy 3"); + // The control for the refusal above: the busy check must not get in the way of + // the ordinary stop. This one runs against the REAL `/api/busy`, with no + // `--force` and nothing faked. expectExit(0, "service", "stop"); await until(`nothing on ${port}`, async () => (await health(port)) === undefined); s = statusJson(3); assert.equal(s.installed, true); assert.equal(s.answering, false); expectExit(0, "service", "logs", "-n", "3"); + // Removed on the way out, so nothing later finds a live-looking instance. The + // supervisor's stop is a real SIGTERM, which is the path that removes it. + await until("runtime.json to go", () => !fs.existsSync(path.join(data, "runtime.json"))); + expectExit(3, "busy"); step("start"); expectExit(0, "service", "start"); @@ -261,9 +410,22 @@ try { statusJson(4); expectExit(4, "service", "logs"); // (the server may add to the data dir as it stops: only what was there before matters) - for (const f of dataBefore) { + // + // `runtime.json` is the ONE documented exception, and it is an exception on + // purpose: it is a liveness marker rather than data, written when the server + // starts listening and removed when it stops (SPEC §4). Keeping it across an + // uninstall would leave a file naming a dead pid for the next `busy` to find + // — "the data and logs stay" is about the user's things, and this is not one + // of them. So it is excluded here and asserted GONE below, which is the + // stronger claim of the two. + for (const f of dataBefore.filter((f) => f !== "runtime.json")) { assert.ok(fs.existsSync(path.join(data, f)), `${f} is still in the data dir`); } + assert.ok(dataBefore.includes("runtime.json"), "it was there to begin with"); + assert.ok( + !fs.existsSync(path.join(data, "runtime.json")), + "runtime.json is removed by the stop inside uninstall, not left naming a dead pid", + ); assert.ok(fs.readFileSync(logFile, "utf8").startsWith(logBefore), "the logs are kept"); expectExit(0, "service", "uninstall"); // idempotent step("ok");