diff --git a/.claude/skills/codetrial-improve/SKILL.md b/.claude/skills/codetrial-improve/SKILL.md index ccafe12eb..5176440c0 100644 --- a/.claude/skills/codetrial-improve/SKILL.md +++ b/.claude/skills/codetrial-improve/SKILL.md @@ -188,7 +188,7 @@ None of this is a gate; each piece names what it can and cannot see. | `scripts/agent-failure-loop.py` (PostToolUse) | Prompts for the shared premise on the second identical gate failure | Know the premise, or block | | `scripts/test.sh` receipts | Records lane, verdict, skipped count and the fingerprint of the tree as the run started, under the git directory | Prove the run was complete, or stop an agent with a shell from writing one; CI recomputes the verdict | | `scripts/agent-closeout-check.py` (Stop) | Tells the user once per tree when it has changes the gate has not passed on in every lane (`all`, or rust, portable and browser) | Block, or see a gate run elsewhere | -| `scripts/ceiling-hotspots.py` | Ranks the files recorded as exceptions to the size limit by commits in a window times lines | Say a file is worth splitting, or whether its tests cover it; it is one input | +| `scripts/ceiling-hotspots.py` | Ranks the Rust files over the default size limit by commits in a window times lines | Say a file is worth splitting, or whether its tests cover it; it is one input | | pre-push note | Notes `src/` changes with no `tests/` change | Judge whether a test was needed | ## Local evidence at closeout diff --git a/.claude/skills/codetrial-improve/rust.md b/.claude/skills/codetrial-improve/rust.md index fcac3f86a..9882190a5 100644 --- a/.claude/skills/codetrial-improve/rust.md +++ b/.claude/skills/codetrial-improve/rust.md @@ -54,17 +54,27 @@ through `clippy.toml`. ## File size -New Rust files stay within 1500 lines. Existing larger files are recorded in -`scripts/rust-style-ceilings.json` and may only shrink: after splitting one by -responsibility, run `scripts/check-rust-style.py --ratchet` to lower its entry. -`scripts/ceiling-hotspots.py` ranks them by churn times size, as one input to -choosing which to split first. Count a file's mutants before splitting it with -`cargo mutants --list -f | wc -l`; `make mutants` downloads a compiler -first. Never raise an entry to land a change; a new module is the answer. The -gate passes `--baseline origin/main`, so a raised entry or a new exemption fails -against what main records, and under CI a baseline ref that was not fetched -fails too, so a shallow checkout cannot skip it. Test bodies live under -`tests/unit/` and answer to the same ceiling. +New Rust files stay within 1500 lines. By default, the checker derives each +limit from the committed file at the same path in HEAD, with a floor of 1500; +working files may shrink freely but larger files may not grow. Without Git +every file gets 1500 and the gate's baseline cannot be read, so a source +archive fails the gate; run it from a clone. +`scripts/ceiling-hotspots.py` ranks the larger files by churn times size, as +one input to choosing which to split first. Count a file's mutants before +splitting it with `cargo mutants --list -f | wc -l`; `make mutants` +downloads a compiler first. + +The gate passes `--baseline origin/main`. For a larger file present in both +that baseline and HEAD, its limit is the smallest size across those two trees +and every reachable committed version at that path in `origin/main..HEAD`, +with a floor of 1500. All reachable commits count, including merged history. +A committed shrink automatically ratchets the limit; a new or renamed path +gets 1500, and deletion retires the allowance. Split a large file by +responsibility rather than grow it. An explicit baseline requires resolvable +HEAD and baseline refs, common history and a complete, non-shallow +repository; an error fails locally and in CI. Diverged refs are supported, +but the branch must still satisfy any smaller baseline limit. Test bodies +live under `tests/unit/` and answer to the same ceiling. ### Splitting a file @@ -106,9 +116,9 @@ What splitting `src/agent/prompts.rs` (3944 lines, six modules under fn name`) fails with "is still defined here" in `cargo test --lib` and not in clippy, and a source digest that hashes the old file (the report evaluation's identity) stops covering the code that left it. -7. **Run `scripts/check-rust-style.py --ratchet`** so the old entry leaves - `scripts/rust-style-ceilings.json`, then clippy at `-D warnings`, then - `./scripts/test.sh rust`. +7. **Commit the split, then run clippy** at `-D warnings` and + `./scripts/test.sh rust`. The old file's limit drops to its committed size + on its own; there is no table to update. A comment added to a file that sits exactly at its ceiling fails the checker, so a note for a ceilinged file goes here or in the split, not in the file. @@ -188,7 +198,7 @@ in [recipes/rust.md](recipes/rust.md). cargo fmt cargo clippy --locked --workspace --all-targets cargo doc --locked --workspace --no-deps -python3 scripts/check-rust-style.py +python3 scripts/check-rust-style.py --baseline origin/main python3 scripts/check-rust-style.py --workspace cargo test --locked ``` diff --git a/.claude/skills/codetrial-review/rules/paths b/.claude/skills/codetrial-review/rules/paths index 631a1b400..f12dedb70 100644 --- a/.claude/skills/codetrial-review/rules/paths +++ b/.claude/skills/codetrial-review/rules/paths @@ -10,6 +10,7 @@ src/gemini* room.md src/gemini* agent.md src/dispatch.rs room.md src/main.rs room.md +src/launch.rs room.md src/config* room.md src/web/setup.rs room.md src/web/token.rs room.md diff --git a/Cargo.lock b/Cargo.lock index 4b805a9a4..63895d55d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,6 +324,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "socket2", "tokio", "tokio-tungstenite", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index 9d7841885..c99687383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,12 +128,15 @@ windows-permissions = "0.2.4" # Independent implementations verify production signatures and digest fixtures. hmac = "0.13.0" sha2 = "0.11.1" +# Borrowed socket inspection for real loopback transport fixtures. +socket2 = "0.6.5" livekit-protocol = "0.8.2" # LiveKit's transport, reached directly to check the vendored proxy path. livekit-net = "0.1.4" # The paused clock, so the Live socket's keepalive and idle limits are tested -# at their real values instead of waiting them out. -tokio = { version = "1.54.1", features = ["test-util"] } +# at their real values instead of waiting them out; signals, so the serving +# fixture can treat TERM as a stop request. +tokio = { version = "1.54.1", features = ["signal", "test-util"] } # The continuity fixture installs a logger to read the SDK's queue-overflow # line; the SDK already depends on this crate. log = "0.4.34" diff --git a/docs/desktop.md b/docs/desktop.md index 09c066d29..0c6b01b5b 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -17,10 +17,11 @@ server in: it runs the `codetrial` executable that ships beside it as a child process, and its window shows the page that child serves on `http://127.0.0.1:3101`. -A child process rather than a library call, because the server's startup lives -in `src/main.rs` (config search, the Setup handover, every refusal it makes -before serving) and moving that into the library would have been a rewrite of -the file the CLI's behavior is pinned to. As a child it is the binary the CLI +A child process rather than a library call, because the server's startup is +the binary's: `src/main.rs` runs the Setup handover and binds the socket, and +builds the server from `src/launch.rs` (config search, every refusal it makes +before serving). Calling that from the app would have meant a second launcher +beside the one the CLI's behavior is pinned to. As a child it is the binary the CLI tests already cover, started with five things a launcher needs that the CLI did not have. Each is decided in the server, so the app keeps no copy of the server's rules: diff --git a/docs/development.md b/docs/development.md index 8c2f59ad4..d9b9063cb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -195,10 +195,21 @@ locks held across await and `mem::forget` are explicit denials too. `unsafe` count at zero — `forbid` rather than `deny`, so a local `#[allow]` cannot reintroduce it. `scripts/check-rust-style.py` holds what a lint cannot ask: that this table stays strict, that no `allow` or bare `#[ignore]` appears, -that no marker or CJK text lands in Rust or `web/`, and that no file passes the -line ceiling in `scripts/rust-style-ceilings.json`, which `--ratchet` only -lowers. The rules and the lints left off on purpose are in the Rust profile of -codetrial-improve, `.claude/skills/codetrial-improve/rust.md`. +that no marker or CJK text lands in Rust or `web/`, and that no file passes its +Git-derived line ceiling. By default, a file may not exceed 1500 lines or its +committed size at the same path in HEAD, whichever is larger. Without Git +every file gets the 1500-line limit and the explicit baseline below cannot be +read, so a source archive fails this gate; run it from a clone. The gate passes +`--baseline origin/main`: larger files present in both the baseline and HEAD +are limited by their smallest size across those two trees and every reachable +commit in `origin/main..HEAD`, with a floor of 1500. Committed shrinking +automatically ratchets the limit; new or renamed paths get 1500, and deletion +retires an allowance. An explicit baseline requires resolvable HEAD and +baseline refs, common history and a complete, non-shallow repository; missing +history fails locally and in CI. Diverged refs are supported, but the branch +must still satisfy any smaller baseline limit. The rules and the lints left +off on purpose are in the Rust profile of codetrial-improve, +`.claude/skills/codetrial-improve/rust.md`. The compiler those lints run on is pinned in `rust-toolchain.toml`, so a new Rust release cannot fail an unchanged tree: Dependabot proposes each bump as @@ -1058,8 +1069,10 @@ python3 scripts/interview-run-budget.py run \ `validate` is free and should always precede `run`. `run` passes the worker `BROWSER_CHECK_RUN_MANIFEST`, `BROWSER_CHECK_RUN_ID`, `BROWSER_CHECK_RUN_MANIFEST_SHA256` (the hash of the manifest bytes), -`BROWSER_CHECK_BUDGET_SOCKET`, `BROWSER_CHECK_BUDGET_TELEMETRY` and -`BROWSER_CHECK_BUDGET_STOP`. `scripts/browser-check.sh` refuses a partial set, a +`BROWSER_CHECK_BUDGET_SOCKET`, `BROWSER_CHECK_BUDGET_TELEMETRY`, +`BROWSER_CHECK_BUDGET_STOP` and `BROWSER_CHECK_BUDGET_REAP_AT` (when its reap +begins, in Unix seconds, which the serving fixture's own deadline is set +before). `scripts/browser-check.sh` refuses a partial set, a run ID or hash that disagrees with the manifest, a binary whose sha256 changed, and `CODETRIAL_WEB_URL`, since an external server is shared and was not built from the pinned executable. With `BROWSER_CHECK_VALIDATE_ENV_ONLY=1` it stops @@ -1121,6 +1134,36 @@ reserve is revalidated before launch against the resolved handover grace at or after the boundary is `late_idle` or `late_terminal_preempted` and never counts toward the two completed rotations a run needs. +`scripts/long-interview-evidence.cjs` reads the acquisition report the test +fixture leaves (receipts, executed batches, interviewer segments and turn +endings, each with its socket and how many of the others preceded it) and gives +`judgeCheckpoint` its evidence. It claims an association only when the facts +leave one reading: a reply cites the calls received on its own socket after the +previous ending, executed before it began, and not received while output was +being discarded. A swallowed or missing ending, a segment sealed by a cut, a +reply that spans a tool call, a socket replacement, a rewritten segment, two +replies in one window, a window edge inside the clock uncertainty, a reply +that began after the question but before its estimated end (a recording can +finish speaking sooner than its word count says) or a latched capture failure +makes the checkpoint `not_assessed` with the reason, never a pass. The +interview is `verified` only when every checkpoint passed. +`tests/fixtures/acquisition-report.json` is real producer output, regenerated +with `CODETRIAL_UPDATE_FIXTURES=1` on the test that writes it. + +What a reply is judged on is what its question asks and no more +(`scripts/long-interview-schedule.cjs`). An inspect question asks about the +duplicate case, so the revision is judged in what the tool read (the answer +key's `tool.result_fact_ids`), and a reply may state the current revision +without penalty but not an earlier one. A board has no marker a model can read +back: `read_board` gives it a count and an image, never text, so a whiteboard +revision is told apart by the stroke count the tool reports (four strokes are +drawn per revision and the board keeps them, so revision n reads `4n strokes`), +its questions do not ask for a revision, and `tests/agent/whiteboard.rs` pins +the wording the judge reads, which the judge's own test takes from +`tests/golden/prompts.json`. The rubrics are lexical and were checked against +the first pilot's real replies. Changing any of this changes the catalogue, so +`SCHEDULE_VERSION`, the pins and every clip manifest move with it. + `run` also samples the owned process tree (RSS, open descriptors and threads for every process, keyed by PID and start time so a reused PID is another process) every 30 seconds into `resource-samples.jsonl`, for either arm of a diff --git a/docs/latency-observability.md b/docs/latency-observability.md index 17c632b5c..da8cb9272 100644 --- a/docs/latency-observability.md +++ b/docs/latency-observability.md @@ -220,14 +220,18 @@ Run the replay and the long-interview report checks without credentials: ```sh python3 scripts/run-python-tests.py tests/test_analyze_latency.py -node --test tests/browser/soak-judge.test.js tests/browser/long-interview-driver.test.js +node --test tests/browser/soak-judge.test.js tests/browser/long-interview-driver.test.js \ + tests/browser/long-interview-evidence.test.js ``` A long-interview report's `assessment_scope` is `structural`, even when `run_kind` declares a qualification workload. Its `qualification` remains -`not_evaluated`: checkpoint `reply_content` and post-swap `decoded_signal` -are `not_assessed` by this driver. Separate evidence is required before -qualifying a build. `exposure_s` ends at the harness's observation boundary; +`not_evaluated`. Checkpoint `reply_content` is `not_assessed` by this driver; +in serving-fixture mode `scripts/browser-check.sh` judges it from the +fixture's report once the fixture has stopped and writes the verdict into the +capture. Post-swap `decoded_signal` (`verified`, `not_required`, `failed` or +`not_assessed`) is reported beside it. Neither changes the qualification or the +exit code. Separate evidence is required before qualifying a build. `exposure_s` ends at the harness's observation boundary; the analyzer's exposure can extend through the terminal socket's end and must not be substituted for it. `end_permit_granted` records permission, while `end_clicked` records the page action; the countdown can win after the permit diff --git a/docs/provider-cost-and-degradation.md b/docs/provider-cost-and-degradation.md index bcf84dcac..b79433927 100644 --- a/docs/provider-cost-and-degradation.md +++ b/docs/provider-cost-and-degradation.md @@ -225,12 +225,13 @@ answers each request on the control topic, `report_retry` with status `accepted` or `early` (with the seconds still to wait, which does not spend the regeneration), and announces `closed` at expiry or when no key can ever answer; a duplicate request during -regeneration gets no answer. The browser waits at most 145 seconds from its -request or the acceptance, whichever came last, so neither a reconnect nor a -lost answer cuts off a regeneration still in progress. The terminal failure -determines eligibility, so a transient burst followed by a schema failure -regenerates as a schema failure. Reloads and process restarts cannot recover -the inputs. +regeneration gets no answer. The browser waits at most 175 seconds from its +request or the acceptance, whichever came last. This covers the configured +generation deadline, one final rejoin grace and every delivery attempt. This +browser bound does not bound arbitrary SDK publish or host scheduling stalls. +The terminal failure determines eligibility, so a transient burst followed by +a schema failure regenerates as a schema failure. Reloads and process +restarts cannot recover the inputs. Every report packet, provisional, regenerated or final, is delivered the same way. The agent publishes those same bytes at most three times, each with a @@ -511,7 +512,11 @@ completed interviews awaiting recovery; admission still refuses excess starts rather than exceeding this bound. A fixed local room refuses a new start while its existing agent is finalizing or awaiting report recovery, as `dispatch_refused ... reason=finalizing` rather than `at_capacity`; it cannot -reuse that agent as an interviewer for another candidate. Provider projects are +reuse that agent as an interviewer for another candidate. A server that has +begun to stop refuses every start that would admit a new room as +`reason=closed`, and the candidate is told it is shutting down; admitted rooms +keep running, and a start for a room still being assessed joins it; a live +room otherwise gets the refusal it had before the stop. Provider projects are rotated and their LiveKit connection-minute quota is refreshed in the background, and known-exhausted projects are skipped. diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index bf30ae363..c8b7c6541 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -59,15 +59,17 @@ const { planFromEnv, validateEnvironment, } = require("./long-interview-plan.cjs"); +const { + judgeSettledSwapAudio, + startSwapMarker, +} = require("./swap-audio-owner.cjs"); const { clockEvidenceLines, completeLines, episodeEvidenceLines, - installedSwapKeys, isRotation, judgeEpisodeReplies, judgeSoak, - judgeSwapAudio, parseClock, parseEpisodes, parseObligations, @@ -510,6 +512,10 @@ const CONNECTION_CAP_SECONDS = 600; /// `GoAway`'s notice plus a stalled reply's twenty seconds. const RESUME_GRACE_MS = 45_000; +// What the shared settle needs from this process: the grace the page is given +// and where its per-episode lines go. +const SETTLE = { graceMs: RESUME_GRACE_MS, log: console.log }; + /// The two runtime lines that are not part of a replacement: the advisory and /// its expiry. Matched literally, printed by src/livekit.rs. What a replacement /// was is read from the fields of its `replacement:` line by @@ -786,6 +792,7 @@ async function longInterview( tokenRequestsMs, joinGuarded, tokenExpiredNow, + probing = false, ) { const plan = longPlan; const countdownStartS = parseCountdown( @@ -849,21 +856,34 @@ async function longInterview( return pageSide.awaitServerDeadline(); }, }; + const swapMarker = probing + ? startSwapMarker({ + page, + roomName, + agentOutput, + newLines: serverLogTail(), + }) + : null; // The schedule's clock starts here: connected, greeted, the join already // dispatched through the budget by the media gate. - const run = await runLongInterview({ - plan, - schedule: prepared.schedule, - playback: prepared.playback, - clipSource: prepared.clipSource, - budget: runBudget, - controller, - joinedAtMs: Date.now(), - interviewStartMs, - joinDispatched: joinGuarded, - observe, - failure: failure() ?? new Promise(() => {}), - }); + let run; + try { + run = await runLongInterview({ + plan, + schedule: prepared.schedule, + playback: prepared.playback, + clipSource: prepared.clipSource, + budget: runBudget, + controller, + joinedAtMs: Date.now(), + interviewStartMs, + joinDispatched: joinGuarded, + observe, + failure: failure() ?? new Promise(() => {}), + }); + } finally { + await swapMarker?.halt(); + } ended = true; refresh(); const clock = parseClock(`${[...evidence.keys()].join("\n")}\n`, roomName); @@ -881,6 +901,33 @@ async function longInterview( ), graceStimuli: 0, }); + const episodeText = `${[...episodeLines.keys()].join("\n")}\n`; + const episodes = parseEpisodes(episodeText, roomName); + // A probe that could not mark or settle leaves the signal not assessed + // rather than failing the run. + let audio = null; + let replies = null; + if (swapMarker) { + try { + await swapMarker.stop(); + replies = judgeEpisodeReplies( + episodes, + parseObligations(episodeText, roomName), + true, + ); + audio = await judgeSettledSwapAudio( + page, + replies, + "long-interview", + SETTLE, + ); + } catch (error) { + console.error(`long-interview: decoded audio not assessed: ${error}`); + } + await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } const log = `${agentOutput.join("")}\n${readServerLog()}`; const serverLog = process.env.SERVER_LOG; const logsComplete = @@ -890,6 +937,7 @@ async function longInterview( log.endsWith("\n"); const finished = finalizeLongRun({ plan, + mode: prepared.mode, schedule: prepared.schedule, playback: prepared.playback, clipSource: prepared.clipSource, @@ -897,10 +945,9 @@ async function longInterview( observation, clock, tokenRequestsMs, - episodes: parseEpisodes( - `${[...episodeLines.keys()].join("\n")}\n`, - roomName, - ).episodes, + episodes: episodes.episodes, + audio, + replies, logs: { truncated: false, evicted: false, complete: logsComplete }, browserDisconnected, tokenExpired: tokenExpiredNow(), @@ -987,8 +1034,6 @@ async function soakInterview( // as it goes past for the same reason: the agent's output is a rolling // window. const evidence = new Map(); - // The installed swaps already handed to the page to sample again. - const marked = new Set(); // One read per call rather than one per signal: this is the whole server // log, and the loop runs for minutes. @@ -1016,24 +1061,6 @@ async function soakInterview( parseObligations(evidenceText(), roomName), final, ); - // Each installed swap is marked in the page once, as soon as its end line is - // read, so the sample that answers it can only open on a Speaking transition - // after this point. Read late, never early: a transition between the install - // and this call is missed and the next one is used instead, which can fail - // a swap whose reply was quicker than a poll but never passes one on audio - // from before it. - const newLines = serverLogTail(); - const markNewSwaps = async () => { - if (!probing) return; - for (const key of installedSwapKeys( - `${agentOutput.join("")}\n${newLines()}`, - roomName, - )) { - if (marked.has(key)) continue; - marked.add(key); - await page.evaluate((swap) => globalThis.__markJimSwap?.(swap), key); - } - }; const failure = (reasons) => new Error(`${reasons.join("; ")} during ${soakSeconds}s soak`); const judge = (final) => @@ -1055,29 +1082,15 @@ async function soakInterview( return soakSummary; }; - // Marked on its own clock, not the loops' below: those wait on log parsing - // and room-service calls, and a swap read late can miss the transition that - // answered it. - let marking = true; - let markError = null; - const marker = (async () => { - while (marking) { - await sleep(250); - if (!marking) break; - try { - await markNewSwaps(); - } catch (error) { - markError = error; - marking = false; - } - } - })(); - const stopMarking = async () => { - marking = false; - await marker; - if (markError) throw markError; - await markNewSwaps(); - }; + const swapMarker = probing + ? startSwapMarker({ + page, + roomName, + agentOutput, + newLines: serverLogTail(), + }) + : null; + const stopMarking = async () => swapMarker?.stop(); try { let nextRefresh = 0; @@ -1169,16 +1182,7 @@ async function soakInterview( // cancels what still waits, and an unsampled swap counts against the soak. let audio = null; if (probing) { - const rearms = await page.evaluate( - (waitMs) => globalThis.__settleJimSwapAudio(waitMs), - RESUME_GRACE_MS, - ); - audio = judgeSwapAudio(replies, rearms); - for (const e of audio.episodes) { - console.log( - `soak episode ${e.episode} audio: ${e.status} ${e.reasons.join("; ")}`.trimEnd(), - ); - } + audio = await judgeSettledSwapAudio(page, replies, "soak", SETTLE); } const result = record(final, replies, audio); @@ -1195,8 +1199,7 @@ async function soakInterview( throw failure(detail.length > 0 ? detail : final.reasons); } } finally { - marking = false; - await marker; + await swapMarker?.halt(); } } @@ -1360,6 +1363,7 @@ async function isolateRustAgent( let bargeReplay = null; let jimAudio = null; let probingAudio = false; + let probingSwapAudio = false; if (process.env.BROWSER_CHECK_FLOW === "barge") { await page.addInitScript( installBargeReplay, @@ -3025,7 +3029,11 @@ PatternLexicon.prototype.search = function(word) { ), ); probingAudio = credentialed && ["voice", "barge", "soak"].includes(flow); - if (probingAudio) { + // The long interview judges the decoded audio of its own swaps inside + // longInterview; the generic end-of-flow assertion on Jim's whole track + // would fail a tone fixture that never makes Jim speak. + probingSwapAudio = credentialed && flow === "long-interview"; + if (probingAudio || probingSwapAudio) { await page.evaluate(mediaProbeSource()); } let joinGuarded = false; @@ -3157,6 +3165,7 @@ PatternLexicon.prototype.search = function(word) { consoleErrors.some((line) => /token.*(expired|invalid)|expired.*token/i.test(line), ), + probingSwapAudio, ); } let testResultText = null; @@ -3326,6 +3335,12 @@ PatternLexicon.prototype.search = function(word) { .evaluate(() => globalThis.__finishJimAudioProbe(true)) .catch(() => null); } + if (probingSwapAudio) { + // Stops the probe's sampling on any failure, whichever step threw. + await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } if ( process.env.BROWSER_CHECK_CAPTURE && (floorLogPath || probingAudio) && diff --git a/scripts/browser-check.sh b/scripts/browser-check.sh index 64a64862c..7936d1e79 100755 --- a/scripts/browser-check.sh +++ b/scripts/browser-check.sh @@ -137,11 +137,56 @@ LOGIN_DB_PATH="$TMP/accounts.db" LOGIN_SESSION_SECRET=browser-check-session CONFIG_PATH="$TMP/codetrial.env.local" +# The in-process serving fixture (scripts/fixture-server.sh): a pinned test +# executable run as the server in place of `codetrial web`, so the interview +# leaves behind what the arming observer saw. Off unless a test name is given. +FIXTURE_TEST=${BROWSER_CHECK_FIXTURE_TEST:-} +FIXTURE_DIR="$TMP/fixture" +FIXTURE_STOP_WAIT_S=${BROWSER_CHECK_FIXTURE_STOP_WAIT_S:-150} + +# The fixture's connection drain, passed to it, and the slack left after its +# settle: the settle is whatever the stop wait has left after both. +FIXTURE_DRAIN_S=10 +FIXTURE_SLACK_S=20 + +# Asked to stop and waited for, not killed: the serving fixture has connections +# to drain, rooms and reports to finish and a report of its own to write, and +# that report is kept before the scratch directory goes. +cleanup_fixture() +{ + fixture_status=0 + "$ROOT/scripts/fixture-server.sh" stop "$FIXTURE_DIR" "$SERVER_PID" "$FIXTURE_STOP_WAIT_S" \ + || fixture_status=1 + wait "$SERVER_PID" 2> /dev/null || { + echo "The serving fixture exited with a failure; its report may say why." >&2 + fixture_status=1 + } + + "$ROOT/scripts/fixture-server.sh" keep "$FIXTURE_DIR" "$BROWSER_CHECK_FIXTURE_OUT" "$SERVER_LOG" \ + || fixture_status=1 + return "$fixture_status" +} + +# The reply content is judged once the serving fixture has stopped and its +# report is kept: the worker finished before that and could not read it. The +# verdict is written into the capture file and never into this script's own +# status, so a run that could not be judged is still the run it was. +judge_reply_content() +{ + [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -f "$BROWSER_CHECK_CAPTURE" ] || return 0 + node "$ROOT/scripts/long-interview-evidence.cjs" \ + --capture "$BROWSER_CHECK_CAPTURE" --fixture "$BROWSER_CHECK_FIXTURE_OUT" \ + || echo "Reply content was not judged; the run's own result stands." >&2 +} + cleanup() { cleanup_status=$? trap - EXIT INT TERM - if [ "${SERVER_PID:-}" ]; then + if [ "${SERVER_PID:-}" ] && [ -n "$FIXTURE_TEST" ]; then + cleanup_fixture || cleanup_status=1 + judge_reply_content + elif [ "${SERVER_PID:-}" ]; then kill "$SERVER_PID" 2> /dev/null || true wait "$SERVER_PID" 2> /dev/null || true fi @@ -278,6 +323,82 @@ if [ -n "$BUDGET_VARS" ]; then --manifest "$BROWSER_CHECK_RUN_MANIFEST" candidate) || exit 2 fi +if [ -n "$FIXTURE_TEST" ]; then + if [ -z "$PREBUILT_BIN" ] || [ "$BROWSER_CHECK_AGENT" != dispatch ]; then + echo "BROWSER_CHECK_FIXTURE_TEST needs a budgeted run (its pinned executable is the" >&2 + echo "fixture) and BROWSER_CHECK_AGENT=dispatch." >&2 + exit 2 + fi + if [ -z "${BROWSER_CHECK_FIXTURE_OUT:-}" ] || [ -e "$BROWSER_CHECK_FIXTURE_OUT" ] \ + || [ -e "$BROWSER_CHECK_FIXTURE_OUT.server.log" ] \ + || [ -L "$BROWSER_CHECK_FIXTURE_OUT.server.log" ] \ + || [ ! -d "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" ] \ + || [ ! -w "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" ]; then + echo "BROWSER_CHECK_FIXTURE_OUT must name a report file that does not exist yet," >&2 + echo "in a directory that does and can be written." >&2 + exit 2 + fi + + # The verdict is written into the capture after the fixture stops, so a + # capture left by an earlier run would be judged against this run's report. + if [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -e "$BROWSER_CHECK_CAPTURE" ]; then + echo "BROWSER_CHECK_CAPTURE must not exist yet when the serving fixture runs." >&2 + exit 2 + fi + + # Neither exists yet, so their directories are compared: the report kept at + # cleanup would otherwise replace the capture the run just wrote. + if [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -d "$(dirname -- "$BROWSER_CHECK_CAPTURE")" ]; then + capture_at="$(cd -P -- "$(dirname -- "$BROWSER_CHECK_CAPTURE")" && pwd)/$(basename -- "$BROWSER_CHECK_CAPTURE")" + out_at="$(cd -P -- "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" && pwd)/$(basename -- "$BROWSER_CHECK_FIXTURE_OUT")" + + # Each also stages files beside it (the log beside the report, the + # judged capture at .partial), so neither may be a prefix of the other. + case $out_at in "$capture_at"*) collides=1 ;; *) collides= ;; esac + case $capture_at in "$out_at"*) collides=1 ;; esac + if [ -n "$collides" ]; then + echo "BROWSER_CHECK_CAPTURE and BROWSER_CHECK_FIXTURE_OUT must name different files." >&2 + exit 2 + fi + fi + + # A leading zero too: shell arithmetic reads it as octal while the tests + # below read it as decimal, so the reserve check would ask for too little. + case $FIXTURE_STOP_WAIT_S in + '' | *[!0-9]* | 0[0-9]*) + echo "BROWSER_CHECK_FIXTURE_STOP_WAIT_S must be a number of seconds." >&2 + exit 2 + ;; + esac + + # One number drives both sides: the fixture settles for this long less its + # connection drain and the slack, so a longer settle cannot make the wait + # below give up on a fixture that is still working. At least 10 s of it. + fixture_min_wait_s=$((FIXTURE_DRAIN_S + FIXTURE_SLACK_S + 10)) + if [ "$FIXTURE_STOP_WAIT_S" -lt "$fixture_min_wait_s" ]; then + echo "BROWSER_CHECK_FIXTURE_STOP_WAIT_S must be at least $fixture_min_wait_s seconds." >&2 + exit 2 + fi + + # The stop waits for a drain and a settle, so the cleanup window has to + # outlast it with the launcher's own cleanup still to come. + reserve=$("$BUDGET_PYTHON" "$BUDGET_TOOL" validate \ + --manifest "$BROWSER_CHECK_RUN_MANIFEST" --print cleanup_reserve_s) || exit 2 + if [ "$reserve" -lt $((FIXTURE_STOP_WAIT_S + 60)) ]; then + echo "cleanup_reserve_s is $reserve; the serving fixture's stop wait of" >&2 + echo "${FIXTURE_STOP_WAIT_S}s needs at least $((FIXTURE_STOP_WAIT_S + 60))s of it." >&2 + exit 2 + fi + + # When the budget runner starts its reap, for the fixture's own deadline. + case ${BROWSER_CHECK_BUDGET_REAP_AT:-} in + '' | *[!0-9]*) + echo "BROWSER_CHECK_FIXTURE_TEST needs BROWSER_CHECK_BUDGET_REAP_AT from the budget runner." >&2 + exit 2 + ;; + esac +fi + # The duration, run kind, end mode and observation target of a long interview # are judged before anything is built or started. The same module answers when # browser-check.cjs is run directly. @@ -292,6 +413,11 @@ if [ "${BROWSER_CHECK_VALIDATE_ENV_ONLY:-}" ]; then exit 0 fi +# Runs the pin, so it waits until validation-only runs are out of the way. +if [ -n "$FIXTURE_TEST" ]; then + "$ROOT/scripts/fixture-server.sh" check "$PREBUILT_BIN" "$FIXTURE_TEST" || exit 2 +fi + # Anything that serves `web/` needs the fetched vendor binaries, and this script # launches its own server rather than going through the Makefile. Without this a # fresh clone serves a MediaPipe loader whose .wasm and model 404, and face @@ -400,10 +526,49 @@ else # mode that a missing dispatch breaks, and nothing else covers it. unset INTERVIEW_ROOM_NAME DISPATCH_MODE=1 - SESSION_SECRET="$LOGIN_SESSION_SECRET" \ - CODETRIAL_DB_PATH="$LOGIN_DB_PATH" \ - "$@" web --config "$CONFIG_PATH" --web-addr "127.0.0.1:$PORT" --web-dir "$ROOT/web" \ - > "$SERVER_LOG" 2>&1 & + if [ -n "$FIXTURE_TEST" ]; then + + # The pinned test executable, running only the fixture. Its + # account database is inside its own directory by construction. + mkdir "$FIXTURE_DIR" + + # In a session of its own where it can be, so a signal sent to + # this script's process group reaches this script, whose trap + # asks the fixture to stop and waits, and not the fixture first: + # an INT would end it without a report, and a TERM in its first + # moments, before it can take one as a stop request, likewise. + # This script has no job control, so setsid runs the fixture in + # place and $! is the fixture; were it to fork, wait_for_server + # would see the launcher exit and fail the run at once. + set -- "$PREBUILT_BIN" --ignored --exact "$FIXTURE_TEST" --nocapture --test-threads=1 + if command -v setsid > /dev/null 2>&1; then + set -- setsid "$@" + fi + + # A backstop only: cleanup asks it to stop first. As late as it + # can be and still leave the stop wait and some margin before + # the runner's reap, counted from now, since the fixture counts + # its deadline from its own start. + FIXTURE_DEADLINE_S=$((BROWSER_CHECK_BUDGET_REAP_AT - $(date +%s) - \ + FIXTURE_STOP_WAIT_S - FIXTURE_SLACK_S)) + if [ "$FIXTURE_DEADLINE_S" -le 0 ]; then + echo "No wall time is left for the serving fixture to run and stop." >&2 + exit 2 + fi + SESSION_SECRET="$LOGIN_SESSION_SECRET" \ + CODETRIAL_FIXTURE_CONFIG="$CONFIG_PATH" \ + CODETRIAL_FIXTURE_DIR="$FIXTURE_DIR" \ + CODETRIAL_FIXTURE_ADDR="127.0.0.1:$PORT" \ + CODETRIAL_FIXTURE_DEADLINE_S="$FIXTURE_DEADLINE_S" \ + CODETRIAL_FIXTURE_DRAIN_S="$FIXTURE_DRAIN_S" \ + CODETRIAL_FIXTURE_SETTLE_S="$((FIXTURE_STOP_WAIT_S - FIXTURE_DRAIN_S - FIXTURE_SLACK_S))" \ + "$@" > "$SERVER_LOG" 2>&1 & + else + SESSION_SECRET="$LOGIN_SESSION_SECRET" \ + CODETRIAL_DB_PATH="$LOGIN_DB_PATH" \ + "$@" web --config "$CONFIG_PATH" --web-addr "127.0.0.1:$PORT" --web-dir "$ROOT/web" \ + > "$SERVER_LOG" 2>&1 & + fi ;; rust) INTERVIEW_ROOM_NAME=${INTERVIEW_ROOM_NAME:-interview-browser-$(date +%s)-$$} diff --git a/scripts/ceiling-hotspots.py b/scripts/ceiling-hotspots.py index b7902c970..1fe06e17d 100644 --- a/scripts/ceiling-hotspots.py +++ b/scripts/ceiling-hotspots.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Rank oversized files by how often they change times how large they are. -`scripts/rust-style-ceilings.json` says which Rust files are over the size -ceiling; it does not say which to split first. A big file nobody touches costs -nothing, and a big file every change goes through costs on every change. This -prints, per top-level directory, each file with its lines, its recorded ceiling, -the commits that touched it in a window, and score = commits x lines. - - default the files named in the ceilings JSON +`scripts/check-rust-style.py` says which Rust files are over the size ceiling; +it does not say which to split first. A big file nobody touches costs nothing, +and a big file every change goes through costs on every change. This prints, +per top-level directory, each file with its lines, its ceiling, the commits +that touched it in a window, and score = commits x lines. + + default every Rust file the style checker sees above its default + ceiling, with the limit it derives from HEAD --under DIR instead, every tracked file under DIR (a directory such as `web`) with at least --min-lines lines; there is no ceiling column to fill, so it shows `-`. `--suffix .js` keeps only @@ -25,16 +26,25 @@ """ import argparse -import json +import importlib.util import subprocess import sys from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -CEILINGS = "scripts/rust-style-ceilings.json" DEFAULT_SINCE = "90.days" +# The gate's own line counts and the limits it derives from HEAD. The gate run +# with `--baseline` can be stricter: it also takes the smallest size since that +# ref, so a file that shrank on a branch and grew back shows a larger ceiling +# here than the one the gate enforces. +_SPEC = importlib.util.spec_from_file_location( + "check_rust_style", Path(__file__).with_name("check-rust-style.py") +) +STYLE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(STYLE) + class GitError(Exception): """git exited non-zero; the message is its standard error.""" @@ -81,20 +91,17 @@ def commit_counts(repo, since, paths): return Counter(name for name in out.split("\0") if name) -def ceiling_rows(repo, ceilings): - """{path: ceiling} for each recorded file that still exists.""" - if not isinstance(ceilings, dict) or not all( - isinstance(name, str) and type(ceiling) is int - for name, ceiling in ceilings.items() - ): - raise ValueError(f"{CEILINGS} is not a map of path to line ceiling") - rows = {} - for name in sorted(ceilings): - if (repo / name).is_file(): - rows[name] = ceilings[name] - else: - print(f"skipped {name}: no such file", file=sys.stderr) - return rows +def ceiling_rows(repo): + """{path: ceiling} for each Rust file over the default ceiling. A file with + no committed version, or one at or under the default in HEAD, gets the + default, as the gate gives it.""" + current = STYLE.sizes(repo) + limits = STYLE.git_ceilings(repo, current) + return { + name: limits.get(name, STYLE.DEFAULT_CEILING) + for name, lines in sorted(current.items()) + if lines > STYLE.DEFAULT_CEILING + } def under_rows(repo, directory, min_lines, suffixes=(), excludes=()): @@ -176,7 +183,7 @@ def main(argv=None): repo, args.under, args.min_lines, args.suffix, args.exclude ) else: - rows = ceiling_rows(repo, json.loads((repo / CEILINGS).read_text())) + rows = ceiling_rows(repo) ranked = rank(repo, rows, args.since) except (GitError, OSError, ValueError) as error: print(f"ceiling-hotspots: {error}", file=sys.stderr) diff --git a/scripts/check-rust-style.py b/scripts/check-rust-style.py index bd211cb6a..a66e68f5b 100644 --- a/scripts/check-rust-style.py +++ b/scripts/check-rust-style.py @@ -17,12 +17,11 @@ no comment naming the untracked working documents. language No CJK in Rust or the browser half; fixtures that need multi-byte text use Greek or emoji. - ceiling A file does not grow past the size recorded for it in - scripts/rust-style-ceilings.json, or past DEFAULT_CEILING when - it has no entry. `--ratchet` lowers entries to what the files - are now and never raises one. `--baseline REF` also refuses an - entry above, or absent from, the one REF records; under CI an unfetched REF - is itself a finding. + ceiling New files stay within DEFAULT_CEILING. Existing larger files + cannot grow beyond their committed size in HEAD. --baseline REF + also uses the smallest committed size since REF; deletion retires + an exemption. Limits come from Git, without a generated table. + An explicit baseline needs complete history and never skips errors. `--workspace` audits Cargo's own membership for lint inheritance and needs Cargo; the default run needs none. Exit status 1 when anything is reported. @@ -31,7 +30,6 @@ import argparse import functools import json -import os import re import subprocess import sys @@ -39,7 +37,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -CEILINGS = "scripts/rust-style-ceilings.json" DEFAULT_CEILING = 1500 RUST_ROOTS = ("src", "tests", "scripts/hook-check") @@ -394,14 +391,6 @@ def check_ceilings(current, recorded): f"{rel}: ceiling: {lines} lines, over {ceiling}; " "split it by responsibility, do not raise the ceiling" ) - for rel, ceiling in sorted(recorded.items()): - if rel not in current: - yield f"{CEILINGS}: {rel} no longer exists; drop its entry" - elif current[rel] < ceiling: - yield ( - f"{CEILINGS}: {rel} is {current[rel]} lines, " - f"under its {ceiling}; run --ratchet so it cannot grow back" - ) def git_ok(root, *args): @@ -418,46 +407,94 @@ def git_ok(root, *args): return result.stdout if result.returncode == 0 else None -def baseline_findings(root, ref, recorded): - """Ceiling findings against `ref`. +def git_ceilings(root, current, baseline=None): + """Derive allowances from committed paths, including every branch version.""" + head = git_ok(root, "rev-parse", "--verify", "-q", "HEAD^{commit}") + if head is None and baseline is None: + return {} - A ref that does not resolve is a finding under CI, where a shallow checkout - would otherwise skip the ratchet without a word, and a note elsewhere. A ref - that has no ceilings file yet, the change that introduces it, is skipped. - """ - if git_ok(root, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}") is None: - if os.environ.get("CI") == "true": - return [f"baseline {ref} is not fetched; fetch it so the ratchet can run"] - print(f"baseline {ref} not found; skipped", file=sys.stderr) - return [] - text = git_ok(root, "show", f"{ref}:{CEILINGS}") - if text is None: - print(f"baseline {ref} has no ceilings file; skipped", file=sys.stderr) - return [] - return list(check_baseline(recorded, json.loads(text))) - - -def check_baseline(recorded, base): - """A ceiling never rises, and no file gains an exemption, since `base`.""" - for rel, ceiling in sorted(recorded.items()): - if rel not in base: - yield f"{CEILINGS}: {rel} is a new exemption; split the file instead" - elif ceiling > base[rel]: - yield f"{CEILINGS}: {rel} rose from {base[rel]} to {ceiling}; split the file instead" - - -def ratchet(current, recorded): - """Lower every entry to the file's size now; add none, raise none.""" - return { - rel: min(ceiling, current[rel]) - for rel, ceiling in recorded.items() - if rel in current and current[rel] > DEFAULT_CEILING + def required(*args): + value = git_ok(root, *args) + if value is None: + raise ValueError( + "cannot read Git size baseline; fetch its complete history" + ) + return value + + if baseline is not None: + ref = git_ok( + root, + "rev-parse", + "--verify", + "-q", + "--end-of-options", + f"{baseline}^{{commit}}", + ) + if ref is None: + raise ValueError( + f"baseline {baseline} is not fetched; fetch its complete history" + ) + if ( + head is None + or required("rev-parse", "--is-shallow-repository").strip() != "false" + ): + raise ValueError( + "Git size baseline needs complete history; fetch --unshallow" + ) + required("merge-base", "--all", ref.strip(), head.strip()) + history = required( + "rev-list", "--parents", "--full-history", f"{ref.strip()}..{head.strip()}" + ) + else: + ref, history = head, "" + + paths = sorted(path for path, size in current.items() if size > DEFAULT_CEILING) + if not paths: + return {} + trees, counts = {}, {} + + def tree(commit): + if commit not in trees: + rows = required( + "--literal-pathspecs", + "ls-tree", + "-rz", + "--full-tree", + commit, + "--", + *paths, + ) + trees[commit] = { + path: meta.split()[2] + for row in rows.split("\0") + if row + for meta, path in [row.split("\t", 1)] + if path in current and path.endswith(".rs") + } + return trees[commit] + + def lines(oid): + if oid not in counts: + counts[oid] = len(required("cat-file", "blob", oid).splitlines()) + return counts[oid] + + base, committed = tree(ref.strip()), tree(head.strip()) + limits = { + path: max(DEFAULT_CEILING, min(lines(base[path]), lines(committed[path]))) + for path, size in current.items() + if size > DEFAULT_CEILING and path in base and path in committed } - - -def load_ceilings(root): - path = root / CEILINGS - return json.loads(path.read_text()) if path.exists() else {} + for row in history.splitlines(): + commit, *parents = row.split() + version = tree(commit) + for path in limits: + if path in version: + limits[path] = min( + limits[path], max(DEFAULT_CEILING, lines(version[path])) + ) + elif any(path in tree(parent) for parent in parents): + limits[path] = DEFAULT_CEILING + return limits def run(root, baseline=None): @@ -467,22 +504,17 @@ def run(root, baseline=None): findings.extend(check_rust(root, path)) for path in web_files(root): findings.extend(check_web(root, path)) - recorded = load_ceilings(root) - findings.extend(check_ceilings(sizes(root), recorded)) - if baseline: - findings.extend(baseline_findings(root, baseline, recorded)) + current = sizes(root) + try: + findings.extend(check_ceilings(current, git_ceilings(root, current, baseline))) + except ValueError as error: + findings.append(f"ceiling: {error}") return findings def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - modes = parser.add_mutually_exclusive_group() - modes.add_argument( - "--ratchet", - action="store_true", - help="lower recorded ceilings to current sizes", - ) - modes.add_argument( + parser.add_argument( "--workspace", action="store_true", help="audit Cargo workspace lint inheritance (requires Cargo and cached dependencies)", @@ -490,16 +522,10 @@ def main(): parser.add_argument( "--baseline", metavar="REF", - help="also refuse ceilings above those REF records; skipped when REF has none", + help="derive size limits from REF and all committed versions since it (needs complete Git history)", ) parser.add_argument("--root", type=Path, default=ROOT, help=argparse.SUPPRESS) args = parser.parse_args() - if args.ratchet: - kept = ratchet(sizes(args.root), load_ceilings(args.root)) - (args.root / CEILINGS).write_text( - json.dumps(dict(sorted(kept.items())), indent=2) + "\n" - ) - return 0 findings = ( list(check_lint_table(args.root)) + list(check_workspace(args.root)) if args.workspace diff --git a/scripts/fixture-server.sh b/scripts/fixture-server.sh new file mode 100755 index 000000000..ef2b0363a --- /dev/null +++ b/scripts/fixture-server.sh @@ -0,0 +1,188 @@ +#!/bin/sh + +# The two steps scripts/browser-check.sh needs around the in-process serving +# fixture (tests/unit/livekit/probe/serving.rs), kept here so they can be run +# against a stand-in executable without a browser. +# +# fixture-server.sh check PIN TEST +# PIN is the pinned test executable. Fails unless it lists TEST as +# exactly one ignored test, so the run cannot start some other test, or +# none, under a name that only looks right. It runs the pin with --list, +# which builds nothing. +# +# fixture-server.sh stop DIR PID WAIT_SECONDS +# Asks the fixture running as PID to stop by creating DIR/stop, then waits +# up to WAIT_SECONDS for it to finish its drain and settle and write its +# report. A fixture that has not exited by then is terminated (killed after +# FIXTURE_SERVER_TERM_GRACE_S, default 5), and that is a failure: its +# report cannot be trusted to be complete. The fixture also treats a TERM +# that reaches it as the same request, but only until it has begun to stop: +# after the stop file it ignores TERM, so the escalation is the kill. +# When DIR/stop cannot be written, the request is a TERM with the same +# wait, and that too is a failure. +# +# fixture-server.sh keep DIR OUT SERVER_LOG +# Keeps what the fixture left before the scratch directory goes: the +# server log beside the report as OUT.server.log (owner only), and DIR/out.json +# as OUT. When the fixture wrote no report, OUT holds one that says so +# and the log tail is printed. Fails when anything could not be kept, and +# when the kept report lacks the shape the judge reads or does not say +# the fixture ended well. + +set -u + +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +usage() +{ + echo "usage: ${0##*/} check PIN TEST | stop DIR PID WAIT_SECONDS | keep DIR OUT SERVER_LOG" >&2 + exit 2 +} + +# A child that has exited but not been waited for still answers kill -0, and the +# caller is the one that waits for it, so once the cheap probe says yes its +# state is asked for. +alive() +{ + kill -0 "$1" 2> /dev/null || return 1 + state=$(ps -o stat= -p "$1" 2> /dev/null) || return 1 + + # Some systems pad the column. + state=${state#"${state%%[! ]*}"} + case $state in + '' | [Zz]*) return 1 ;; + *) return 0 ;; + esac +} + +# Whether $1 exited within $2 seconds. +exited_within() +{ + waited=0 + while alive "$1" && [ "$waited" -lt "$2" ]; do + sleep 1 + waited=$((waited + 1)) + done + ! alive "$1" +} + +check() +{ + [ $# -eq 2 ] || usage + pin=$1 + test_name=$2 + case $test_name in + '' | *[[:space:]]*) + echo "fixture-server: the test name must be one exact name" >&2 + return 2 + ;; + esac + if [ ! -x "$pin" ]; then + echo "fixture-server: $pin is not an executable" >&2 + return 2 + fi + listed=$("$pin" --list --ignored 2> /dev/null) || { + echo "fixture-server: $pin would not list its ignored tests" >&2 + return 2 + } + count=$(printf '%s\n' "$listed" | grep -cxF "$test_name: test") + if [ "$count" != 1 ]; then + echo "fixture-server: $pin lists $count ignored tests named $test_name, not 1" >&2 + return 2 + fi +} + +stop() +{ + [ $# -eq 3 ] || usage + dir=$1 + pid=$2 + wait_seconds=$3 + + # Not 0 or a leading zero, since `kill 0` signals this script's whole + # process group rather than the fixture, and not 1, a container's init. + case $pid in + '' | *[!0-9]* | 0* | 1) usage ;; + esac + case $wait_seconds in + '' | *[!0-9]*) usage ;; + esac + grace=${FIXTURE_SERVER_TERM_GRACE_S:-5} + + # Not `:`, whose failed redirection would end this script, a special + # built-in's, and leave the caller waiting on a fixture nothing asked to + # stop. + if ! true > "$dir/stop"; then + echo "fixture-server: could not write the stop file; terminating the fixture" >&2 + + # While it serves, the fixture takes a TERM as the same request as the + # stop file, so the TERM gets the whole wait before the kill. + grace=$wait_seconds + elif exited_within "$pid" "$wait_seconds"; then + return 0 + else + echo "fixture-server: the fixture had not stopped after ${wait_seconds}s; terminating it" >&2 + fi + kill -TERM "$pid" 2> /dev/null || true + exited_within "$pid" "$grace" || kill -KILL "$pid" 2> /dev/null || true + return 1 +} + +keep() +{ + [ $# -eq 3 ] || usage + dir=$1 + out=$2 + server_log=$3 + kept_status=0 + + # Beside the report, because the scratch directory goes next and the + # server's own log is the only account of why a fixture misbehaved. Every + # write here refuses a file that is already there (set -C), so nothing kept + # can replace another run's file or the capture this one just wrote. + if ! (umask 077 && set -C && cat "$server_log" > "$out.server.log") 2> /dev/null; then + echo "fixture-server: could not keep the server log beside the report" >&2 + kept_status=1 + fi + if [ ! -f "$dir/out.json" ]; then + echo "fixture-server: the serving fixture left no report; the end of its log:" >&2 + tail -n 40 "$server_log" >&2 || true + + # A report that says so, so a reader of the report path never finds + # nothing where a verdict was owed. + (umask 077 && set -C && printf '%s\n' '{"outcome":"the serving fixture left no report","arming_failure":null,"capture":null}' \ + > "$out") 2> /dev/null \ + || echo "fixture-server: could not write that at $out; a file already there is not this run's report" >&2 + return 1 + fi + + # Owner only, like the log: it holds the interviewer's replies and the tool + # results. + if ! (umask 077 && set -C && cat "$dir/out.json" > "$out"); then + + # The scratch directory goes next, and this is the only copy. + echo "fixture-server: could not keep the serving fixture's report; it said:" >&2 + cat "$dir/out.json" >&2 + return 1 + fi + + # Accepted only if it has the shape the judge reads and says it ended well. + node "$here/long-interview-evidence.cjs" --check-fixture "$out" || kept_status=1 + return "$kept_status" +} + +case ${1:-} in + check) + shift + check "$@" + ;; + stop) + shift + stop "$@" + ;; + keep) + shift + keep "$@" + ;; + *) usage ;; +esac diff --git a/scripts/interview-run-budget.py b/scripts/interview-run-budget.py index e7936f811..2773883f0 100644 --- a/scripts/interview-run-budget.py +++ b/scripts/interview-run-budget.py @@ -1385,6 +1385,9 @@ def launch(args, command): os.chmod(socket_dir, 0o700) socket_path = os.path.join(socket_dir, "permit.sock") t0 = time.monotonic() + # When the reap begins, on the wall clock, for a worker that must be done + # before it (the serving fixture's own deadline). + reap_at = int(time.time() + reap_start) faults = [] service = PermitService( manifest, @@ -1405,6 +1408,7 @@ def launch(args, command): BROWSER_CHECK_BUDGET_SOCKET=socket_path, BROWSER_CHECK_BUDGET_TELEMETRY=telemetry_path, BROWSER_CHECK_BUDGET_STOP=stop_file, + BROWSER_CHECK_BUDGET_REAP_AT=str(reap_at), ) # A launcher that is told to stop must stop the worker with it: the worker # sits in its own session and would otherwise keep spending. @@ -1698,6 +1702,8 @@ def command_validate(args): print(digest) elif args.print_field == "run_id": print(manifest["run_id"]) + elif args.print_field == "cleanup_reserve_s": + print(manifest["cleanup_reserve_s"]) else: print(f"ok {manifest['run_id']} {digest}") return 0 @@ -1725,7 +1731,9 @@ def main(argv=None): validate = commands.add_parser("validate", help="check a manifest") validate.add_argument("--manifest", required=True) validate.add_argument("--verify-binaries", action="store_true") - validate.add_argument("--print", dest="print_field", choices=("sha", "run_id")) + validate.add_argument( + "--print", dest="print_field", choices=("sha", "run_id", "cleanup_reserve_s") + ) validate.add_argument( "--min-cleanup-reserve-s", type=int, default=MIN_CLEANUP_RESERVE_S ) diff --git a/scripts/live-sdk-fixture.sh b/scripts/live-sdk-fixture.sh index 7ee1da6a3..1182f66ac 100755 --- a/scripts/live-sdk-fixture.sh +++ b/scripts/live-sdk-fixture.sh @@ -126,9 +126,9 @@ launch() || die "the pinned binary changed since build" artifacts=$dir/artifacts-$letter [ ! -e "$artifacts" ] || die "$artifacts exists; a scenario runs once per directory" - name=$("$bin" --list --ignored 2> /dev/null \ - | sed -n "s/: test\$//p" | grep "^${live_test}\$") \ - || die "the pinned binary lists no live fixture" + name=$live_test + sh "$ROOT/scripts/fixture-server.sh" check "$bin" "$name" \ + || die "the pinned binary does not list $name exactly once" writer=$("$bin" --list 2> /dev/null \ | sed -n "s/: test\$//p" | grep "^${TEST_FILTER}live_write_manifest\$") \ || die "the pinned binary lists no manifest writer" diff --git a/scripts/long-interview-driver.cjs b/scripts/long-interview-driver.cjs index 5ec6a3753..5458afc51 100644 --- a/scripts/long-interview-driver.cjs +++ b/scripts/long-interview-driver.cjs @@ -30,6 +30,7 @@ const path = require("path"); const sched = require("./long-interview-schedule.cjs"); const timing_ = require("./long-interview-timing.cjs"); const { BudgetDenied } = require("./run-budget-guard.cjs"); +const { summarizeSwapAudio } = require("./soak-judge.cjs"); const CLIP_DIR_ENV = "BROWSER_CHECK_LONG_CLIP_DIR"; const TONE_FIXTURE_ENV = "BROWSER_CHECK_LONG_TONE_FIXTURE"; @@ -435,9 +436,40 @@ function judgeDispatch(schedule, playback, run) { return errors; } +// A swap row is a verdict and at most four small samples, so the rows cost a +// fixed amount per swap; the cap keeps a runaway page from growing the report, +// and what it cuts is counted. +const DECODED_ROWS_KEPT = 512; + +/// The judged swap rows with the samples behind them, for a reader that has +/// only the report. By default the capture file sits beside the artifact +/// directory, not in it, so these do not draw on the archive bound; a capture +/// placed inside that directory counts against it like any other file. +function decodedSignalRows(audio) { + const rows = (Array.isArray(audio?.episodes) ? audio.episodes : []).filter( + (row) => row && typeof row === "object", + ); + return { + decoded_signal_rows: rows + .slice(0, DECODED_ROWS_KEPT) + .map(({ swap, episode, status, reasons, ...detail }) => ({ + swap, + episode, + status, + reasons, + samples: detail.samples ?? [], + samples_dropped: detail.samples_dropped ?? 0, + ...(detail.covered_by ? { covered_by: detail.covered_by } : {}), + })), + decoded_signal_rows_dropped: Math.max(0, rows.length - DECODED_ROWS_KEPT), + }; +} + /// The run's verdict, its failure classes and the exit code the process /// should leave with. `observation` is judgeObservation's result and -/// `episodes` the parsed replacement episodes of the room. +/// `episodes` the parsed replacement episodes of the room. `audio` is +/// judgeSwapAudio's result and `replies` the reply judge's, when the page ran +/// the decoded-audio probe. function finalizeLongRun(o) { const { plan, schedule, playback, run, clipSource, observation } = o; const timing = timingForPlan(plan); @@ -517,6 +549,7 @@ function finalizeLongRun(o) { ? "failed" : resolved.status; + const signal = summarizeSwapAudio(o.audio, o.replies); const speechClips = clipSource.kind === CLIP_SOURCE.speech; const covered = !run.aborted && dispatchErrors.length === 0 && run.uncovered.length === 0; @@ -538,7 +571,11 @@ function finalizeLongRun(o) { assessment_scope: "structural", qualification: "not_evaluated", reply_content: "not_assessed", - decoded_signal: "not_assessed", + decoded_signal: signal.status, + ...decodedSignalRows(o.audio), + ...(signal.status === "failed" + ? { decoded_signal_reasons: signal.reasons } + : {}), end_mode: plan.endMode, duration_min: plan.durationMin, observation_target_s: plan.observationTargetS, @@ -558,6 +595,12 @@ function finalizeLongRun(o) { cleanup_assessed: cleanupAssessed, end_permit_granted: run.endPermitGranted, end_clicked: run.endClicked, + // Where the schedule's clock began, so a reader after teardown can put + // the interview clock's readings on it. + schedule_origin_unix_ms: run.joinedAtMs, + // The surface the schedule was generated for, which the plan alone does + // not name and a reader rebuilding the schedule needs. + schedule_mode: o.mode, schedule_start_delay_s: (run.joinedAtMs - run.interviewStartMs) / 1000, exposure_s: stamps.exposureS, timestamps: stamps.timestamps, @@ -575,7 +618,11 @@ function finalizeLongRun(o) { failure_classes: resolved.artifacts.map((a) => a.class), not_exercised: [ "reply content: checkpoint answers are not judged by this driver", - "decoded signal: native capture records do not verify a post-swap decoded waveform", + ...(signal.status === "not_assessed" + ? [ + "decoded signal: native capture records do not verify a post-swap decoded waveform", + ] + : []), ...(cleanupAssessed ? [] : ["cleanup: nothing active after teardown was checked by the worker"]), diff --git a/scripts/long-interview-evidence.cjs b/scripts/long-interview-evidence.cjs new file mode 100644 index 000000000..94f9b0e06 --- /dev/null +++ b/scripts/long-interview-evidence.cjs @@ -0,0 +1,600 @@ +"use strict"; + +// Turns the fixture's acquisition report into the evidence `judgeCheckpoint` +// reads, and says so plainly when it cannot. The report holds ordered facts +// only (receipts, batches, reply segments and turn endings, each with the +// socket it came from and how many of the others preceded it); this module +// is where they become a claim, and a claim is made only when the facts leave +// one reading. Anything else is `not_assessed` with the reason, never a pass +// and never a guess at a failure. + +const fs = require("node:fs"); + +const sched = require("./long-interview-schedule.cjs"); + +const { generateSchedule, judgeCheckpoint } = sched; + +// A reading whose sampling uncertainty exceeds this cannot place an event +// against a window edge. +const MAX_CLOCK_UNCERTAINTY_MS = 1000; + +// The lists a capture holds, which the judge and the acceptance check both read. +const CAPTURE_LISTS = ["receipts", "batches", "replies", "boundaries"]; + +const isCount = (value) => Number.isSafeInteger(value) && value >= 0; + +/// Why the report cannot be read at all, or an empty list. +function reportProblems(report) { + const problems = []; + if (!report || typeof report !== "object") return ["no report"]; + if (report.arming_failure != null) { + problems.push(`the observer failed: ${report.arming_failure}`); + } + const capture = report.capture; + if (!capture || typeof capture !== "object") { + problems.push("no capture was admitted"); + return problems; + } + if (capture.failure != null) { + problems.push(`the capture latched ${capture.failure}`); + } + for (const name of CAPTURE_LISTS) { + if (!Array.isArray(capture[name])) problems.push(`${name} is missing`); + } + if (problems.length) return problems; + // A row the judge walks is readable when it is there, carries its + // identity and passes its own check; one that is not makes the capture one + // it cannot judge, never an exception halfway through the merge. + const readable = (rows, ok) => + rows.every((row) => row && row.identity && ok(row)); + const counted = (rows, fields) => + readable(rows, (row) => fields.every((field) => isCount(row[field]))); + if ( + !counted(capture.replies, [ + "receipts_before", + "batches_before", + "receipts_last", + "batches_last", + ]) + ) { + problems.push("a reply has no positions"); + } + if ( + !counted(capture.boundaries, [ + "receipts_before", + "batches_before", + "replies_before", + ]) + ) { + problems.push("a boundary has no positions"); + } + const hasId = (row) => row && typeof row.id === "string"; + if ( + !readable( + capture.batches, + (batch) => Array.isArray(batch.calls) && batch.calls.every(hasId), + ) + ) { + problems.push("a batch has no calls"); + } + if (!readable(capture.receipts, hasId)) { + problems.push("a receipt has no id"); + } + return problems; +} + +/// A reading of the interview clock as seconds since the schedule began with +/// its own sampling uncertainty, or null when it cannot place an event. +function readClock(identity, scheduleStartUnixMs, maxUncertaintyMs) { + const clock = identity?.clock; + if (!clock || !Number.isFinite(clock.emitted_unix_ms)) return null; + const uncertainty = clock.clock_uncertainty_ms; + if (!Number.isFinite(uncertainty) || uncertainty > maxUncertaintyMs) { + return null; + } + return { + s: (clock.emitted_unix_ms - scheduleStartUnixMs) / 1000, + u: uncertainty / 1000, + }; +} + +/// Where a reading sits against a threshold: -1 before it, 1 after it, 0 when +/// its uncertainty reaches across, which is no answer. +function sideOf(reading, threshold) { + if (reading.s + reading.u < threshold) return -1; + return reading.s - reading.u > threshold ? 1 : 0; +} + +/// Every string a tool result carries, one per line. The fact matchers look +/// for tokens in text, and JSON escaping would hide a token that spans a +/// newline or holds a quote. +function resultText(value) { + if (typeof value === "string") return value; + if (Array.isArray(value)) return value.map(resultText).join("\n"); + if (value && typeof value === "object") { + return Object.values(value).map(resultText).join("\n"); + } + return ""; +} + +/// Where the turn reply `index` belongs to began: the last ending that the +/// loop took on its socket and that arrived before the reply did. An ending of +/// another generation (swallowed, or from another socket) that falls inside the +/// turn leaves ownership of everything in it unknown. +function turnStart(capture, index) { + const socket = capture.replies[index].identity.socket; + let receipts = 0; + let replies = 0; + const strays = []; + for (const boundary of capture.boundaries) { + if (boundary.replies_before > index) continue; + if (boundary.accepted && boundary.identity.socket === socket) { + receipts = Math.max(receipts, boundary.receipts_before); + replies = Math.max(replies, boundary.replies_before); + } else { + strays.push(boundary); + } + } + const stray = strays.some((boundary) => boundary.receipts_before > receipts); + return { receipts, replies, stray }; +} + +/// The segments of `socket` with index in [from, to). +const segmentsOf = (capture, socket, from, to) => + capture.replies + .slice(from, to) + .filter((reply) => reply.identity.socket === socket); + +/// How reply `index` ended: "complete" or "interrupted" when the first +/// provider ending after it, on its socket and in its generation, was taken +/// and nothing else sealed it first, otherwise "unknown" with the reason. +function endingOf(capture, index) { + const reply = capture.replies[index]; + const socket = reply.identity.socket; + for (const boundary of capture.boundaries) { + if (boundary.identity.socket !== socket) continue; + if (boundary.replies_before <= index) continue; + if ( + segmentsOf(capture, socket, index + 1, boundary.replies_before).length + ) { + return { + kind: "unknown", + why: "a cut or rollover sealed the reply before any provider ending", + }; + } + if (!boundary.accepted) { + return { + kind: "unknown", + why: "its ending was swallowed by the interruption guard", + }; + } + if ( + reply.identity.trace_turn == null || + boundary.identity.trace_turn !== reply.identity.trace_turn + ) { + return { + kind: "unknown", + why: "the ending belongs to another generation than the reply", + }; + } + return { + kind: boundary.ending === "Interrupted" ? "interrupted" : "complete", + }; + } + return { kind: "unknown", why: "no provider ending followed the reply" }; +} + +/// The tool calls reply `index` may cite, or the reason the facts do not say. +/// Calls count only when they were received on the reply's own socket and +/// generation after the previous ending, executed before the reply began, and +/// not received while output was being discarded. +function citedCalls(capture, index) { + const reply = capture.replies[index]; + const socket = reply.identity.socket; + if ( + reply.receipts_before !== reply.receipts_last || + reply.batches_before !== reply.batches_last + ) { + return { why: "the reply continued across a tool call" }; + } + const turn = turnStart(capture, index); + if (turn.stray) { + return { why: "an ending of another generation lies inside the turn" }; + } + if (segmentsOf(capture, socket, turn.replies, index).length) { + return { why: "an earlier segment of the same turn may own the calls" }; + } + const window = capture.receipts.slice(turn.receipts, reply.receipts_before); + if (window.some((receipt) => receipt.identity.socket !== socket)) { + return { + why: "a socket replacement lies between the question and the reply", + }; + } + const calls = []; + for (const receipt of window) { + if (receipt.discarded) continue; + if (receipt.batch == null || receipt.batch >= reply.batches_before) { + continue; + } + if ( + reply.identity.trace_turn == null || + receipt.identity.trace_turn !== reply.identity.trace_turn + ) { + return { why: "a call belongs to another generation than the reply" }; + } + calls.push(receipt); + } + return { calls }; +} + +/// The evidence for one cited call, or the reason it cannot be read. +function toolCall(capture, receipt, cp, key, options) { + const batch = capture.batches.find((b) => b.sequence === receipt.batch); + const call = batch?.calls.find((candidate) => candidate.id === receipt.id); + if (!call) return { why: `call ${receipt.id} has no executed batch` }; + if (batch.local_send_completed == null) { + return { why: `the send of call ${receipt.id} was not observed` }; + } + const reading = readClock( + batch.identity, + options.scheduleStartUnixMs, + options.maxClockUncertaintyMs, + ); + if (!reading) { + return { why: `call ${receipt.id} has no usable clock reading` }; + } + const earliest = Math.max( + cp.at_s, + ...(key.tools ?? [key.tool]) + .filter((need) => need.name === call.name) + .map((need) => need.not_before_s), + ); + const latest = cp.stimulus_end_s + cp.response_window_s; + if (sideOf(reading, earliest) === 0 || sideOf(reading, latest) === 0) { + return { + why: `call ${receipt.id} lies within the clock uncertainty of a limit`, + }; + } + const failed = + call.result && typeof call.result === "object" && "error" in call.result; + return { + call: { + call_id: call.id, + name: call.name, + at_s: reading.s, + ok: !failed && batch.local_send_completed === true, + result_text: resultText(call.result), + }, + }; +} + +const notAssessed = (checkpointId, reasons) => ({ + checkpoint_id: checkpointId, + verdict: "not_assessed", + reasons, +}); + +const judge = (bundle, evidence) => ({ + checkpoint_id: evidence.checkpoint_id, + ...judgeCheckpoint(bundle, evidence), +}); + +/// The one reply that started inside the checkpoint's window as +/// {index, at}, null when none did, or a reason when the clock cannot say. +function replyInWindow(capture, from, to, settings) { + const inside = []; + for (const [index, reply] of capture.replies.entries()) { + const reading = readClock( + reply.identity, + settings.scheduleStartUnixMs, + settings.maxClockUncertaintyMs, + ); + if (!reading) return { why: "a reply has no usable clock reading" }; + const [start, end] = [sideOf(reading, from), sideOf(reading, to)]; + if (start === 0 || end === 0) { + return { + why: "a reply lies within the clock uncertainty of the window edge", + }; + } + if (start === 1 && end === -1) inside.push({ index, at: reading.s }); + } + if (inside.length > 1) { + return { why: `${inside.length} replies started in the window` }; + } + return { reply: inside[0] ?? null }; +} + +/// One checkpoint's verdict: "pass", "fail" or "not_assessed", with reasons. +function assessCheckpoint(bundle, report, checkpointId, options) { + const settings = { + maxClockUncertaintyMs: MAX_CLOCK_UNCERTAINTY_MS, + ...options, + }; + const problems = reportProblems(report); + if (problems.length) return notAssessed(checkpointId, problems); + if (!Number.isFinite(settings.scheduleStartUnixMs)) { + return notAssessed(checkpointId, ["the schedule origin is unknown"]); + } + const cp = bundle.schedule.checkpoints.find( + (c) => c.checkpoint_id === checkpointId, + ); + const key = bundle.answer_key.checkpoints.find( + (c) => c.checkpoint_id === checkpointId, + ); + if (!cp || !key) return notAssessed(checkpointId, ["unknown checkpoint"]); + const capture = report.capture; + + // The window opens at the question's estimated end, from its word count; a + // recording that finished speaking sooner can be answered before that. Such + // a reply cannot be told from one that talked over the question, so neither + // is held against the reply as a missing answer. + const early = replyInWindow(capture, cp.at_s, cp.stimulus_end_s, settings); + if (early.why) return notAssessed(checkpointId, [early.why]); + if (early.reply) { + return notAssessed(checkpointId, [ + "a reply began before the question's estimated end", + ]); + } + + const found = replyInWindow( + capture, + cp.stimulus_end_s, + cp.stimulus_end_s + cp.response_window_s, + settings, + ); + if (found.why) return notAssessed(checkpointId, [found.why]); + if (!found.reply) return judge(bundle, { checkpoint_id: checkpointId }); + + const { index, at } = found.reply; + const reply = capture.replies[index]; + if (reply.revised) { + return notAssessed(checkpointId, [ + "the reply's text was rewritten while it was shown", + ]); + } + if (!reply.final_segment) { + return notAssessed(checkpointId, ["the reply was never sealed"]); + } + const ending = endingOf(capture, index); + if (ending.kind === "unknown") return notAssessed(checkpointId, [ending.why]); + const cited = citedCalls(capture, index); + if (!cited.calls) return notAssessed(checkpointId, [cited.why]); + const calls = []; + for (const receipt of cited.calls) { + const read = toolCall(capture, receipt, cp, key, settings); + if (!read.call) return notAssessed(checkpointId, [read.why]); + calls.push(read.call); + } + return judge(bundle, { + checkpoint_id: checkpointId, + reply: { + text: reply.text, + started_at_s: at, + completed: ending.kind === "complete", + interrupted: ending.kind === "interrupted", + tool_call_ids: calls.map((call) => call.call_id), + }, + tool_calls: calls, + }); +} + +/// "verified" only when every checkpoint was assessed and passed; "failed" when +/// any assessed one failed; otherwise "not_assessed". +function summarize(checkpoints) { + const failed = checkpoints.some((c) => c.verdict === "fail"); + const verified = + checkpoints.length > 0 && checkpoints.every((c) => c.verdict === "pass"); + return { + status: failed ? "failed" : verified ? "verified" : "not_assessed", + checkpoints, + uncovered: checkpoints + .filter((c) => c.verdict === "not_assessed") + .map((c) => c.checkpoint_id), + }; +} + +function judgeInterview(bundle, report, options) { + return summarize( + bundle.schedule.checkpoints.map((cp) => + assessCheckpoint(bundle, report, cp.checkpoint_id, options), + ), + ); +} + +const MAX_REASONS = 20; + +/// Why a fixture report that is there cannot vouch for the run, read the same +/// by the keep check and by the judge: unreadable, or ended other than well. +function fixtureRefusal(report) { + if (report?.unreadable) { + return `the serving fixture's report could not be read: ${report.unreadable}`; + } + if (typeof report?.outcome === "string" && report.outcome !== "ok") { + return `the serving fixture ended with ${report.outcome}`; + } + return null; +} + +/// Why the serving fixture's report is not one to accept, from its shape and +/// the outcome it names alone. A failure the observer or the capture reports +/// about itself is not a reason here: the judge says not_assessed for it, and +/// the run keeps its own result. A report that holds neither a capture nor an +/// observer failure is malformed, and refuses the run. +function fixtureProblems(report) { + const refusal = fixtureRefusal(report); + if (refusal) return [refusal]; + if (!report || typeof report !== "object" || Array.isArray(report)) { + return ["the serving fixture's report is not an object"]; + } + if (typeof report.outcome !== "string") { + return ["the serving fixture's report names no outcome"]; + } + const problems = []; + for (const key of ["arming_failure", "bound"]) { + if (!(key in report)) problems.push(`the report has no ${key}`); + } + const capture = report.capture; + // An observer that failed admits no capture and says so itself; that is the + // judge's to name. A report with neither is not one an interview left. + if (capture == null && report.arming_failure != null) return problems; + if (!capture || typeof capture !== "object") { + return [...problems, "the report holds no capture"]; + } + if (!("failure" in capture)) + problems.push("the capture has no failure field"); + for (const name of CAPTURE_LISTS) { + if (!Array.isArray(capture[name])) + problems.push(`the capture has no ${name} list`); + } + return problems; +} + +/// Judges a capture against its fixture report, or says in every +/// checkpoint's reason why it could not. +function judgeCapture(capture, fixture) { + const { report } = capture; + let bundle; + try { + bundle = generateSchedule(report.schedule_mode, { + durationMin: capture.plan.durationMin, + observationTargetS: capture.plan.observationTargetS, + }); + } catch (error) { + return summarize([ + notAssessed("interview", [ + `the schedule cannot be rebuilt: ${error.message}`, + ]), + ]); + } + const refuse = (reason) => + summarize( + bundle.schedule.checkpoints.map((cp) => + notAssessed(cp.checkpoint_id, [reason]), + ), + ); + if (report.exit_code !== 0 || report.status !== "complete") { + return refuse("the run did not complete cleanly"); + } + // A tone fixture asked the model nothing, so there is no content to judge. + if (report.validation_fixture !== false) { + return refuse("the run played tone fixtures, not spoken questions"); + } + if (report.anchor?.source !== "server_clock") { + return refuse("the interview clock was not anchored to the server"); + } + if (!fixture) return refuse("the serving fixture left no report"); + const refusal = + fixtureRefusal(fixture) ?? + (fixture.outcome === "ok" + ? null + : "the serving fixture's report names no outcome"); + if (refusal) return refuse(refusal); + if (bundle.schedule.schedule_sha256 !== report.schedule_sha256) { + return refuse("the schedule that ran is not the one rebuilt from the plan"); + } + return judgeInterview(bundle, fixture, { + scheduleStartUnixMs: report.schedule_origin_unix_ms, + }); +} + +/// The three report fields a verdict sets. The reasons are always present, +/// empty for a verified interview, so that rewriting a report with a better +/// verdict does not keep an earlier one's. +function verdictFields(verdict) { + const counts = { pass: 0, fail: 0, not_assessed: 0 }; + for (const checkpoint of verdict.checkpoints) counts[checkpoint.verdict] += 1; + const reasons = verdict.checkpoints + .filter((checkpoint) => checkpoint.verdict !== "pass") + .map( + (checkpoint) => `${checkpoint.checkpoint_id}: ${checkpoint.reasons[0]}`, + ); + return { + reply_content: verdict.status, + reply_content_checkpoints: counts, + reply_content_reasons: reasons.slice(0, MAX_REASONS), + }; +} + +/// The capture file with its long-run report's `reply_content` filled in from +/// the serving fixture's report. Nothing else in the report moves: the exit +/// code and `qualification` were settled before this ran, and a verdict here +/// does not reopen them. +function mergeVerdict(capture, fixture) { + const report = capture?.report; + if (capture?.flow !== "long-interview" || !report) return null; + return { + ...capture, + report: { ...report, ...verdictFields(judgeCapture(capture, fixture)) }, + }; +} + +/// A JSON file, null when it is not there, or why it cannot be read: a report +/// that is there but unreadable is a different failure from one the fixture +/// never wrote, and the verdict says which. +const readJson = (file) => { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + // The kind of failure and no more: a parse error's message quotes the + // text around it, which can be the interviewer's reply. + if (error.code === "ENOENT") return null; + return { unreadable: error.code ?? error.name }; + } +}; + +/// `--capture FILE --fixture FILE`: rewrites the capture with the verdict. +/// Exits 0 when it merged or had nothing to judge, 2 when it could not read +/// the capture or write it back; the caller treats either as no change to the +/// run's own result. +function main(argv, { out = console.log, err = console.error } = {}) { + if (argv[0] === "--check-fixture") { + const problems = fixtureProblems(readJson(argv[1] ?? "")); + problems.forEach((problem) => err(`fixture report: ${problem}`)); + return problems.length ? 1 : 0; + } + const flag = (name) => { + const at = argv.indexOf(name); + return at < 0 ? undefined : argv[at + 1]; + }; + const [captureFile, fixtureFile] = [flag("--capture"), flag("--fixture")]; + if (!captureFile || !fixtureFile) { + err("usage: long-interview-evidence.cjs --capture FILE --fixture FILE"); + return 2; + } + const capture = readJson(captureFile); + if (!capture || capture.unreadable) { + const why = capture?.unreadable ? `: ${capture.unreadable}` : ""; + err(`cannot read the capture ${captureFile}${why}`); + return 2; + } + const merged = mergeVerdict(capture, readJson(fixtureFile)); + if (!merged) return 0; + const partial = `${captureFile}.partial`; + try { + fs.writeFileSync(partial, `${JSON.stringify(merged, null, 2)}\n`); + fs.renameSync(partial, captureFile); + } catch (error) { + err(`cannot write ${captureFile}: ${error.message}`); + return 2; + } + const { reply_content: status, reply_content_checkpoints: counts } = + merged.report; + out( + `long-interview reply_content=${status} pass=${counts.pass} fail=${counts.fail} not_assessed=${counts.not_assessed}`, + ); + return 0; +} + +if (require.main === module) process.exit(main(process.argv.slice(2))); + +module.exports = { + MAX_CLOCK_UNCERTAINTY_MS, + assessCheckpoint, + fixtureProblems, + judgeInterview, + main, + mergeVerdict, + reportProblems, + summarize, + verdictFields, +}; diff --git a/scripts/long-interview-page.cjs b/scripts/long-interview-page.cjs index e547d194c..1ccc6281e 100644 --- a/scripts/long-interview-page.cjs +++ b/scripts/long-interview-page.cjs @@ -172,15 +172,16 @@ function pageController(page, { selectLanguage, mode = "coding" }) { } /// Draws one closed shape per entry of the revision's strokes, in a row of its -/// own, so each revision leaves a board that differs from the last. The board -/// has no text tool, so the marker cannot be written on it: a revision is told -/// apart by position and shape count, not by a legible marker. +/// own, so each revision leaves a board that differs from the last. The marker +/// is not drawn: read_board gives the model a stroke count and an image, never +/// text, so a revision is told apart by the board's running stroke count, which +/// is four more after each revision (the judge reads it in the tool's text). async function drawBoardRevision(page, fixture, index) { const bounds = await page.locator("#board").boundingBox(); if (!bounds) throw new Error("the visible board had no drawing bounds"); const row = bounds.y + 30 + ((index - 1) % 6) * 70; let column = bounds.x + 30; - for (let i = 0; i < fixture.strokes.length; i++) { + for (let i = 0; i < fixture.strokes; i++) { await page.mouse.move(column, row); await page.mouse.down(); await page.mouse.move(column + 80, row, { steps: 4 }); diff --git a/scripts/long-interview-schedule.cjs b/scripts/long-interview-schedule.cjs index 2b2440172..171510271 100644 --- a/scripts/long-interview-schedule.cjs +++ b/scripts/long-interview-schedule.cjs @@ -29,7 +29,7 @@ const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); -const SCHEDULE_VERSION = 1; +const SCHEDULE_VERSION = 3; const CYCLE_S = 300; /// The end and report reservation; no stimulus is scheduled inside it. const STOP_BEFORE_END_S = 120; @@ -124,12 +124,21 @@ const AUTHORED = Object.freeze({ sentence: "My constraints are at most ten thousand numbers, exactly one valid pair, and never reuse the same element.", review: { - id: "constraints-v1", + id: "constraints-v2", reviewer: "schedule author", status: "reviewed", }, concepts: [ - ["ten thousand", "10000", "10,000", "10^4"], + [ + "ten thousand", + "ten-thousand", + "10 thousand", + "10k", + "10000", + "10,000", + "10 000", + "10^4", + ], ["exactly one", "one valid pair", "single valid pair", "one pair"], [ "same element", @@ -142,7 +151,7 @@ const AUTHORED = Object.freeze({ }, duplicate_case: { review: { - id: "duplicate-v1", + id: "duplicate-v2", reviewer: "schedule author", status: "reviewed", }, @@ -150,8 +159,15 @@ const AUTHORED = Object.freeze({ [ "different indices", "distinct indices", + "different index", + "distinct index", + "different position", + "distinct position", + "different positions", + "distinct positions", "two different", "two distinct", + "two separate", "separate indices", "not the same index", ], @@ -176,11 +192,21 @@ const AUTHORED = Object.freeze({ prime: "Could you compare the map approach with checking every pair?", barge: "Let me clarify the example first. I am still explaining my reasoning.", - state_question: - "What are the original constraints, our current stage, and the latest {thing}?", - final_question: - "Before we wrap up, what is the question I still have open, and what is the latest {thing}?", - things: { coding: "source revision", whiteboard: "diagram revision" }, + // What each question asks for is all its reply is held to. A board's + // revision cannot be asked for: read_board gives the model a count and an + // image, never text, and a marker drawn in the image is not something to + // judge a reply on. The whiteboard questions do not ask for a revision; it is + // judged in what read_board reports, the stroke count. + state_question: { + coding: + "What are the original constraints, our current stage, and the latest source revision?", + whiteboard: "What are the original constraints, and our current stage?", + }, + final_question: { + coding: + "Before we wrap up, what is the question I still have open, and what is the latest source revision?", + whiteboard: "Before we wrap up, what is the question I still have open?", + }, }); const canonicalize = (value) => { @@ -198,7 +224,7 @@ const hashOf = (value) => sha256(canonicalize(value)); /// Pinned. Edit the catalogue above, bump SCHEDULE_VERSION, and replace this. const AUTHORED_SHA256 = - "323220654cdf1c5ce18e4bd3a739c9b45557f891e94536a92720035c1deceb1f"; + "55bfd3f2315601dd598a42bb9ceb0c9e3f0d0acabcecb82d2aceeb7a20f3468b"; function checkAuthoredPin(expected = AUTHORED_SHA256) { const actual = hashOf(AUTHORED); @@ -226,6 +252,9 @@ function nonceFor(mode, checkpointId) { return `${first[digest[0] % first.length]} ${second[digest[1] % second.length]}`; } +/// Shapes the page controller draws per board revision, one stroke each. +const STROKES_PER_REVISION = 4; + function markerFor(mode, index) { return `rev${pad2(index)}-${sha256(`${SCHEDULE_VERSION}|${mode}|rev|${index}`).slice(0, 6)}`; } @@ -251,11 +280,12 @@ function revisionFixture(mode, index) { ].join("\n"); return { marker, kind: "editor", language: "javascript", source }; } + // The board keeps every revision's strokes, and its reading is the count. return { kind: "board", marker, - label: `complement map example ${marker}`, - strokes: ["box nums", "arrow need", "box seen", `label ${marker}`], + strokes: STROKES_PER_REVISION, + strokes_total: STROKES_PER_REVISION * index, }; } @@ -324,6 +354,7 @@ const rubricMatch = (spec) => ({ /// The schedule and its answer key for `mode`, derived from `plan` /// ({durationMin, observationTargetS}, as long-interview-plan.cjs validated). function generateSchedule(mode, plan) { + const coding = mode === "coding"; checkAuthoredPin(); if (!TOOL_BY_MODE[mode]) { throw new Error(`mode must be coding or whiteboard, not ${mode}`); @@ -334,7 +365,6 @@ function generateSchedule(mode, plan) { observationTargetS, ); const tool = TOOL_BY_MODE[mode]; - const thing = AUTHORED.things[mode]; const recalls = recallPlan(cycles); const steps = [{ action_id: JOIN_ACTION_ID, at_s: 0, kind: "join" }]; @@ -403,12 +433,22 @@ function generateSchedule(mode, plan) { `We are in the ${phase} stage.`, { mode: "folded", tokens: [phase] }, ); - const revisionFact = factOf( - `revision_${c}`, - base + 110, - `The latest ${thing} is ${revision.marker}.`, - { mode: "exact", tokens: [revision.marker] }, - ); + // A coding revision is a marker a reply states; a board's is the count + // read_board reports, judged in what the tool read and never in a reply, so + // its id is not revision_* (the prefix a reply may not state a stale one of). + const revisionFact = coding + ? factOf( + `revision_${c}`, + base + 110, + `The latest source revision is ${revision.marker}.`, + { mode: "exact", tokens: [revision.marker] }, + ) + : factOf( + `board_${c}`, + base + 110, + `The board holds ${revision.strokes_total} strokes.`, + { mode: "exact", tokens: [`${revision.strokes_total} strokes`] }, + ); facts.push(phaseFact, revisionFact); const planParts = [`Stage ${cycle}. We are now in the ${phase} stage.`]; @@ -506,7 +546,7 @@ function generateSchedule(mode, plan) { addClip(id("barge"), "barge", [AUTHORED.barge], base + 175); const recallId = recalls.get(cycle); - const stateQuestion = AUTHORED.state_question.replace("{thing}", thing); + const stateQuestion = AUTHORED.state_question[mode]; const recallSeed = recallId ? AUTHORED.seeds.find((s) => s.id === recallId) : null; @@ -536,6 +576,8 @@ function generateSchedule(mode, plan) { response_window_s: nextInput(inspectEnd, base + 175), clip_id: id("ask_inspect"), cycle, + // What was seeded for this question. The key below asks less of the + // reply: the revision is judged in what the tool read. source_fact_ids: ["duplicate_case", revisionFact.id], required_tool: { name: tool, call_id_required: true }, required_tools: [tool], @@ -543,7 +585,9 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: id("cp_inspect"), nonce: inspectNonce, - element_fact_ids: ["duplicate_case", revisionFact.id], + // Only what the question asks. The revision is checked in what the tool + // read, below, and a reply may state it without being faulted. + element_fact_ids: ["duplicate_case"], tool: { name: tool, result_fact_ids: [revisionFact.id], @@ -596,7 +640,9 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: id("cp_state"), nonce: stateNonce, - element_fact_ids: stateFacts, + element_fact_ids: stateFacts.filter( + (id) => coding || id !== revisionFact.id, + ), tools, }); } @@ -610,15 +656,14 @@ function generateSchedule(mode, plan) { ); facts.push(pendingFact); const finalNonce = nonceFor(mode, "cp_final"); - const finalText = `${AUTHORED.final_question.replace("{thing}", thing)} Please also say the code words ${finalNonce}.`; + const finalText = `${AUTHORED.final_question[mode]} Please also say the code words ${finalNonce}.`; const finalSpeech = addClip( "final_checkpoint", "checkpoint", [finalText], finalAt, ); - const lastRevisionFact = `revision_${pad2(cycles)}`; - const finalPhaseFact = `phase_${pad2(cycles)}`; + const lastRevisionFact = `${coding ? "revision" : "board"}_${pad2(cycles)}`; steps.push({ action_id: "final_checkpoint", at_s: finalAt, @@ -626,7 +671,11 @@ function generateSchedule(mode, plan) { clip_id: "final_checkpoint", checkpoint_id: "cp_final", }); - const finalFacts = [pendingFact.id, finalPhaseFact, lastRevisionFact]; + // What the final question asks for, and no more: the open question and, for + // a coding interview, the latest revision; never the stage. A board's + // revision is judged in what the tool read, so it is a source, not an element. + const finalFacts = [pendingFact.id, lastRevisionFact]; + const finalElements = coding ? finalFacts : [pendingFact.id]; checkpoints.push({ checkpoint_id: "cp_final", kind: "final", @@ -642,7 +691,7 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: "cp_final", nonce: finalNonce, - element_fact_ids: finalFacts, + element_fact_ids: finalElements, tools: [ { name: tool, @@ -909,13 +958,14 @@ function judgeCheckpoint(bundle, evidence, options = {}) { const misses = matchFact(fact.match, reply.text); if (misses.length) fail(`reply lacks ${id}: ${misses[0]}`); } - // A reply that names an earlier revision is not about the current one. + // A reply that names an earlier revision is not about the current one. The + // current one is whichever the checkpoint asks for or its tool must show. + const current = new Set([ + ...key.element_fact_ids, + ...(key.tools ?? [key.tool]).flatMap((t) => t.result_fact_ids), + ]); for (const fact of answerKey.facts) { - if ( - !/^revision_\d+$/.test(fact.id) || - key.element_fact_ids.includes(fact.id) - ) - continue; + if (!/^revision_\d+$/.test(fact.id) || current.has(fact.id)) continue; if (factIntegrity(fact) && matchFact(fact.match, reply.text).length === 0) { fail(`reply states the stale revision ${fact.id}`); } diff --git a/scripts/rust-style-ceilings.json b/scripts/rust-style-ceilings.json deleted file mode 100644 index ec7689e5f..000000000 --- a/scripts/rust-style-ceilings.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "src/agent.rs": 2004, - "src/agent/evidence.rs": 2135, - "src/agent/problem_variants.rs": 1819, - "src/agent/report.rs": 1730, - "src/gemini.rs": 1870, - "src/livekit.rs": 1935, - "src/livekit/session.rs": 2946, - "src/livekit/turn.rs": 2333, - "tests/agent/framework.rs": 4640, - "tests/agent/prompts.rs": 2380, - "tests/agent/runtime.rs": 2605, - "tests/cli.rs": 3183, - "tests/interview_behavior.rs": 1706, - "tests/recording.rs": 3864, - "tests/unit/accounts/schema.migration_tests.rs": 1505, - "tests/unit/agent/evidence.rs": 3761, - "tests/unit/gemini.report_eval.rs": 2691, - "tests/unit/gemini.rs": 5001, - "tests/unit/livekit.rs": 6946, - "tests/unit/livekit/cost.rs": 2179, - "tests/unit/livekit/report.rs": 2821, - "tests/unit/livekit/session.rs": 6023, - "tests/unit/livekit/turn.rs": 3405, - "tests/unit/tasks/session.rs": 1861, - "tests/web/accounts.rs": 1808, - "tests/web/recording.rs": 2496, - "tests/web/token.rs": 2151 -} diff --git a/scripts/soak-judge.cjs b/scripts/soak-judge.cjs index 77877b854..1d98c47f8 100644 --- a/scripts/soak-judge.cjs +++ b/scripts/soak-judge.cjs @@ -793,6 +793,52 @@ function isFreshDecodedSample(r) { const SWAP_AUDIO_OK = ["not_required", "verified", "covered_by_later_swap"]; +/// What a judged row keeps of each sample behind it: the fields the judgment +/// read, and nothing the page might add. A sample is a handful of numbers, so +/// a row costs a fixed, small amount however long the interview runs. +const SAMPLE_FIELDS = [ + "seq", + "status", + "reason", + "edgesAtMark", + "windowEdge", + "counterPairs", + "bytesReceived", + "packetsReceived", + "peakRms", +]; +const MAX_SAMPLES_PER_SWAP = 4; + +const MAX_SAMPLE_TEXT = 200; + +/// A page-supplied reason is cut, so a long one cannot grow a row. +const keptValue = (value) => + typeof value === "string" ? value.slice(0, MAX_SAMPLE_TEXT) : value; + +const keptSample = (sample) => + Object.fromEntries( + SAMPLE_FIELDS.filter((field) => field in sample).map((field) => [ + field, + keptValue(sample[field]), + ]), + ); + +/// The samples a row keeps: the one that decided it (the first fresh one, +/// else the first that was ok) always, then the rest in the order taken, up +/// to the bound. A cap that cut by position alone could keep four stale +/// samples and drop the fresh one the verdict rests on. +function keptSamples(own) { + const decisive = + own.find(isFreshDecodedSample) ?? own.find((r) => r.status === "ok"); + const kept = own.slice(0, MAX_SAMPLES_PER_SWAP); + // Past the bound, the decisive sample takes the last slot, which keeps the + // rest in the order they were taken. + if (decisive && !kept.includes(decisive)) { + kept[MAX_SAMPLES_PER_SWAP - 1] = decisive; + } + return kept.map(keptSample); +} + /// Whether the audio the browser decoded after each swap that answered a reply /// is on record. `replies` is the result of judgeEpisodeReplies and `rearms` /// the per-swap samples the page took after marking each installed swap @@ -807,13 +853,25 @@ const SWAP_AUDIO_OK = ["not_required", "verified", "covered_by_later_swap"]; /// nothing about physical audibility or the delay before it began. function judgeSwapAudio(replies, rearms = []) { const rows = []; + // Not Map.groupBy, which needs Node 21; the checks run on Node 18. + const bySwap = new Map(); + for (const r of rearms) { + if (!bySwap.has(r.swap)) bySwap.set(r.swap, []); + bySwap.get(r.swap).push(r); + } for (const e of replies.episodes) { const swap = `${e.pid}:${e.session}:${e.episode}`; + const own = bySwap.get(swap) ?? []; + // Kept with the verdict it supports, so the rows can be audited later + // without the page; dropped ones are counted, never silently lost. + const kept = keptSamples(own); const row = { swap, episode: e.episode, status: "not_required", reasons: [], + samples: kept, + samples_dropped: own.length - kept.length, }; rows.push(row); if (e.status !== "pass" || e.class !== "answered") continue; @@ -821,7 +879,6 @@ function judgeSwapAudio(replies, rearms = []) { row.status = status; row.reasons.push(reason); }; - const own = rearms.filter((r) => r.swap === swap); if (own.length === 0) { fail( "missing_sample", @@ -843,6 +900,13 @@ function judgeSwapAudio(replies, rearms = []) { const silent = own.find((r) => /silent/.test(r.status)); if (superseded && covering) { row.status = "covered_by_later_swap"; + // The sample that covers it belongs to another swap's row, whose cap + // can drop it, so it is carried here as well. + row.covered_by = { + swap: covering.swap, + seq: covering.seq, + sample: keptSample(covering), + }; } else if (silent) { fail( "silent", @@ -864,6 +928,41 @@ function judgeSwapAudio(replies, rearms = []) { }; } +/// The decoded-signal status of a judgeSwapAudio result, read from its episode +/// rows alone: `failed` with their reasons when any row is not an accepted +/// status, `verified` when a sample was owed and taken, `not_required` when +/// none was owed, and `not_assessed` when the rows are not usable, never a +/// pass. +/// +/// `replies` is the reply judge's result for the same episodes. The audio +/// judge skips an episode the reply judge could not place (no reply record, an +/// unfinished line), so those read as `not_required`, and one verified +/// neighbor would hide the gap: when the replies are not all placed, a +/// would-be `verified` or `not_required` is `not_assessed` instead. A signal +/// that was demonstrated to fail stays `failed`. +function summarizeSwapAudio(audio, replies) { + const rows = audio?.episodes; + const usable = + Array.isArray(rows) && + rows.every( + (e) => + e && + typeof e.status === "string" && + Array.isArray(e.reasons) && + e.reasons.every((r) => typeof r === "string"), + ); + if (!usable) return { status: "not_assessed", reasons: [] }; + const failed = rows.filter((e) => !SWAP_AUDIO_OK.includes(e.status)); + if (failed.length) { + return { status: "failed", reasons: failed.flatMap((e) => e.reasons) }; + } + if (replies && replies.ok === false) { + return { status: "not_assessed", reasons: [] }; + } + const owed = rows.some((e) => e.status !== "not_required"); + return { status: owed ? "verified" : "not_required", reasons: [] }; +} + /// What a soak writes about itself. Every field is the verdict of a part, so a /// reader cannot mistake a pilot that completed for a qualification: in pilot /// mode `qualification` is `not_evaluated` whatever `verdict` says. @@ -927,6 +1026,8 @@ module.exports = { judgeEpisodes, judgeSoak, judgeSwapAudio, + MAX_SAMPLES_PER_SWAP, + summarizeSwapAudio, judgeEpisodeReplies, parseClock, parseVerdictMode, diff --git a/scripts/swap-audio-owner.cjs b/scripts/swap-audio-owner.cjs new file mode 100644 index 000000000..6fdccddd9 --- /dev/null +++ b/scripts/swap-audio-owner.cjs @@ -0,0 +1,89 @@ +"use strict"; + +// The one owner of the decoded-audio marker and its settle, shared by the soak, +// voice and long-interview flows. It lives apart from browser-check.cjs so its +// ordering, cancellation and page-failure behavior can be driven with a page +// that is only an object with `evaluate`. + +const { installedSwapKeys, judgeSwapAudio } = require("./soak-judge.cjs"); + +const POLL_MS = 250; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/// Marks each installed swap in the page once, as soon as its end line is +/// read, so the sample that answers it can only open on a Speaking transition +/// after that point. Read late, never early: a transition between the install +/// and this call is missed and the next one is used instead, which can fail a +/// swap whose reply was quicker than a poll but never passes one on audio from +/// before it. It runs on its own clock, not the callers' loops: those wait on +/// log parsing and room-service calls, and a swap read late can miss the +/// transition that answered it. `halt` ends the loop on a failure path; +/// `stop` ends it, rethrows a marking failure and marks what is left. +/// +/// `newLines` returns the log gained since its last call and `agentOutput` is +/// the agent's retained output; the swaps are read from both. `keysOf` and +/// `wait` are the real parser and timer unless a caller says otherwise. +function startSwapMarker({ + page, + roomName, + agentOutput, + newLines, + keysOf = (text) => installedSwapKeys(text, roomName), + wait = sleep, + pollMs = POLL_MS, +}) { + const marked = new Set(); + let marking = true; + let markError = null; + const markNewSwaps = async () => { + for (const key of keysOf(`${agentOutput.join("")}\n${newLines()}`)) { + if (marked.has(key)) continue; + marked.add(key); + await page.evaluate((swap) => globalThis.__markJimSwap?.(swap), key); + } + }; + const loop = (async () => { + while (marking) { + await wait(pollMs); + if (!marking) break; + try { + await markNewSwaps(); + } catch (error) { + markError = error; + marking = false; + } + } + })(); + const halt = async () => { + marking = false; + await loop; + }; + return { + halt, + async stop() { + await halt(); + if (markError) throw markError; + await markNewSwaps(); + }, + }; +} + +/// The decoded audio each answered swap owes: the page is given the rest of +/// the grace for the first Speaking transition after the last swap, then +/// cancels what still waits, and the result is judged against the episodes. +async function judgeSettledSwapAudio(page, replies, label, { graceMs, log }) { + const rearms = await page.evaluate( + (waitMs) => globalThis.__settleJimSwapAudio(waitMs), + graceMs, + ); + const audio = judgeSwapAudio(replies, rearms); + for (const e of audio.episodes) { + log( + `${label} episode ${e.episode} audio: ${e.status} ${e.reasons.join("; ")}`.trimEnd(), + ); + } + return audio; +} + +module.exports = { judgeSettledSwapAudio, startSwapMarker }; diff --git a/src/dispatch.rs b/src/dispatch.rs index 1bcbed7fc..9428bdc4e 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -21,6 +21,9 @@ pub struct DispatchState { /// Whose each reserved room is and whether they have arrived, so a page /// that gives up its start can end the wait it began and nobody else's. starts: HashMap, + /// Set once by `close_admission`: no new room is admitted after it, under + /// the same lock that decides every admission and `is_idle`. + closed: bool, } struct Start { @@ -65,6 +68,55 @@ pub struct LocalDispatcher { pub max_concurrent: usize, } +/// Test-only observer of the assignment boundary: told each room the dispatcher +/// admits as new, after admission and before its task is spawned, so a fixture +/// can name the room it will see without choosing the name. It changes no +/// dispatch behavior. +#[cfg(test)] +pub(crate) type AssignmentHook = Arc; +#[cfg(test)] +static ASSIGNED: Mutex> = Mutex::new(None); + +#[cfg(test)] +pub(crate) fn set_assignment_hook(hook: Option) { + *ASSIGNED.lock().unwrap_or_else(|error| error.into_inner()) = hook; +} + +/// Whether `hook` is the one installed. Compares the allocations, not the +/// vtables. +#[cfg(test)] +pub(crate) fn assignment_hook_is(hook: &AssignmentHook) -> bool { + ASSIGNED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .is_some_and(|now| std::ptr::addr_eq(Arc::as_ptr(now), Arc::as_ptr(hook))) +} + +/// Removes `hook` only if it is still the installed one, so a teardown never +/// clears a successor's. +#[cfg(test)] +pub(crate) fn clear_assignment_hook(hook: &AssignmentHook) { + let mut slot = ASSIGNED.lock().unwrap_or_else(|error| error.into_inner()); + if slot + .as_ref() + .is_some_and(|now| std::ptr::addr_eq(Arc::as_ptr(now), Arc::as_ptr(hook))) + { + *slot = None; + } +} + +#[cfg(test)] +pub(crate) fn notify_assigned(room: &str) { + let hook = ASSIGNED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if let Some(hook) = hook { + hook(room); + } +} + impl RoomDispatcher for LocalDispatcher { /// A room stays in `live` from admission until its task ends, so one whose /// report is still being written counts too. @@ -77,6 +129,13 @@ impl RoomDispatcher for LocalDispatcher { true } + fn close_admission(&self) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .closed = true; + } + fn candidate_video(&self) -> Option { Some(self.config.gemini_candidate_video_enabled) } @@ -162,9 +221,15 @@ impl LocalDispatcher { eprintln!("codetrial dispatch_refused room={room_name} reason=finalizing"); return Err(DispatchRefusal::Finalizing); } + Reservation::Closed => { + eprintln!("codetrial dispatch_refused room={room_name} reason=closed"); + return Err(DispatchRefusal::Stopping); + } Reservation::New(slot) => slot, }; let config = agent_config_for(&self.config, provider); + #[cfg(test)] + notify_assigned(&slot.room_name); // Spawned, not awaited: this runs on the request path, and the task // outlives the response by the length of the interview or attempt. @@ -262,6 +327,7 @@ enum Reservation { Full { limit: usize, guest_cap: bool }, GuestRoomInUse, Finalizing, + Closed, } impl LocalDispatcher { @@ -300,6 +366,10 @@ impl LocalDispatcher { }; } + if state.closed { + return Reservation::Closed; + } + if state.live.len() >= self.max_concurrent { return Reservation::Full { limit: self.max_concurrent, diff --git a/src/gemini/lifecycle.rs b/src/gemini/lifecycle.rs index d2ff7e3c7..713c7f541 100644 --- a/src/gemini/lifecycle.rs +++ b/src/gemini/lifecycle.rs @@ -66,6 +66,14 @@ impl LiveReaderInspection { } impl GeminiLiveSession { + /// Limits one authored loopback connection without replacing its transport. + #[cfg(test)] + pub(crate) fn loopback_send_limit_for_test( + endpoint: &str, + ) -> std::io::Result { + super::socket::loopback::install(endpoint) + } + /// Leaves a fail-fast session in the room while its owner retires the /// transport. Metadata and unconsumed usage stay with the returned socket. pub(crate) fn retire(&mut self) -> Self { diff --git a/src/gemini/report_calls.rs b/src/gemini/report_calls.rs index 3fdba0dc1..2e6f2df4b 100644 --- a/src/gemini/report_calls.rs +++ b/src/gemini/report_calls.rs @@ -25,10 +25,15 @@ pub(crate) async fn generate_report_with_keys( rounds: crate::agent::ReportRounds, run: ReportRun<'_>, ) -> Result> { + #[cfg(test)] + let url = tests::scoped_report_endpoint(run.scope) + .unwrap_or_else(|| gemini_generate_content_url(model)); + #[cfg(not(test))] + let url = gemini_generate_content_url(model); generate_report_with_keys_at( ReportCalls { keys, - url: gemini_generate_content_url(model), + url, material, communication_assessed: rounds.communication_assessed(), budget: ReportCallBudget::new(), diff --git a/src/gemini/socket.rs b/src/gemini/socket.rs index 6a8a96114..8c302fe91 100644 --- a/src/gemini/socket.rs +++ b/src/gemini/socket.rs @@ -28,6 +28,11 @@ pub(super) async fn connect(url: &str) -> Result { let connector = Some(Connector::Rustls( std::sync::Arc::clone(crate::tls_config()), )); + #[cfg(test)] + if let Some(stream) = loopback::connect(url).await? { + let (socket, _) = client_async_tls_with_config(request, stream, None, connector).await?; + return Ok(socket); + } let Some(proxy) = proxy_for(request.uri(), &Matcher::from_env()) else { let (socket, _) = connect_async_tls_with_config(request, None, false, connector).await?; return Ok(socket); @@ -73,3 +78,7 @@ async fn tunnel(proxy: &Intercept, target: &Uri) -> Result #[cfg(test)] #[path = "../../tests/unit/gemini/socket.rs"] mod tests; + +#[cfg(test)] +#[path = "socket/loopback.rs"] +pub(super) mod loopback; diff --git a/src/gemini/socket/loopback.rs b/src/gemini/socket/loopback.rs new file mode 100644 index 000000000..6e74b41e7 --- /dev/null +++ b/src/gemini/socket/loopback.rs @@ -0,0 +1,118 @@ +//! A scoped socket-option fixture, retaining the real TCP and WebSocket path. + +use std::io; +use std::net::{Ipv4Addr, SocketAddrV4}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::net::{TcpSocket, TcpStream}; + +static ACTIVE: Mutex>> = Mutex::new(None); +const REQUESTED: u32 = 1024; +const MAX_BUFFER: u32 = 16384; + +struct State { + endpoint: String, + address: SocketAddrV4, + claimed: AtomicBool, + buffers: Mutex>, +} + +/// Owned by one isolated fixture until its real session and peer are joined. +pub(crate) struct Registration { + state: Arc, +} + +pub(in crate::gemini) fn install(endpoint: &str) -> io::Result { + let parsed = reqwest::Url::parse(endpoint).map_err(io::Error::other)?; + if parsed.scheme() != "ws" + || parsed.host_str() != Some("127.0.0.1") + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + || parsed.query() != Some("key=sequence-test-key") + || parsed.fragment().is_some() + || parsed.as_str() != endpoint + || parsed.port().is_none_or(|port| port == 0) + { + return Err(io::Error::other( + "socket fixture requires its exact authored loopback URL", + )); + } + let state = Arc::new(State { + endpoint: parsed.to_string(), + address: SocketAddrV4::new(Ipv4Addr::LOCALHOST, parsed.port().unwrap_or(0)), + claimed: AtomicBool::new(false), + buffers: Mutex::new(None), + }); + let mut active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + if active.is_some() { + return Err(io::Error::other("a socket fixture is already installed")); + } + *active = Some(Arc::clone(&state)); + Ok(Registration { state }) +} + +fn claim(endpoint: &str) -> io::Result>> { + let active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + let Some(state) = active.as_ref().filter(|state| state.endpoint == endpoint) else { + return Ok(None); + }; + if state.claimed.swap(true, Ordering::SeqCst) { + return Err(io::Error::other("the socket fixture was already consumed")); + } + Ok(Some(Arc::clone(state))) +} + +fn admitted(bytes: u32) -> io::Result { + if (1..=MAX_BUFFER).contains(&bytes) { + Ok(bytes) + } else { + Err(io::Error::other( + "socket buffer exceeds the declared fixture bound", + )) + } +} + +pub(super) async fn connect(endpoint: &str) -> io::Result> { + let Some(state) = claim(endpoint)? else { + return Ok(None); + }; + let socket = TcpSocket::new_v4()?; + socket.set_send_buffer_size(REQUESTED)?; + let before = admitted(socket.send_buffer_size()?)?; + let stream = socket.connect(state.address.into()).await?; + let connected = admitted( + u32::try_from(socket2::SockRef::from(&stream).send_buffer_size()?) + .map_err(io::Error::other)?, + )?; + *state + .buffers + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some((before, connected)); + Ok(Some(stream)) +} + +impl Registration { + pub(crate) fn sender_buffers(&self) -> Option<(u32, u32)> { + *self + .state + .buffers + .lock() + .unwrap_or_else(|error| error.into_inner()) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + if active + .as_ref() + .is_some_and(|state| Arc::ptr_eq(state, &self.state)) + { + *active = None; + } + } +} + +#[path = "../../../tests/unit/gemini/socket_loopback.rs"] +mod tests; diff --git a/src/launch.rs b/src/launch.rs new file mode 100644 index 000000000..f9f646e56 --- /dev/null +++ b/src/launch.rs @@ -0,0 +1,1084 @@ +//! How a launch turns its flags, environment and config file into a server. +//! +//! Lives in the library, not the binary, so a test fixture builds the server +//! from the same constructors `codetrial web` does instead of a copy that only +//! resembles them. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +mod peer; + +pub use peer::{PeerServices, peer_services}; + +use crate::config::{DEFAULT_DURATION_MIN, DEFAULT_ROOM_PREFIX, DEFAULT_WEB_DIR}; +use crate::web::WebServerConfig; + +/// One installation keeps its config file and its account database together in +/// `config/`, under whichever root it was installed as: a checkout, or the +/// folder a released binary was unpacked into. Beside the executable is not +/// that root for a checkout -- the executable is in `target/`, which +/// `make clean` deletes -- so the two are named separately and joined per case. +const CONFIG_DIR: &str = "config"; + +const CONFIG_FILE_NAME: &str = "codetrial.env.local"; + +/// Tracked, so it exists in a checkout and in no release archive. That makes it +/// the one thing that tells a `config/` belonging to this project apart from a +/// directory of the same name that happens to sit where the binary was run. +const CONFIG_EXAMPLE_NAME: &str = "codetrial.env.example"; + +const DEFAULT_ACCOUNT_DB_PATH: &str = "codetrial.db"; + +pub const DEFAULT_SESSION_SECRET: &str = "codetrial-local-session"; + +/// Where a `--data-dir` install keeps the key its session cookies are signed +/// with, when its config file does not set `SESSION_SECRET`. +pub const SESSION_SECRET_FILE: &str = "session-secret"; + +#[derive(Debug, Default, Clone)] +pub struct CliOptions { + pub config_path: Option, + pub data_dir: Option, + pub import_config: Option, + pub web_addr: Option, + pub web_dir: Option, + pub allowed_hosts: Option, + pub room_prefix: Option, + pub duration_min: Option, + pub task_mode: bool, + pub no_update_check: bool, +} + +/// Validates operator settings before a listener or background task starts. +pub fn web_server_config( + values: &std::collections::BTreeMap, + pool: crate::config::ProviderPool, + production: bool, + options: &CliOptions, +) -> Result { + // After the pool is complete, so the "is this project in the pool" check + // sees the pool the server will actually serve, and before the listener + // does any work: a half-configured recording block is a startup failure, + // never a surprise at the moment a candidate's interview ends. + let recording = + crate::config::load_recording(values, &pool, interview_duration_min(values), production) + .map_err(|error| error.to_string())?; + recording_needs_verified_identity( + recording.is_some(), + nonempty(values, "GITHUB_CLIENT_ID").is_some() + && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), + )?; + recording_can_deliver(recording.as_ref())?; + let (peer, peer_observer) = peer::peer_services( + values, + nonempty(values, "GITHUB_CLIENT_ID").is_some() + && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), + )? + .unzip(); + + let config = WebServerConfig { + tasks: None, + task_mode_requested: options.task_mode, + web_dir: match nonempty(values, "CODETRIAL_WEB_DIR") { + Some(dir) => PathBuf::from(dir), + + // Under the data directory rather than the working one, which a + // launcher does not choose: a `web/` that happens to sit where the + // app was started would otherwise replace the embedded pages. + None => match &options.data_dir { + Some(dir) => dir.join(DEFAULT_WEB_DIR), + None => PathBuf::from(DEFAULT_WEB_DIR), + }, + }, + github_client_id: nonempty(values, "GITHUB_CLIENT_ID"), + github_client_secret: nonempty(values, "GITHUB_CLIENT_SECRET"), + session_secret: Some(value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET)), + db_path: Some(account_db_path(values, options)), + github_oauth_base_url: None, + github_api_base_url: None, + room_prefix: value_or(values, "CODETRIAL_ROOM_PREFIX", DEFAULT_ROOM_PREFIX), + fixed_room_name: nonempty(values, "INTERVIEW_ROOM_NAME"), + production, + trusted_proxy_hops: trusted_proxy_hops(values), + allowed_hosts: allowed_hosts(values)?, + compiler_explorer_enabled: compiler_explorer_enabled(values), + guest_mode_enabled: guest_mode_enabled(values), + recording, + peer_observer, + peer, + pool, + probe_provider_quota: true, + + // Not a function of its own: off macOS this is always `None`, so no + // test on the Linux CI could tell a replaced body from this one. + update_check_commit: crate::config::update_check_commit( + values.get("CODETRIAL_UPDATE_CHECK").map(String::as_str), + env!("CODETRIAL_RELEASE_COMMIT"), + cfg!(target_os = "macos"), + ), + provision_rooms: true, + }; + if let Some(observer) = &config.peer_observer { + observer.validate_audio_storage(&config.web_dir)?; + } + Ok(config) +} + +/// The pool this deployment will serve. +/// +/// Built straight from the values rather than through `load_from_pairs`: the +/// web mode serves HTTP and mints LiveKit tokens, and has no use for the Gemini +/// key that a full agent config insists on. Demanding it here turned a web-only +/// deployment into an empty pool, silently. +fn web_provider_pool( + values: &BTreeMap, + options: &CliOptions, + production: bool, +) -> Result { + let mut pool = crate::config::ProviderPool::default(); + let (url, api_key, api_secret) = match ( + nonempty(values, "LIVEKIT_URL"), + nonempty(values, "LIVEKIT_API_KEY"), + nonempty(values, "LIVEKIT_API_SECRET"), + ) { + (Some(url), Some(api_key), Some(api_secret)) => (url, api_key, api_secret), + + // Names the file that was actually read. A fixed `config/` in this + // message sends anyone running with `--config` or a bare + // `./codetrial.env.local` to edit a file the server never opened. + _ => { + return Err(format!( + "missing required LiveKit credentials: set LIVEKIT_URL, LIVEKIT_API_KEY and LIVEKIT_API_SECRET in {}", + primary_config_path(options)?.display() + )); + } + }; + crate::config::validate_livekit_url(&url, production)?; + pool.providers.push(crate::config::Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url, + api_key, + api_secret, + google_api_keys: crate::config::google_api_keys(values)?, + }); + extend_pool( + &mut pool, + production, + &config_dir(options), + &value_or(values, crate::config::PROVIDER_ORDER_KEY, ""), + ); + Ok(pool) +} + +/// Why a production launch may not sign session cookies, or `None` where it +/// may. The built-in default is a published string, so in production it is not +/// a weak signing key, it is a known one. +/// +/// A function rather than an inline check because `serve_setup` asks it too. +/// The verdict needs nothing from the submission, and a check that could have +/// been made before the page was served but is made after it has taken +/// credentials, written them and answered 200 is a launch that exits with the +/// config already on disk -- which is what stops the next launch being a cold +/// start, so the page never comes back to say why. +pub fn production_secret_refusal(values: &BTreeMap) -> Option { + if crate::config::is_production(values) + && value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) == DEFAULT_SESSION_SECRET + { + return Some( + "SESSION_SECRET must be set when NODE_ENV=production; the built-in \ + default is a published value that would let anyone forge a session cookie" + .to_string(), + ); + } + None +} + +/// The environment and the flags, which in a cold start is everything +/// `load_values` has: the file it reads between them is the file that does not +/// exist yet. +/// +/// Its own function so `run_web` does not name the process environment. Reading +/// a deployment key straight from it there is what +/// `binary_web_reads_deployment_keys_from_the_config_file` refuses, because a +/// key set in the config file was once silently a no-op. +pub fn environment_and_flags(options: &CliOptions) -> BTreeMap { + let mut values = environment(options); + apply_options(&mut values, options); + values +} + +/// A first `--data-dir` launch started where the credentials Setup asks for +/// are already exported writes them into the folder's config, as Setup would +/// have, instead of asking for them again; every later launch then finds the +/// file, including one started from a desktop that exports nothing. +/// +/// The one exception to `environment` ignoring the process environment, and a +/// narrow one: only Setup's own keys, only while the folder has no config, and +/// written down rather than read live, so the file stays the whole of the +/// install's configuration. +pub fn data_dir_config_from_environment( + options: &CliOptions, + production: bool, +) -> Result<(), String> { + let Some(dir) = &options.data_dir else { + return Ok(()); + }; + let path = dir.join(CONFIG_FILE_NAME); + if path.exists() { + return Ok(()); + } + let exported = std::env::vars().collect(); + let Some(contents) = + crate::web::config_from_values(&exported, production).map_err(|error| { + format!("the exported Setup keys cannot be written to a config: {error}") + })? + else { + return Ok(()); + }; + std::fs::create_dir_all(dir) + .and_then(|()| crate::create_owner_only(&path, contents.as_bytes())) + .map_err(|error| format!("cannot write {}: {error}", path.display()))?; + println!( + "codetrial: wrote {} from the exported LiveKit and Google keys", + path.display() + ); + Ok(()) +} + +/// The process environment, or nothing for a `--data-dir` launch. +/// +/// A launcher that names the folder is saying the install's configuration is +/// in it, and an environment it was started from is not part of that: a shell +/// that exported `CODETRIAL_DB_PATH` or `NODE_ENV` for a CLI deployment would +/// otherwise move this install's reports, or change its rules, depending on +/// how it was started. Decided here, once, rather than by every launcher +/// keeping its own list of the keys this program reads. +pub fn environment(options: &CliOptions) -> BTreeMap { + if options.data_dir.is_some() { + return BTreeMap::new(); + } + std::env::vars().collect() +} + +pub fn load_values(options: &CliOptions) -> Result, String> { + let mut values = environment(options); + let path = primary_config_path(options)?; + crate::config::apply_config_file(&mut values, crate::config::read_config_file(&path)?); + apply_options(&mut values, options); + if let Some(dir) = &options.data_dir + && nonempty(&values, "SESSION_SECRET").is_none() + { + let secret = data_dir_session_secret(&dir.join(SESSION_SECRET_FILE))?; + values.insert("SESSION_SECRET".to_string(), secret); + } + Ok(values) +} + +/// The key a `--data-dir` install signs session cookies with, made on its first +/// launch and reused, so a relaunch keeps everyone signed in. +/// +/// Not the built-in default: that one is published, and any program on the +/// machine can reach a loopback port to present a cookie signed with it. +/// Owner-only on Unix for the same reason. A file that is not a key this wrote +/// is refused rather than replaced, because replacing it signs everyone out +/// and hides whatever damaged it. +pub fn data_dir_session_secret(path: &std::path::Path) -> Result { + match std::fs::read_to_string(path) { + Ok(text) => { + let secret = text.trim(); + if secret.len() >= 32 + && secret + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') + { + Ok(secret.to_string()) + } else { + Err(format!( + "{} is not a session key this program wrote. Move it aside to make a new \ + one; everyone will be signed out.", + path.display() + )) + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let secret = crate::accounts::random_token(32) + .map_err(|error| format!("cannot generate a session key: {error}"))?; + crate::create_owner_only(path, secret.as_bytes()) + .map_err(|error| format!("cannot write {}: {error}", path.display()))?; + Ok(secret) + } + Err(error) => Err(format!("cannot read {}: {error}", path.display())), + } +} + +/// The flags, over whatever the environment and the config file said. +/// +/// Split out because a cold start has to build the same map without the file +/// it does not have yet: `serve_setup` decides where to bind and what a +/// submission must survive, and both answers have to be the ones `run_web` +/// will reach a moment later. Spelled twice, this precedence is free to drift. +pub fn apply_options(values: &mut BTreeMap, options: &CliOptions) { + if let Some(value) = &options.web_addr { + values.insert("CODETRIAL_WEB_ADDR".to_string(), value.clone()); + } + if let Some(value) = &options.web_dir { + values.insert("CODETRIAL_WEB_DIR".to_string(), value.clone()); + } + if let Some(value) = &options.allowed_hosts { + values.insert("CODETRIAL_ALLOWED_HOSTS".to_string(), value.clone()); + } + if let Some(value) = &options.room_prefix { + values.insert("CODETRIAL_ROOM_PREFIX".to_string(), value.clone()); + } + if let Some(value) = options.duration_min { + values.insert("CODETRIAL_DURATION_MIN".to_string(), value.to_string()); + } + + // Over the config file as well: the desktop app is published as its own + // release, so the one the check reads never names its build. + if options.no_update_check { + values.insert("CODETRIAL_UPDATE_CHECK".to_string(), "false".to_string()); + } +} + +pub fn primary_config_path(options: &CliOptions) -> Result { + // The one place searched, and not one place among several: the working + // directory and the executable's folder are whatever the launcher left them + // as, so a config found there would belong to some other install. + if let Some(dir) = &options.data_dir { + let path = dir.join(CONFIG_FILE_NAME); + if path.is_file() { + return Ok(path); + } + return Err(format!( + "required configuration file is missing: {}; write one with LIVEKIT_URL, \ + LIVEKIT_API_KEY and LIVEKIT_API_SECRET", + path.display() + )); + } + if let Some(path) = &options.config_path { + let path = PathBuf::from(path); + if !path.is_file() { + return Err(format!( + "required configuration file is missing: {}", + path.display() + )); + } + return Ok(path); + } + + // The working directory first, which is a checkout's `config/` and an + // operator's deployment directory, then the folder the executable sits in. + // A release binary is unpacked into a folder of its own and its config is + // written there, so it has to be findable from a shortcut or a terminal + // opened somewhere else, not only from a double-click. + // + // Within each of those two roots, `config/` before the bare filename. The + // bare ones are where a release used to keep its config and where an + // operator may still keep one; they are searched, not written. + for path in [ + PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME), + PathBuf::from(CONFIG_FILE_NAME), + crate::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME), + crate::exe_dir().join(CONFIG_FILE_NAME), + ] { + if path.is_file() { + return Ok(path); + } + } + + // The keys, not a file to copy. `config/codetrial.env.example` exists in a + // checkout and nowhere else: the release archives hold the executable and + // nothing beside it, so telling someone who just unpacked one to copy it + // sends them looking for a file they were never given. Naming what the file + // must contain is an instruction both audiences can act on. + Err(format!( + "required configuration file is missing: ./{CONFIG_DIR}/{CONFIG_FILE_NAME}, \ + ./{CONFIG_FILE_NAME}, or {CONFIG_DIR}/{CONFIG_FILE_NAME} beside the \ + executable; write one with LIVEKIT_URL, LIVEKIT_API_KEY and \ + LIVEKIT_API_SECRET ({CONFIG_DIR}/{CONFIG_EXAMPLE_NAME} lists the \ + optional keys in a checkout)" + )) +} + +/// The directory of the config file this launch actually read, which is where +/// everything else belonging to the installation lives: the provider files it +/// discovers, and the account database it writes. +/// +/// So `--config /etc/codetrial/prod.env` discovers +/// `/etc/codetrial/codetrial.env.` rather than whatever `config/` happens +/// to sit in the current directory. Both halves of a deployment are launched +/// with the same `--config`, and that is what makes them agree on the pool. +/// +/// The database is written here rather than read, which is the one asymmetry: +/// a `--config` pointing somewhere a process may not write is a startup +/// failure that names the directory, and `CODETRIAL_DB_PATH` is the way out. +pub fn config_dir(options: &CliOptions) -> PathBuf { + if let Some(dir) = &options.data_dir { + return dir.clone(); + } + + // The search order comes from primary_config_path rather than being written + // out again here. Spelled twice, it was free to drift, and the drift is + // silent: the server keeps reading its own config while the pool quietly + // collapses to the primary project. + // + // A named file still decides, even when it is missing. The operator said + // where the config lives, and load_values is what refuses a path that is + // not there; answering with a different directory would be this function + // second-guessing that. + let path = options + .config_path + .as_deref() + .map(PathBuf::from) + .or_else(|| primary_config_path(options).ok()) + .unwrap_or_else(|| PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME)); + match path.parent() { + // A bare filename has an empty parent, and its siblings are in the + // working directory, not in `config/`. + Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(), + _ => PathBuf::from("."), + } +} + +/// The pool this process routes with. Discovery happens here, once, and never +/// inside `load_from_pairs`: a config loader that reads the current directory +/// gives two processes started from two directories two different pools, and a +/// positional index into those pools then names two different LiveKit projects. +pub fn extend_pool( + pool: &mut crate::config::ProviderPool, + production: bool, + config_dir: &std::path::Path, + order: &str, +) { + let (providers, warnings) = crate::config::discover_providers(config_dir, production); + for warning in warnings { + eprintln!("{warning}"); + } + pool.providers.extend(providers); + + // Outside the `discover` guard: an order naming the environment provider is + // still an order, and a deployment with discovery off should not silently + // ignore the one it was given. + for warning in crate::config::order_providers(&mut pool.providers, order) { + eprintln!("{warning}"); + } + + for line in pool_report(&pool.providers) { + eprintln!("{line}"); + } +} + +/// What startup says about the pool, in rotation order, and nothing for an +/// empty one. Every provider question asked of a running server so far has been +/// answerable only by reading the config directory and the argv back to itself +/// and guessing which won. +fn pool_report(providers: &[crate::config::Provider]) -> Vec { + if providers.is_empty() { + return Vec::new(); + } + let (summary, warnings) = crate::config::pool_summary(providers); + std::iter::once(summary) + .chain( + warnings + .into_iter() + .map(|warning| format!("WARNING: {warning}")), + ) + .collect() +} + +/// Refuses to start on a database that will not migrate, before anything is +/// served. The listener is already bound by this point, so the socket exists +/// and its backlog fills, but nothing accepts from it and the process exits +/// non-zero instead. The router runs the same migration again on the connection +/// it keeps, where the only thing it can do about a failure is answer 503. +pub fn initialize_accounts(config: &WebServerConfig) -> Result<(), String> { + let Some(login) = crate::web::login_config(config) else { + return Ok(()); + }; + + // The router opens the long-lived connection and performs the startup sweep + // on it; this first pass makes migration failure fatal before the server + // starts accepting. + crate::web::initialize_account_database(&login.db_path) + .map_err(|error| format!("account database {}: {error}", login.db_path.display()))?; + Ok(()) +} + +/// The dispatcher cap, with anything unusable about it said out loud. +/// +/// One helper for both modes: a warning printed by web and swallowed by serve +/// would be the same misconfiguration reported twice differently. +fn read_max_concurrent(values: &BTreeMap) -> usize { + let mut warnings = Vec::new(); + let max_concurrent = crate::config::max_concurrent_interviews(values, &mut warnings); + for warning in &warnings { + eprintln!("{warning}"); + } + max_concurrent +} + +/// Zero unless the operator states how many proxies front this server, so the +/// default never trusts a forwarded client address. +/// Recording without an OAuth app is a server that refuses every interview. +/// +/// The delivery target is the primary verified address GitHub returns, and +/// without the OAuth flow every account is self-declared, so `/api/token` +/// answers 403 to all of them. Said at startup, where an operator can act on +/// it, rather than one candidate at a time. +/// +/// `oauth` is both credentials, not just the id. `login_config` enables OAuth +/// only when it has the pair, so an id with no secret is exactly as unable to +/// verify anyone as no id at all, and would have started a server that refused +/// every interview while looking configured. +/// A deployment that records has to be able to hand the file over. +/// +/// The key is parsed here, before the listener does any work, because the +/// alternative is a server that records interviews and discovers at the first +/// delivery that it can never sign for one. Configuration already checked the +/// shape; this is the credential itself. +pub fn recording_can_deliver( + recording: Option<&crate::config::RecordingConfig>, +) -> Result<(), String> { + let Some(recording) = recording else { + return Ok(()); + }; + crate::delivery::GoogleDelivery::new( + &recording.service_account_json, + &recording.gcs_bucket, + &recording.drive_id, + std::sync::Arc::new(|| 0), + ) + .map(|_| ()) + .map_err(|error| format!("CODETRIAL_RECORDING_SERVICE_ACCOUNT_JSON: {error}")) +} + +pub fn recording_needs_verified_identity(recording: bool, oauth: bool) -> Result<(), String> { + if recording && !oauth { + return Err( + "CODETRIAL_RECORDING_ENABLED=true needs GITHUB_CLIENT_ID and \ + GITHUB_CLIENT_SECRET: a recording is delivered to the verified address \ + GitHub returns, and without the OAuth app every account is self-declared \ + and every interview would be refused" + .to_string(), + ); + } + Ok(()) +} + +/// The configured interview length, which is what a recording's maximum +/// duration has to clear. Read here rather than through `load_from_pairs` +/// because `codetrial web` deliberately runs without a full agent config. +fn interview_duration_min(values: &BTreeMap) -> u32 { + nonempty(values, "CODETRIAL_DURATION_MIN") + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_DURATION_MIN) +} + +/// `CODETRIAL_ALLOWED_HOSTS`, a comma-separated list of `Host` values. +/// +/// Unset or empty means no check. A value that holds only separators is +/// refused instead: it was meant to name a host, and reading it as no list +/// would answer every `Host` the person believes is refused. +pub fn allowed_hosts(values: &BTreeMap) -> Result, String> { + let hosts: Vec = value_or(values, "CODETRIAL_ALLOWED_HOSTS", "") + .split(',') + .map(str::trim) + .filter(|host| !host.is_empty()) + .map(str::to_string) + .collect(); + if hosts.is_empty() && nonempty(values, "CODETRIAL_ALLOWED_HOSTS").is_some() { + return Err("CODETRIAL_ALLOWED_HOSTS names no Host".to_string()); + } + Ok(hosts) +} + +fn trusted_proxy_hops(values: &BTreeMap) -> u32 { + nonempty(values, "CODETRIAL_TRUSTED_PROXY_HOPS") + .and_then(|value| value.parse().ok()) + .unwrap_or(0) +} + +/// What a `codetrial web` start without `NODE_ENV=production` is allowing, in +/// the operator's words rather than the code's. +/// +/// Every guard this reports on is keyed on `NODE_ENV`, so the one mistake none +/// of them can catch is the deployment that sets every credential correctly and +/// never sets `NODE_ENV` at all. The refusal in `run_web` fires only once the +/// operator has already said this is production; a server that was meant to be +/// production and does not say so gets the local defaults and no complaint. +/// This is the only signal it ever gets. +/// +/// `config::is_production` matches the exact string, so a `NODE_ENV=Production` +/// typo lands here too, which is the other way a deployment ends up local +/// without meaning to. +pub fn relaxed_for_local_use(values: &BTreeMap) -> Vec { + if crate::config::is_production(values) { + return Vec::new(); + } + + let mut warnings = vec![ + "NODE_ENV is not production: the page's CSP allows loopback origins and a \ + plaintext ws:// LiveKit URL is accepted. Set NODE_ENV=production to deploy." + .to_string(), + ]; + if nonempty(values, "SESSION_SECRET").is_none() { + warnings.push(format!( + "SESSION_SECRET is unset: signing session cookies with the built-in \ + {DEFAULT_SESSION_SECRET:?}, which is published in this repository and lets \ + anyone forge a session." + )); + } + warnings +} + +/// Why the built-in `SESSION_SECRET` may not sign cookies on `bound`, or `None` +/// where it may. +/// +/// Split out for the reason `relaxed_for_local_use` is: the caller can only +/// return the string, and a refusal reachable solely by starting a process that +/// otherwise never exits is a branch no test can assert on without hanging. +/// +/// `NODE_ENV` is a claim; the address this process bound is a fact. A server +/// other machines can reach is not the local run the published default exists +/// for, whether or not anyone remembered to say so, which is the mistake the +/// warning in `relaxed_for_local_use` can name but not prevent. +/// +/// Unspecified addresses (`0.0.0.0`, `::`) are not loopback and must not be +/// treated as such: binding every interface is how a container serves the +/// world. +/// +/// `to_canonical` before the test, because `Ipv6Addr::is_loopback` is true of +/// `::1` and of nothing else: a dual-stack socket that landed on +/// `::ffff:127.0.0.1`, which is a name for the same interface, would otherwise +/// be refused an address only this machine can reach. +/// How a listener on `bound` can be reached from somewhere other than this +/// machine, or `None` where it cannot. +/// +/// Shared, because two guards ask it and a third answer added to one of them +/// would drift silently -- in the direction that leaves the page taking +/// credentials open. What each guard says about it is not shared: the remedies +/// differ, an address is passed on the command line and a declared proxy is a +/// variable in the environment. +#[derive(Clone, Copy)] +pub enum Reachable { + /// The bind address is not loopback. + Address, + /// `CODETRIAL_TRUSTED_PROXY_HOPS` is the operator saying requests arrive + /// through something in front, which is the same admission: a server on + /// loopback behind nginx or a tunnel is as public as one on `0.0.0.0`. + DeclaredProxy, +} + +pub fn reachable_from_elsewhere( + values: &BTreeMap, + bound: std::net::SocketAddr, +) -> Option { + if !bound.ip().to_canonical().is_loopback() { + Some(Reachable::Address) + } else if trusted_proxy_hops(values) > 0 { + Some(Reachable::DeclaredProxy) + } else { + None + } +} + +pub fn task_mode_allowed(reachable: Option, fixed_room: bool, recording: bool) -> bool { + task_mode_refusal(reachable, fixed_room, recording).is_none() +} + +/// Why task mode may not run on this server, in the words of what to change, or +/// `None` where it may. +pub fn task_mode_refusal( + reachable: Option, + fixed_room: bool, + recording: bool, +) -> Option<&'static str> { + match (reachable, fixed_room, recording) { + (Some(Reachable::Address), _, _) => Some( + "it is bound to an address other than loopback; start it with --web-addr 127.0.0.1:3000", + ), + (Some(Reachable::DeclaredProxy), _, _) => { + Some("CODETRIAL_TRUSTED_PROXY_HOPS says requests arrive through a proxy; unset it") + } + (None, true, _) => Some("INTERVIEW_ROOM_NAME shares one fixed room; unset it"), + (None, false, true) => { + Some("recording is on; task mode does not run on a recording server") + } + (None, false, false) => None, + } +} + +/// Task mode is on when `--task-mode` asked for it and the server is one it may +/// run on; either alone is not enough. +pub fn task_mode_enabled( + requested: bool, + reachable: Option, + fixed_room: bool, + recording: bool, +) -> bool { + requested && task_mode_allowed(reachable, fixed_room, recording) +} + +pub fn published_secret_refusal( + values: &BTreeMap, + bound: std::net::SocketAddr, +) -> Option { + // What the cookie signer will actually use, not whether the key was + // mentioned. `value_or` trims, so `SESSION_SECRET=" codetrial-local-session + // "` is the published key spelled with whitespace rather than a secret of + // the operator's own, and a guard that only asked whether the variable was + // set would wave it through. + if value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) != DEFAULT_SESSION_SECRET { + return None; + } + + let reason = match reachable_from_elsewhere(values, bound)? { + Reachable::Address => format!("bind {bound}"), + Reachable::DeclaredProxy => "serve from behind a declared proxy".to_string(), + }; + + Some(format!( + "SESSION_SECRET must be set to {reason}: {DEFAULT_SESSION_SECRET:?} is published in \ + this repository, so every session cookie this server signs would be forgeable by \ + anyone who can reach it" + )) +} + +fn compiler_explorer_enabled(values: &BTreeMap) -> bool { + crate::config::compiler_explorer_enabled( + values + .get("CODETRIAL_COMPILER_EXPLORER_ENABLED") + .map(String::as_str), + ) +} + +pub fn guest_mode_recording_conflict(guest_mode_enabled: bool, recording_enabled: bool) -> bool { + guest_mode_enabled && recording_enabled +} + +pub fn guest_mode_enabled(values: &BTreeMap) -> bool { + crate::config::guest_mode_enabled( + values + .get("CODETRIAL_GUEST_MODE_ENABLED") + .map(String::as_str), + ) +} + +pub fn nonempty(values: &BTreeMap, key: &str) -> Option { + values + .get(key) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +pub fn value_or(values: &BTreeMap, key: &str, default: &str) -> String { + nonempty(values, key).unwrap_or_else(|| default.to_string()) +} + +/// One place decides where accounts live, because `web` and `serve` both open +/// the same database and disagreeing about it would split a person's reports. +/// +/// Beside the config file rather than in the working directory: the database +/// has to be the same file on the next launch, and the directory a process was +/// started from is not. It used to be a bare filename, which meant a launch +/// from a shortcut or from another terminal found the config it wrote and then +/// opened an empty database next to wherever it had been started. +pub fn account_db_path(values: &BTreeMap, options: &CliOptions) -> PathBuf { + match nonempty(values, "CODETRIAL_DB_PATH") { + Some(path) => PathBuf::from(path), + None => config_dir(options).join(DEFAULT_ACCOUNT_DB_PATH), + } +} + +/// Where a cold start writes the config it just took, which is the same +/// `config/` the search above looks in first for the root this is installed as. +/// +/// A checkout is told apart by the template it ships, not by `config/` merely +/// existing: a released binary run from a directory that happens to hold one +/// would otherwise write the credentials there and lose them the moment it was +/// next started from somewhere else. +pub fn cold_start_config_path(options: &CliOptions) -> PathBuf { + if let Some(dir) = &options.data_dir { + return dir.join(CONFIG_FILE_NAME); + } + let in_checkout = PathBuf::from(CONFIG_DIR); + if in_checkout.join(CONFIG_EXAMPLE_NAME).is_file() { + return in_checkout.join(CONFIG_FILE_NAME); + } + crate::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME) +} + +/// What a launch settles before it binds a socket: the refusals and warnings +/// that depend only on the values, the provider pool, the server config and +/// the account database. +pub struct Prepared { + pub config: WebServerConfig, + pub max_concurrent: usize, +} + +pub fn prepare( + values: &BTreeMap, + options: &CliOptions, +) -> Result { + if let Some(refusal) = production_secret_refusal(values) { + return Err(refusal); + } + for warning in relaxed_for_local_use(values) { + eprintln!("{warning}"); + } + + // Beside the other startup warnings and not next to the dispatcher that + // consumes it. A cap the operator wrote and this cannot honor is worth + // saying before the first thing that can fail: read late, a bad bind or a + // half-configured pool ends the process first and the operator never learns + // the number in their config file was ignored. + let max_concurrent = read_max_concurrent(values); + + let production = crate::config::is_production(values); + let pool = web_provider_pool(values, options, production)?; + + let config = web_server_config(values, pool, production, options)?; + + if guest_mode_recording_conflict(config.guest_mode_enabled, config.recording.is_some()) { + eprintln!( + "WARNING: CODETRIAL_GUEST_MODE_ENABLED=true is ignored while recording is enabled; \\ + guest mode is unavailable" + ); + } + + initialize_accounts(&config)?; + Ok(Prepared { + config, + max_concurrent, + }) +} + +/// Who started this server. A launcher that owns it (the desktop app, a test +/// fixture) offers no Setup page; an operator who started it is offered one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LaunchedBy { + Parent, + Operator, +} + +/// What a launch settles once its socket is bound and its address is known: +/// the published-secret refusal, the Setup page and task mode, both of which +/// depend on where the server is reachable. +pub fn finish( + mut config: WebServerConfig, + values: &BTreeMap, + options: &CliOptions, + bound: std::net::SocketAddr, + launched_by: LaunchedBy, +) -> Result<(WebServerConfig, Option), String> { + if let Some(refusal) = published_secret_refusal(values, bound) { + return Err(refusal); + } + let setup = crate::web::SetupContext::new( + primary_config_path(options)?, + values.clone(), + bound, + launched_by == LaunchedBy::Operator, + config.trusted_proxy_hops, + )?; + // Same reason `serve_setup` prints this: say where to go. + println!("codetrial: open http://{bound} in your browser"); + if let Some(setup) = &setup { + println!( + "codetrial: to change keys at http://{bound}/setup, enter {}", + setup.code() + ); + } + + // Task mode is off unless `--task-mode` asks for it. It serves the learner + // at this machine and nobody else: it holds decrypted reference solutions + // in memory and trusts its page to report the test rules. Reachable from + // anywhere else, or set up to record or to share one fixed room, its routes + // say why they are off. A declared proxy makes a loopback bind as public as + // any other. + let reachable = reachable_from_elsewhere(values, bound); + let (fixed_room, recording) = (config.fixed_room_name.is_some(), config.recording.is_some()); + config.tasks = task_mode_enabled(config.task_mode_requested, reachable, fixed_room, recording) + .then(crate::tasks::access::TaskService::new); + + // Said at startup, because the address printed above is the interview lobby + // on any other server, and a learner who asked for task mode would not + // otherwise know what to open or why nothing changed. + if config.tasks.is_some() { + println!( + "codetrial: task mode is on; open http://{bound} for the default task, or the assignment link your instructor sent" + ); + } else if config.task_mode_requested + && let Some(reason) = task_mode_refusal(reachable, fixed_room, recording) + { + eprintln!("codetrial: --task-mode is off: {reason}"); + } + + Ok((config, setup)) +} + +pub fn local_agent_config( + values: BTreeMap, +) -> Result, String> { + // Whether this process also hosts interviewers. A full agent config, which + // is a Gemini key on top of what the web side needs, means yes; without it + // this is the web half of a split deployment and agents arrive from + // elsewhere. Deciding it by what is configured, rather than by a flag, + // keeps a single-host production deployment working out of the box while + // leaving the split one exactly as it was. No `extend_pool` here on + // purpose. The dispatcher never looks a provider up: it is handed the one + // the token was minted from, so a second scan could only introduce a pool + // that disagrees with the web side's. + // + // Only keys that are absent make it the web half. An entry that is present + // but invalid is a mistake in this host's config, and serving on without + // the local interviewer would leave every interview waiting forever. + let agent_config = match crate::config::load_from_pairs(values) { + Ok(config) => Some(config), + Err(error) if error.invalid_entries.is_empty() => None, + Err(error) => return Err(error.to_string()), + }; + if agent_config.is_none() { + eprintln!( + "no GOOGLE_API_KEY or GOOGLE_API_KEYS: serving the web side only. Interviews will wait forever unless \ + an agent joins each room from elsewhere." + ); + } + Ok(agent_config) +} + +/// How long a stopping server may wait on what it still holds. +#[derive(Clone, Copy, Debug, Default)] +pub struct Bounds { + /// Open connections holding the shutdown up once it has begun (`None` + /// waits for them): a client holding a half-sent request would otherwise + /// keep a fixture from ever reporting. + pub drain: Option, + /// Admitted rooms and the reports they are still writing, after the last + /// connection has gone and nothing new can be admitted (`None` does not + /// wait, and ends them when the runtime drops). + pub settle: Option, +} + +/// Serves the web service, and the local interviewer when there is an agent +/// config, until `shutdown` resolves and the connections in flight have +/// finished. `codetrial web` passes a future that never resolves and no +/// bounds; a fixture passes its stop signal and bounds. The listener must +/// already be non-blocking. The runtime is dropped before this returns, which +/// ends any room task still running. +pub fn serve( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + agent_config: Option, + max_concurrent: usize, + shutdown: impl std::future::Future + Send + 'static, + bounds: Bounds, +) -> Result<(), String> { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + let dispatcher = agent_config.map(|config| { + std::sync::Arc::new(crate::dispatch::LocalDispatcher { + config, + runtime: runtime.handle().clone(), + state: std::sync::Arc::default(), + max_concurrent, + }) as std::sync::Arc + }); + runtime.block_on(run(listener, config, setup, dispatcher, shutdown, bounds)) +} + +/// `serve` with the dispatcher supplied, for a caller that has its own. +pub fn serve_with( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + dispatcher: Option>, + shutdown: impl std::future::Future + Send + 'static, + bounds: Bounds, +) -> Result<(), String> { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + runtime.block_on(run(listener, config, setup, dispatcher, shutdown, bounds)) +} + +async fn run( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + dispatcher: Option>, + shutdown: impl std::future::Future + Send + 'static, + bounds: Bounds, +) -> Result<(), String> { + let listener = tokio::net::TcpListener::from_std(listener) + .map_err(|error| format!("web server failed: {error}"))?; + let watched = dispatcher.clone(); + let closing = dispatcher.clone(); + let (begun, shutting_down) = tokio::sync::oneshot::channel::<()>(); + let served = axum::serve( + listener, + crate::web::web_service_with_setup(config, dispatcher, setup), + ) + .with_graceful_shutdown(async move { + shutdown.await; + + // Closing the listener does not end requests already in flight, and one + // of them could still admit a room while the settle wait below is + // looking at an idle dispatcher. + if let Some(dispatcher) = closing { + dispatcher.close_admission(); + } + let _ = begun.send(()); + }); + let overdue = async { + match (shutting_down.await, bounds.drain) { + (Ok(()), Some(bound)) => tokio::time::sleep(bound).await, + _ => std::future::pending().await, + } + }; + let served = tokio::select! { + result = served => result, + () = overdue => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "connections were still open when the shutdown bound passed", + )), + } + .map_err(|error| format!("web server failed: {error}")); + + // Nothing new is admitted now. Whatever was is allowed to finish, report + // included, before the runtime drops and ends it. + match (bounds.settle, watched) { + (Some(limit), Some(dispatcher)) if !settled(&*dispatcher, limit).await => { + Err(unfinished(served)) + } + _ => served, + } +} + +/// Whether the dispatcher went idle within `limit`. +async fn settled(dispatcher: &dyn crate::web::RoomDispatcher, limit: std::time::Duration) -> bool { + let began = tokio::time::Instant::now(); + while !dispatcher.is_idle() { + if began.elapsed() >= limit { + return false; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + true +} + +/// The error of a stop that found rooms or reports still running at the settle +/// bound: that, and the shutdown's own error as well when it had one, since a +/// drain timeout must not hide that evidence was cut off. +fn unfinished(served: Result<(), String>) -> String { + const UNFINISHED: &str = "rooms or reports were still running when the settle bound passed"; + match served { + Ok(()) => UNFINISHED.into(), + Err(error) => format!("{error}; {UNFINISHED}"), + } +} + +#[cfg(test)] +#[path = "../tests/unit/launch.rs"] +mod tests; diff --git a/src/main_peer.rs b/src/launch/peer.rs similarity index 87% rename from src/main_peer.rs rename to src/launch/peer.rs index 349ac2aa4..66736c666 100644 --- a/src/main_peer.rs +++ b/src/launch/peer.rs @@ -1,14 +1,14 @@ //! Startup rules for reciprocal practice, offered once its prerequisites exist. -use codetrial::config::{ +use crate::config::{ MAX_CAPTURE_SECONDS, PeerConfig, PeerObserverConfig, load_peer, load_peer_observer, }; use std::collections::BTreeMap; -pub(super) type PeerServices = Option<(PeerConfig, PeerObserverConfig)>; +pub type PeerServices = Option<(PeerConfig, PeerObserverConfig)>; /// Reciprocal practice needs GitHub sign-in for member identity and the Google /// keys for the silent observer. Without either it is simply not offered. -pub(super) fn peer_services( +pub fn peer_services( values: &BTreeMap, oauth: bool, ) -> Result { diff --git a/src/lib.rs b/src/lib.rs index 917987c19..b3d49d751 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod config; pub mod delivery; pub mod dispatch; pub mod gemini; +pub mod launch; pub mod livekit; mod peer_audio; pub mod recording; @@ -12,6 +13,10 @@ pub mod tasks; pub mod token; pub mod web; +#[cfg(test)] +#[path = "../tests/unit/isolation.rs"] +mod test_isolation; + /// Lives here rather than in one of the modules because `accounts` stamps rows /// with it and `web` stamps tokens with it, and neither should have to depend /// on the other for a clock. diff --git a/src/livekit/latency.rs b/src/livekit/latency.rs index 3178181be..23082b64f 100644 --- a/src/livekit/latency.rs +++ b/src/livekit/latency.rs @@ -348,6 +348,8 @@ impl ReplyLatency { self.session = session; self.socket = socket; self.silence_window_ms = silence_window_ms; + #[cfg(test)] + super::probe::acquisition::bound(room, session); } /// A microphone frame was dequeued at `now`. @@ -387,6 +389,17 @@ impl ReplyLatency { self.anchor.map(|anchor| offset_ms(now, anchor)) } + #[cfg(test)] + pub(super) fn acquisition_identity(&self) -> super::probe::acquisition::IdentityView<'_> { + super::probe::acquisition::IdentityView { + room: &self.room, + session: self.session, + socket: self.socket, + trace_turn: self.trace.as_ref().map(|trace| trace.turn), + clock: self.observation_clock(), + } + } + /// A fresh wall/monotonic pair for joining local harness observations. /// Sampling delay and whole-millisecond rounding are uncertainty, not /// drift. diff --git a/src/livekit/probe.rs b/src/livekit/probe.rs index 2efb7018d..9de7280ea 100644 --- a/src/livekit/probe.rs +++ b/src/livekit/probe.rs @@ -20,6 +20,12 @@ //! frame that arrives next, so the hold starts within about one frame (10 ms) //! of being asked for. +pub(super) mod acquisition; + +#[cfg(unix)] +#[path = "../../tests/unit/livekit/probe/serving.rs"] +mod serving; + use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Instant; diff --git a/src/livekit/probe/acquisition.rs b/src/livekit/probe/acquisition.rs new file mode 100644 index 000000000..6ad7f73f3 --- /dev/null +++ b/src/livekit/probe/acquisition.rs @@ -0,0 +1,926 @@ +//! Bounded private execution evidence for authored acquisition fixtures. + +pub(in crate::livekit) mod owner; + +use std::io::{self, Write}; +use std::sync::{Arc, Mutex, PoisonError}; + +use serde::ser::{Serialize, SerializeSeq, Serializer}; +use serde_json::Value; + +use crate::gemini::GeminiFunctionCall; +use crate::livekit::latency::ReplyLatency; + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Identity { + pub(in crate::livekit) room: String, + pub(in crate::livekit) session: u64, + pub(in crate::livekit) socket: u64, + pub(in crate::livekit) trace_turn: Option, + pub(in crate::livekit) clock: Value, +} + +impl Identity { + fn from_view(view: &IdentityView<'_>) -> Self { + Self { + room: view.room.into(), + session: view.session, + socket: view.socket, + trace_turn: view.trace_turn, + clock: view.clock.clone(), + } + } +} + +#[derive(serde::Serialize)] +pub(in crate::livekit) struct IdentityView<'a> { + pub(in crate::livekit) room: &'a str, + pub(in crate::livekit) session: u64, + pub(in crate::livekit) socket: u64, + pub(in crate::livekit) trace_turn: Option, + pub(in crate::livekit) clock: Value, +} + +#[derive(Clone, Copy)] +pub(in crate::livekit) struct Limits { + pub(in crate::livekit) batches: usize, + pub(in crate::livekit) calls: usize, + /// Serialized JSON bytes, punctuation included, that each retained array + /// (batches, receipts, replies and turn endings) is held to on its own, so + /// a report can hold up to four times this. `calls` likewise bounds the + /// receipts, the replies and the turn endings each. + pub(in crate::livekit) bytes: usize, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Call { + pub(in crate::livekit) id: String, + pub(in crate::livekit) name: String, + pub(in crate::livekit) result: Value, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Batch { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) sequence: usize, + pub(in crate::livekit) calls: Vec, + /// None means no local send completion was observed, not a failed delivery. + pub(in crate::livekit) local_send_completed: Option, +} + +/// A tool call as the handler received it, before the acknowledgement and +/// board awaits that can still cancel the batch. Received, executed and +/// locally sent are three separate facts, and none of them is the provider's +/// receipt. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Receipt { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) id: String, + pub(in crate::livekit) name: String, + /// Delivered after a hold or discard had cut its generation off. Such a + /// call is still answered, so it is kept, but it is never to be tied to a + /// later reply. + pub(in crate::livekit) discarded: bool, + /// Sequence of the executed batch that answered it; `None` for a call that + /// was received and never executed. Stands in for a stage: received, + /// executed and locally sent are one lifecycle that the reply capture will + /// fold into a single per-call record. + pub(in crate::livekit) batch: Option, +} + +/// The newest view of one interviewer transcript segment, as queued for the +/// page (the writer may coalesce waiting views, so `views` can exceed what the +/// page received). Only ordered facts are kept: how many receipts and batches +/// existed +/// when the segment first appeared says what could have preceded it, and +/// nothing here says what it was about. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Reply { + /// Where the segment first appeared; later views keep it. + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) id: String, + pub(in crate::livekit) text: String, + pub(in crate::livekit) final_segment: bool, + /// A view was not an extension of the one before it, or a final segment + /// was published as open again: what the page showed was rewritten. + pub(in crate::livekit) revised: bool, + pub(in crate::livekit) views: usize, + pub(in crate::livekit) receipts_before: usize, + pub(in crate::livekit) batches_before: usize, + /// The same counts at the newest view. A turn spans tool calls, so a + /// segment whose two counts differ straddles them and says nothing about + /// which call its later words follow. + pub(in crate::livekit) receipts_last: usize, + pub(in crate::livekit) batches_last: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) enum Ending { + Complete, + Interrupted, +} + +/// A turn ending the provider sent, and whether the room loop took it. A +/// swallowed ending belongs to a generation already cut off, so it closes +/// nothing the page or the model saw; it is kept because the boundary a reply +/// would otherwise be bounded by is then missing. Local cuts and socket +/// replacements are not endings: they show as a sealed segment or a new socket +/// in the identity. An ending the old socket sent while a speech handoff +/// collected its events is consumed there and never seen here, so a socket +/// change between two replies is an unknown boundary, never an absent one. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Boundary { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) ending: Ending, + pub(in crate::livekit) accepted: bool, + pub(in crate::livekit) receipts_before: usize, + pub(in crate::livekit) batches_before: usize, + pub(in crate::livekit) replies_before: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(in crate::livekit) enum Failure { + // Room/session admission or an opaque ticket's batch association differs. + Scope, + Limit, + // Missing or reused provider IDs cannot be correlated safely. + AmbiguousId, +} + +#[derive(Clone, Debug, PartialEq)] +pub(in crate::livekit) struct Snapshot { + pub(in crate::livekit) batches: Vec, + pub(in crate::livekit) failure: Option, + pub(in crate::livekit) retained_calls: usize, + pub(in crate::livekit) retained_bytes: usize, + pub(in crate::livekit) receipts: Vec, + /// Serialized length of the receipt array, brackets and commas included, + /// held to the same byte limit as the batches but counted apart from them. + pub(in crate::livekit) retained_receipt_bytes: usize, + pub(in crate::livekit) replies: Vec, + pub(in crate::livekit) boundaries: Vec, + /// Serialized length of each retained reply and of the two arrays, + /// brackets and commas included, so a newer view of a reply, which + /// arrives for every transcript chunk, is charged for its own change + /// rather than by serializing every reply again under the lock. + reply_bytes: Vec, + retained_reply_bytes: usize, + retained_boundary_bytes: usize, +} + +impl Snapshot { + /// The receipt a call executed under `identity` answers: same session and + /// same socket, so a call received on one socket and executed on another + /// is left unlinked as evidence of a replacement. + fn receipt_mut(&mut self, identity: &IdentityView<'_>, id: &str) -> Option<&mut Receipt> { + self.receipts.iter_mut().find(|receipt| { + receipt.id == id + && receipt.identity.session == identity.session + && receipt.identity.socket == identity.socket + }) + } +} + +pub(in crate::livekit) struct Capture { + room: String, + session: u64, + limits: Limits, + inner: Mutex, +} + +impl Capture { + pub(in crate::livekit) fn new( + room: &str, + session: u64, + limits: Limits, + ) -> Result, &'static str> { + if room.is_empty() || session == 0 { + return Err("acquisition requires an assigned room and nonzero session"); + } + if limits.batches == 0 || limits.calls == 0 || limits.bytes < 2 { + return Err("acquisition requires positive retention limits"); + } + + // Bound the admitted scope before retaining its room string. Full batch + // accounting still happens at record time, including its actual clock. + let scope = IdentityView { + room, + session, + socket: 0, + trace_turn: None, + clock: Value::Null, + }; + let mut counter = BoundedBytes { + remaining: limits.bytes - 2, + written: 0, + }; + if serde_json::to_writer(&mut counter, &scope).is_err() { + return Err("acquisition scope metadata exceeds byte limit"); + } + Ok(Arc::new(Self { + room: room.into(), + session, + limits, + inner: Mutex::new(Snapshot { + batches: Vec::new(), + failure: None, + retained_calls: 0, + retained_bytes: 2, + receipts: Vec::new(), + // The empty array, so the count is the serialized length. + retained_receipt_bytes: 2, + replies: Vec::new(), + boundaries: Vec::new(), + reply_bytes: Vec::new(), + retained_reply_bytes: 2, + retained_boundary_bytes: 2, + }), + })) + } + + /// Whether the identity is the room and session this capture was admitted + /// for; the socket may change under it. + fn admits(&self, identity: &IdentityView<'_>) -> bool { + identity.room == self.room && identity.session == self.session + } + + /// Whether an array that serialized to `retained` bytes still fits once + /// `value` is added to it, and its length if so. + fn charged(&self, retained: usize, added: usize, value: &impl Serialize) -> Option { + let mut counter = BoundedBytes { + remaining: self.limits.bytes.checked_sub(retained + added)?, + written: 0, + }; + serde_json::to_writer(&mut counter, value).ok()?; + Some(counter.written) + } + + pub(in crate::livekit) fn snapshot(&self) -> Snapshot { + self.inner + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// Records the calls of one batch at the handler's entry. Scoped like a + /// batch, bounded by the call and byte limits, and all or nothing: a batch + /// that does not fit latches the failure and keeps none of its receipts. + pub(in crate::livekit) fn receive( + &self, + identity: IdentityView<'_>, + calls: &[GeminiFunctionCall], + discarded: bool, + ) { + if calls.is_empty() { + return; + } + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + let failure = if !self.admits(&identity) { + Some(Failure::Scope) + } else if calls.len() > self.limits.calls.saturating_sub(snapshot.receipts.len()) { + Some(Failure::Limit) + } else if calls.iter().enumerate().any(|(index, call)| { + call.id.is_empty() + || calls[..index].iter().any(|earlier| earlier.id == call.id) + || snapshot.receipts.iter().any(|old| old.id == call.id) + }) { + Some(Failure::AmbiguousId) + } else { + None + }; + if let Some(failure) = failure { + snapshot.failure = Some(failure); + return; + } + let mut retained = snapshot.retained_receipt_bytes; + let mut fresh = Vec::with_capacity(calls.len()); + for (index, call) in calls.iter().enumerate() { + // A comma before every receipt but the very first. + let comma = usize::from(snapshot.receipts.len() + index != 0); + let receipt = Receipt { + identity: Identity::from_view(&identity), + id: call.id.clone(), + name: call.name.clone(), + discarded, + batch: None, + }; + let Some(bytes) = self.charged(retained, comma, &receipt) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + retained += comma + bytes; + fresh.push(receipt); + } + snapshot.retained_receipt_bytes = retained; + snapshot.receipts.extend(fresh); + } + + /// Notes a published interviewer segment. A segment is identified by its + /// session, socket and id, and only its newest text is kept, so the byte + /// limit bounds the serialized array of newest views. A view that does not + /// fit latches the failure and leaves the earlier ones as they were. + pub(in crate::livekit) fn reply( + &self, + identity: IdentityView<'_>, + id: &str, + text: &str, + final_segment: bool, + ) { + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if !self.admits(&identity) { + snapshot.failure = Some(Failure::Scope); + return; + } + let at = snapshot.replies.iter().position(|reply| { + reply.id == id + && reply.identity.session == identity.session + && reply.identity.socket == identity.socket + }); + let (reply, old) = if let Some(at) = at { + // Built anew rather than cloned: the old text, the largest field, + // is about to be replaced. + let was = &snapshot.replies[at]; + let reply = Reply { + identity: was.identity.clone(), + id: was.id.clone(), + text: text.into(), + final_segment, + revised: was.revised + || !text.starts_with(&was.text) + || (was.final_segment && !final_segment), + views: was.views + 1, + receipts_before: was.receipts_before, + batches_before: was.batches_before, + receipts_last: snapshot.receipts.len(), + batches_last: snapshot.batches.len(), + }; + (reply, snapshot.reply_bytes[at]) + } else if snapshot.replies.len() >= self.limits.calls { + snapshot.failure = Some(Failure::Limit); + return; + } else { + let reply = Reply { + identity: Identity::from_view(&identity), + id: id.into(), + text: text.into(), + final_segment, + revised: false, + views: 1, + receipts_before: snapshot.receipts.len(), + batches_before: snapshot.batches.len(), + receipts_last: snapshot.receipts.len(), + batches_last: snapshot.batches.len(), + }; + (reply, 0) + }; + // A comma before every reply but the first. + let comma = usize::from(at.is_none() && !snapshot.replies.is_empty()); + let rest = snapshot.retained_reply_bytes - old; + let Some(bytes) = self.charged(rest, comma, &reply) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + snapshot.retained_reply_bytes = rest + comma + bytes; + if let Some(at) = at { + snapshot.replies[at] = reply; + snapshot.reply_bytes[at] = bytes; + } else { + snapshot.replies.push(reply); + snapshot.reply_bytes.push(bytes); + } + } + + /// Notes a turn ending in arrival order. Bounded like the replies: by the + /// call limit in count and the byte limit in serialized size. + pub(in crate::livekit) fn boundary( + &self, + identity: IdentityView<'_>, + ending: Ending, + accepted: bool, + ) { + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if !self.admits(&identity) { + snapshot.failure = Some(Failure::Scope); + return; + } + let boundary = Boundary { + identity: Identity::from_view(&identity), + ending, + accepted, + receipts_before: snapshot.receipts.len(), + batches_before: snapshot.batches.len(), + replies_before: snapshot.replies.len(), + }; + if snapshot.boundaries.len() >= self.limits.calls { + snapshot.failure = Some(Failure::Limit); + return; + } + let comma = usize::from(!snapshot.boundaries.is_empty()); + let Some(bytes) = self.charged(snapshot.retained_boundary_bytes, comma, &boundary) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + snapshot.retained_boundary_bytes += comma + bytes; + snapshot.boundaries.push(boundary); + } + + fn record( + &self, + identity: IdentityView<'_>, + answers: &[(GeminiFunctionCall, Value)], + ) -> Option { + if answers.is_empty() { + return None; + } + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return None; + } + let failure = if !self.admits(&identity) { + Some(Failure::Scope) + } else if snapshot.batches.len() >= self.limits.batches + || answers.len() > self.limits.calls.saturating_sub(snapshot.retained_calls) + { + Some(Failure::Limit) + } else if answers.iter().enumerate().any(|(index, (call, _))| { + call.id.is_empty() + || answers[..index] + .iter() + .any(|(earlier, _)| earlier.id == call.id) + || snapshot + .batches + .iter() + .any(|batch| batch.calls.iter().any(|old| old.id == call.id)) + }) { + Some(Failure::AmbiguousId) + } else { + None + }; + if let Some(failure) = failure { + snapshot.failure = Some(failure); + return None; + } + let sequence = snapshot.batches.len(); + let comma = usize::from(sequence != 0); + let view = BatchView { + identity: &identity, + sequence, + calls: CallsView(answers), + local_send_completed: None, + }; + let Some(bytes) = self.charged(snapshot.retained_bytes, comma, &view) else { + snapshot.failure = Some(Failure::Limit); + return None; + }; + let batch = Batch { + identity: Identity::from_view(&identity), + sequence, + calls: answers + .iter() + .map(|(call, result)| Call { + id: call.id.clone(), + name: call.name.clone(), + result: result.clone(), + }) + .collect(), + local_send_completed: None, + }; + snapshot.retained_bytes += comma + bytes; + snapshot.retained_calls += answers.len(); + snapshot.batches.push(batch); + + // Linked only to a receipt from this same socket: a call received on + // one socket and executed on another is evidence of a replacement, not + // of an answer. + let digits = sequence.to_string().len(); + let mut receipt_bytes = snapshot.retained_receipt_bytes; + for (call, _) in answers { + if let Some(receipt) = snapshot.receipt_mut(&identity, &call.id) { + receipt.batch = Some(sequence); + // `null` (four bytes) becomes the sequence number. + receipt_bytes = (receipt_bytes + digits).saturating_sub(4); + } + } + snapshot.retained_receipt_bytes = receipt_bytes; + Some(sequence) + } +} + +/// Admission fixes the batch's identity permanently: batches only append. +/// An original capture and its sequence associate completion without copying +/// another clock object or looking up whichever capture is installed later. +pub(in crate::livekit) struct Ticket { + capture: Arc, + sequence: usize, +} + +impl Ticket { + pub(in crate::livekit) fn complete(self, sent: bool) { + let mut snapshot = self + .capture + .inner + .lock() + .unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if snapshot + .batches + .get(self.sequence) + .is_none_or(|batch| batch.local_send_completed.is_some()) + { + snapshot.failure = Some(Failure::Scope); + return; + } + let extra = usize::from(!sent); + if extra + > self + .capture + .limits + .bytes + .saturating_sub(snapshot.retained_bytes) + { + snapshot.failure = Some(Failure::Limit); + return; + } + // null and true have equal JSON width; false needs one more byte. + snapshot.batches[self.sequence].local_send_completed = Some(sent); + snapshot.retained_bytes += extra; + } +} + +#[derive(serde::Serialize)] +struct BatchView<'a> { + identity: &'a IdentityView<'a>, + sequence: usize, + calls: CallsView<'a>, + local_send_completed: Option, +} + +#[derive(serde::Serialize)] +struct CallView<'a> { + id: &'a str, + name: &'a str, + result: &'a Value, +} + +struct CallsView<'a>(&'a [(GeminiFunctionCall, Value)]); +impl Serialize for CallsView<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(self.0.len()))?; + for (call, result) in self.0 { + sequence.serialize_element(&CallView { + id: &call.id, + name: &call.name, + result, + })?; + } + sequence.end() + } +} + +struct BoundedBytes { + remaining: usize, + written: usize, +} +impl Write for BoundedBytes { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.len() > self.remaining { + return Err(io::Error::other("acquisition byte limit")); + } + self.remaining -= bytes.len(); + self.written += bytes.len(); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// Why an arming observer stopped trusting its scope. Latched: a fixture that +/// saw any of these has no evidence to report. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(in crate::livekit) enum ArmFailure { + /// A bind arrived before any assignment named a room. + BindBeforeAssignment, + /// A second, different room was assigned or bound. + UnrelatedRoom, + /// A bind carried session zero, or a different session than the first. + Session, + /// The bound scope could not be admitted by `Capture::new`. + Scope, + /// Another capture already held the single installation slot. + Install, +} + +/// Dispatch-to-bind handoff for a fixture that cannot know the room name or +/// session id in advance. It starts unarmed and admits nothing: +/// +/// - the dispatcher's accepted assignment names the one expected room; +/// - the first bind for that room latches its nonzero session, builds the +/// `Capture` and installs it, so a tool batch can never arrive at an unbound +/// scope; +/// - a bind before any assignment, a second room, session zero, a different +/// later session or an occupied installation slot fails the observer for +/// good, which also uninstalls the capture: evidence from a scope it doubted +/// would mislead. +/// +/// The observer owns the install guard, so removing the observer (`retire`) +/// or dropping it takes the capture out with it, and a callback that was +/// already in flight finds a retired observer and does nothing. +pub(in crate::livekit) struct Arming { + limits: Limits, + /// Whether a bound scope is installed for `observe` to find. Off only for + /// the detached form, which exists so the state machine can be tested + /// without touching the process-wide slot. + publish: bool, + state: Mutex, +} + +enum ArmState { + Unarmed, + Expecting(String), + Bound { + capture: Arc, + _installed: Option, + }, + Failed(ArmFailure), + Retired, +} + +impl ArmState { + fn room(&self) -> Option<&str> { + match self { + Self::Expecting(room) => Some(room), + Self::Bound { capture, .. } => Some(&capture.room), + _ => None, + } + } + + fn is_terminal(&self) -> bool { + matches!(self, Self::Failed(_) | Self::Retired) + } +} + +impl Arming { + pub(in crate::livekit) fn new(limits: Limits) -> Arc { + Self::build(limits, true) + } + + /// The same rules without installing anything. + pub(in crate::livekit) fn detached(limits: Limits) -> Arc { + Self::build(limits, false) + } + + fn build(limits: Limits, publish: bool) -> Arc { + Arc::new(Self { + limits, + publish, + state: Mutex::new(ArmState::Unarmed), + }) + } + + fn state(&self) -> std::sync::MutexGuard<'_, ArmState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(in crate::livekit) fn on_assignment(&self, room: &str) { + let mut state = self.state(); + if state.is_terminal() { + return; + } + match state.room() { + None => *state = ArmState::Expecting(room.into()), + Some(expected) if expected == room => {} + Some(_) => *state = ArmState::Failed(ArmFailure::UnrelatedRoom), + } + } + + pub(in crate::livekit) fn on_bind(&self, room: &str, session: u64) { + let mut state = self.state(); + if state.is_terminal() { + return; + } + if let Some(next) = self.after_bind(&state, room, session) { + *state = next; + } + } + + /// The state a bind moves to, or `None` when it changes nothing. + fn after_bind(&self, state: &ArmState, room: &str, session: u64) -> Option { + let fail = |failure| Some(ArmState::Failed(failure)); + match state { + ArmState::Unarmed => fail(ArmFailure::BindBeforeAssignment), + _ if state.room() != Some(room) => fail(ArmFailure::UnrelatedRoom), + ArmState::Bound { capture, .. } if capture.session != session => { + fail(ArmFailure::Session) + } + ArmState::Expecting(_) if session == 0 => fail(ArmFailure::Session), + ArmState::Expecting(_) => { + let Ok(capture) = Capture::new(room, session, self.limits) else { + return fail(ArmFailure::Scope); + }; + let installed = if self.publish { + match install(&capture) { + Ok(installed) => Some(installed), + Err(_) => return fail(ArmFailure::Install), + } + } else { + None + }; + Some(ArmState::Bound { + capture, + _installed: installed, + }) + } + _ => None, + } + } + + /// Ends the observer: whatever it installed is removed and later events + /// are ignored. + fn retire(&self) { + *self.state() = ArmState::Retired; + } + + /// The admitted scope, once bound and for as long as nothing has failed. + pub(in crate::livekit) fn capture(&self) -> Option> { + match &*self.state() { + ArmState::Bound { capture, .. } => Some(Arc::clone(capture)), + _ => None, + } + } + + /// What this observer saw, for a fixture to leave behind: the failure that + /// ended its trust, if any, and the admitted scope's retained evidence. + pub(in crate::livekit) fn report(&self) -> serde_json::Value { + let capture = self.capture().map(|capture| { + let snapshot = capture.snapshot(); + serde_json::json!({ + "batches": snapshot.batches, + "failure": snapshot.failure.map(|failure| format!("{failure:?}")), + "retained_calls": snapshot.retained_calls, + "retained_bytes": snapshot.retained_bytes, + "receipts": snapshot.receipts, + "retained_receipt_bytes": snapshot.retained_receipt_bytes, + "replies": snapshot.replies, + "boundaries": snapshot.boundaries, + }) + }); + serde_json::json!({ + "arming_failure": self.failure().map(|failure| format!("{failure:?}")), + "capture": capture, + }) + } + + pub(in crate::livekit) fn failure(&self) -> Option { + match &*self.state() { + ArmState::Failed(failure) => Some(*failure), + _ => None, + } + } +} + +static ARMING: Mutex>> = Mutex::new(None); +static ACTIVE: Mutex>> = Mutex::new(None); + +/// Uninstall stops new lookups, but admitted tickets can still complete. +/// Fixtures must join admitted writers before taking a final snapshot. +pub(in crate::livekit) struct Installed(Arc); +impl Drop for Installed { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &self.0)) { + *active = None; + } + } +} + +pub(in crate::livekit) fn install(capture: &Arc) -> Result { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.is_some() { + return Err("an acquisition capture is already installed"); + } + *active = Some(Arc::clone(capture)); + Ok(Installed(Arc::clone(capture))) +} + +/// Removes the arming observer and anything it installed when dropped. +pub(in crate::livekit) struct ArmingInstalled { + arming: Arc, + hook: crate::dispatch::AssignmentHook, +} +impl Drop for ArmingInstalled { + fn drop(&mut self) { + // One critical section, the same lock `install_arming` takes: a + // replacement observer can only install once this one is entirely gone, + // so this teardown never clears the replacement's hook or meets this + // observer's capture still installed. + let mut arming = ARMING.lock().unwrap_or_else(PoisonError::into_inner); + if arming + .as_ref() + .is_some_and(|now| Arc::ptr_eq(now, &self.arming)) + { + *arming = None; + } + crate::dispatch::clear_assignment_hook(&self.hook); + self.arming.retire(); + } +} + +/// Starts watching the dispatcher's assignments and the latency binds for the +/// one room the fixture will be given. Nothing is captured until both agree. +pub(in crate::livekit) fn install_arming( + arming: &Arc, +) -> Result { + let mut slot = ARMING.lock().unwrap_or_else(PoisonError::into_inner); + if slot.is_some() { + return Err("an acquisition arming observer is already installed"); + } + *slot = Some(Arc::clone(arming)); + let watching = Arc::clone(arming); + let hook: crate::dispatch::AssignmentHook = Arc::new(move |room: &str| { + watching.on_assignment(room); + }); + crate::dispatch::set_assignment_hook(Some(Arc::clone(&hook))); + Ok(ArmingInstalled { + arming: Arc::clone(arming), + hook, + }) +} + +/// The latency bind boundary: called with every room and session the loop +/// binds. +pub(in crate::livekit) fn bound(room: &str, session: u64) { + let arming = ARMING + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + if let Some(arming) = arming { + arming.on_bind(room, session); + } +} + +fn active() -> Option> { + ACTIVE + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(Arc::clone) +} + +pub(in crate::livekit) fn observe( + latency: &ReplyLatency, + answers: &[(GeminiFunctionCall, Value)], +) -> Option { + let capture = active()?; + let sequence = capture.record(latency.acquisition_identity(), answers)?; + Some(Ticket { capture, sequence }) +} + +/// The handler's entry, before anything that can cancel the batch. +pub(in crate::livekit) fn receive( + latency: &ReplyLatency, + calls: &[GeminiFunctionCall], + discarded: bool, +) { + if let Some(capture) = active() { + capture.receive(latency.acquisition_identity(), calls, discarded); + } +} + +/// A turn ending as the room loop decided it; other events are not endings. +pub(in crate::livekit) fn boundary( + latency: &ReplyLatency, + event: &crate::gemini::GeminiEvent, + accepted: bool, +) { + use crate::gemini::GeminiEvent; + let ending = match event { + GeminiEvent::TurnComplete => Ending::Complete, + GeminiEvent::Interrupted => Ending::Interrupted, + _ => return, + }; + if let Some(capture) = active() { + capture.boundary(latency.acquisition_identity(), ending, accepted); + } +} + +/// An interviewer segment as the room loop publishes it. +pub(in crate::livekit) fn reply(latency: &ReplyLatency, id: &str, text: &str, final_segment: bool) { + if let Some(capture) = active() { + capture.reply(latency.acquisition_identity(), id, text, final_segment); + } +} + +#[path = "../../../tests/unit/livekit/probe/acquisition.rs"] +mod tests; diff --git a/src/livekit/probe/acquisition/owner.rs b/src/livekit/probe/acquisition/owner.rs new file mode 100644 index 000000000..7bafbcd3d --- /dev/null +++ b/src/livekit/probe/acquisition/owner.rs @@ -0,0 +1,129 @@ +//! Controlled owner-await gates, distinct from physical transport pressure. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use tokio::sync::Notify; + +use super::Capture; +use crate::agent::ThinkingHold; +use crate::livekit::session::GeminiEventContext; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::livekit) enum Stage { + Acknowledgement, + BoardSettlement, + BoardResend, + ToolSend, +} + +#[derive(Clone, Copy)] +pub(in crate::livekit) enum Transition { + Hold(bool), + Discard(bool), +} + +pub(in crate::livekit) struct Gate { + scope: Arc, + socket: u64, + stage: Stage, + transition: Option, + entered: Notify, + release: Notify, +} + +impl Gate { + pub(in crate::livekit) fn new( + scope: &Arc, + socket: u64, + stage: Stage, + transition: Option, + ) -> Result, &'static str> { + if socket == 0 { + return Err("owner gate requires an assigned socket"); + } + if transition.is_some() && stage != Stage::ToolSend { + return Err("owner transition requires the scoped tool-send boundary"); + } + Ok(Arc::new(Self { + scope: Arc::clone(scope), + socket, + stage, + transition, + entered: Notify::new(), + release: Notify::new(), + })) + } + + pub(in crate::livekit) async fn entered(&self) -> Result<(), tokio::time::error::Elapsed> { + tokio::time::timeout(Duration::from_secs(2), self.entered.notified()).await + } + + pub(in crate::livekit) fn release(&self) { + self.release.notify_one(); + } +} + +static ACTIVE: Mutex>> = Mutex::new(None); + +pub(in crate::livekit) struct Installed(Arc); +impl Drop for Installed { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &self.0)) { + *active = None; + } + } +} + +pub(in crate::livekit) fn install(gate: &Arc) -> Result { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.is_some() { + return Err("an owner gate is already installed"); + } + *active = Some(Arc::clone(gate)); + Ok(Installed(Arc::clone(gate))) +} + +pub(in crate::livekit) async fn before(stage: Stage, context: &mut GeminiEventContext<'_>) { + let gate = ACTIVE + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(Arc::clone); + let Some(gate) = gate else { return }; + let matches = { + let identity = context.activity.latency.acquisition_identity(); + gate.stage == stage + && gate.scope.room == identity.room + && gate.scope.session == identity.session + && gate.socket == identity.socket + }; + if !matches { + return; + } + { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if !active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &gate)) { + return; + } + *active = None; + } + gate.entered.notify_one(); + gate.release.notified().await; + match gate.transition { + Some(Transition::Hold(held)) => { + context.state.thinking_hold = if held { + ThinkingHold::Held { + since: Instant::now(), + } + } else { + ThinkingHold::Off + }; + } + Some(Transition::Discard(discarded)) => { + context.activity.socket.discarding_output = discarded; + } + None => {} + } +} diff --git a/src/livekit/recovery.rs b/src/livekit/recovery.rs index 3f268fa3b..d0a08bf41 100644 --- a/src/livekit/recovery.rs +++ b/src/livekit/recovery.rs @@ -30,7 +30,7 @@ pub(super) enum Terminal { #[cfg(test)] impl Terminal { - pub(super) fn name(&self) -> &'static str { + fn name(&self) -> &'static str { match self { Self::End { .. } => "End", Self::Deadline => "Deadline", @@ -44,9 +44,9 @@ impl Terminal { pub(super) struct Cancelled; pub(super) struct Packet { - pub(super) topic: Option, - pub(super) sender: Option, - pub(super) payload: Vec, + topic: Option, + sender: Option, + payload: Vec, } /// How many unsent context summaries a recovery keeps, and their total size. @@ -181,29 +181,28 @@ struct Action { /// A test-only barrier after input preparation and before transport polling. #[cfg(test)] pub(super) struct WriteBarrier { - pub(super) phase: &'static str, - pub(super) entered: std::sync::Arc, - pub(super) release: std::sync::Arc, + phase: &'static str, + entered: std::sync::Arc, + release: std::sync::Arc, } /// A test-only action run in the poll that completes a labelled write, before /// the driver can observe any later input. #[cfg(test)] pub(super) struct AfterWrite { - pub(super) phase: &'static str, - pub(super) action: Box, + phase: &'static str, + action: Box, } pub(super) struct Driver<'r, 'c, 'i> { - pub(super) room: &'r RoomLink, - pub(super) context: &'r mut GeminiEventContext<'c>, - pub(super) loops: &'r mut RoomLoop, - pub(super) interview: InterviewContext<'i>, - pub(super) ingress: Ingress, - pub(super) revision: u64, - pub(super) terminal: Option, + room: &'r RoomLink, + context: &'r mut GeminiEventContext<'c>, + loops: &'r mut RoomLoop, + interview: InterviewContext<'i>, + ingress: Ingress, + revision: u64, + terminal: Option, held_contexts: VecDeque, - held_bytes: usize, held_omitted: usize, held_omitted_total: usize, held_steps: Vec<&'static str>, @@ -236,7 +235,6 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { revision: 0, terminal: None, held_contexts: VecDeque::new(), - held_bytes: 0, held_omitted: 0, held_omitted_total: 0, held_steps: Vec::new(), @@ -343,17 +341,17 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { self.held_omitted_total += 1; return; } + let mut held_bytes = self.held_contexts.iter().map(String::len).sum::(); while self.held_contexts.len() >= MAX_HELD_CONTEXTS - || self.held_bytes + context.len() > MAX_HELD_BYTES + || held_bytes + context.len() > MAX_HELD_BYTES { let Some(old) = self.held_contexts.pop_front() else { break; }; - self.held_bytes -= old.len(); + held_bytes -= old.len(); self.held_omitted += 1; self.held_omitted_total += 1; } - self.held_bytes += context.len(); self.held_contexts.push_back(context); } @@ -505,7 +503,7 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { } } - pub(super) async fn wait(&mut self, op: impl Future) -> Result { + async fn wait(&mut self, op: impl Future) -> Result { tokio::pin!(op); loop { if self.terminal.is_some() { @@ -635,7 +633,7 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { } } - pub(super) async fn flush_notices(&mut self) -> Result<(), Cancelled> { + async fn flush_notices(&mut self) -> Result<(), Cancelled> { if let Some(round) = self.round { let message = serde_json::json!({"type":"round_state", "round":"behavioral", "status":round}); @@ -703,7 +701,6 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { async fn flush_contexts(&mut self, session: &mut GeminiLiveSession) -> Result { while !self.held_contexts.is_empty() || self.held_omitted != 0 { let contexts = std::mem::take(&mut self.held_contexts); - self.held_bytes = 0; let steps = std::mem::take(&mut self.held_steps); let omitted = std::mem::take(&mut self.held_omitted); let mut body = contexts.into_iter().collect::>().join("\n"); diff --git a/src/livekit/report.rs b/src/livekit/report.rs index 78e597de3..04710cc01 100644 --- a/src/livekit/report.rs +++ b/src/livekit/report.rs @@ -125,6 +125,8 @@ pub(super) async fn generate_report_bounded( seed: i64, refused: &std::sync::atomic::AtomicBool, ) -> GeneratedReport { + #[cfg(test)] + crate::gemini::tests::note_bounded_report(boot.room_name, seed); tokio::time::timeout( REPORT_TIMEOUT, generate_report_with_keys( diff --git a/src/livekit/session.rs b/src/livekit/session.rs index b7e4b3019..8a3954ff9 100644 --- a/src/livekit/session.rs +++ b/src/livekit/session.rs @@ -16,16 +16,20 @@ mod floor; mod observations; mod recovery; +mod tool_calls; use floor::write_floor; pub(super) use floor::{observe_floor, reconcile_floor}; -use observations::{HintCounters, ToolTiming, execute_timed_tool_batch, known_tool_name}; +use observations::{ + HintCounters, ToolTiming, execute_timed_tool_batch, known_tool_name, silent_completion_kind, +}; use recovery::log_socket_pressure; pub(super) use recovery::{ apply_recovery_transcript, drain_retired_usage, prepare_input, prepare_listening_handoff, prepare_plain_context, }; +use tool_calls::on_tool_calls; use std::collections::HashMap; use std::time::{Duration, Instant}; @@ -722,15 +726,6 @@ fn observe_latency_event( None } -/// What a completion with nothing in it answered, if this event is one. -fn silent_completion_kind(event: &GeminiEvent, activity: &RuntimeActivity) -> Option<&'static str> { - if matches!(event, GeminiEvent::TurnComplete) && !activity.generating { - activity.debt_silence_answers() - } else { - None - } -} - /// `accept_event_with_reply_debt`, plus the silence the event answered. Both /// reads come before the accept, because the accept clears the waits they /// read. A silent completion that settles the reply still closes the turn's @@ -828,6 +823,8 @@ pub(super) async fn handle_gemini_event( let discarding = context.activity.socket.discarding_output; let (accepted, interruption_debt, silent_owed) = accept_event_noting_silence(&event, context.activity, held, Instant::now()); + #[cfg(test)] + super::probe::acquisition::boundary(&context.activity.latency, &event, accepted); let handled = if accepted { if let Some(silent_owed) = silent_owed { eprintln!( @@ -1013,155 +1010,6 @@ pub(super) fn settle_hold_at_turn_end(state: &mut RuntimeState, activity: &mut R } } -/// Answers every call in the batch in one message, and republishes the -/// checklist once when any of them moved it. -async fn on_tool_calls( - room: &RoomLink, - context: &mut GeminiEventContext<'_>, - calls: Vec, -) -> Result<(), Box> { - if calls.is_empty() { - return Ok(()); - } - // A filler that fails to queue is not worth the tool round it precedes. - if let Err(error) = play_tool_ack(context, &calls).await { - eprintln!("tool acknowledgement not played: {error}"); - } - if context.state.example_drawing_enabled - && !context.state.interview_mode.is_whiteboard() - && calls.iter().any(|call| call.name == TOOL_READ_BOARD) - { - context.board.settle_for_read(context.state).await; - } - let timed = !context.state.interviewer_silenced() && !context.activity.socket.discarding_output; - let reviewed_revision = context.activity.substantive_revision_at_last_review; - let tool_started = Instant::now(); - let hints_before = HintCounters::capture(context.state); - let answers = execute_timed_tool_batch( - context.state, - context.activity, - calls, - reviewed_revision, - timed, - ); - let tool_execute_ms = crate::agent::saturating_ms(tool_started.elapsed()); - let editor_review_revision = answers - .iter() - .any(|(_, answer)| answer.get("editorReview").is_some()) - .then_some(context.state.evidence_ledger.code.substantive_revision); - - // What `read_board` could not put in its own response. Once per batch, so a - // batch that asked twice puts the board up once, and before the responses - // rather than after: Gemini starts its reply as soon as it has them, so a - // board sent after would arrive under an answer already being spoken about - // the board before it, while the tool has said this one is in front of it. - // - // Not `?`: an image the socket would not take is worth a line and a turn - // that answers from the board it already had, not an interview ended on the - // write. - let board_started = Instant::now(); - if std::mem::take(&mut context.state.board_resend_requested) - && let Err(error) = board::resend(context.board, context.gemini).await - { - eprintln!("board resend failed ({error}); waiting for the close to be reported"); - } - let board_resend_ms = crate::agent::saturating_ms(board_started.elapsed()); - - // Not `?`. A response that did not go out is owed nothing back, so the flag - // stays down, and the socket it failed on is replaced when its close is - // reported; the checklist below still reflects the calls. - let send_started = Instant::now(); - let sent = context.gemini.send_tool_responses(&answers).await; - let tool_send_ms = crate::agent::saturating_ms(send_started.elapsed()); - if timed { - context - .activity - .turn_timing - .note_tool_response(send_started, Instant::now(), sent.is_ok()); - } - - // Gemini resumes the moment the responses land, so serializing them again - // for the lines below waits until after the send. - if !context.activity.socket.discarding_output && !context.state.interviewer_silenced() { - let timing = observations::tool_timing( - context.activity, - context.state, - &answers, - hints_before, - ToolTiming { - batch_ms: crate::agent::saturating_ms(tool_started.elapsed()), - execute_ms: tool_execute_ms, - board_ms: board_resend_ms, - send_ms: tool_send_ms, - sent: sent.is_ok(), - }, - ); - latency::log(&timing); - } - for (call, response) in &answers { - let name = known_tool_name(&call.name); - eprintln!( - "codetrial live_tool_response room={} tool={name} bytes={} accepted={}", - room.name(), - response.to_string().len(), - response.get("error").is_none(), - ); - - // The phase as the checklist spells it, and only when it is one: the - // argument is model text, and the rest of the call stays out of the log - // for the reason the checklist carries no summary. - if call.name == TOOL_RECORD_FRAMEWORK_EVIDENCE { - let phase = call - .args - .get("phase") - .and_then(serde_json::Value::as_str) - .filter(|phase| { - crate::agent::REACTO_PHASE_IDS.contains(phase) - || crate::agent::STAR_PHASE_IDS.contains(phase) - }) - .unwrap_or("?"); - eprintln!( - "evidence: at={} phase={phase} accepted={} room={}", - log_clock(context.state), - response.get("error").is_none(), - room.name() - ); - } - } - - if let Some(revision) = editor_review_revision { - eprintln!( - "codetrial live_editor_review room={} revision={revision} sent={}", - room.name(), - sent.is_ok(), - ); - } - context.activity.settle_tool_editor_review( - editor_review_revision, - sent.is_ok(), - Instant::now(), - ); - - // A reminder of open earlier steps in a response that never went out has - // not been read, so it may be named again. - context.state.settle_earlier_steps(sent.is_ok()); - match sent { - // Gemini now owes a generation for this, and will deliver it on this - // socket or not at all. - Ok(()) => { - context.gemini.input_cause = Some(TurnCause::Tool.label()); - context.activity.note_tool_response(Instant::now()); - } - Err(error) => { - eprintln!( - "Gemini tool response failed ({error}); waiting for the close to be reported" - ); - } - } - flush_framework_progress(room, context.state).await; - Ok(()) -} - /// A tool round holds back the reply a candidate is waiting for by a model /// round trip; when enabled, a short acknowledgement in Jim's voice fills it. /// Queued, not counted as the reply: the watchdog and the latency trace still @@ -1233,7 +1081,7 @@ fn on_output_transcript(room: &RoomLink, context: &mut GeminiEventContext<'_>, t final_segment: false, identity: None, }; - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } @@ -2824,7 +2672,7 @@ pub(super) fn close_turns(room: &RoomLink, context: &mut GeminiEventContext<'_>) eprintln!("{line}"); } for segment in closing { - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } @@ -2838,7 +2686,7 @@ fn close_interviewer_turn(room: &RoomLink, context: &mut GeminiEventContext<'_>) crate::current_epoch_millis(), None, ) { - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } diff --git a/src/livekit/session/observations.rs b/src/livekit/session/observations.rs index 6dbffcd0f..19fbb53d4 100644 --- a/src/livekit/session/observations.rs +++ b/src/livekit/session/observations.rs @@ -1,16 +1,21 @@ -//! Content-free tool context for the observational replay harness. +//! Production tool telemetry stays content-free; cfg(test) acquisition +//! separately +//! captures private authored-fixture answers before subsequent asynchronous +//! work. use std::time::Instant; use serde_json::Value; use crate::agent::RuntimeState; -use crate::gemini::GeminiFunctionCall; +use crate::gemini::{GeminiEvent, GeminiFunctionCall}; use crate::livekit::turn::RuntimeActivity; use crate::runtime::{ TOOL_END_INTERVIEW, TOOL_LOG_HINT, TOOL_READ_BOARD, TOOL_READ_CONVERSATION, TOOL_READ_EDITOR, TOOL_RECORD_FRAMEWORK_EVIDENCE, }; +use super::{GeminiEventContext, RoomLink, Segment}; + /// A declared tool's name, never the model's text for an undeclared one. pub(super) fn known_tool_name(name: &str) -> &'static str { [ @@ -32,8 +37,9 @@ pub(super) fn execute_timed_tool_batch( calls: Vec, reviewed_revision: u64, timed: bool, + #[cfg(test)] ticket: &mut Option, ) -> Vec<(GeminiFunctionCall, Value)> { - super::execute_tool_batch_observed( + let answers = super::execute_tool_batch_observed( state, calls, reviewed_revision, @@ -46,7 +52,45 @@ pub(super) fn execute_timed_tool_batch( activity.latency.closing_requested(started, finished); } }, - ) + ); + #[cfg(test)] + { + *ticket = super::super::probe::acquisition::observe(&activity.latency, &answers); + } + answers +} + +/// Publishes a transcript segment, noting the interviewer's for the +/// acquisition capture on the way. +pub(super) fn publish_segment( + room: &RoomLink, + context: &mut GeminiEventContext<'_>, + segment: Segment, +) { + #[cfg(test)] + // The id, not the identity: a candidate's final is published as the agent + // when the candidate has already left. + if segment.identity.is_none() && segment.id.starts_with("interviewer-") { + super::super::probe::acquisition::reply( + &context.activity.latency, + &segment.id, + &segment.text, + segment.final_segment, + ); + } + context.transcripts.publish(room, segment); +} + +/// What a completion with nothing in it answered, if this event is one. +pub(super) fn silent_completion_kind( + event: &GeminiEvent, + activity: &RuntimeActivity, +) -> Option<&'static str> { + if matches!(event, GeminiEvent::TurnComplete) && !activity.generating { + activity.debt_silence_answers() + } else { + None + } } pub(super) struct HintCounters { diff --git a/src/livekit/session/tool_calls.rs b/src/livekit/session/tool_calls.rs new file mode 100644 index 000000000..4bbb6319c --- /dev/null +++ b/src/livekit/session/tool_calls.rs @@ -0,0 +1,184 @@ +//! Executes and sends one interview tool batch. + +use super::{ + GeminiEventContext, GeminiFunctionCall, HintCounters, Instant, RoomLink, TOOL_READ_BOARD, + TOOL_RECORD_FRAMEWORK_EVIDENCE, ToolTiming, TurnCause, board, execute_timed_tool_batch, + flush_framework_progress, known_tool_name, latency, log_clock, observations, play_tool_ack, +}; + +#[cfg(test)] +use crate::livekit::probe::acquisition::owner::{self, Stage}; + +/// Answers every call in the batch in one message, and republishes the +/// checklist once when any of them moved it. +pub(super) async fn on_tool_calls( + room: &RoomLink, + context: &mut GeminiEventContext<'_>, + calls: Vec, +) -> Result<(), Box> { + if calls.is_empty() { + return Ok(()); + } + // Before every await below, any of which can cancel the batch. + #[cfg(test)] + crate::livekit::probe::acquisition::receive( + &context.activity.latency, + &calls, + context.activity.socket.discarding_output || context.state.interviewer_silenced(), + ); + // A filler that fails to queue is not worth the tool round it precedes. + #[cfg(test)] + owner::before(Stage::Acknowledgement, context).await; + if let Err(error) = play_tool_ack(context, &calls).await { + eprintln!("tool acknowledgement not played: {error}"); + } + if context.state.example_drawing_enabled + && !context.state.interview_mode.is_whiteboard() + && calls.iter().any(|call| call.name == TOOL_READ_BOARD) + { + #[cfg(test)] + owner::before(Stage::BoardSettlement, context).await; + context.board.settle_for_read(context.state).await; + } + let timed = !context.state.interviewer_silenced() && !context.activity.socket.discarding_output; + let reviewed_revision = context.activity.substantive_revision_at_last_review; + let tool_started = Instant::now(); + let hints_before = HintCounters::capture(context.state); + #[cfg(test)] + let mut ticket = None; + let answers = execute_timed_tool_batch( + context.state, + context.activity, + calls, + reviewed_revision, + timed, + #[cfg(test)] + &mut ticket, + ); + let tool_execute_ms = crate::agent::saturating_ms(tool_started.elapsed()); + let editor_review_revision = answers + .iter() + .any(|(_, answer)| answer.get("editorReview").is_some()) + .then_some(context.state.evidence_ledger.code.substantive_revision); + + // What `read_board` could not put in its own response. Once per batch, so a + // batch that asked twice puts the board up once, and before the responses + // rather than after: Gemini starts its reply as soon as it has them, so a + // board sent after would arrive under an answer already being spoken about + // the board before it, while the tool has said this one is in front of it. + // + // Not `?`: an image the socket would not take is worth a line and a turn + // that answers from the board it already had, not an interview ended on the + // write. + let board_started = Instant::now(); + if std::mem::take(&mut context.state.board_resend_requested) + && let Err(error) = { + #[cfg(test)] + owner::before(Stage::BoardResend, context).await; + board::resend(context.board, context.gemini).await + } + { + eprintln!("board resend failed ({error}); waiting for the close to be reported"); + } + let board_resend_ms = crate::agent::saturating_ms(board_started.elapsed()); + + // Not `?`. A response that did not go out is owed nothing back, so the flag + // stays down, and the socket it failed on is replaced when its close is + // reported; the checklist below still reflects the calls. + #[cfg(test)] + owner::before(Stage::ToolSend, context).await; + let send_started = Instant::now(); + let sent = context.gemini.send_tool_responses(&answers).await; + let tool_send_ms = crate::agent::saturating_ms(send_started.elapsed()); + #[cfg(test)] + if let Some(ticket) = ticket { + ticket.complete(sent.is_ok()); + } + if timed { + context + .activity + .turn_timing + .note_tool_response(send_started, Instant::now(), sent.is_ok()); + } + + // Gemini resumes the moment the responses land, so serializing them again + // for the lines below waits until after the send. + if !context.activity.socket.discarding_output && !context.state.interviewer_silenced() { + let timing = observations::tool_timing( + context.activity, + context.state, + &answers, + hints_before, + ToolTiming { + batch_ms: crate::agent::saturating_ms(tool_started.elapsed()), + execute_ms: tool_execute_ms, + board_ms: board_resend_ms, + send_ms: tool_send_ms, + sent: sent.is_ok(), + }, + ); + latency::log(&timing); + } + for (call, response) in &answers { + let name = known_tool_name(&call.name); + eprintln!( + "codetrial live_tool_response room={} tool={name} bytes={} accepted={}", + room.name(), + response.to_string().len(), + response.get("error").is_none(), + ); + + // The phase as the checklist spells it, and only when it is one: the + // argument is model text, and the rest of the call stays out of the log + // for the reason the checklist carries no summary. + if call.name == TOOL_RECORD_FRAMEWORK_EVIDENCE { + let phase = call + .args + .get("phase") + .and_then(serde_json::Value::as_str) + .filter(|phase| { + crate::agent::REACTO_PHASE_IDS.contains(phase) + || crate::agent::STAR_PHASE_IDS.contains(phase) + }) + .unwrap_or("?"); + eprintln!( + "evidence: at={} phase={phase} accepted={} room={}", + log_clock(context.state), + response.get("error").is_none(), + room.name() + ); + } + } + + if let Some(revision) = editor_review_revision { + eprintln!( + "codetrial live_editor_review room={} revision={revision} sent={}", + room.name(), + sent.is_ok(), + ); + } + context.activity.settle_tool_editor_review( + editor_review_revision, + sent.is_ok(), + Instant::now(), + ); + + // A reminder of open earlier steps in a response that never went out has + // not been read, so it may be named again. + context.state.settle_earlier_steps(sent.is_ok()); + match sent { + // Gemini now owes a generation for this, and will deliver it on this + // socket or not at all. + Ok(()) => { + context.gemini.input_cause = Some(TurnCause::Tool.label()); + context.activity.note_tool_response(Instant::now()); + } + Err(error) => { + eprintln!( + "Gemini tool response failed ({error}); waiting for the close to be reported" + ); + } + } + flush_framework_progress(room, context.state).await; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index dabb37b14..426b4bbe4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,44 +1,22 @@ -mod main_peer; - use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; -use codetrial::config::{ - AgentConfig, DEFAULT_DURATION_MIN, DEFAULT_ROOM_PREFIX, DEFAULT_WEB_ADDR, DEFAULT_WEB_DIR, +use codetrial::config::{AgentConfig, DEFAULT_WEB_ADDR, is_production}; +use codetrial::launch::{ + Bounds, CliOptions, LaunchedBy, Reachable, cold_start_config_path, config_dir, + data_dir_config_from_environment, environment_and_flags, extend_pool, finish, load_values, + local_agent_config, nonempty, prepare, primary_config_path, production_secret_refusal, + reachable_from_elsewhere, serve, value_or, +}; +#[cfg(test)] +use codetrial::launch::{ + DEFAULT_SESSION_SECRET, SESSION_SECRET_FILE, account_db_path, allowed_hosts, apply_options, + data_dir_session_secret, environment, guest_mode_enabled, guest_mode_recording_conflict, + published_secret_refusal, recording_can_deliver, recording_needs_verified_identity, + relaxed_for_local_use, serve_with, task_mode_allowed, task_mode_enabled, task_mode_refusal, + web_server_config, }; -use codetrial::web::{RoomDispatcher, WebServerConfig}; - -/// One installation keeps its config file and its account database together in -/// `config/`, under whichever root it was installed as: a checkout, or the -/// folder a released binary was unpacked into. Beside the executable is not -/// that root for a checkout -- the executable is in `target/`, which -/// `make clean` deletes -- so the two are named separately and joined per case. -const CONFIG_DIR: &str = "config"; -const CONFIG_FILE_NAME: &str = "codetrial.env.local"; -/// Tracked, so it exists in a checkout and in no release archive. That makes it -/// the one thing that tells a `config/` belonging to this project apart from a -/// directory of the same name that happens to sit where the binary was run. -const CONFIG_EXAMPLE_NAME: &str = "codetrial.env.example"; -const DEFAULT_ACCOUNT_DB_PATH: &str = "codetrial.db"; -const DEFAULT_SESSION_SECRET: &str = "codetrial-local-session"; -/// Where a `--data-dir` install keeps the key its session cookies are signed -/// with, when its config file does not set `SESSION_SECRET`. -const SESSION_SECRET_FILE: &str = "session-secret"; - -#[derive(Debug, Default, Clone)] -struct CliOptions { - config_path: Option, - data_dir: Option, - import_config: Option, - web_addr: Option, - web_dir: Option, - allowed_hosts: Option, - room_prefix: Option, - duration_min: Option, - task_mode: bool, - no_update_check: bool, -} /// Every mode this binary has: its name, how many positionals it takes counting /// the mode word, how it is called, and what runs it. One table and not one @@ -300,92 +278,17 @@ fn bind_web_listener(addr: &str) -> Result { Ok(listener) } -/// Validates operator settings before a listener or background task starts. -fn web_server_config( - values: &std::collections::BTreeMap, - pool: codetrial::config::ProviderPool, - production: bool, - options: &CliOptions, -) -> Result { - // After the pool is complete, so the "is this project in the pool" check - // sees the pool the server will actually serve, and before the listener - // does any work: a half-configured recording block is a startup failure, - // never a surprise at the moment a candidate's interview ends. - let recording = codetrial::config::load_recording( - values, - &pool, - interview_duration_min(values), - production, - ) - .map_err(|error| error.to_string())?; - recording_needs_verified_identity( - recording.is_some(), - nonempty(values, "GITHUB_CLIENT_ID").is_some() - && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), - )?; - recording_can_deliver(recording.as_ref())?; - let (peer, peer_observer) = main_peer::peer_services( - values, - nonempty(values, "GITHUB_CLIENT_ID").is_some() - && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), - )? - .unzip(); - - let config = WebServerConfig { - tasks: None, - task_mode_requested: options.task_mode, - web_dir: match nonempty(values, "CODETRIAL_WEB_DIR") { - Some(dir) => PathBuf::from(dir), - - // Under the data directory rather than the working one, which a - // launcher does not choose: a `web/` that happens to sit where the - // app was started would otherwise replace the embedded pages. - None => match &options.data_dir { - Some(dir) => dir.join(DEFAULT_WEB_DIR), - None => PathBuf::from(DEFAULT_WEB_DIR), - }, - }, - github_client_id: nonempty(values, "GITHUB_CLIENT_ID"), - github_client_secret: nonempty(values, "GITHUB_CLIENT_SECRET"), - session_secret: Some(value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET)), - db_path: Some(account_db_path(values, options)), - github_oauth_base_url: None, - github_api_base_url: None, - room_prefix: value_or(values, "CODETRIAL_ROOM_PREFIX", DEFAULT_ROOM_PREFIX), - fixed_room_name: nonempty(values, "INTERVIEW_ROOM_NAME"), - production, - trusted_proxy_hops: trusted_proxy_hops(values), - allowed_hosts: allowed_hosts(values)?, - compiler_explorer_enabled: compiler_explorer_enabled(values), - guest_mode_enabled: guest_mode_enabled(values), - recording, - peer_observer, - peer, - pool, - probe_provider_quota: true, - - // Not a function of its own: off macOS this is always `None`, so no - // test on the Linux CI could tell a replaced body from this one. - update_check_commit: codetrial::config::update_check_commit( - values.get("CODETRIAL_UPDATE_CHECK").map(String::as_str), - env!("CODETRIAL_RELEASE_COMMIT"), - cfg!(target_os = "macos"), - ), - provision_rooms: true, - }; - if let Some(observer) = &config.peer_observer { - observer.validate_audio_storage(&config.web_dir)?; - } - Ok(config) -} - fn run_web(options: CliOptions) -> Result<(), String> { // Falls through: once `serve_setup` returns, the config exists and the rest // of this function is an ordinary launch, on the socket Setup served on. // Why that socket is carried here rather than rebound: `serve_setup`. Read // once, so the question and the page it opens answer from the same map. let before_config = environment_and_flags(&options); - let launched_as_child = codetrial::exit_when_stdin_closes(); + let launched_by = if codetrial::exit_when_stdin_closes() { + LaunchedBy::Parent + } else { + LaunchedBy::Operator + }; data_dir_config_from_environment(&options, is_production(&before_config))?; let handed_over = if is_cold_start(&options, &before_config) { Some(serve_setup(&before_config, &options)?) @@ -394,33 +297,9 @@ fn run_web(options: CliOptions) -> Result<(), String> { }; let values = load_values(&options)?; - if let Some(refusal) = production_secret_refusal(&values) { - return Err(refusal); - } - for warning in relaxed_for_local_use(&values) { - eprintln!("{warning}"); - } - - // Beside the other startup warnings and not next to the dispatcher that - // consumes it. A cap the operator wrote and this cannot honor is worth - // saying before the first thing that can fail: read late, a bad bind or a - // half-configured pool ends the process first and the operator never learns - // the number in their config file was ignored. - let max_concurrent = read_max_concurrent(&values); - - let production = is_production(&values); - let pool = web_provider_pool(&values, &options, production)?; - - let mut config = web_server_config(&values, pool, production, &options)?; - - if guest_mode_recording_conflict(config.guest_mode_enabled, config.recording.is_some()) { - eprintln!( - "WARNING: CODETRIAL_GUEST_MODE_ENABLED=true is ignored while recording is enabled; \\ - guest mode is unavailable" - ); - } - - initialize_accounts(&config)?; + let prepared = prepare(&values, &options)?; + let max_concurrent = prepared.max_concurrent; + let config = prepared.config; // The address is not re-read for a cold start: `serve_setup` read it from // the same environment and flags, and the config file written since holds @@ -430,15 +309,16 @@ fn run_web(options: CliOptions) -> Result<(), String> { None => bind_web_listener(&value_or(&values, "CODETRIAL_WEB_ADDR", DEFAULT_WEB_ADDR))?, }; - // The other half of the `SESSION_SECRET` guard above, and the half that - // does not depend on the operator having said anything. + // The other half of the `SESSION_SECRET` guard in `launch::prepare`, and + // the half that does not depend on the operator having said anything. // // After the bind rather than before it, because the address may be a // hostname: `CODETRIAL_WEB_ADDR=interviews.example:3000` is resolved by - // `ToSocketAddrs` and only the socket knows what it landed on. That puts it - // beside `initialize_accounts`, which refuses after binding for the same - // reason and with the same consequence: the socket exists and its backlog - // fills, nothing accepts from it, and the process exits non-zero. + // `ToSocketAddrs` and only the socket knows what it landed on. A refusal + // here therefore comes after the bind, with the consequence that follows: + // the socket exists and its backlog fills, nothing accepts from it, and the + // process exits non-zero. (`launch::prepare`, with `initialize_accounts`, + // runs before the bind and has no such cost.) // // Fails closed when the socket cannot name itself, which a just-bound // listener does not do. Skipping the check on an unreadable address would @@ -447,48 +327,7 @@ fn run_web(options: CliOptions) -> Result<(), String> { let bound = listener .local_addr() .map_err(|error| format!("bound listener has no address to check: {error}"))?; - if let Some(refusal) = published_secret_refusal(&values, bound) { - return Err(refusal); - } - let setup = codetrial::web::SetupContext::new( - primary_config_path(&options)?, - values.clone(), - bound, - !launched_as_child, - config.trusted_proxy_hops, - )?; - // Same reason `serve_setup` prints this: say where to go. - println!("codetrial: open http://{bound} in your browser"); - if let Some(setup) = &setup { - println!( - "codetrial: to change keys at http://{bound}/setup, enter {}", - setup.code() - ); - } - - // Task mode is off unless `--task-mode` asks for it. It serves the learner - // at this machine and nobody else: it holds decrypted reference solutions - // in memory and trusts its page to report the test rules. Reachable from - // anywhere else, or set up to record or to share one fixed room, its routes - // say why they are off. A declared proxy makes a loopback bind as public as - // any other. - let reachable = reachable_from_elsewhere(&values, bound); - let (fixed_room, recording) = (config.fixed_room_name.is_some(), config.recording.is_some()); - config.tasks = task_mode_enabled(config.task_mode_requested, reachable, fixed_room, recording) - .then(codetrial::tasks::access::TaskService::new); - - // Said at startup, because the address printed above is the interview lobby - // on any other server, and a learner who asked for task mode would not - // otherwise know what to open or why nothing changed. - if config.tasks.is_some() { - println!( - "codetrial: task mode is on; open http://{bound} for the default task, or the assignment link your instructor sent" - ); - } else if config.task_mode_requested - && let Some(reason) = task_mode_refusal(reachable, fixed_room, recording) - { - eprintln!("codetrial: --task-mode is off: {reason}"); - } + let (config, setup) = finish(config, &values, &options, bound, launched_by)?; // Past every startup check, so whatever a previous run left in the log is // no longer true. Written on failure and removed on none, the file outlives @@ -498,96 +337,17 @@ fn run_web(options: CliOptions) -> Result<(), String> { // at nothing else. clear_web_error_log(&options); - // Whether this process also hosts interviewers. A full agent config, which - // is a Gemini key on top of what the web side needs, means yes; without it - // this is the web half of a split deployment and agents arrive from - // elsewhere. Deciding it by what is configured, rather than by a flag, - // keeps a single-host production deployment working out of the box while - // leaving the split one exactly as it was. No `extend_pool` here on - // purpose. The dispatcher never looks a provider up: it is handed the one - // the token was minted from, so a second scan could only introduce a pool - // that disagrees with the web side's. - // - // Only keys that are absent make it the web half. An entry that is present - // but invalid is a mistake in this host's config, and serving on without - // the local interviewer would leave every interview waiting forever. - let agent_config = match codetrial::config::load_from_pairs(values) { - Ok(config) => Some(config), - Err(error) if error.invalid_entries.is_empty() => None, - Err(error) => return Err(error.to_string()), - }; - if agent_config.is_none() { - eprintln!( - "no GOOGLE_API_KEY or GOOGLE_API_KEYS: serving the web side only. Interviews will wait forever unless \ - an agent joins each room from elsewhere." - ); - } - - let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); - runtime - .block_on(async { - let listener = tokio::net::TcpListener::from_std(listener)?; - let dispatcher = agent_config.map(|config| { - Arc::new(codetrial::dispatch::LocalDispatcher { - config, - runtime: tokio::runtime::Handle::current(), - state: Arc::default(), - max_concurrent, - }) as Arc - }); - axum::serve( - listener, - codetrial::web::web_service_with_setup(config, dispatcher, setup), - ) - .await - }) - .map_err(|error| format!("web server failed: {error}")) -} + let agent_config = local_agent_config(values)?; -/// The pool this deployment will serve. -/// -/// Built straight from the values rather than through `load_from_pairs`: the -/// web mode serves HTTP and mints LiveKit tokens, and has no use for the Gemini -/// key that a full agent config insists on. Demanding it here turned a web-only -/// deployment into an empty pool, silently. -fn web_provider_pool( - values: &BTreeMap, - options: &CliOptions, - production: bool, -) -> Result { - let mut pool = codetrial::config::ProviderPool::default(); - let (url, api_key, api_secret) = match ( - nonempty(values, "LIVEKIT_URL"), - nonempty(values, "LIVEKIT_API_KEY"), - nonempty(values, "LIVEKIT_API_SECRET"), - ) { - (Some(url), Some(api_key), Some(api_secret)) => (url, api_key, api_secret), - - // Names the file that was actually read. A fixed `config/` in this - // message sends anyone running with `--config` or a bare - // `./codetrial.env.local` to edit a file the server never opened. - _ => { - return Err(format!( - "missing required LiveKit credentials: set LIVEKIT_URL, LIVEKIT_API_KEY and LIVEKIT_API_SECRET in {}", - primary_config_path(options)?.display() - )); - } - }; - codetrial::config::validate_livekit_url(&url, production)?; - pool.providers.push(codetrial::config::Provider { - id: codetrial::config::PRIMARY_PROVIDER_ID.to_string(), - url, - api_key, - api_secret, - google_api_keys: codetrial::config::google_api_keys(values)?, - }); - extend_pool( - &mut pool, - production, - &config_dir(options), - &value_or(values, codetrial::config::PROVIDER_ORDER_KEY, ""), - ); - Ok(pool) + serve( + listener, + config, + setup, + agent_config, + max_concurrent, + std::future::pending(), + Bounds::default(), + ) } /// The solo self-serve cold start: no `--config` given, none of the searched @@ -695,29 +455,6 @@ fn serve_setup( Ok(retained) } -/// Why a production launch may not sign session cookies, or `None` where it -/// may. The built-in default is a published string, so in production it is not -/// a weak signing key, it is a known one. -/// -/// A function rather than an inline check because `serve_setup` asks it too. -/// The verdict needs nothing from the submission, and a check that could have -/// been made before the page was served but is made after it has taken -/// credentials, written them and answered 200 is a launch that exits with the -/// config already on disk -- which is what stops the next launch being a cold -/// start, so the page never comes back to say why. -fn production_secret_refusal(values: &BTreeMap) -> Option { - if is_production(values) - && value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) == DEFAULT_SESSION_SECRET - { - return Some( - "SESSION_SECRET must be set when NODE_ENV=production; the built-in \ - default is a published value that would let anyone forge a session cookie" - .to_string(), - ); - } - None -} - /// Why Setup may not serve on `bound`, or `None` where it may. /// /// Setup writes LiveKit credentials to disk for whoever posts them and has @@ -878,613 +615,6 @@ fn load_agent_config(options: &CliOptions) -> Result { Ok(config) } -/// The environment and the flags, which in a cold start is everything -/// `load_values` has: the file it reads between them is the file that does not -/// exist yet. -/// -/// Its own function so `run_web` does not name the process environment. Reading -/// a deployment key straight from it there is what -/// `binary_web_reads_deployment_keys_from_the_config_file` refuses, because a -/// key set in the config file was once silently a no-op. -fn environment_and_flags(options: &CliOptions) -> BTreeMap { - let mut values = environment(options); - apply_options(&mut values, options); - values -} - -/// A first `--data-dir` launch started where the credentials Setup asks for -/// are already exported writes them into the folder's config, as Setup would -/// have, instead of asking for them again; every later launch then finds the -/// file, including one started from a desktop that exports nothing. -/// -/// The one exception to `environment` ignoring the process environment, and a -/// narrow one: only Setup's own keys, only while the folder has no config, and -/// written down rather than read live, so the file stays the whole of the -/// install's configuration. -fn data_dir_config_from_environment(options: &CliOptions, production: bool) -> Result<(), String> { - let Some(dir) = &options.data_dir else { - return Ok(()); - }; - let path = dir.join(CONFIG_FILE_NAME); - if path.exists() { - return Ok(()); - } - let exported = std::env::vars().collect(); - let Some(contents) = - codetrial::web::config_from_values(&exported, production).map_err(|error| { - format!("the exported Setup keys cannot be written to a config: {error}") - })? - else { - return Ok(()); - }; - std::fs::create_dir_all(dir) - .and_then(|()| codetrial::create_owner_only(&path, contents.as_bytes())) - .map_err(|error| format!("cannot write {}: {error}", path.display()))?; - println!( - "codetrial: wrote {} from the exported LiveKit and Google keys", - path.display() - ); - Ok(()) -} - -/// The process environment, or nothing for a `--data-dir` launch. -/// -/// A launcher that names the folder is saying the install's configuration is -/// in it, and an environment it was started from is not part of that: a shell -/// that exported `CODETRIAL_DB_PATH` or `NODE_ENV` for a CLI deployment would -/// otherwise move this install's reports, or change its rules, depending on -/// how it was started. Decided here, once, rather than by every launcher -/// keeping its own list of the keys this program reads. -fn environment(options: &CliOptions) -> BTreeMap { - if options.data_dir.is_some() { - return BTreeMap::new(); - } - std::env::vars().collect() -} - -fn load_values(options: &CliOptions) -> Result, String> { - let mut values = environment(options); - let path = primary_config_path(options)?; - codetrial::config::apply_config_file(&mut values, codetrial::config::read_config_file(&path)?); - apply_options(&mut values, options); - if let Some(dir) = &options.data_dir - && nonempty(&values, "SESSION_SECRET").is_none() - { - let secret = data_dir_session_secret(&dir.join(SESSION_SECRET_FILE))?; - values.insert("SESSION_SECRET".to_string(), secret); - } - Ok(values) -} - -/// The key a `--data-dir` install signs session cookies with, made on its first -/// launch and reused, so a relaunch keeps everyone signed in. -/// -/// Not the built-in default: that one is published, and any program on the -/// machine can reach a loopback port to present a cookie signed with it. -/// Owner-only on Unix for the same reason. A file that is not a key this wrote -/// is refused rather than replaced, because replacing it signs everyone out -/// and hides whatever damaged it. -fn data_dir_session_secret(path: &std::path::Path) -> Result { - match std::fs::read_to_string(path) { - Ok(text) => { - let secret = text.trim(); - if secret.len() >= 32 - && secret - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') - { - Ok(secret.to_string()) - } else { - Err(format!( - "{} is not a session key this program wrote. Move it aside to make a new \ - one; everyone will be signed out.", - path.display() - )) - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let secret = codetrial::accounts::random_token(32) - .map_err(|error| format!("cannot generate a session key: {error}"))?; - codetrial::create_owner_only(path, secret.as_bytes()) - .map_err(|error| format!("cannot write {}: {error}", path.display()))?; - Ok(secret) - } - Err(error) => Err(format!("cannot read {}: {error}", path.display())), - } -} - -/// The flags, over whatever the environment and the config file said. -/// -/// Split out because a cold start has to build the same map without the file -/// it does not have yet: `serve_setup` decides where to bind and what a -/// submission must survive, and both answers have to be the ones `run_web` -/// will reach a moment later. Spelled twice, this precedence is free to drift. -fn apply_options(values: &mut BTreeMap, options: &CliOptions) { - if let Some(value) = &options.web_addr { - values.insert("CODETRIAL_WEB_ADDR".to_string(), value.clone()); - } - if let Some(value) = &options.web_dir { - values.insert("CODETRIAL_WEB_DIR".to_string(), value.clone()); - } - if let Some(value) = &options.allowed_hosts { - values.insert("CODETRIAL_ALLOWED_HOSTS".to_string(), value.clone()); - } - if let Some(value) = &options.room_prefix { - values.insert("CODETRIAL_ROOM_PREFIX".to_string(), value.clone()); - } - if let Some(value) = options.duration_min { - values.insert("CODETRIAL_DURATION_MIN".to_string(), value.to_string()); - } - - // Over the config file as well: the desktop app is published as its own - // release, so the one the check reads never names its build. - if options.no_update_check { - values.insert("CODETRIAL_UPDATE_CHECK".to_string(), "false".to_string()); - } -} - -fn primary_config_path(options: &CliOptions) -> Result { - // The one place searched, and not one place among several: the working - // directory and the executable's folder are whatever the launcher left them - // as, so a config found there would belong to some other install. - if let Some(dir) = &options.data_dir { - let path = dir.join(CONFIG_FILE_NAME); - if path.is_file() { - return Ok(path); - } - return Err(format!( - "required configuration file is missing: {}; write one with LIVEKIT_URL, \ - LIVEKIT_API_KEY and LIVEKIT_API_SECRET", - path.display() - )); - } - if let Some(path) = &options.config_path { - let path = PathBuf::from(path); - if !path.is_file() { - return Err(format!( - "required configuration file is missing: {}", - path.display() - )); - } - return Ok(path); - } - - // The working directory first, which is a checkout's `config/` and an - // operator's deployment directory, then the folder the executable sits in. - // A release binary is unpacked into a folder of its own and its config is - // written there, so it has to be findable from a shortcut or a terminal - // opened somewhere else, not only from a double-click. - // - // Within each of those two roots, `config/` before the bare filename. The - // bare ones are where a release used to keep its config and where an - // operator may still keep one; they are searched, not written. - for path in [ - PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME), - PathBuf::from(CONFIG_FILE_NAME), - codetrial::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME), - codetrial::exe_dir().join(CONFIG_FILE_NAME), - ] { - if path.is_file() { - return Ok(path); - } - } - - // The keys, not a file to copy. `config/codetrial.env.example` exists in a - // checkout and nowhere else: the release archives hold the executable and - // nothing beside it, so telling someone who just unpacked one to copy it - // sends them looking for a file they were never given. Naming what the file - // must contain is an instruction both audiences can act on. - Err(format!( - "required configuration file is missing: ./{CONFIG_DIR}/{CONFIG_FILE_NAME}, \ - ./{CONFIG_FILE_NAME}, or {CONFIG_DIR}/{CONFIG_FILE_NAME} beside the \ - executable; write one with LIVEKIT_URL, LIVEKIT_API_KEY and \ - LIVEKIT_API_SECRET ({CONFIG_DIR}/{CONFIG_EXAMPLE_NAME} lists the \ - optional keys in a checkout)" - )) -} - -/// The directory of the config file this launch actually read, which is where -/// everything else belonging to the installation lives: the provider files it -/// discovers, and the account database it writes. -/// -/// So `--config /etc/codetrial/prod.env` discovers -/// `/etc/codetrial/codetrial.env.` rather than whatever `config/` happens -/// to sit in the current directory. Both halves of a deployment are launched -/// with the same `--config`, and that is what makes them agree on the pool. -/// -/// The database is written here rather than read, which is the one asymmetry: -/// a `--config` pointing somewhere a process may not write is a startup -/// failure that names the directory, and `CODETRIAL_DB_PATH` is the way out. -fn config_dir(options: &CliOptions) -> PathBuf { - if let Some(dir) = &options.data_dir { - return dir.clone(); - } - - // The search order comes from primary_config_path rather than being written - // out again here. Spelled twice, it was free to drift, and the drift is - // silent: the server keeps reading its own config while the pool quietly - // collapses to the primary project. - // - // A named file still decides, even when it is missing. The operator said - // where the config lives, and load_values is what refuses a path that is - // not there; answering with a different directory would be this function - // second-guessing that. - let path = options - .config_path - .as_deref() - .map(PathBuf::from) - .or_else(|| primary_config_path(options).ok()) - .unwrap_or_else(|| PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME)); - match path.parent() { - // A bare filename has an empty parent, and its siblings are in the - // working directory, not in `config/`. - Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(), - _ => PathBuf::from("."), - } -} - -/// The pool this process routes with. Discovery happens here, once, and never -/// inside `load_from_pairs`: a config loader that reads the current directory -/// gives two processes started from two directories two different pools, and a -/// positional index into those pools then names two different LiveKit projects. -fn extend_pool( - pool: &mut codetrial::config::ProviderPool, - production: bool, - config_dir: &std::path::Path, - order: &str, -) { - let (providers, warnings) = codetrial::config::discover_providers(config_dir, production); - for warning in warnings { - eprintln!("{warning}"); - } - pool.providers.extend(providers); - - // Outside the `discover` guard: an order naming the environment provider is - // still an order, and a deployment with discovery off should not silently - // ignore the one it was given. - for warning in codetrial::config::order_providers(&mut pool.providers, order) { - eprintln!("{warning}"); - } - - // Said once, at startup, in rotation order. Every provider question asked - // of a running server so far has been answerable only by reading the config - // directory and the argv back to itself and guessing which won. - if !pool.providers.is_empty() { - let (summary, warnings) = codetrial::config::pool_summary(&pool.providers); - eprintln!("{summary}"); - for warning in warnings { - eprintln!("WARNING: {warning}"); - } - } -} - -/// Refuses to start on a database that will not migrate, before anything is -/// served. The listener is already bound by this point, so the socket exists -/// and its backlog fills, but nothing accepts from it and the process exits -/// non-zero instead. The router runs the same migration again on the connection -/// it keeps, where the only thing it can do about a failure is answer 503. -fn initialize_accounts(config: &WebServerConfig) -> Result<(), String> { - let Some(login) = codetrial::web::login_config(config) else { - return Ok(()); - }; - - // The router opens the long-lived connection and performs the startup sweep - // on it; this first pass makes migration failure fatal before the server - // starts accepting. - codetrial::web::initialize_account_database(&login.db_path) - .map_err(|error| format!("account database {}: {error}", login.db_path.display()))?; - Ok(()) -} - -/// The dispatcher cap, with anything unusable about it said out loud. -/// -/// One helper for both modes: a warning printed by web and swallowed by serve -/// would be the same misconfiguration reported twice differently. -fn read_max_concurrent(values: &BTreeMap) -> usize { - let mut warnings = Vec::new(); - let max_concurrent = codetrial::config::max_concurrent_interviews(values, &mut warnings); - for warning in &warnings { - eprintln!("{warning}"); - } - max_concurrent -} - -fn is_production(values: &BTreeMap) -> bool { - codetrial::config::is_production(values) -} - -/// Zero unless the operator states how many proxies front this server, so the -/// default never trusts a forwarded client address. -/// Recording without an OAuth app is a server that refuses every interview. -/// -/// The delivery target is the primary verified address GitHub returns, and -/// without the OAuth flow every account is self-declared, so `/api/token` -/// answers 403 to all of them. Said at startup, where an operator can act on -/// it, rather than one candidate at a time. -/// -/// `oauth` is both credentials, not just the id. `login_config` enables OAuth -/// only when it has the pair, so an id with no secret is exactly as unable to -/// verify anyone as no id at all, and would have started a server that refused -/// every interview while looking configured. -/// A deployment that records has to be able to hand the file over. -/// -/// The key is parsed here, before the listener does any work, because the -/// alternative is a server that records interviews and discovers at the first -/// delivery that it can never sign for one. Configuration already checked the -/// shape; this is the credential itself. -fn recording_can_deliver( - recording: Option<&codetrial::config::RecordingConfig>, -) -> Result<(), String> { - let Some(recording) = recording else { - return Ok(()); - }; - codetrial::delivery::GoogleDelivery::new( - &recording.service_account_json, - &recording.gcs_bucket, - &recording.drive_id, - std::sync::Arc::new(|| 0), - ) - .map(|_| ()) - .map_err(|error| format!("CODETRIAL_RECORDING_SERVICE_ACCOUNT_JSON: {error}")) -} - -fn recording_needs_verified_identity(recording: bool, oauth: bool) -> Result<(), String> { - if recording && !oauth { - return Err( - "CODETRIAL_RECORDING_ENABLED=true needs GITHUB_CLIENT_ID and \ - GITHUB_CLIENT_SECRET: a recording is delivered to the verified address \ - GitHub returns, and without the OAuth app every account is self-declared \ - and every interview would be refused" - .to_string(), - ); - } - Ok(()) -} - -/// The configured interview length, which is what a recording's maximum -/// duration has to clear. Read here rather than through `load_from_pairs` -/// because `codetrial web` deliberately runs without a full agent config. -fn interview_duration_min(values: &BTreeMap) -> u32 { - nonempty(values, "CODETRIAL_DURATION_MIN") - .and_then(|value| value.parse().ok()) - .unwrap_or(DEFAULT_DURATION_MIN) -} - -/// `CODETRIAL_ALLOWED_HOSTS`, a comma-separated list of `Host` values. -/// -/// Unset or empty means no check. A value that holds only separators is -/// refused instead: it was meant to name a host, and reading it as no list -/// would answer every `Host` the person believes is refused. -fn allowed_hosts(values: &BTreeMap) -> Result, String> { - let hosts: Vec = value_or(values, "CODETRIAL_ALLOWED_HOSTS", "") - .split(',') - .map(str::trim) - .filter(|host| !host.is_empty()) - .map(str::to_string) - .collect(); - if hosts.is_empty() && nonempty(values, "CODETRIAL_ALLOWED_HOSTS").is_some() { - return Err("CODETRIAL_ALLOWED_HOSTS names no Host".to_string()); - } - Ok(hosts) -} - -fn trusted_proxy_hops(values: &BTreeMap) -> u32 { - nonempty(values, "CODETRIAL_TRUSTED_PROXY_HOPS") - .and_then(|value| value.parse().ok()) - .unwrap_or(0) -} - -/// What a `codetrial web` start without `NODE_ENV=production` is allowing, in -/// the operator's words rather than the code's. -/// -/// Every guard this reports on is keyed on `NODE_ENV`, so the one mistake none -/// of them can catch is the deployment that sets every credential correctly and -/// never sets `NODE_ENV` at all. The refusal in `run_web` fires only once the -/// operator has already said this is production; a server that was meant to be -/// production and does not say so gets the local defaults and no complaint. -/// This is the only signal it ever gets. -/// -/// `config::is_production` matches the exact string, so a `NODE_ENV=Production` -/// typo lands here too, which is the other way a deployment ends up local -/// without meaning to. -fn relaxed_for_local_use(values: &BTreeMap) -> Vec { - if is_production(values) { - return Vec::new(); - } - - let mut warnings = vec![ - "NODE_ENV is not production: the page's CSP allows loopback origins and a \ - plaintext ws:// LiveKit URL is accepted. Set NODE_ENV=production to deploy." - .to_string(), - ]; - if nonempty(values, "SESSION_SECRET").is_none() { - warnings.push(format!( - "SESSION_SECRET is unset: signing session cookies with the built-in \ - {DEFAULT_SESSION_SECRET:?}, which is published in this repository and lets \ - anyone forge a session." - )); - } - warnings -} - -/// Why the built-in `SESSION_SECRET` may not sign cookies on `bound`, or `None` -/// where it may. -/// -/// Split out for the reason `relaxed_for_local_use` is: the caller can only -/// return the string, and a refusal reachable solely by starting a process that -/// otherwise never exits is a branch no test can assert on without hanging. -/// -/// `NODE_ENV` is a claim; the address this process bound is a fact. A server -/// other machines can reach is not the local run the published default exists -/// for, whether or not anyone remembered to say so, which is the mistake the -/// warning in `relaxed_for_local_use` can name but not prevent. -/// -/// Unspecified addresses (`0.0.0.0`, `::`) are not loopback and must not be -/// treated as such: binding every interface is how a container serves the -/// world. -/// -/// `to_canonical` before the test, because `Ipv6Addr::is_loopback` is true of -/// `::1` and of nothing else: a dual-stack socket that landed on -/// `::ffff:127.0.0.1`, which is a name for the same interface, would otherwise -/// be refused an address only this machine can reach. -/// How a listener on `bound` can be reached from somewhere other than this -/// machine, or `None` where it cannot. -/// -/// Shared, because two guards ask it and a third answer added to one of them -/// would drift silently -- in the direction that leaves the page taking -/// credentials open. What each guard says about it is not shared: the remedies -/// differ, an address is passed on the command line and a declared proxy is a -/// variable in the environment. -#[derive(Clone, Copy)] -enum Reachable { - /// The bind address is not loopback. - Address, - /// `CODETRIAL_TRUSTED_PROXY_HOPS` is the operator saying requests arrive - /// through something in front, which is the same admission: a server on - /// loopback behind nginx or a tunnel is as public as one on `0.0.0.0`. - DeclaredProxy, -} - -fn reachable_from_elsewhere( - values: &BTreeMap, - bound: std::net::SocketAddr, -) -> Option { - if !bound.ip().to_canonical().is_loopback() { - Some(Reachable::Address) - } else if trusted_proxy_hops(values) > 0 { - Some(Reachable::DeclaredProxy) - } else { - None - } -} - -fn task_mode_allowed(reachable: Option, fixed_room: bool, recording: bool) -> bool { - task_mode_refusal(reachable, fixed_room, recording).is_none() -} - -/// Why task mode may not run on this server, in the words of what to change, or -/// `None` where it may. -fn task_mode_refusal( - reachable: Option, - fixed_room: bool, - recording: bool, -) -> Option<&'static str> { - match (reachable, fixed_room, recording) { - (Some(Reachable::Address), _, _) => Some( - "it is bound to an address other than loopback; start it with --web-addr 127.0.0.1:3000", - ), - (Some(Reachable::DeclaredProxy), _, _) => { - Some("CODETRIAL_TRUSTED_PROXY_HOPS says requests arrive through a proxy; unset it") - } - (None, true, _) => Some("INTERVIEW_ROOM_NAME shares one fixed room; unset it"), - (None, false, true) => { - Some("recording is on; task mode does not run on a recording server") - } - (None, false, false) => None, - } -} - -/// Task mode is on when `--task-mode` asked for it and the server is one it may -/// run on; either alone is not enough. -fn task_mode_enabled( - requested: bool, - reachable: Option, - fixed_room: bool, - recording: bool, -) -> bool { - requested && task_mode_allowed(reachable, fixed_room, recording) -} - -fn published_secret_refusal( - values: &BTreeMap, - bound: std::net::SocketAddr, -) -> Option { - // What the cookie signer will actually use, not whether the key was - // mentioned. `value_or` trims, so `SESSION_SECRET=" codetrial-local-session - // "` is the published key spelled with whitespace rather than a secret of - // the operator's own, and a guard that only asked whether the variable was - // set would wave it through. - if value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) != DEFAULT_SESSION_SECRET { - return None; - } - - let reason = match reachable_from_elsewhere(values, bound)? { - Reachable::Address => format!("bind {bound}"), - Reachable::DeclaredProxy => "serve from behind a declared proxy".to_string(), - }; - - Some(format!( - "SESSION_SECRET must be set to {reason}: {DEFAULT_SESSION_SECRET:?} is published in \ - this repository, so every session cookie this server signs would be forgeable by \ - anyone who can reach it" - )) -} - -fn compiler_explorer_enabled(values: &BTreeMap) -> bool { - codetrial::config::compiler_explorer_enabled( - values - .get("CODETRIAL_COMPILER_EXPLORER_ENABLED") - .map(String::as_str), - ) -} - -fn guest_mode_recording_conflict(guest_mode_enabled: bool, recording_enabled: bool) -> bool { - guest_mode_enabled && recording_enabled -} - -fn guest_mode_enabled(values: &BTreeMap) -> bool { - codetrial::config::guest_mode_enabled( - values - .get("CODETRIAL_GUEST_MODE_ENABLED") - .map(String::as_str), - ) -} - -fn nonempty(values: &BTreeMap, key: &str) -> Option { - values - .get(key) - .map(|value| value.trim()) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn value_or(values: &BTreeMap, key: &str, default: &str) -> String { - nonempty(values, key).unwrap_or_else(|| default.to_string()) -} - -/// One place decides where accounts live, because `web` and `serve` both open -/// the same database and disagreeing about it would split a person's reports. -/// -/// Beside the config file rather than in the working directory: the database -/// has to be the same file on the next launch, and the directory a process was -/// started from is not. It used to be a bare filename, which meant a launch -/// from a shortcut or from another terminal found the config it wrote and then -/// opened an empty database next to wherever it had been started. -fn account_db_path(values: &BTreeMap, options: &CliOptions) -> PathBuf { - match nonempty(values, "CODETRIAL_DB_PATH") { - Some(path) => PathBuf::from(path), - None => config_dir(options).join(DEFAULT_ACCOUNT_DB_PATH), - } -} - -/// Where a cold start writes the config it just took, which is the same -/// `config/` the search above looks in first for the root this is installed as. -/// -/// A checkout is told apart by the template it ships, not by `config/` merely -/// existing: a released binary run from a directory that happens to hold one -/// would otherwise write the credentials there and lose them the moment it was -/// next started from somewhere else. -fn cold_start_config_path(options: &CliOptions) -> PathBuf { - if let Some(dir) = &options.data_dir { - return dir.join(CONFIG_FILE_NAME); - } - let in_checkout = PathBuf::from(CONFIG_DIR); - if in_checkout.join(CONFIG_EXAMPLE_NAME).is_file() { - return in_checkout.join(CONFIG_FILE_NAME); - } - codetrial::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME) -} - #[cfg(test)] // `bin.rs`, not `main.rs`: cargo reads `tests//main.rs` as an // integration-test target of its own, which would compile this file a second diff --git a/src/web/mod.rs b/src/web/mod.rs index c9ee8a9f7..f0aff159c 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -833,6 +833,13 @@ pub trait RoomDispatcher: Send + Sync + 'static { false } + /// Stops admitting new rooms; those already admitted keep running. A + /// stopping server calls it as its shutdown begins, so "nothing live" can + /// mean nothing will be. The default does nothing, for a dispatcher that + /// cannot refuse; with `is_idle` also left at its default, a server that + /// bounds its stop waits out the whole bound and reports rooms running. + fn close_admission(&self) {} + fn ensure_guest_agent( &self, _room_name: &str, @@ -856,13 +863,14 @@ pub enum AgentJob { }, } -/// Why no interviewer will come. The two answer the candidate differently: a +/// Why no interviewer will come. Each answers the candidate differently: a /// full server frees up as any interview ends, a finalizing room only when its -/// own report is out. +/// own report is out, and a stopping server not until it is started again. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DispatchRefusal { AtCapacity, Finalizing, + Stopping, } pub fn web_service(config: WebServerConfig) -> IntoMakeServiceWithConnectInfo { diff --git a/src/web/token.rs b/src/web/token.rs index 4d5991729..080836e27 100644 --- a/src/web/token.rs +++ b/src/web/token.rs @@ -632,6 +632,9 @@ async fn dispatch_or_release_consent( super::DispatchRefusal::Finalizing => { "This room is still finishing the previous interview's report. Try again in a few minutes." } + super::DispatchRefusal::Stopping => { + "The server is shutting down and is not starting new interviews. Try again later." + } }; Some(json_response( diff --git a/tests/agent/whiteboard.rs b/tests/agent/whiteboard.rs index bb48a623a..c2220eb32 100644 --- a/tests/agent/whiteboard.rs +++ b/tests/agent/whiteboard.rs @@ -206,6 +206,27 @@ fn a_board_reports_its_own_age_and_an_empty_one_says_so() { assert_eq!(board_age_seconds(&state), Some(0)); } +/// What the long-interview judge reads. The fixture draws no text, so a +/// revision is told apart by the stroke count `read_board` reports, and the +/// judge (`scripts/long-interview-schedule.cjs`) looks for " strokes" in +/// this text. Its test reads the frozen output in `tests/golden/prompts.json`, +/// so a rewording breaks it there too; this pins the phrase on this side. +#[test] +fn read_board_states_the_stroke_count_the_whiteboard_judge_matches() { + for strokes in [4, 20, 68] { + let text = read_board_text(strokes, 3, Some(12), 30); + assert!( + text.contains(&format!( + "image: {strokes} strokes, as the candidate last left it" + )), + "{text}" + ); + } + // An undrawn board says there is nothing to count, and so cannot match one. + let empty = read_board_text(0, 0, None, 30); + assert!(!empty.contains("strokes"), "{empty}"); +} + /// The reviewer of a whiteboard interview is pointed at the board and at /// nothing that does not exist. /// diff --git a/tests/browser/face-detector.test.js b/tests/browser/face-detector.test.js index 1e8b41896..b36e1e464 100644 --- a/tests/browser/face-detector.test.js +++ b/tests/browser/face-detector.test.js @@ -12,13 +12,7 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { createServer } from "node:net"; -import { - existsSync, - mkdirSync, - mkdtempSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { launchChromium, root } from "./source.js"; @@ -42,7 +36,10 @@ async function freePort() { let PORT = 0; let BASE = ""; -const binary = resolve(root, "target/debug/codetrial"); +const targetDir = resolve( + process.env.CARGO_TARGET_DIR || resolve(root, "target"), +); +const binary = join(targetDir, "debug/codetrial"); let server = null; let browser = null; @@ -79,12 +76,7 @@ before(async () => { server = null; return; } - // `target/` is missing on a clean checkout, and `mkdtemp` needs its parent. - // Creating it keeps a missing build a skipped suite, which is what the - // chromium guard above already does, rather than an ENOENT thrown out of - // `before`. - mkdirSync(resolve(root, "target"), { recursive: true }); - workDir = mkdtempSync(join(root, "target", "face-detector-")); + workDir = mkdtempSync(join(targetDir, "face-detector-")); const configPath = join(workDir, "codetrial.env.local"); writeFileSync( configPath, @@ -322,7 +314,10 @@ test("a VideoFrame is not a frame this detector can read", async (t) => { /// and the check reads "unavailable" as ready: no face is ever required. /// Detection has to go through `face-worker.js`, the one response granted eval. test("the setup face check detects a face under the page's own policy", async (t) => { - if (!browser) return t.skip("playwright chromium unavailable"); + if (!browser || !server) { + t.skip("playwright browser or built server binary unavailable"); + return; + } const page = await browser.newPage(); try { await page.goto(BASE, { waitUntil: "domcontentloaded" }); diff --git a/tests/browser/long-interview-driver.test.js b/tests/browser/long-interview-driver.test.js index 2a5d4870f..d9d42a29a 100644 --- a/tests/browser/long-interview-driver.test.js +++ b/tests/browser/long-interview-driver.test.js @@ -539,6 +539,12 @@ test("the duration is claimed only from the server clock", async () => { assert.equal(viaToken.report.full_interview, false); assert.equal(finalize(played).report.anchor.source, "server_clock"); assert.equal(finalize(played).report.schedule_start_delay_s, 5); + assert.equal(finalize(played).report.schedule_mode, played.prepared.mode); + // The origin a later reader needs to read the interview clock against. + assert.equal( + finalize(played).report.schedule_origin_unix_ms, + played.joinedAtMs, + ); }); test("a stimulus whose page action throws still spends its timestamp", async () => { @@ -1084,6 +1090,33 @@ test("a run the page's countdown ended is reported as such, and its capture is k assert.match(cjs, /longCaptureWritten = true;\s*fs\.writeFileSync/); }); +test("the long interview shares the soak's swap marker and keeps the generic Jim audio assertion off", () => { + const cjs = fs.readFileSync( + path.join(ROOT, "scripts", "browser-check.cjs"), + "utf8", + ); + // The probe is installed for the long flow, but only the soak, voice and + // barge flows assert Jim's whole track. + assert.match( + cjs, + /probingSwapAudio = credentialed && flow === "long-interview"/, + ); + assert.match(cjs, /\["voice", "barge", "soak"\]\.includes\(flow\)/); + // One marker and one settle-and-judge owner serve both flows. + // It lives in scripts/swap-audio-owner.cjs (tested in swap-audio-owner.test.js); + // the two flows call it and neither keeps a copy of its own. + assert.match(cjs, /require\("\.\/swap-audio-owner\.cjs"\)/); + assert.match(cjs, /startSwapMarker\(\{/); + assert.match(cjs, /judgeSettledSwapAudio\(/); + assert.equal(cjs.match(/__markJimSwap|__settleJimSwapAudio/g), null); + // The verdict reaches the report, and any failure still stops the probe. + assert.match(cjs, /\n\s+audio,\n\s+replies,\n\s+logs: \{ truncated: false/); + assert.match( + cjs, + /if \(probingSwapAudio\) \{[^}]*__finishJimAudioProbe\(true\)/, + ); +}); + test("a completed long interview reports structural scope without claiming qualification", async () => { const finished = finalize(await play(SMOKE), { cleanup: undefined }); assert.equal(finished.exitCode, EXIT.pass); @@ -1101,6 +1134,145 @@ test("a completed long interview reports structural scope without claiming quali ); }); +const swapRow = (status, reasons = []) => ({ + swap: "1:2:1", + episode: 1, + status, + reasons, +}); + +test("a decoded-signal verdict from the media probe is reported apart from the structural result", async () => { + const played = await play(SMOKE); + const cases = [ + [{ ok: true, episodes: [swapRow("verified")], reasons: [] }, "verified"], + [ + { ok: true, episodes: [swapRow("covered_by_later_swap")], reasons: [] }, + "verified", + ], + [ + { ok: true, episodes: [swapRow("not_required")], reasons: [] }, + "not_required", + ], + [{ ok: true, episodes: [], reasons: [] }, "not_required"], + ]; + for (const [audio, expected] of cases) { + const finished = finalize(played, { audio }); + assert.equal(finished.report.decoded_signal, expected); + assert.equal(finished.exitCode, EXIT.pass); + assert.equal(finished.report.qualification, "not_evaluated"); + assert.equal(finished.report.reply_content, "not_assessed"); + assert.ok( + !finished.report.not_exercised.some((n) => /^decoded signal:/.test(n)), + ); + } +}); + +test("the judged swap rows and their samples are kept in the report, bounded", async () => { + const played = await play(SMOKE); + const sample = { seq: 1, status: "ok", peakRms: 0.2 }; + const rows = Array.from({ length: 600 }, (_, i) => ({ + ...swapRow("verified"), + episode: i + 1, + samples: [sample], + samples_dropped: 3, + extra: "not copied", + })); + const kept = finalize(played, { + audio: { ok: true, episodes: rows, reasons: [] }, + }).report; + assert.equal(kept.decoded_signal_rows.length, 512); + assert.equal(kept.decoded_signal_rows_dropped, 88); + assert.deepEqual(kept.decoded_signal_rows[0], { + swap: "1:2:1", + episode: 1, + status: "verified", + reasons: [], + samples: [sample], + samples_dropped: 3, + }); + const covered = finalize(played, { + audio: { + ok: true, + episodes: [ + { + ...swapRow("covered_by_later_swap"), + covered_by: { swap: "1:2:2", seq: 2 }, + }, + ], + reasons: [], + }, + }).report; + assert.deepEqual(covered.decoded_signal_rows[0].covered_by, { + swap: "1:2:2", + seq: 2, + }); + const none = finalize(played, { audio: undefined }).report; + assert.deepEqual(none.decoded_signal_rows, []); + assert.equal(none.decoded_signal_rows_dropped, 0); +}); + +test("a missing or failed decoded sample is a failed signal, never a pass", async () => { + const played = await play(SMOKE); + const reasons = ["no post-swap audio sample was taken for episode 1"]; + const finished = finalize(played, { + audio: { + ok: false, + episodes: [swapRow("missing_sample", reasons)], + reasons, + }, + }); + assert.equal(finished.report.decoded_signal, "failed"); + assert.deepEqual(finished.report.decoded_signal_reasons, reasons); + assert.equal(finished.report.qualification, "not_evaluated"); + assert.equal( + finished.report.full_interview, + finalize(played).report.full_interview, + ); +}); + +test("an unplaced reply keeps a clean-looking signal from being reported", async () => { + const played = await play(SMOKE); + const audio = { episodes: [swapRow("verified")] }; + for (const [replies, expected] of [ + [{ ok: false, episodes: [] }, "not_assessed"], + [{ ok: true, episodes: [] }, "verified"], + [null, "verified"], + ]) { + const finished = finalize(played, { audio, replies }); + assert.equal(finished.report.decoded_signal, expected); + } +}); + +test("a status the judge does not accept is a failed signal", async () => { + const played = await play(SMOKE); + const finished = finalize(played, { + audio: { episodes: [swapRow("bogus", ["odd"])] }, + }); + assert.equal(finished.report.decoded_signal, "failed"); + assert.deepEqual(finished.report.decoded_signal_reasons, ["odd"]); +}); + +test("an unusable decoded-signal verdict leaves the signal not assessed", async () => { + const played = await play(SMOKE); + for (const audio of [ + null, + {}, + { ok: true }, + { ok: true, episodes: "x" }, + { ok: true, episodes: [{}], reasons: [] }, + { ok: true, episodes: [null], reasons: [] }, + { ok: true, episodes: [{ status: 1, reasons: [] }] }, + { ok: true, episodes: [{ status: "verified", reasons: [7] }] }, + { ok: true, episodes: [{ status: "verified" }] }, + ]) { + const finished = finalize(played, { audio }); + assert.equal(finished.report.decoded_signal, "not_assessed"); + assert.ok( + finished.report.not_exercised.some((n) => /^decoded signal:/.test(n)), + ); + } +}); + for (const plan of [ SMOKE, { ...SMOKE, runKind: "pilot", durationMin: 60, observationTargetS: 3600 }, diff --git a/tests/browser/long-interview-evidence.test.js b/tests/browser/long-interview-evidence.test.js new file mode 100644 index 000000000..5e9b0987f --- /dev/null +++ b/tests/browser/long-interview-evidence.test.js @@ -0,0 +1,723 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +import evidence_ from "../../scripts/long-interview-evidence.cjs"; +import sched from "../../scripts/long-interview-schedule.cjs"; + +const { + assessCheckpoint, + fixtureProblems, + judgeInterview, + main, + mergeVerdict, + reportProblems, + summarize, + verdictFields, +} = evidence_; +const { answerFor, generateSchedule } = sched; + +const ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const BUNDLE = generateSchedule("coding", { + durationMin: 90, + observationTargetS: 5400, +}); +const CP = BUNDLE.schedule.checkpoints[0]; +const ANSWER = answerFor(BUNDLE, CP.checkpoint_id); +const START = 1_700_000_000_000; + +// What the Rust producer wrote: three replies (the first cites an executed +// read_editor call), then a completed, an interrupted one and a swallowed +// ending. Structure is the producer's; only the clocks and the authored +// content are put back, because a fixture can hold neither. +const FIXTURE = JSON.parse( + fs.readFileSync(`${ROOT}/tests/fixtures/acquisition-report.json`, "utf8"), +); + +function at(identity, seconds, uncertainty = 4) { + identity.clock.emitted_unix_ms = START + Math.round(seconds * 1000); + identity.clock.clock_uncertainty_ms = uncertainty; +} + +/// The fixture as a faithful run of checkpoint 1: reply A answers it after +/// reading the editor, replies B and C come long after the window. +function faithful(edit = () => {}) { + const report = structuredClone(FIXTURE); + const { replies, batches, receipts } = report.capture; + replies[0].text = ANSWER.reply.text; + batches[0].calls[0].name = ANSWER.tool_calls[0].name; + batches[0].calls[0].result = ANSWER.tool_calls[0].result_text; + receipts[0].name = ANSWER.tool_calls[0].name; + at(batches[0].identity, ANSWER.tool_calls[0].at_s); + at(replies[0].identity, ANSWER.reply.started_at_s); + at(replies[1].identity, 400); + at(replies[2].identity, 500); + edit(report); + return report; +} + +const OPTIONS = { scheduleStartUnixMs: START }; +const assess = (report, options = OPTIONS) => + assessCheckpoint(BUNDLE, report, CP.checkpoint_id, options); + +/// One test per row: the edit, applied to a faithful run, must give `verdict` +/// with a reason matching the pattern. +function rows(table, verdict, suffix) { + for (const [name, edit, pattern] of table) { + test(`${name} ${suffix}`, () => { + const result = assess(faithful(edit)); + assert.equal(result.verdict, verdict, reasonsOf(result)); + assert.match(reasonsOf(result), pattern); + }); + } +} + +function reasonsOf(result) { + return result.reasons.join(" | "); +} + +test("a faithful reply that cites an executed read passes", () => { + const result = assess(faithful()); + assert.deepEqual(result, { + checkpoint_id: CP.checkpoint_id, + verdict: "pass", + reasons: [], + }); +}); + +test("the raw producer report cannot place anything in time", () => { + const result = assess(structuredClone(FIXTURE)); + assert.equal(result.verdict, "not_assessed"); + assert.match(reasonsOf(result), /clock/); +}); + +const UNREADABLE = [ + ["an observer failure", (r) => (r.arming_failure = "Scope"), /observer/], + ["a latched capture", (r) => (r.capture.failure = "Limit"), /latched Limit/], + ["no capture", (r) => (r.capture = null), /no capture/], + [ + "a reply without positions", + (r) => delete r.capture.replies[0].receipts_last, + /positions/, + ], + ["a batch without calls", (r) => delete r.capture.batches[0].calls, /calls/], + [ + "a batch with an unreadable call", + (r) => (r.capture.batches[0].calls = [null]), + /calls/, + ], + [ + "a receipt without an id", + (r) => delete r.capture.receipts[0].id, + /receipt/, + ], +]; +rows(UNREADABLE, "not_assessed", "makes every checkpoint not_assessed"); + +test("an unknown schedule origin or checkpoint is not assessed", () => { + assert.equal(assess(faithful(), {}).verdict, "not_assessed"); + assert.equal( + assessCheckpoint(BUNDLE, faithful(), "nope", OPTIONS).verdict, + "not_assessed", + ); +}); + +test("a clock reading too uncertain to place an event is unusable", () => { + const result = assess( + faithful((r) => at(r.capture.replies[0].identity, 140, 5000)), + ); + assert.equal(result.verdict, "not_assessed"); +}); + +const UNKNOWN = [ + [ + "a swallowed ending", + (r) => (r.capture.boundaries[0].accepted = false), + /swallowed/, + ], + [ + "a reply sealed by a cut before any ending", + (r) => (r.capture.boundaries[0].replies_before = 2), + /sealed the reply/, + ], + [ + "no ending at all", + (r) => (r.capture.boundaries = []), + /no provider ending/, + ], + [ + "a reply that continued across a tool call", + (r) => (r.capture.replies[0].receipts_last = 2), + /continued across/, + ], + [ + "a call received on another socket", + (r) => (r.capture.receipts[0].identity.socket = 2), + /replacement/, + ], + [ + "a rewritten reply", + (r) => (r.capture.replies[0].revised = true), + /rewritten/, + ], + [ + "a reply never sealed", + (r) => (r.capture.replies[0].final_segment = false), + /never sealed/, + ], + [ + "a second reply inside the window", + (r) => at(r.capture.replies[1].identity, 150), + /2 replies/, + ], + [ + "a reply just before the window, within the uncertainty", + (r) => at(r.capture.replies[1].identity, CP.stimulus_end_s - 0.5, 1000), + /window edge/, + ], + [ + "a reply just after the window, within the uncertainty", + (r) => + at( + r.capture.replies[1].identity, + CP.stimulus_end_s + CP.response_window_s + 0.5, + 1000, + ), + /window edge/, + ], + [ + "a reply within the clock uncertainty of the question's start", + (r) => at(r.capture.replies[2].identity, CP.at_s + 0.2, 1000), + /window edge/, + ], + [ + "the faithful reply, after a recording that ended its speech early", + (r) => at(r.capture.replies[0].identity, CP.stimulus_end_s - 2), + /before the question's estimated end/, + ], + [ + "the faithful reply too close to the start of the window", + (r) => at(r.capture.replies[0].identity, CP.stimulus_end_s + 0.2, 1000), + /window edge/, + ], + [ + "a generation other than the reply's ended it", + (r) => (r.capture.boundaries[0].identity.trace_turn = 9), + /another generation/, + ], + [ + "a call from another generation", + (r) => (r.capture.receipts[0].identity.trace_turn = 9), + /another generation/, + ], + [ + "an ending of another generation inside the turn", + (r) => { + const stray = structuredClone(r.capture.boundaries[0]); + stray.accepted = false; + stray.replies_before = 0; + r.capture.boundaries.unshift(stray); + }, + /another generation lies inside/, + ], + [ + "an ending taken on another socket inside the turn", + (r) => { + const other = structuredClone(r.capture.boundaries[0]); + other.identity.socket = 2; + other.replies_before = 0; + r.capture.boundaries.unshift(other); + }, + /another generation lies inside/, + ], + [ + "a send nobody observed", + (r) => (r.capture.batches[0].local_send_completed = null), + /not observed/, + ], + [ + "a call with no usable clock reading", + (r) => at(r.capture.batches[0].identity, 136, 5000), + /usable clock/, + ], + [ + "a call too close to the limit it must clear", + (r) => at(r.capture.batches[0].identity, 125.2, 1000), + /uncertainty of a limit/, + ], + [ + "a cited call with no executed batch", + (r) => (r.capture.batches[0].calls[0].id = "other"), + /no executed batch/, + ], + [ + "a reply that continued across an executed batch", + (r) => (r.capture.replies[0].batches_last = 2), + /continued across/, + ], + [ + "an earlier segment of the same turn", + (r) => { + // The faithful reply is B, which follows A with no ending between. + const [a, b] = r.capture.replies; + b.text = a.text; + at(b.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + r.capture.boundaries[0].replies_before = 0; + }, + /same turn/, + ], +]; +rows(UNKNOWN, "not_assessed", "is not assessed, never a pass or a fail"); + +const FAILS = [ + [ + "no reply in the window", + (r) => at(r.capture.replies[0].identity, 300), + /no reply was recorded/, + ], + [ + "a reply without the nonce", + (r) => (r.capture.replies[0].text = "something unrelated"), + /nonce/, + ], + [ + "an interrupted reply", + (r) => { + // The faithful reply is the interrupted one (reply C). + const [a, , c] = r.capture.replies; + c.text = a.text; + at(c.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + }, + /interrupted/, + ], + [ + "a call received while output was being discarded", + (r) => (r.capture.receipts[0].discarded = true), + /no read_editor call/, + ], + [ + "a call that was received and never executed", + (r) => (r.capture.receipts[0].batch = null), + /no read_editor call/, + ], + [ + "a call executed after the reply began", + (r) => { + r.capture.replies[0].batches_before = 0; + r.capture.replies[0].batches_last = 0; + }, + /no read_editor call/, + ], + [ + "a response that was not sent", + (r) => (r.capture.batches[0].local_send_completed = false), + /did not succeed/, + ], + [ + "a tool that answered with an error", + (r) => (r.capture.batches[0].calls[0].result = { error: "denied" }), + /did not succeed/, + ], + [ + "a result that lacks the revision", + (r) => (r.capture.batches[0].calls[0].result = "nothing useful"), + /lacks/, + ], +]; +rows(FAILS, "fail", "fails the checkpoint"); + +test("a tool result is matched on its strings, not its JSON escaping", () => { + const report = faithful((r) => { + r.capture.batches[0].calls[0].result = { + frameworkState: { phases: [] }, + result: `line one\n"${ANSWER.tool_calls[0].result_text}"\n`, + }; + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a reply that began before the question was asked does not count against it", () => { + // The previous question's answer, say: before at_s, so outside both the + // early span and the window. + const report = faithful((r) => + at(r.capture.replies[2].identity, CP.at_s - 5), + ); + assert.equal(assess(report).verdict, "pass"); +}); + +test("an ending from another socket does not close the reply", () => { + const report = faithful((r) => { + const other = structuredClone(r.capture.boundaries[0]); + other.identity.socket = 2; + other.ending = "Interrupted"; + r.capture.boundaries.unshift(other); + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a reply on another socket between a reply and its ending is not its sealer", () => { + const report = faithful((r) => { + r.capture.replies[1].identity.socket = 2; + r.capture.boundaries[0].replies_before = 2; + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a later reply does not inherit the previous turn's call", () => { + const report = faithful((r) => { + const [a, b] = r.capture.replies; + b.text = a.text; + at(b.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + }); + const result = assess(report); + assert.equal(result.verdict, "fail", reasonsOf(result)); + assert.match(reasonsOf(result), /no read_editor call/); +}); + +test("a report missing its parts is named", () => { + for (const [edit, pattern] of [ + [(r) => delete r.capture.boundaries, /boundaries is missing/], + [ + (r) => delete r.capture.boundaries[0].replies_before, + /boundary has no positions/, + ], + [(r) => delete r.capture.replies[0].identity, /reply has no positions/], + ]) { + const report = structuredClone(FIXTURE); + edit(report); + assert.match(reportProblems(report).join(" "), pattern); + } +}); + +test("the interview is verified only when every checkpoint passed", () => { + const pass = { checkpoint_id: "a", verdict: "pass", reasons: [] }; + const fail = { checkpoint_id: "b", verdict: "fail", reasons: ["x"] }; + const unknown = { + checkpoint_id: "c", + verdict: "not_assessed", + reasons: ["y"], + }; + assert.equal(summarize([pass, pass]).status, "verified"); + assert.equal(summarize([pass, unknown]).status, "not_assessed"); + assert.deepEqual(summarize([pass, unknown]).uncovered, ["c"]); + assert.equal(summarize([pass, fail, unknown]).status, "failed"); + assert.equal(summarize([]).status, "not_assessed"); +}); + +test("judging the interview covers every scheduled checkpoint", () => { + const verdict = judgeInterview(BUNDLE, faithful(), { + scheduleStartUnixMs: START, + }); + assert.equal(verdict.checkpoints.length, BUNDLE.schedule.checkpoints.length); + assert.equal(verdict.checkpoints[0].verdict, "pass"); + // The run holds one faithful reply, so every other checkpoint found none. + assert.equal(verdict.status, "failed"); + const broken = judgeInterview( + BUNDLE, + faithful((r) => (r.capture.failure = "Limit")), + { + scheduleStartUnixMs: START, + }, + ); + assert.equal(broken.status, "not_assessed"); + assert.equal(broken.uncovered.length, BUNDLE.schedule.checkpoints.length); +}); + +test("an unreadable report is named, not thrown on", () => { + assert.deepEqual(reportProblems(null), ["no report"]); + assert.deepEqual(reportProblems(faithful()), []); +}); + +// The capture file the browser check leaves, with the long-run report fields +// the merge reads. +function capture() { + const plan = { durationMin: 90, observationTargetS: 5400 }; + return { + flow: "long-interview", + plan, + report: { + status: "complete", + exit_code: 0, + anchor: { source: "server_clock" }, + validation_fixture: false, + schedule_mode: "coding", + qualification: "not_evaluated", + reply_content: "not_assessed", + schedule_sha256: BUNDLE.schedule.schedule_sha256, + schedule_origin_unix_ms: START, + }, + }; +} + +/// The serving fixture's report: the acquisition report it left, ended ok. +const servingReport = () => ({ + outcome: "ok", + bound: "127.0.0.1:0", + ...faithful(), +}); + +function merged(edit = () => {}) { + const [file, fixture] = [capture(), servingReport()]; + edit(file, fixture); + return mergeVerdict(file, fixture); +} + +test("the merge fills in reply_content and nothing else", () => { + const result = merged(); + assert.equal(result.report.reply_content, "failed"); + assert.deepEqual(result.report.reply_content_checkpoints, { + pass: 1, + fail: BUNDLE.schedule.checkpoints.length - 1, + not_assessed: 0, + }); + assert.equal(result.report.qualification, "not_evaluated"); + assert.equal(result.report.status, "complete"); + assert.match(result.report.reply_content_reasons[0], /no reply was recorded/); +}); + +const UNJUDGED = [ + [ + "a fixture that did not end ok", + (file, fixture) => (fixture.outcome = "shutdown bound"), + /ended with shutdown bound/, + ], + [ + "a schedule other than the planned one", + (file) => (file.report.schedule_sha256 = "0"), + /not the one rebuilt/, + ], + [ + "an unknown schedule origin", + (file) => delete file.report.schedule_origin_unix_ms, + /origin/, + ], + [ + "a latched capture", + (file, fixture) => (fixture.capture.failure = "Limit"), + /latched/, + ], + [ + "a plan that builds no schedule", + (file) => (file.plan.durationMin = 1), + /cannot be rebuilt/, + ], + [ + "a surface that is not one", + (file) => (file.report.schedule_mode = "dispatch"), + /cannot be rebuilt/, + ], + [ + "a run that failed", + (file) => (file.report.exit_code = 1), + /did not complete cleanly/, + ], + [ + "a run that did not finish", + (file) => (file.report.status = "failed"), + /did not complete cleanly/, + ], + [ + "a clock not anchored to the server", + (file) => (file.report.anchor.source = "first_token_request"), + /not anchored/, + ], + [ + "a run that played tone fixtures", + (file) => (file.report.validation_fixture = true), + /tone fixtures/, + ], + [ + "a capture that does not say what it played", + (file) => delete file.report.validation_fixture, + /tone fixtures/, + ], +]; +for (const [name, edit, pattern] of UNJUDGED) { + test(`${name} leaves reply_content not_assessed with the reason`, () => { + const result = merged(edit); + assert.equal(result.report.reply_content, "not_assessed"); + assert.match(result.report.reply_content_reasons.join(" "), pattern); + }); +} + +test("only the three verdict fields move", () => { + const file = capture(); + const after = mergeVerdict(file, servingReport()); + const VERDICT = [ + "reply_content", + "reply_content_checkpoints", + "reply_content_reasons", + ]; + const without = (report) => + Object.fromEntries( + Object.entries(report).filter(([key]) => !VERDICT.includes(key)), + ); + assert.deepEqual(without(after.report), without(file.report)); + assert.deepEqual( + Object.keys(after.report).filter((key) => key.startsWith("reply_content")), + VERDICT, + ); +}); + +test("a verified interview carries an empty reasons list, not a stale one", () => { + const pass = { checkpoint_id: "a", verdict: "pass", reasons: [] }; + const fields = verdictFields(summarize([pass, pass])); + assert.equal(fields.reply_content, "verified"); + assert.deepEqual(fields.reply_content_reasons, []); + assert.deepEqual(fields.reply_content_checkpoints, { + pass: 2, + fail: 0, + not_assessed: 0, + }); +}); + +test("a missing fixture report is named, and other flows are left alone", () => { + const file = capture(); + const result = mergeVerdict(file, null); + assert.equal(result.report.reply_content, "not_assessed"); + assert.match(result.report.reply_content_reasons[0], /left no report/); + assert.equal(mergeVerdict({ ...file, flow: "soak" }, null), null); +}); + +function withFiles(run) { + const dir = fs.mkdtempSync(`${os.tmpdir()}/evidence-`); + try { + fs.writeFileSync(`${dir}/capture.json`, JSON.stringify(capture())); + fs.writeFileSync(`${dir}/out.json`, JSON.stringify(servingReport())); + run(dir); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("the command rewrites the capture in place and reports one line", () => { + withFiles((dir) => { + const lines = []; + const code = main( + ["--capture", `${dir}/capture.json`, "--fixture", `${dir}/out.json`], + { out: (line) => lines.push(line) }, + ); + assert.equal(code, 0); + assert.match(lines[0], /^long-interview reply_content=failed pass=1 /); + const after = JSON.parse(fs.readFileSync(`${dir}/capture.json`, "utf8")); + assert.equal(after.report.reply_content, "failed"); + }); +}); + +test("a fixture report that is there but unreadable is not called missing", () => { + withFiles((dir) => { + const run = (fixture) => { + main(["--capture", `${dir}/capture.json`, "--fixture", fixture], { + out: () => {}, + }); + const after = JSON.parse(fs.readFileSync(`${dir}/capture.json`, "utf8")); + return after.report.reply_content_reasons[0]; + }; + fs.writeFileSync(`${dir}/cut.json`, '{"outcome": "ok", "secret reply te'); + const reason = run(`${dir}/cut.json`); + assert.match(reason, /report could not be read: SyntaxError$/); + assert.doesNotMatch(reason, /secret/); + assert.match(run(`${dir}/never-written.json`), /left no report/); + }); +}); + +test("a capture that is there but unreadable exits 2 and is named so", () => { + withFiles((dir) => { + fs.writeFileSync(`${dir}/capture.json`, "{"); + const errs = []; + const code = main( + ["--capture", `${dir}/capture.json`, "--fixture", `${dir}/out.json`], + { err: (line) => errs.push(line) }, + ); + assert.equal(code, 2); + assert.match(errs.join(" "), /SyntaxError/); + assert.equal(fs.readFileSync(`${dir}/capture.json`, "utf8"), "{"); + }); +}); + +test("the fixture check names a report that is there but unreadable", () => { + withFiles((dir) => { + fs.writeFileSync(`${dir}/cut.json`, '{"outcome"'); + const errs = []; + assert.equal( + main(["--check-fixture", `${dir}/cut.json`], { + err: (line) => errs.push(line), + }), + 1, + ); + assert.match(errs.join(" "), /could not be read: SyntaxError/); + }); +}); + +test("a command line or capture it cannot use exits 2 and writes nothing", () => { + withFiles((dir) => { + const before = fs.readFileSync(`${dir}/capture.json`, "utf8"); + const err = () => {}; + for (const argv of [ + [], + ["--capture", `${dir}/capture.json`], + ["--fixture", `${dir}/out.json`], + ["--capture", `${dir}/nope`, "--fixture", `${dir}/out.json`], + ]) { + assert.equal(main(argv, { err }), 2, argv.join(" ")); + } + assert.equal(fs.readFileSync(`${dir}/capture.json`, "utf8"), before); + }); +}); + +test("a serving fixture report is accepted by its shape and its outcome", () => { + const without = (key) => { + const copy = servingReport(); + delete copy[key]; + return copy; + }; + assert.deepEqual(fixtureProblems(servingReport()), []); + const rows = [ + [null, /not an object/], + [[], /not an object/], + [{}, /no outcome/], + [{ outcome: "shutdown bound" }, /ended with shutdown bound/], + [without("arming_failure"), /no arming_failure/], + [without("bound"), /no bound/], + [{ ...servingReport(), capture: null }, /no capture/], + ]; + for (const [report, pattern] of rows) { + assert.match(fixtureProblems(report).join(" "), pattern); + } + const lacking = servingReport(); + delete lacking.capture.boundaries; + delete lacking.capture.failure; + assert.match(fixtureProblems(lacking).join(" "), /no failure field/); + assert.match(fixtureProblems(lacking).join(" "), /no boundaries list/); + // An observer that failed says so itself, and the judge names it. + const failed = { ...servingReport(), arming_failure: "Scope", capture: null }; + assert.deepEqual(fixtureProblems(failed), []); + // A latched failure inside a well-formed report is the judge's to name. + const latched = servingReport(); + latched.capture.failure = "Limit"; + assert.deepEqual(fixtureProblems(latched), []); +}); + +test("the check command exits 1 on a report it would not accept", () => { + withFiles((dir) => { + const err = []; + const sink = (line) => err.push(line); + assert.equal( + main(["--check-fixture", `${dir}/out.json`], { err: sink }), + 0, + ); + fs.writeFileSync(`${dir}/bad.json`, JSON.stringify({ outcome: "boom" })); + assert.equal( + main(["--check-fixture", `${dir}/bad.json`], { err: sink }), + 1, + ); + assert.match(err.join(" "), /ended with boom/); + assert.equal(main(["--check-fixture", `${dir}/missing`], { err: sink }), 1); + assert.equal(main(["--check-fixture"], { err: sink }), 1); + }); +}); diff --git a/tests/browser/long-interview-schedule.test.js b/tests/browser/long-interview-schedule.test.js index 0b5ba55ee..085ce9867 100644 --- a/tests/browser/long-interview-schedule.test.js +++ b/tests/browser/long-interview-schedule.test.js @@ -53,9 +53,9 @@ const bundles = { // Pinned on purpose: a change to the authored questions, the timing or the // answer key moves these, and the change then has to be made on purpose. const PINS = { - coding: "1a926968c26adbf61aeea60a5d10dd5a461180fc12686d196d30e94f7b2313a1", + coding: "c8099dc2a945e42b147f431525557a481faba3373a9ee507307f9669f1b35cc9", whiteboard: - "d5851de32e3f63803d0adaab9a93e72ab732aa65a9d05f213f444bb6540c9b93", + "84e1a9fe28e0a0a3eced4ec2b8ba3b2551d98abc5a8291575bac4fc1e5952900", }; test("the authored catalogue is hash pinned", () => { @@ -216,8 +216,20 @@ test("every 90 minute schedule recalls facts seeded 30 minutes earlier", () => { test("the nonce is never the whole answer in the answer key", () => { for (const mode of ["coding", "whiteboard"]) { for (const k of bundles[mode].answer_key.checkpoints) { - assert.ok(k.element_fact_ids.length >= 2); + assert.ok(k.element_fact_ids.length >= 1); assert.ok((k.tools ?? [k.tool]).length >= 1); + // Nothing is required of the reply that its question did not ask for. + if (k.checkpoint_id === "cp_final") { + const last = bundles[mode].answer_key.facts + .map((f) => f.id) + .filter((id) => /^revision_\d+$/.test(id)) + .at(-1); + assert.deepEqual( + k.element_fact_ids, + mode === "coding" ? ["pending_question", last] : ["pending_question"], + mode, + ); + } } } }); @@ -310,6 +322,19 @@ test("interview-run-budget.py validate consumes the generated manifest fields", // ---------------------------------------------------------------- the judge +test("an inspect question asks about the duplicate case, not the revision", () => { + // The first paid pilot's c04 reply answered what was asked and said the code + // words, and failed because the key also wanted a marker the question never + // asked for. The revision is checked where it belongs: in what the tool read. + for (const mode of ["coding", "whiteboard"]) { + for (const k of bundles[mode].answer_key.checkpoints) { + if (!k.checkpoint_id.endsWith("_cp_inspect")) continue; + assert.deepEqual(k.element_fact_ids, ["duplicate_case"], k.checkpoint_id); + assert.match(k.tool.result_fact_ids.join(), /^(revision|board)_\d+$/); + } + } +}); + const CP_STATE = "c07_cp_state"; // a recall: two tool reads, five facts const CP_INSPECT = "c05_cp_inspect"; @@ -333,17 +358,30 @@ test("a faithful answer passes every checkpoint of both schedules", () => { } }); +const PASS = { verdict: "pass", reasons: [] }; +const reasonsOf = (mode, evidence) => + judgeCheckpoint(bundles[mode], evidence).reasons.join("\n"); const key = (mode, id) => bundles[mode].answer_key.checkpoints.find((k) => k.checkpoint_id === id); const factsOf = (mode) => bundles[mode].answer_key.facts; -const marker = (mode, n) => - factsOf(mode).find((f) => f.id === `revision_${String(n).padStart(2, "0")}`) +const factToken = (mode, prefix, n) => + factsOf(mode).find((f) => f.id === `${prefix}_${String(n).padStart(2, "0")}`) .match.tokens[0]; +const boardToken = (n) => factToken("whiteboard", "board", n); +const marker = (mode, n) => factToken(mode, "revision", n); // Each fixture names the reason it must fail. Every one of them keeps the // nonce, and all but the first few keep the expected tokens, so a judge that // stops at the nonce or at the words passes them. const NEGATIVES = [ + { + name: "an inspect reply that states an earlier revision", + cp: CP_INSPECT, + reason: /stale revision revision_04/, + edit: (e) => { + e.reply.text += ` ${marker("coding", 4)}`; + }, + }, { name: "nonce only, tool evidence valid", cp: CP_STATE, @@ -372,21 +410,21 @@ const NEGATIVES = [ }, { name: "wrong revision stated in the reply", - cp: CP_INSPECT, - reason: /lacks revision_05|stale revision/, + cp: CP_STATE, + reason: /lacks revision_07|stale revision/, edit: (e) => { e.reply.text = e.reply.text.replace( - marker("coding", 5), - marker("coding", 4), + marker("coding", 7), + marker("coding", 6), ); }, }, { name: "current and stale revision both stated", - cp: CP_INSPECT, - reason: /stale revision revision_04/, + cp: CP_STATE, + reason: /stale revision revision_06/, edit: (e) => { - e.reply.text += ` ${marker("coding", 4)}`; + e.reply.text += ` ${marker("coding", 6)}`; }, }, { @@ -533,7 +571,7 @@ const NEGATIVES = [ ]; test("the negative fixtures are a fixed set and every one fails", () => { - assert.equal(NEGATIVES.length, 22); + assert.equal(NEGATIVES.length, 23); for (const fixture of NEGATIVES) { const evidence = evidenceFor("coding", fixture.cp); // Each starts from a passing answer, so the edit alone is what fails it. @@ -597,15 +635,95 @@ test("a judge key that drops the schedule's tool requirement is refused", () => ); }); +// What read_board really answers with: the frozen output tests/agent/prompts.rs +// pins to the Rust that writes it, with its count swapped. The count is the +// part of a board a reply cannot read off the fixture's drawing with no text on +// it, so it is what tells one revision from the next. +const GOLDEN = JSON.parse( + fs.readFileSync(path.join(ROOT, "tests", "golden", "prompts.json"), "utf8"), +); +assert.match(GOLDEN.readBoard, /: 22 strokes,/); +const boardText = (strokes) => + GOLDEN.readBoard.replace(": 22 strokes,", `: ${strokes} strokes,`); + +test("a whiteboard revision is told apart by the stroke count read_board reports", () => { + // Four strokes are drawn per revision and the board keeps them, so revision n + // reads 4n strokes. No marker is drawn: read_board gives a count and an image. + for (const n of [1, 5, 11, 17]) { + assert.equal(boardToken(n), `${4 * n} strokes`); + } + const evidence = evidenceFor("whiteboard", CP_INSPECT); + evidence.tool_calls[0].result_text = boardText(20); + assert.deepEqual(judgeCheckpoint(bundles.whiteboard, evidence), PASS); + // The count of the revision before, a count that merely contains this one, + // and a board that was never drawn on are not the current board. + for (const stale of [ + boardText(16), + boardText(120), + boardText(2), + GOLDEN.readBoardEmpty, + ]) { + evidence.tool_calls[0].result_text = stale; + assert.match( + reasonsOf("whiteboard", evidence), + /result lacks board_05/, + stale, + ); + } +}); + +test("every whiteboard checkpoint passes on the real read_board text and is held to nothing more", () => { + for (const c of bundles.whiteboard.schedule.checkpoints) { + const k = key("whiteboard", c.checkpoint_id); + const need = (k.tools ?? [k.tool]).find((t) => t.name === "read_board"); + const board = need.result_fact_ids.find((id) => id.startsWith("board_")); + const count = factsOf("whiteboard").find((f) => f.id === board).match + .tokens[0]; + const evidence = evidenceFor("whiteboard", c.checkpoint_id); + const call = evidence.tool_calls.find((x) => x.name === "read_board"); + call.result_text = boardText(Number.parseInt(count, 10)); + assert.deepEqual( + judgeCheckpoint(bundles.whiteboard, evidence), + PASS, + c.checkpoint_id, + ); + // A reply may talk about counts without being faulted for a stale one. + evidence.reply.text += + " Each revision adds 4 strokes, so the board is much fuller now."; + assert.deepEqual( + judgeCheckpoint(bundles.whiteboard, evidence), + PASS, + c.checkpoint_id, + ); + } +}); + +test("a whiteboard question never asks for a revision no one can read", () => { + const { schedule, answer_key: answerKey } = bundles.whiteboard; + for (const clip of schedule.clips) { + assert.ok(!/revision/i.test(clip.source_text), clip.clip_id); + } + for (const k of answerKey.checkpoints) { + assert.ok( + !k.element_fact_ids.some((id) => /^(revision|board)_/.test(id)), + k.checkpoint_id, + ); + // It is still judged: in what the tool read. + assert.ok( + (k.tools ?? [k.tool]).some((t) => + t.result_fact_ids.some((id) => id.startsWith("board_")), + ), + k.checkpoint_id, + ); + } +}); + test("the same fixtures fail on the whiteboard schedule", () => { - // Board revisions carry the same markers through read_board. + // Board revisions are judged from the stroke count read_board reports. const evidence = evidenceFor("whiteboard", CP_INSPECT); assert.equal(evidence.tool_calls[0].name, "read_board"); - evidence.tool_calls[0].result_text = marker("whiteboard", 4); - assert.match( - judgeCheckpoint(bundles.whiteboard, evidence).reasons.join("\n"), - /result lacks revision_05/, - ); + evidence.tool_calls[0].result_text = boardToken(4); + assert.match(reasonsOf("whiteboard", evidence), /result lacks board_05/); const coding = evidenceFor("whiteboard", CP_INSPECT); coding.tool_calls[0].name = "read_editor"; assert.match( @@ -659,15 +777,82 @@ test("an unreviewed rubric cannot certify a reply", () => { assert.match(verdict.reasons.join("\n"), /rubric is not reviewed/); }); -test("exact tokens are case sensitive and folded tokens are not", () => { +test("an inspect reply may state the current revision without being faulted for it", () => { const evidence = evidenceFor("coding", CP_INSPECT); + evidence.reply.text += ` The latest source revision is ${marker("coding", 5)}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS); +}); + +test("the constraints rubric accepts the ways a model says ten thousand", () => { + for (const said of [ + "between 2 and 10 thousand transactions", + "ten-thousand numbers", + "up to 10k", + "at most 10\u00a0000 numbers", + ]) { + const evidence = evidenceFor("coding", CP_STATE); + const before = evidence.reply.text; + evidence.reply.text = before.replace(/ten thousand/i, said); + assert.notEqual(evidence.reply.text, before, "the answer states the bound"); + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS, said); + } + // A number that merely contains the token is not the bound. + for (const not of [ + "110k numbers", + "210 thousand numbers", + "model10kweights", + ]) { + const evidence = evidenceFor("coding", CP_STATE); + evidence.reply.text = evidence.reply.text.replace(/ten thousand/i, not); + assert.match( + reasonsOf("coding", evidence), + /rubric concept 1 is missing/, + not, + ); + } +}); + +test("the duplicate-case rubric accepts the plain ways of saying two different indices", () => { + for (const said of [ + "it uses a different index each time", + "the two positions are different positions", + "it takes two separate indices", + "each element has a distinct index", + // The pilot's own c01 reply, which the first rubric missed. + "you'll always be getting the index of a previous, different position", + "the second one is a distinct position", + ]) { + const evidence = evidenceFor("coding", CP_INSPECT); + evidence.reply.text = `${said}. ${key("coding", CP_INSPECT).nonce}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS, said); + } +}); + +test("the final question is answered by its pending question and revision alone", () => { + const k = key("coding", "cp_final"); + const last = k.element_fact_ids.at(-1); + const number = Number(last.replace("revision_", "")); + const evidence = evidenceFor("coding", "cp_final"); + const pending = factsOf("coding").find((f) => f.id === "pending_question"); + evidence.reply.text = `${k.nonce}. The open question is whether to return them in ${pending.match.tokens[0]}, and the latest source revision is ${marker("coding", number)}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS); + const stale = structuredClone(evidence); + stale.reply.text += ` ${marker("coding", number - 1)}`; + assert.match( + reasonsOf("coding", stale), + new RegExp(`stale revision revision_${number - 1}`), + ); +}); + +test("exact tokens are case sensitive and folded tokens are not", () => { + const evidence = evidenceFor("coding", CP_STATE); evidence.reply.text = evidence.reply.text.replace( - marker("coding", 5), - marker("coding", 5).toUpperCase(), + marker("coding", 7), + marker("coding", 7).toUpperCase(), ); assert.match( judgeCheckpoint(bundles.coding, evidence).reasons.join("\n"), - /reply lacks revision_05/, + /reply lacks revision_07/, ); const folded = evidenceFor("coding", CP_STATE); folded.reply.text = folded.reply.text.replace( diff --git a/tests/browser/report-recovery.test.js b/tests/browser/report-recovery.test.js index b1d72fff6..ae5aa0468 100644 --- a/tests/browser/report-recovery.test.js +++ b/tests/browser/report-recovery.test.js @@ -396,6 +396,27 @@ test("page finalizes a regeneration once, cancelling provisional timers", async ); }); +test("page keeps the room for a report held through the rejoin grace", async () => { + const page = pageHarness(); + await page.receiveReport(page.room, packet(raw)); + page.advance(30_000); + assert.equal(page.reportRecovery.retry(), true); + page.receiveControl(control(accepted)); + // A 124-second generation can be held until a rejoin at 152 seconds. + page.advance(152_000); + await settle(); + assert.equal(page.scope.state.phase, "report_recovery"); + assert.equal(page.scope.state.room, page.room); + assert.ok(!page.events.includes("disconnect")); + await page.receiveReport( + page.room, + packet({ incomplete: true, summary: "retained regeneration" }), + ); + await settle(); + assert.equal(page.scope.state.phase, "report"); + assert.equal(page.saves.at(-1).summary, "retained regeneration"); +}); + test("the agent's closing notice finalizes the page through its control topic", async () => { const page = pageHarness(); await page.receiveReport(page.room, packet(raw)); diff --git a/tests/browser/review-workbench.test.js b/tests/browser/review-workbench.test.js index 33316162d..27de6733c 100644 --- a/tests/browser/review-workbench.test.js +++ b/tests/browser/review-workbench.test.js @@ -33,7 +33,26 @@ async function start( manifestFiles = null, ) { const bundle = mkdtempSync(join(tmpdir(), "codetrial-workbench-test-")); - t.after(() => rmSync(bundle, { recursive: true, force: true })); + const writers = []; + const own = (child) => { + const closed = new Promise((resolve) => child.once("close", resolve)); + writers.push({ child, closed }); + return child; + }; + let page; + t.after(async () => { + try { + await page?.close(); + } finally { + for (const { child, closed } of writers.reverse()) { + if (child.pid && child.exitCode === null && child.signalCode === null) + child.kill(); + await closed; + } + // The reviewer can write presence files until these processes close. + rmSync(bundle, { recursive: true, force: true }); + } + }); const files = ["src/example.rs", "tests/example.rs"]; writeFileSync( join(bundle, "manifest.json"), @@ -85,30 +104,24 @@ Path(sys.argv[sys.argv.index('--output-last-message')+1]).write_text(json.dumps( env.PATH = `${bin}:${env.PATH}`; args.push("--reviewer", "codex"); } - const server = spawn(python, [helper, "serve", "--port", "0", ...args], { - cwd: root, - env, - stdio: ["ignore", "pipe", "pipe"], - }); - t.after(async () => { - if (server.exitCode === null) { - const exited = once(server, "exit"); - server.kill(); - await exited; - } - }); + const server = own( + spawn(python, [helper, "serve", "--port", "0", ...args], { + cwd: root, + env, + stdio: ["ignore", "pipe", "pipe"], + }), + ); const lines = createInterface({ input: server.stdout }); const [url] = await once(lines, "line", { signal: t.signal }); lines.close(); - const page = await browser.newPage(); - t.after(() => page.close()); + page = await browser.newPage(); await page.goto(url); await page.locator("#workspace").waitFor({ state: "visible" }); const inspect = () => JSON.parse( execFileSync(python, [helper, "inspect", ...args], { encoding: "utf8" }), ); - return { bundle, args, page, url, inspect, files, server }; + return { bundle, args, page, url, inspect, files, server, own }; } test( @@ -116,7 +129,7 @@ test( { timeout: 30000 }, async (t) => { if (!browser) return t.skip("playwright chromium unavailable"); - const { page, args, bundle, inspect } = await start(t); + const { page, args, bundle, inspect, own } = await start(t); await page.getByText(/Commit threshold reached .*at least 200/).waitFor(); assert.equal(inspect().total, 4); assert.equal(await page.locator("#answers-form textarea").count(), 1); @@ -134,18 +147,17 @@ test( await page.getByText("Saved on this computer", { exact: true }).waitFor(); const saved = inspect(); assert.equal(saved.status, "ready_to_send"); - const waiting = spawn(python, [ - helper, - "wait", - ...args, - "--after", - String(saved.revision), - "--timeout", - "10", - ]); - t.after(() => { - if (waiting.exitCode === null) waiting.kill(); - }); + const waiting = own( + spawn(python, [ + helper, + "wait", + ...args, + "--after", + String(saved.revision), + "--timeout", + "10", + ]), + ); let received = ""; waiting.stdout.on("data", (chunk) => { received += chunk; diff --git a/tests/browser/soak-judge.test.js b/tests/browser/soak-judge.test.js index a3750599f..669779005 100644 --- a/tests/browser/soak-judge.test.js +++ b/tests/browser/soak-judge.test.js @@ -874,6 +874,42 @@ test("a swap that owed no audio is not made to speak, whatever its sample says", assert.deepEqual(statuses(held), [[1, "not_required"]]); }); +test("a covering sample its own row's cap drops is still carried by the row it covers", () => { + const log = owedSwap(1) + owedSwap(2); + const superseded = rearm(1, { + seq: 1, + status: "superseded", + reason: "superseded-by-later-swap", + windowEdge: null, + counterPairs: 0, + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }); + // Swap 2's own verdict rests on its first sample (seq 0, before swap 1's); + // the one that covers swap 1 is its fifth, past the cap of four. + const stale = { windowEdge: 4 }; + const second = [ + rearm(2, { seq: 0 }), + rearm(2, { seq: 2, ...stale }), + rearm(2, { seq: 3, ...stale }), + rearm(2, { seq: 4, ...stale }), + rearm(2, { seq: 5, pageSecret: "not kept", reason: "r".repeat(300) }), + ]; + const audio = audioOf(log, [superseded, ...second]); + const [first, other] = audio.episodes; + assert.equal(first.status, "covered_by_later_swap"); + assert.equal(first.covered_by.seq, 5); + assert.equal(first.covered_by.sample.seq, 5); + // Carried through the same whitelist and cut as a row's own samples. + assert.ok(!("pageSecret" in first.covered_by.sample)); + assert.equal(first.covered_by.sample.reason.length, 200); + assert.ok( + !other.samples.some((s) => s.seq === 5), + "the cap dropped it there", + ); +}); + test("a swap superseded by a later swap is covered only by that swap's fresh sample", () => { const log = owedSwap(1) + owedSwap(2); const superseded = rearm(1, { @@ -887,6 +923,13 @@ test("a swap superseded by a later swap is covered only by that swap's fresh sam }); const covered = audioOf(log, [superseded, rearm(2)]); assert.equal(covered.ok, true); + // The row names the other swap's sample that covers it, and carries it. + assert.deepEqual(covered.episodes[0].covered_by, { + swap: "7:1:2", + seq: 2, + sample: covered.episodes[1].samples[0], + }); + assert.equal(covered.episodes[1].covered_by, undefined); assert.deepEqual(statuses(covered), [ [1, "covered_by_later_swap"], [2, "verified"], @@ -1166,6 +1209,50 @@ test("the carried reply in the series rejects audio accepted on the old socket", ); }); +test("a judged row keeps the samples behind it, whitelisted and bounded", () => { + const replies = seriesReplies(); + const samples = [1, 2, 3, 4, 5, 6].map((n) => + rearm(2, { seq: n, pageSecret: "not kept", windowEdge: 5 + n }), + ); + const row = judge + .judgeSwapAudio(replies, samples) + .episodes.find((e) => e.episode === 2); + assert.equal(row.status, "verified"); + assert.equal(row.samples.length, judge.MAX_SAMPLES_PER_SWAP); + assert.equal(row.samples_dropped, 2); + assert.deepEqual( + row.samples.map((sample) => sample.seq), + [1, 2, 3, 4], + ); + assert.ok(row.samples.every((sample) => !("pageSecret" in sample))); + assert.equal(row.samples[0].peakRms, 0.2); + // The sample a verdict rests on survives the cap even when it comes last. + const stale = [1, 2, 3, 4].map((n) => + rearm(2, { seq: n, windowEdge: 4, status: "ok" }), + ); + const fifth = judge + .judgeSwapAudio(replies, [...stale, rearm(2, { seq: 5 })]) + .episodes.find((e) => e.episode === 2); + assert.equal(fifth.status, "verified"); + assert.ok(fifth.samples.some((sample) => sample.seq === 5)); + assert.equal(fifth.samples.length, judge.MAX_SAMPLES_PER_SWAP); + assert.deepEqual( + fifth.samples.map((sample) => sample.seq), + [1, 2, 3, 5], + ); + // A page-supplied reason is cut rather than carried whole. + const long = judge + .judgeSwapAudio(replies, [rearm(2, { reason: "x".repeat(5000) })]) + .episodes.find((e) => e.episode === 2); + assert.equal(long.samples[0].reason.length, 200); + // A swap that owed nothing keeps what was sampled for it too, and one with no + // sample keeps none rather than inventing them. + const none = judge + .judgeSwapAudio(replies, []) + .episodes.find((e) => e.episode === 2); + assert.deepEqual([none.samples, none.samples_dropped], [[], 0]); +}); + test("native capture records in the series do not prove fresh decoded audio", () => { const replies = seriesReplies(); assert.equal(replies.ok, true); @@ -1187,3 +1274,29 @@ test("native capture records in the series do not prove fresh decoded audio", () 5, ); }); + +test("a swap whose reply left no record is not assessed, never read as no sample owed", () => { + const gap = + fixtures.start(1, { obligation: 3, origin: "prompt" }) + + fixtures.end(1, "installed", { obligation: 3 }); + const replies = replyTo(gap + owedSwap(2)); + assert.equal(replies.ok, false); + // The audio judge skips what the reply judge could not place, so on its own + // it reports the gap as nothing owed and a verified neighbor hides it. + const audio = judge.judgeSwapAudio(replies, [rearm(2)]); + assert.deepEqual(statuses(audio), [ + [1, "not_required"], + [2, "verified"], + ]); + assert.equal(judge.summarizeSwapAudio(audio).status, "verified"); + assert.equal(judge.summarizeSwapAudio(audio, replies).status, "not_assessed"); + // A demonstrated signal failure is still a failure, whatever else is missing. + const unsampled = judge.judgeSwapAudio(replies, []); + assert.equal(judge.summarizeSwapAudio(unsampled, replies).status, "failed"); + // With every reply placed the replies argument changes nothing. + const clean = replyTo(owedSwap(1)); + assert.equal( + judge.summarizeSwapAudio(audioOf(owedSwap(1), [rearm(1)]), clean).status, + "verified", + ); +}); diff --git a/tests/browser/swap-audio-owner.test.js b/tests/browser/swap-audio-owner.test.js new file mode 100644 index 000000000..5758c6d88 --- /dev/null +++ b/tests/browser/swap-audio-owner.test.js @@ -0,0 +1,258 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import owner from "../../scripts/swap-audio-owner.cjs"; +import * as fixtures from "./soak-judge-fixtures.js"; + +const { judgeSettledSwapAudio, startSwapMarker } = owner; + +/// A page that is only `evaluate`: it runs the function it is given against a +/// stand-in global, so what the marker and the settle ask of the page is +/// observable without a browser. +function fakePage({ failOn = null } = {}) { + const calls = []; + const page = { + calls, + async evaluate(fn, arg) { + const marks = []; + const stand = { + __markJimSwap: (key) => { + if (key === failOn) throw new Error(`page refused ${key}`); + marks.push(key); + }, + __settleJimSwapAudio: async (waitMs) => { + calls.push(["settle", waitMs]); + return page.rearms ?? []; + }, + }; + const real = globalThis.__markJimSwap; + const realSettle = globalThis.__settleJimSwapAudio; + Object.assign(globalThis, stand); + try { + const out = await fn(arg); + marks.forEach((key) => calls.push(["mark", key])); + return out; + } finally { + globalThis.__markJimSwap = real; + globalThis.__settleJimSwapAudio = realSettle; + } + }, + }; + return page; +} + +/// A clock the test steps by hand instead of waiting 250 ms for. +function manualClock() { + const waiting = []; + const delays = []; + return { + delays, + wait: (ms) => { + delays.push(ms); + return new Promise((resolve) => waiting.push(resolve)); + }, + async tick() { + const due = waiting.splice(0); + due.forEach((resolve) => resolve()); + // Let the loop run its body before the test looks. + await new Promise((resolve) => setImmediate(resolve)); + }, + }; +} + +// Swap keys are the words "swap:KEY" in the text; the parser under test is the +// soak judge's, covered by its own tests. +const keysOf = (text) => + [...text.matchAll(/swap:(\S+)/g)].map((match) => match[1]); + +/// Ends the loop the way production does: its pending wait runs out, which the +/// test's clock does by hand. +async function ending(clock, promise) { + await clock.tick(); + return promise; +} + +function marker(overrides = {}) { + const clock = manualClock(); + const agentOutput = []; + let pending = ""; + const page = overrides.page ?? fakePage(); + const handle = startSwapMarker({ + page, + roomName: "room", + agentOutput, + newLines: () => { + const text = pending; + pending = ""; + return text; + }, + keysOf, + wait: clock.wait, + ...overrides.options, + }); + return { + clock, + page, + handle, + agentOutput, + log: (text) => { + pending += text; + }, + }; +} + +test("each installed swap is marked once, in the order it was read", async () => { + const { clock, page, handle, log, agentOutput } = marker(); + agentOutput.push("swap:7:1:1\n"); + log("swap:7:1:2\nswap:7:1:1\n"); + await clock.tick(); + agentOutput.push("swap:7:1:3\n"); + await clock.tick(); + await clock.tick(); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [ + ["mark", "7:1:1"], + ["mark", "7:1:2"], + ["mark", "7:1:3"], + ]); +}); + +test("the loop polls on its own 250 ms clock", async () => { + const { clock, handle } = marker(); + await clock.tick(); + await ending(clock, handle.stop()); + assert.ok(clock.delays.length >= 1); + assert.ok( + clock.delays.every((ms) => ms === 250), + clock.delays.join(), + ); +}); + +test("with the real parser, an installed swap line in the output is marked", async () => { + const clock = manualClock(); + const page = fakePage(); + const handle = startSwapMarker({ + page, + roomName: fixtures.ROOM, + agentOutput: [fixtures.swapped(1, 3, "answered", 2)], + newLines: () => "", + wait: clock.wait, + }); + await clock.tick(); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [["mark", "7:1:1"]]); +}); + +test("stop marks what the loop had not yet read before it returns", async () => { + const { clock, page, handle, log } = marker(); + log("swap:7:1:9\n"); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [["mark", "7:1:9"]]); +}); + +test("a page that fails to mark stops the loop and is rethrown by stop", async () => { + const page = fakePage({ failOn: "7:1:2" }); + const { clock, handle, log } = marker({ page }); + log("swap:7:1:1\nswap:7:1:2\nswap:7:1:3\n"); + await clock.tick(); + // The loop gave up at the failure and reads nothing more on its own. + log("swap:7:1:4\n"); + await clock.tick(); + await clock.tick(); + const rejected = assert.rejects(handle.stop(), /page refused 7:1:2/); + await clock.tick(); + await rejected; + // Nothing after the failure was marked, and the loop did not carry on. + assert.deepEqual(page.calls, [["mark", "7:1:1"]]); +}); + +test("halt after a failed mark neither rethrows nor marks again", async () => { + const page = fakePage({ failOn: "7:1:1" }); + const { clock, handle, log } = marker({ page }); + log("swap:7:1:1\n"); + await clock.tick(); + await ending(clock, handle.halt()); + assert.deepEqual(page.calls, []); + // A halted loop reads nothing more, however long the log grows. + log("swap:7:1:2\n"); + await clock.tick(); + assert.deepEqual(page.calls, []); +}); + +test("halt waits for the loop to finish, and a wake after it marks nothing", async () => { + const { clock, page, handle, log } = marker(); + log("swap:7:1:5\n"); + let finished = false; + const halting = handle.halt().then(() => { + finished = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(finished, false, "halt returned before the loop did"); + await clock.tick(); + await halting; + assert.equal(finished, true); + assert.deepEqual(page.calls, []); +}); + +test("the settle hands the page its grace, judges its samples and reports each episode", async () => { + const page = fakePage(); + // Episode 1 has its fresh sample; episode 2 has none. + page.rearms = [ + { + swap: "7:1:1", + seq: 1, + status: "ok", + reason: null, + edgesAtMark: 4, + windowEdge: 5, + counterPairs: 3, + bytesReceived: 900, + packetsReceived: 9, + peakRms: 0.2, + }, + ]; + const lines = []; + const answered = (episode) => ({ + pid: 7, + session: 1, + episode, + status: "pass", + class: "answered", + }); + const audio = await judgeSettledSwapAudio( + page, + { episodes: [answered(1), answered(2)] }, + "soak", + { graceMs: 1234, log: (line) => lines.push(line) }, + ); + assert.deepEqual(page.calls, [["settle", 1234]]); + assert.deepEqual( + audio.episodes.map((e) => [e.episode, e.status]), + [ + [1, "verified"], + [2, "missing_sample"], + ], + ); + assert.equal(lines.length, 2); + // A row with no reasons carries no trailing space. + assert.equal(lines[0], "soak episode 1 audio: verified"); + assert.match( + lines[1], + /^soak episode 2 audio: missing_sample no post-swap audio sample/, + ); +}); + +test("a page that fails to settle propagates, so the caller can still stop the probe", async () => { + const page = { + evaluate: async () => { + throw new Error("page closed"); + }, + }; + await assert.rejects( + judgeSettledSwapAudio(page, { episodes: [] }, "soak", { + graceMs: 1, + log: () => {}, + }), + /page closed/, + ); +}); diff --git a/tests/cli.rs b/tests/cli.rs index cb7ef30f2..0e3417219 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2175,8 +2175,9 @@ fn binary_web_serves_in_production_with_a_session_secret() { /// startup refusal read from the same values, for real. #[test] fn binary_web_reads_deployment_keys_from_the_config_file() { - let source = std::fs::read_to_string("src/main.rs") - .unwrap() + let source = ["src/main.rs", "src/launch.rs"] + .map(|path| std::fs::read_to_string(path).unwrap()) + .concat() .replace("\r\n", "\n"); // Ends at the closing brace in column zero rather than at whatever item @@ -2193,21 +2194,14 @@ fn binary_web_reads_deployment_keys_from_the_config_file() { .to_string() }; - // Both halves of the startup path. The configuration is assembled in - // `web_server_config` and the socket is bound in `run_web`, and the - // property is about the pair: what this deployment runs on comes out of the - // config file the operator wrote, never out of the ambient environment. - // Splitting the function moved the first assertion's subject and would have - // left this reading an empty haystack had it kept naming only one of them. - let serve = body("run_web"); - let configure = body("web_server_config"); - - assert!( - configure.contains("trusted_proxy_hops(values)"), - "{configure}" - ); - assert!(!serve.contains("std::env::var"), "{serve}"); - assert!(!configure.contains("std::env::var"), "{configure}"); + // Every phase of the startup path, in src/launch.rs and src/main.rs, takes + // its deployment from the operator's config file, never the environment. + assert!(body("web_server_config").contains("trusted_proxy_hops(values)")); + let phases = "run_web prepare finish local_agent_config web_server_config \ + serve serve_with run"; + for name in phases.split(' ') { + assert!(!body(name).contains("std::env::var"), "{name}"); + } } /// The log is the one place the error text sends a reader with no console, so diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 3191b9f89..e7ff0c6dc 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -19,6 +19,7 @@ only thing in the repo that crosses that boundary. | `integrity-chain.json` | `integrityEventPayload` | `apply_integrity` | | `task-action.json`, `task-edit.json`, `task-revision.json`, `task-run.json`, `task-start.json`, `task-end.json`, `task-connected.json`, `task-thinking.json` | `taskActionPayload`, `taskEditPayload`, `taskRevisionPayload`, `taskRunPayload`, `taskStartPayload`, `taskEndPayload`, `taskConnectedPayload`, `taskThinkingPayload` in `web/lib.js` | the typed messages `TaskSession::receive` parses, checked in `tests/unit/tasks/session.rs` | | `task-state.json`, `task-error.json`, `task-review.json` | written by the generator; the Rust side that must agree is `TaskSession::snapshot` (state), `ErrorMessage` (error) and `tasks::report` (review) | `web/task.js`, driven from these files by `tests/browser/task-workspace.test.js` and `task-controller.test.js`; Rust reads them in `tests/unit/tasks/session.rs` and `report.rs` | +| `acquisition-report.json` | `Arming::report` in `src/livekit/probe/acquisition.rs`, written by `acquisition_report_is_the_wire_fixture_the_consumer_reads` (regenerate with `CODETRIAL_UPDATE_FIXTURES=1 cargo test --lib acquisition_report_is_the_wire`) | `scripts/long-interview-evidence.cjs`, read by `tests/browser/long-interview-evidence.test.js` | `tests/fixtures/task-mode/` is not wire data: it is the packaged task set, its encrypted package and the patches both the server and the packaging tool must diff --git a/tests/fixtures/acquisition-report.json b/tests/fixtures/acquisition-report.json new file mode 100644 index 000000000..16a3d9efd --- /dev/null +++ b/tests/fixtures/acquisition-report.json @@ -0,0 +1,208 @@ +{ + "arming_failure": null, + "capture": { + "batches": [ + { + "calls": [ + { + "id": "call-1", + "name": "read_editor", + "result": { + "frameworkState": { + "phases": [] + }, + "result": "BEGIN UNTRUSTED EDITOR (python)\n(the editor is currently empty)\nEND UNTRUSTED EDITOR\nBEGIN UNTRUSTED TEST RUN\nNo test run was recorded; tests may not have been attempted or may not have been available for the selected language/problem yet.\nEND UNTRUSTED TEST RUN\nTIMER: about 45 minutes remain on the candidate's countdown." + } + } + ], + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "local_send_completed": true, + "sequence": 0 + } + ], + "boundaries": [ + { + "accepted": true, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "receipts_before": 1, + "replies_before": 1 + }, + { + "accepted": true, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 2 + }, + "receipts_before": 1, + "replies_before": 2 + }, + { + "accepted": true, + "batches_before": 1, + "ending": "Interrupted", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 3 + }, + "receipts_before": 1, + "replies_before": 3 + }, + { + "accepted": false, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": null + }, + "receipts_before": 1, + "replies_before": 3 + } + ], + "failure": null, + "receipts": [ + { + "batch": 0, + "discarded": false, + "id": "call-1", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "name": "read_editor" + } + ], + "replies": [ + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-0", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-A one. two.", + "views": 3 + }, + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-1", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 2 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-B.", + "views": 2 + }, + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-2", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 3 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-C cut", + "views": 2 + } + ], + "retained_bytes": 0, + "retained_calls": 1, + "retained_receipt_bytes": 0 + } +} diff --git a/tests/fixtures/report-recovery.json b/tests/fixtures/report-recovery.json index 9265471ac..5df7992b7 100644 --- a/tests/fixtures/report-recovery.json +++ b/tests/fixtures/report-recovery.json @@ -2,7 +2,7 @@ "expiresInSeconds": 300, "retryAfterSeconds": 30, "quotaRetryAfterSeconds": 60, - "retryWaitSeconds": 145, + "retryWaitSeconds": 175, "closeGraceSeconds": 15, "retryStatuses": [ "accepted", diff --git a/tests/stubs/interview-run-budget/fake_worker.py b/tests/stubs/interview-run-budget/fake_worker.py index d09b1b2c4..b394b1d1d 100644 --- a/tests/stubs/interview-run-budget/fake_worker.py +++ b/tests/stubs/interview-run-budget/fake_worker.py @@ -78,6 +78,7 @@ def raw_permit_request(guard, action_id, seq): signal.signal(signal.SIGTERM, signal.SIG_IGN) note("pids", str(os.getpid())) +note("reap_at", os.environ["BROWSER_CHECK_BUDGET_REAP_AT"]) telemetry(event="hb") guard = irb.WorkerGuard.from_env() stop = os.environ["BROWSER_CHECK_BUDGET_STOP"] diff --git a/tests/stubs/interview-run-budget/stand-in-fixture.py b/tests/stubs/interview-run-budget/stand-in-fixture.py new file mode 100644 index 000000000..c8a6da887 --- /dev/null +++ b/tests/stubs/interview-run-budget/stand-in-fixture.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import json +import os +import signal +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +if "--list" in sys.argv: + print("module::serve: test") + sys.exit(0) +directory = os.environ["CODETRIAL_FIXTURE_DIR"] +address = os.environ["CODETRIAL_FIXTURE_ADDR"] +host, port = address.rsplit(":", 1) + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def do_POST(self): + self.send_response(200) + self.send_header("Set-Cookie", "codetrial_session=stand-in; Path=/") + self.end_headers() + + def log_message(self, *args): + pass + + +server = HTTPServer((host, int(port)), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() +with open(os.path.join(directory, "ready"), "w") as ready: + ready.write(address) +# The two lines the script looks for to know a room got an interviewer. +print("codetrial dispatch room=stand-in", flush=True) +print("starting interview: room=stand-in", flush=True) +terms = [] +signal.signal(signal.SIGTERM, lambda *args: terms.append(1)) +with open(os.environ["STAND_IN_PIDFILE"], "w") as pidfile: + pidfile.write(str(os.getpid())) +# A deadline of its own: a run that never reaches the stop file must not leave +# this behind. +deadline = time.time() + 120 +while ( + not os.path.exists(os.path.join(directory, "stop")) + and not terms + and time.time() < deadline +): + time.sleep(0.05) +print("stand-in fixture stopping", flush=True) +if os.environ.get("STAND_IN_WRITES_REPORT"): + report = { + "outcome": "ok", + "bound": address, + "arming_failure": None, + # What the launcher passed, so the test can hold it to the manifest. + "launched_with": { + name: os.environ.get(f"CODETRIAL_FIXTURE_{name}") + for name in ("DEADLINE_S", "DRAIN_S", "SETTLE_S") + }, + "capture": { + "failure": None, + "receipts": [], + "batches": [], + "replies": [], + "boundaries": [], + }, + } + with open(os.path.join(directory, "out.json"), "w") as out: + json.dump(report, out) diff --git a/tests/stubs/interview-run-budget/stand-in-node.sh b/tests/stubs/interview-run-budget/stand-in-node.sh new file mode 100644 index 000000000..8926817ab --- /dev/null +++ b/tests/stubs/interview-run-budget/stand-in-node.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Stands in for node on browser-check.cjs only: leaves the capture a finished +# long interview would, then exits as told. Everything else runs the real node. +case $1 in + */scripts/browser-check.cjs) + printf '%s' "$STAND_IN_CAPTURE" > "$BROWSER_CHECK_CAPTURE" + exit "${STAND_IN_WORKER_EXIT:-0}" + ;; +esac +exec "$REAL_NODE" "$@" diff --git a/tests/test_ceiling_hotspots.py b/tests/test_ceiling_hotspots.py index 5e446577d..0504a4e7c 100644 --- a/tests/test_ceiling_hotspots.py +++ b/tests/test_ceiling_hotspots.py @@ -1,7 +1,6 @@ """The hotspot ranking orders files by commits times lines, inside the window.""" import importlib.util -import json import os import subprocess import sys @@ -16,6 +15,9 @@ # A commit far outside any window the tests ask for. LONG_AGO = "2001-01-01T00:00:00 +0000" +# The style checker's default ceiling; only files above it are ranked by default. +OVER = 1501 + def run(*args): return subprocess.run( @@ -76,11 +78,6 @@ def commit(self, files, date=None): self.git("add", "-A") self.git("commit", "-q", "-m", "change", date=date) - def ceilings(self, entries): - path = self.repo / "scripts/rust-style-ceilings.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(entries)) - def test_score_is_commits_times_lines_and_orders_the_table(self): self.commit({"src/big.rs": 10, "src/busy.rs": 5, "src/idle.rs": 100}) self.commit({"src/busy.rs": 6}) @@ -162,29 +159,31 @@ def test_under_filters_by_size_suffix_and_prefix(self): self.assertEqual(rows, {"web/big.js": None}) def test_cli_prints_one_table_per_directory_with_the_revision(self): - self.commit({"src/a.rs": 3, "tests/t.rs": 2}) - self.ceilings({"src/a.rs": 3, "tests/t.rs": 2}) + self.commit({"src/a.rs": OVER, "tests/t.rs": OVER + 1}) done = run("--repo", str(self.repo)) self.assertEqual(done.returncode, 0, done.stderr) self.assertIn("# history at ", done.stdout) self.assertIn("## src/ (1 files)", done.stdout) self.assertIn("## tests/ (1 files)", done.stdout) - def test_a_ceiling_for_a_missing_file_is_reported_and_skipped(self): - self.commit({"src/a.rs": 3}) - self.ceilings({"src/a.rs": 3, "src/gone.rs": 9}) + def test_a_file_at_the_default_ceiling_is_not_ranked(self): + self.commit({"src/a.rs": OVER, "src/small.rs": OVER - 1, "web/a.js": OVER}) done = run("--repo", str(self.repo)) self.assertEqual(done.returncode, 0, done.stderr) - self.assertIn("skipped src/gone.rs", done.stderr) - self.assertNotIn("src/gone.rs", done.stdout) - - def test_a_ceilings_file_of_the_wrong_shape_exits_two(self): - self.commit({"src/a.rs": 3}) - for shape in ([1, 2], {"src/a.rs": "3"}, {"src/a.rs": True}): - self.ceilings(shape) - done = run("--repo", str(self.repo)) - self.assertEqual(done.returncode, 2, (shape, done.stderr)) - self.assertIn("not a map of path to line ceiling", done.stderr) + self.assertIn("src/a.rs", done.stdout) + self.assertNotIn("src/small.rs", done.stdout) + self.assertNotIn("web/a.js", done.stdout) + + def test_the_ceiling_is_the_committed_size_not_the_working_one(self): + self.commit({"src/a.rs": OVER + 9}) + (self.repo / "src/a.rs").write_text("x\n" * (OVER + 20)) + (self.repo / "src/new.rs").write_text("x\n" * OVER) + done = run("--repo", str(self.repo)) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + row_of(done.stdout, "src/a.rs")[1:3], [str(OVER + 20), str(OVER + 9)] + ) + self.assertEqual(row_of(done.stdout, "src/new.rs")[1:3], [str(OVER), "1500"]) def test_cli_under_shows_a_dash_where_no_ceiling_exists(self): self.commit({"web/big.js": 20}) @@ -195,13 +194,13 @@ def test_cli_under_shows_a_dash_where_no_ceiling_exists(self): ) def test_a_file_with_no_commit_in_the_window_scores_zero(self): - self.commit({"src/a.rs": 4}, date=LONG_AGO) - self.ceilings({"src/a.rs": 4}) + self.commit({"src/a.rs": OVER}, date=LONG_AGO) done = run("--repo", str(self.repo), "--since", "30.days") self.assertEqual(done.returncode, 0, done.stderr) self.assertIn("since 30.days", done.stdout) self.assertEqual( - row_of(done.stdout, "src/a.rs"), ["src/a.rs", "4", "4", "0", "0"] + row_of(done.stdout, "src/a.rs"), + ["src/a.rs", str(OVER), str(OVER), "0", "0"], ) def test_under_with_nothing_to_list_exits_two(self): @@ -216,23 +215,19 @@ def test_outside_a_repository_exits_two(self): self.assertEqual(done.returncode, 2) self.assertIn("ceiling-hotspots:", done.stderr) - def test_the_real_tree_lists_every_recorded_ceiling_that_exists(self): - recorded = json.loads((ROOT / "scripts/rust-style-ceilings.json").read_text()) - present = [name for name in recorded if (ROOT / name).is_file()] + def test_the_real_tree_lists_every_file_over_the_default_ceiling(self): + mod = self.mod.STYLE + over = [ + n for n, lines in mod.sizes(ROOT).items() if lines > mod.DEFAULT_CEILING + ] done = run("--repo", str(ROOT)) self.assertEqual(done.returncode, 0, done.stderr) listed = [ line.split()[0] for line in done.stdout.splitlines() - if line.split() and line.split()[0] in recorded + if line.split() and line.split()[0] in over ] - self.assertEqual(sorted(listed), sorted(present)) - # The gate fails any file above its entry, so a row over its ceiling - # would mean this script counts lines differently from the gate. - for line in done.stdout.splitlines(): - cells = line.split() - if cells and cells[0] in recorded: - self.assertLessEqual(int(cells[1]), int(cells[2]), cells[0]) + self.assertEqual(sorted(listed), sorted(over)) if __name__ == "__main__": diff --git a/tests/test_check_rust_style.py b/tests/test_check_rust_style.py index 0007aa90b..43242859e 100644 --- a/tests/test_check_rust_style.py +++ b/tests/test_check_rust_style.py @@ -9,7 +9,6 @@ import unittest from pathlib import Path from types import SimpleNamespace -from unittest import mock from unittest.mock import Mock SCRIPT = Path(__file__).resolve().parents[1] / "scripts/check-rust-style.py" @@ -257,39 +256,167 @@ def test_cjk_is_refused_in_rust_and_the_browser_half(self): self.findings("src/lib.rs", f'const S: &str = "{greek}";\n'), [] ) - def test_a_file_cannot_pass_its_ceiling(self): - body = "fn f() {}\n" * 1501 - self.assertFires("ceiling", "src/big.rs", body) - self.write("scripts/rust-style-ceilings.json", json.dumps({"src/big.rs": 1501})) - self.assertEqual(run(self.root).stdout, "") - self.write("src/big.rs", body + "fn g() {}\n") - grown = run(self.root) - self.assertIn("src/big.rs: ceiling", grown.stdout) - self.assertEqual(grown.returncode, 1) - - def test_a_ceiling_above_the_file_is_refused_until_ratcheted(self): - self.write("src/big.rs", "fn f() {}\n" * 1600) - self.write("scripts/rust-style-ceilings.json", json.dumps({"src/big.rs": 1700})) - self.assertIn("run --ratchet", run(self.root).stdout) - run(self.root, "--ratchet") - self.assertEqual(run(self.root).stdout, "") - - def test_a_ceiling_for_a_deleted_file_is_refused(self): - self.write( - "scripts/rust-style-ceilings.json", json.dumps({"src/gone.rs": 2000}) + def test_a_new_file_cannot_exceed_the_default_ceiling(self): + self.assertEqual(self.findings("src/big.rs", "// line\n" * 1500), []) + self.assertFires("ceiling", "src/big.rs", "// line\n" * 1501) + + def start_history(self): + self.git("init", "-q") + self.git("symbolic-ref", "HEAD", "refs/heads/main") + self.commit_tree("base") + self.git("tag", "base") + + def commit_tree(self, message): + self.git("add", "-A") + self.git("commit", "-qm", message) + + def test_git_history_provides_ceilings_without_a_table(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.assertFalse((self.root / "scripts/rust-style-ceilings.json").exists()) + self.assertEqual(self.git("status", "--porcelain").stdout, b"") + self.write("src/big.rs", "// line\n" * 1601) + self.assertTrue(any("over 1600" in x for x in self.style.run(self.root))) + + def test_committed_shrink_cannot_grow_back(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.write("src/big.rs", "// line\n" * 1550) + self.assertEqual(self.style.run(self.root, "base"), []) + self.commit_tree("shrink") + self.write("src/big.rs", "// line\n" * 1560) + self.commit_tree("regrow") + self.assertEqual( + self.style.run(self.root), [] + ) # HEAD checks uncommitted growth. + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "base")) ) - self.assertIn("src/gone.rs no longer exists", run(self.root).stdout) - def test_ratchet_lowers_and_never_raises(self): - self.write("src/big.rs", "fn f() {}\n" * 1600) - self.write( - "scripts/rust-style-ceilings.json", - json.dumps({"src/big.rs": 1700, "src/small.rs": 1800}), + def test_new_committed_and_renamed_paths_get_no_exemption(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("mv", "src/big.rs", "src/renamed.rs") + self.write("src/new.rs", "// line\n" * 1600) + self.commit_tree("new paths") + findings = self.style.run(self.root, "base") + for name in ["renamed.rs", "new.rs"]: + self.assertTrue(any(name in x and "over 1500" in x for x in findings)) + + def test_deletion_retires_the_allowance(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + (self.root / "src/big.rs").unlink() + self.assertEqual(self.style.run(self.root, "base"), []) + self.commit_tree("delete") + self.write("src/big.rs", "// line\n" * 1600) + self.commit_tree("recreate") + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "base")) + ) + + def test_shrinking_below_default_keeps_the_general_default(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.write("src/big.rs", "// line\n" * 1400) + self.commit_tree("shrink") + self.write("src/big.rs", "// line\n" * 1500) + self.assertEqual(self.style.run(self.root, "base"), []) + # One past the default is checked against the default, rather than + # against the 1600 the base allowed. + self.write("src/big.rs", "// line\n" * 1501) + findings = self.style.run(self.root, "base") + self.assertTrue(any("over 1500" in x for x in findings), findings) + + def test_merge_history_cannot_hide_a_smaller_version(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/big.rs", "// line\n" * 1550) + self.commit_tree("shrink on side") + self.git("checkout", "main") + self.git("merge", "--no-ff", "-s", "ours", "side", "-m", "merge") + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "base")) + ) + + def test_a_merge_deletion_retires_the_allowance(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/other.rs", "// other\n") + self.commit_tree("side changes") + self.git("checkout", "main") + self.git("merge", "--no-ff", "--no-commit", "side") + self.git("rm", "src/big.rs") + self.commit_tree("merge deletes") + self.write("src/big.rs", "// line\n" * 1600) + self.commit_tree("recreate") + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "base")) + ) + + def test_unrelated_history_cannot_supply_an_allowance(self): + self.start_history() + self.git("checkout", "--orphan", "unrelated") + self.commit_tree("other root") + self.assertTrue( + any( + "cannot read Git size baseline" in x + for x in self.style.run(self.root, "base") + ) + ) + + def test_diverged_baseline_keeps_branch_history(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/big.rs", "// line\n" * 1550) + self.commit_tree("shrink") + self.git("checkout", "main") + self.write("src/other.rs", "// other\n") + self.commit_tree("main advances") + self.git("checkout", "side") + self.write("src/big.rs", "// line\n" * 1560) + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "main")) + ) + + def test_git_and_worktree_count_newlines_the_same_way(self): + name = "src/large name " + chr(0x3B1) + ".rs" + self.write(name, "// line\r\n" * 1500 + "// last") + self.start_history() + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.write(name, "// line\n" * 1500 + "// last\n") + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.write(name, "// line\n" * 1500 + "// last\n// added") + self.assertTrue( + any("over 1501" in x for x in self.style.run(self.root, "HEAD")) + ) + + def test_a_descendant_baseline_cannot_relax_head(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "ahead") + self.write("src/big.rs", "// line\n" * 2000) + self.commit_tree("ahead grows") + self.git("checkout", "main") + self.write("src/big.rs", "// line\n" * 1900) + self.assertTrue( + any("over 1600" in x for x in self.style.run(self.root, "ahead")) + ) + + def test_a_baseline_only_path_is_still_new_in_the_worktree(self): + self.start_history() + self.git("checkout", "-qb", "ahead") + self.write("src/new.rs", "// line\n" * 1600) + self.commit_tree("ahead adds") + self.git("checkout", "main") + self.write("src/new.rs", "// line\n" * 1600) + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "ahead")) ) - self.write("src/small.rs", "fn f() {}\n") - run(self.root, "--ratchet") - kept = json.loads((self.root / "scripts/rust-style-ceilings.json").read_text()) - self.assertEqual(kept, {"src/big.rs": 1600}) def test_every_required_lint_refuses_weakening_or_removal(self): for group, lints in self.style.REQUIRED_LINTS.items(): @@ -445,36 +572,28 @@ def git(self, *args): check=True, ) - def test_a_ceiling_cannot_rise_or_appear_since_the_baseline(self): - body = "fn f() {}\n" * 1600 - ceilings = "scripts/rust-style-ceilings.json" - self.write("src/big.rs", body) - self.write(ceilings, json.dumps({"src/big.rs": 1600})) - self.git("init", "-q") - self.git("add", "-A") - self.git("commit", "-qm", "base") - self.assertEqual(self.style.run(self.root, "HEAD"), []) - self.write("src/big.rs", body + "fn g() {}\n" * 100) - self.write(ceilings, json.dumps({"src/big.rs": 1700})) + def test_explicit_missing_baseline_fails(self): + # The checker reads no CI variable, so there is one answer. The test + # does not set one: the runner executes cases in threads, and a changed + # os.environ under a thread that is spawning git makes the spawn fail + # with EFAULT. A repository with history, so the refusal is of the + # absent ref and not of the directory. + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() self.assertTrue( - any( - "rose from 1600 to 1700" in x for x in self.style.run(self.root, "HEAD") - ) + any("not fetched" in x for x in self.style.run(self.root, "no-such-ref")) ) - self.write("src/big.rs", body) - self.write("src/new.rs", body) - self.write(ceilings, json.dumps({"src/big.rs": 1600, "src/new.rs": 1600})) - self.assertTrue( - any("new exemption" in x for x in self.style.run(self.root, "HEAD")) - ) - with mock.patch.dict(os.environ, {"CI": ""}): - self.assertEqual(self.style.run(self.root, "no-such-ref"), []) - with mock.patch.dict(os.environ, {"CI": "true"}): - self.assertTrue( - any( - "not fetched" in x for x in self.style.run(self.root, "no-such-ref") - ) - ) + + def test_shallow_history_cannot_claim_a_ratchet(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + with tempfile.TemporaryDirectory() as clone: + self.git("clone", "-q", "--depth", "1", self.root.as_uri(), clone) + findings = self.style.run(Path(clone), "HEAD") + self.assertTrue(any("complete history" in x for x in findings)) + + def test_legacy_ratchet_command_is_removed(self): + self.assertEqual(run(self.root, "--ratchet").returncode, 2) if __name__ == "__main__": diff --git a/tests/test_interview_run_budget.py b/tests/test_interview_run_budget.py index 7ba83208e..e7f4126f4 100644 --- a/tests/test_interview_run_budget.py +++ b/tests/test_interview_run_budget.py @@ -650,10 +650,15 @@ def test_bounded_cleanup_gives_up_on_cleanup_that_hangs(self): class LauncherRunTests(Scratch): def test_a_clean_run_completes_with_every_action_granted_and_dispatched_once(self): + started = time.time() result, _ = self.launch("normal", env={"FAKE_OBSERVED": "1"}) self.assertEqual(result.returncode, 0, result.stderr) report = self.result() self.assertEqual(report["status"], "complete") + # The worker is told when the reap begins: before the wall cap. + reap_at = int((self.fake / "reap_at").read_text()) + self.assertGreater(reap_at, started - 1) + self.assertLess(reap_at, started + self.manifest()["outer_wall_cap_s"]) self.assertEqual(self.dispatched(), ACTIONS + ["end"]) self.assertEqual(self.grants(), ACTIONS + ["end"]) hard = report["hard"]["scripted_actions"] @@ -1168,6 +1173,526 @@ def test_a_live_run_needs_the_launchers_socket(self): self.assertEqual(result.returncode, 2, result.stderr) self.assertIn("live BROWSER_CHECK_BUDGET_SOCKET", result.stderr) + def dummy_config(self): + """The credentials a dispatch run asks for, from a throwaway config + rather than whatever this machine keeps.""" + config = self.root / "dummy.env.local" + config.write_text( + "LIVEKIT_URL=wss://primary.example\nLIVEKIT_API_KEY=k\n" + "LIVEKIT_API_SECRET=s\nGOOGLE_API_KEY=g\n" + ) + return config + + def test_the_serving_fixture_needs_a_budgeted_dispatch_run_and_a_fresh_report(self): + config = self.dummy_config() + report = self.root / "fixture-report.json" + # A reserve that outlasts the 150 s default stop wait with a minute over. + manifest = self.write_manifest( + outer_wall_cap_s=3600, cleanup_reserve_s=300, selected_duration_s=600 + ) + base = { + "CODETRIAL_CONFIG_ENV": str(config), + "BROWSER_CHECK_RUN_MANIFEST": str(manifest), + "BROWSER_CHECK_RUN_MANIFEST_SHA256": sha256(manifest), + "BROWSER_CHECK_FIXTURE_TEST": "module::serve", + "BROWSER_CHECK_AGENT": "dispatch", + "BROWSER_CHECK_FIXTURE_OUT": str(report), + "BROWSER_CHECK_BUDGET_REAP_AT": str(int(time.time()) + 3600), + } + result = self.check(**base) + self.assertEqual(result.returncode, 0, result.stderr) + # A capture path that is free is fine. + free = self.check(**base, BROWSER_CHECK_CAPTURE=str(self.root / "capture.json")) + self.assertEqual(free.returncode, 0, free.stderr) + no_run = { + "BROWSER_CHECK_RUN_MANIFEST": None, + "BROWSER_CHECK_RUN_ID": None, + "BROWSER_CHECK_RUN_MANIFEST_SHA256": None, + "BROWSER_CHECK_BUDGET_SOCKET": None, + } + existing = self.root / "already-there.json" + existing.write_text("{}") + linked = self.root / "linked" + linked.symlink_to(self.root, target_is_directory=True) + rows = [ + ({**base, "BROWSER_CHECK_AGENT": "home"}, "needs a budgeted run"), + ({**base, **no_run}, "needs a budgeted run"), + ({**base, "BROWSER_CHECK_FIXTURE_OUT": None}, "must name a report file"), + ({**base, "BROWSER_CHECK_BUDGET_REAP_AT": None}, "BUDGET_REAP_AT"), + ({**base, "BROWSER_CHECK_BUDGET_REAP_AT": "soon"}, "BUDGET_REAP_AT"), + ( + {**base, "BROWSER_CHECK_CAPTURE": str(report)}, + "must name different files", + ), + ( + # Through a symlinked directory: the paths are compared once + # resolved, not as they are spelled. + {**base, "BROWSER_CHECK_CAPTURE": str(linked / report.name)}, + "must name different files", + ), + ( + {**base, "BROWSER_CHECK_CAPTURE": f"{report}.server.log"}, + "must name different files", + ), + ( + { + **base, + "BROWSER_CHECK_CAPTURE": str(report.with_suffix("")), + "BROWSER_CHECK_FIXTURE_OUT": f"{report.with_suffix('')}.partial", + }, + "must name different files", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_OUT": str(existing)}, + "must name a report file", + ), + ( + { + **base, + "BROWSER_CHECK_FIXTURE_OUT": str( + self.root / "no-such-dir" / "r.json" + ), + }, + "in a directory that does and can be written", + ), + ( + {**base, "BROWSER_CHECK_CAPTURE": str(existing)}, + "BROWSER_CHECK_CAPTURE must not exist yet", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "soon"}, + "number of seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "0150"}, + "number of seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "39"}, + "at least 40 seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "241"}, + "needs at least 301s", + ), + ] + for env, expected in rows: + with self.subTest(env=env): + result = self.check(**env) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(expected, result.stderr) + + def test_the_stop_wait_must_fit_inside_the_manifests_cleanup_reserve(self): + config = self.dummy_config() + # The reserve the shared manifest here carries is the accepted minimum, + # far short of a stop wait of 150 s. + result = self.check( + CODETRIAL_CONFIG_ENV=str(config), + BROWSER_CHECK_AGENT="dispatch", + BROWSER_CHECK_FIXTURE_TEST="module::serve", + BROWSER_CHECK_FIXTURE_OUT=str(self.root / "report.json"), + ) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("cleanup_reserve_s is 60", result.stderr) + self.assertIn("needs at least 210s", result.stderr) + + def test_the_fixture_is_started_in_a_session_of_its_own(self): + script = (ROOT / "scripts" / "browser-check.sh").read_text() + # A TERM sent to the script's process group must reach the script, whose + # trap stops the fixture gracefully, and not the fixture first. The + # launch itself needs a browser, so this reads the script. + self.assertRegex(script, r'set -- setsid "\$@"') + self.assertRegex( + script, r'--ignored --exact "\$FIXTURE_TEST" --nocapture --test-threads=1' + ) + + def kill_pidfile(self, path): + if path.exists(): + self.kill(int(path.read_text())) + + GOOD_CAPTURE = { + "flow": "long-interview", + "mode": "dispatch", + "plan": {"durationMin": 90, "observationTargetS": 5400}, + "report": { + "status": "complete", + "exit_code": 0, + "anchor": {"source": "server_clock"}, + "validation_fixture": False, + "schedule_mode": "coding", + "qualification": "not_evaluated", + "reply_content": "not_assessed", + "schedule_origin_unix_ms": 1700000000000, + }, + } + + def good_capture(self): + """The capture a finished long interview leaves, with the schedule hash + the judge will rebuild from the plan.""" + sha = subprocess.run( + [ + shutil.which("node"), + "-e", + "console.log(require('./scripts/long-interview-schedule.cjs')" + ".generateSchedule('coding',{durationMin:90,observationTargetS:5400})" + ".schedule.schedule_sha256)", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + capture = json.loads(json.dumps(self.GOOD_CAPTURE)) + capture["report"]["schedule_sha256"] = sha + return json.dumps(capture) + + def run_fixture_path(self, writes_report=True, worker_exit=0, capture_text=None): + """browser-check.sh from the manifest to the exit trap, with a stand-in + fixture server and a stand-in worker: no browser, no credentials.""" + fixture = self.root / "stand-in-fixture" + support.write_executable( + fixture, support.stub("interview-run-budget/stand-in-fixture.py") + ) + manifest = self.write_manifest( + outer_wall_cap_s=3600, + cleanup_reserve_s=300, + selected_duration_s=600, + binaries={"candidate": {"path": str(fixture), "sha256": sha256(fixture)}}, + ) + shim = self.root / "shim" + shim.mkdir() + # The script keeps its floor and latency logs outside its own scratch + # directory by design; here they go inside this test's. + tmp = self.root / "tmp" + tmp.mkdir() + support.write_executable( + shim / "node", support.stub("interview-run-budget/stand-in-node.sh") + ) + socket_path = self.root / "budget.sock" + # Bound and never listened on: the script only asks that it be a socket. + listener = socket.socket(socket.AF_UNIX) + listener.bind(str(socket_path)) + self.addCleanup(listener.close) + report = self.root / "kept-report.json" + capture = self.root / "capture.json" + pidfile = self.root / "stand-in.pid" + self.addCleanup(self.kill_pidfile, pidfile) + result = self.check( + BROWSER_CHECK_VALIDATE_ENV_ONLY=None, + BROWSER_CHECK_AGENT="dispatch", + BROWSER_CHECK_FIXTURE_TEST="module::serve", + BROWSER_CHECK_FIXTURE_OUT=str(report), + BROWSER_CHECK_FIXTURE_STOP_WAIT_S="40", + BROWSER_CHECK_CAPTURE=str(capture), + BROWSER_CHECK_RUN_MANIFEST=str(manifest), + BROWSER_CHECK_RUN_MANIFEST_SHA256=sha256(manifest), + BROWSER_CHECK_BUDGET_SOCKET=str(socket_path), + BROWSER_CHECK_BUDGET_REAP_AT=str(int(time.time()) + 3600), + CODETRIAL_CONFIG_ENV=str(self.dummy_config()), + PATH=f"{shim}:{os.environ['PATH']}", + TMPDIR=str(tmp), + PORT=None, + INTERVIEW_ROOM_NAME=None, + STAND_IN_PIDFILE=str(pidfile), + REAL_NODE=shutil.which("node"), + STAND_IN_CAPTURE=capture_text or self.good_capture(), + STAND_IN_WORKER_EXIT=str(worker_exit), + STAND_IN_WRITES_REPORT="1" if writes_report else None, + ) + return result, report, capture + + def test_the_fixture_path_runs_from_start_to_a_judged_capture(self): + result, report, capture = self.run_fixture_path() + self.assertEqual(result.returncode, 0, result.stderr) + # The report was kept with a private copy of the server's log. + kept_report = json.loads(report.read_text()) + self.assertEqual(kept_report["outcome"], "ok") + # Its drain and settle add up to the stop wait less the slack, and its + # own deadline leaves the stop wait and the slack before the runner's + # reap (an hour from the launch here), less what the launch used. + launched = kept_report["launched_with"] + self.assertEqual((launched["DRAIN_S"], launched["SETTLE_S"]), ("10", "10")) + ceiling = 3600 - 40 - 20 + self.assertIn(int(launched["DEADLINE_S"]), range(ceiling - 60, ceiling + 1)) + log = Path(str(report) + ".server.log") + self.assertIn("stand-in fixture stopping", log.read_text()) + self.assertEqual(log.stat().st_mode & 0o777, 0o600) + # And judged after it stopped: the verdict is in the capture, the rest + # of the report is as the worker left it. + kept = json.loads(capture.read_text())["report"] + self.assertEqual(kept["reply_content"], "failed", kept) + self.assertEqual(kept["qualification"], "not_evaluated") + self.assertEqual(kept["status"], "complete") + self.assertIn("long-interview reply_content=failed", result.stdout) + + def test_a_judgment_never_turns_a_failed_run_green(self): + failed, _, capture = self.run_fixture_path(worker_exit=1) + self.assertNotEqual(failed.returncode, 0, failed.stderr) + self.assertEqual( + json.loads(capture.read_text())["report"]["reply_content"], "failed" + ) + + def test_a_judgment_that_cannot_run_leaves_a_good_run_good(self): + result, _, capture = self.run_fixture_path(capture_text="not a capture") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("cannot read the capture", result.stderr) + self.assertIn("run's own result stands", result.stderr) + self.assertEqual(capture.read_text(), "not a capture") + + def test_a_fixture_that_wrote_no_report_fails_the_run_and_says_so(self): + result, report, capture = self.run_fixture_path(writes_report=False) + self.assertNotEqual(result.returncode, 0, result.stderr) + self.assertIn("left no report", json.loads(report.read_text())["outcome"]) + kept = json.loads(capture.read_text())["report"] + self.assertEqual(kept["reply_content"], "not_assessed", kept) + + def test_the_sdk_fixture_launcher_pins_its_test_name_through_the_shared_check(self): + script = (ROOT / "scripts" / "live-sdk-fixture.sh").read_text() + self.assertIn('fixture-server.sh" check "$bin" "$name"', script) + + +class FixtureServerScriptTests(Scratch): + """scripts/fixture-server.sh against stand-in executables.""" + + SCRIPT = ROOT / "scripts" / "fixture-server.sh" + + def run_script(self, *args, **env): + environment = dict(os.environ) + environment.update(env) + return subprocess.run( + ["sh", str(self.SCRIPT), *map(str, args)], + env=environment, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + def pin(self, listing): + pin = self.root / "pin" + support.write_executable( + pin, + '#!/bin/sh\nif [ "$1" = --list ] && [ "$2" = --ignored ]; then\n' + f"cat <<'EOF'\n{listing}EOF\nfi\n", + ) + return pin + + GOOD_REPORT = { + "outcome": "ok", + "bound": "127.0.0.1:1", + "arming_failure": None, + "capture": { + "failure": None, + "receipts": [], + "batches": [], + "replies": [], + "boundaries": [], + }, + } + + def keep(self, report, log="server output\n", before=None): + """Runs `keep` over a stand-in fixture directory; returns the result and + the path the report was kept at.""" + directory = Path(tempfile.mkdtemp(dir=self.root)) + if report is not None: + (directory / "out.json").write_text(json.dumps(report)) + server_log = directory / "web.log" + server_log.write_text(log) + out = directory / "kept.json" + if before: + before(out) + result = self.run_script("keep", directory, out, server_log) + return result, out + + def test_keep_holds_a_good_report_and_a_private_copy_of_the_log(self): + result, out = self.keep(self.GOOD_REPORT) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(out.read_text())["outcome"], "ok") + # Both private: the report holds the interviewer's replies and the + # tool results. + self.assertEqual(out.stat().st_mode & 0o777, 0o600) + kept = Path(str(out) + ".server.log") + self.assertEqual(kept.read_text(), "server output\n") + self.assertEqual(kept.stat().st_mode & 0o777, 0o600) + + def test_keep_never_replaces_a_file_already_at_the_report_path(self): + # A capture written there by this run, or another run's report, is + # left as it was and the keep fails. + result, out = self.keep( + self.GOOD_REPORT, before=lambda out: out.write_text("the capture\n") + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(out.read_text(), "the capture\n") + + def test_keep_replaces_neither_a_log_nor_a_report_already_there(self): + result, out = self.keep( + self.GOOD_REPORT, + before=lambda out: Path(str(out) + ".server.log").write_text("earlier\n"), + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(Path(str(out) + ".server.log").read_text(), "earlier\n") + result, out = self.keep(None, before=lambda out: out.write_text("earlier\n")) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(out.read_text(), "earlier\n") + self.assertIn("not this run's report", result.stderr) + + def test_keep_leaves_an_envelope_when_the_fixture_wrote_nothing(self): + result, out = self.keep(None) + self.assertEqual(result.returncode, 1, result.stderr) + envelope = json.loads(out.read_text()) + self.assertEqual(out.stat().st_mode & 0o777, 0o600) + self.assertIn("left no report", envelope["outcome"]) + self.assertIsNone(envelope["capture"]) + self.assertIn("server output", result.stderr) + + def test_keep_refuses_a_report_the_judge_could_not_read(self): + reports = ( + {**self.GOOD_REPORT, "outcome": "shutdown bound"}, + {k: v for k, v in self.GOOD_REPORT.items() if k != "bound"}, + {**self.GOOD_REPORT, "capture": None}, + ) + for report in reports: + with self.subTest(report=report): + result, out = self.keep(report) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertTrue(out.exists()) + + def test_keep_fails_when_the_log_cannot_be_kept(self): + # A directory sits where the log copy has to go. + result, _ = self.keep( + self.GOOD_REPORT, before=lambda out: Path(str(out) + ".server.log").mkdir() + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("could not keep the server log", result.stderr) + + def test_the_pin_must_list_the_test_exactly_once(self): + pin = self.pin("a::one: test\nmodule::serve: test\n") + self.assertEqual(self.run_script("check", pin, "module::serve").returncode, 0) + rows = [ + ("module::missing", "lists 0"), + ("serve", "lists 0"), + ] + for name, expected in rows: + with self.subTest(name=name): + result = self.run_script("check", pin, name) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(expected, result.stderr) + twice = self.pin("module::serve: test\nmodule::serve: test\n") + result = self.run_script("check", twice, "module::serve") + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("lists 2", result.stderr) + + def test_a_pin_that_is_not_runnable_or_a_name_that_is_not_one_name_is_refused(self): + plain = self.root / "plain" + plain.write_text("not executable") + failing = self.root / "failing" + support.write_executable(failing, "#!/bin/sh\nexit 1\n") + for args in [ + (plain, "module::serve"), + (failing, "module::serve"), + (self.pin("module::serve: test\n"), ""), + (self.pin("module::serve: test\n"), "two names"), + ]: + with self.subTest(args=args): + self.assertEqual(self.run_script("check", *args).returncode, 2) + + def test_a_fixture_that_stops_on_request_is_waited_for_and_leaves_its_report(self): + fixture = self.root / "fixture" + support.write_executable( + fixture, + '#!/bin/sh\nwhile [ ! -e "$1/stop" ]; do sleep 0.1; done\n' + 'sleep 1\necho \'{"outcome":"ok"}\' > "$1/out.json"\n', + ) + directory = self.root / "fixture-dir" + directory.mkdir() + process = subprocess.Popen([str(fixture), str(directory)]) + self.spawned.append(process.pid) + result = self.run_script("stop", directory, process.pid, 20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((directory / "stop").exists()) + # It finished its own work first: the stop did not cut it short. The + # process is still unreaped here, which the script must not mistake for + # running. + self.assertEqual( + (directory / "out.json").read_text().strip(), '{"outcome":"ok"}' + ) + self.assertEqual(process.wait(timeout=10), 0) + + def test_a_fixture_that_will_not_stop_is_terminated_and_that_is_a_failure(self): + stubborn = self.root / "stubborn" + support.write_executable( + stubborn, "#!/bin/sh\ntrap '' TERM\nwhile :; do sleep 1; done\n" + ) + directory = self.root / "stubborn-dir" + directory.mkdir() + process = subprocess.Popen([str(stubborn)]) + self.spawned.append(process.pid) + result = self.run_script( + "stop", directory, process.pid, 0, FIXTURE_SERVER_TERM_GRACE_S="1" + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("had not stopped after 0s", result.stderr) + process.wait(timeout=10) + self.assertIsNotNone(process.returncode) + + def test_a_stop_file_that_cannot_be_written_falls_back_to_a_bounded_term(self): + # TERM is a stop request to the fixture: it finishes and exits cleanly + # inside the wait, past the kill grace, so only the wait lets it. + fixture = self.root / "on-term" + support.write_executable( + fixture, + "#!/bin/sh\ntrap 'sleep 2; exit 0' TERM\nwhile :; do sleep 0.1; done\n", + ) + process = subprocess.Popen([str(fixture)]) + self.spawned.append(process.pid) + time.sleep(0.5) + started = time.monotonic() + result = self.run_script( + "stop", + self.root / "no-such-dir", + process.pid, + 20, + FIXTURE_SERVER_TERM_GRACE_S="1", + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("could not write the stop file", result.stderr) + self.assertEqual(process.wait(timeout=10), 0, "it was asked, not killed") + self.assertLess(time.monotonic() - started, 15) + + def test_a_padded_zombie_state_from_ps_still_counts_as_exited(self): + # A system that pads the state column (" Z") must not leave the stop + # waiting out its whole limit for a process that has already exited. + bin_dir = self.root / "bin" + bin_dir.mkdir() + support.write_executable(bin_dir / "ps", "#!/bin/sh\necho ' Z+'\n") + sleeper = subprocess.Popen(["sleep", "300"]) + self.spawned.append(sleeper.pid) + directory = self.root / "padded-dir" + directory.mkdir() + started = time.monotonic() + result = self.run_script( + "stop", + directory, + sleeper.pid, + 30, + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertLess(time.monotonic() - started, 10) + + def test_stop_refuses_a_pid_or_wait_that_is_not_a_number(self): + for args in [ + ("stop", self.root, "abc", 1), + ("stop", self.root, 0, 1), + ("stop", self.root, 1, 1), + ("stop", self.root, "007", 1), + ("stop", self.root, 1, "soon"), + ("stop", self.root), + ("bogus",), + ]: + with self.subTest(args=args): + self.assertEqual(self.run_script(*args).returncode, 2) + class WorkerGuardNodeTests(Scratch): """scripts/run-budget-guard.cjs against the real Python permit service.""" diff --git a/tests/unit/bin.rs b/tests/unit/bin.rs index ea9a41808..095508d14 100644 --- a/tests/unit/bin.rs +++ b/tests/unit/bin.rs @@ -847,12 +847,12 @@ fn task_mode_runs_only_on_a_private_server_that_neither_records_nor_shares_a_roo #[test] fn peer_is_offered_once_oauth_and_observer_keys_exist() { let oauth = |settings: &[(&str, &str)]| { - super::main_peer::peer_services(&values(settings), true).unwrap() + codetrial::launch::peer_services(&values(settings), true).unwrap() }; assert!(oauth(&[]).is_none()); assert!(oauth(&[("GOOGLE_API_KEY", "test-key")]).is_some()); assert!( - super::main_peer::peer_services(&values(&[("GOOGLE_API_KEY", "test-key")]), false) + codetrial::launch::peer_services(&values(&[("GOOGLE_API_KEY", "test-key")]), false) .unwrap() .is_none() ); @@ -899,7 +899,7 @@ fn peer_observer_rejects_sessions_longer_than_its_two_hour_capture() { ("CODETRIAL_PEER_HARD_MAX_MINUTES", "150"), ]); assert!( - super::main_peer::peer_services(&long, true) + codetrial::launch::peer_services(&long, true) .unwrap_err() .contains("at most 120") ); @@ -907,5 +907,247 @@ fn peer_observer_rejects_sessions_longer_than_its_two_hour_capture() { ("GOOGLE_API_KEY", "test-key"), ("CODETRIAL_PEER_HARD_MAX_MINUTES", "120"), ]); - assert!(super::main_peer::peer_services(&limit, true).is_ok()); + assert!(codetrial::launch::peer_services(&limit, true).is_ok()); +} + +/// A folder with an empty config file in it and values pointing the account +/// database into it, so `prepare` touches nothing outside the test. +fn launch_fixture(label: &str) -> (std::path::PathBuf, super::CliOptions) { + let dir = std::env::temp_dir().join(format!( + "codetrial-bin-launch-{label}-{}-{}", + std::process::id(), + codetrial::current_epoch_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + let config = dir.join("codetrial.env.local"); + std::fs::write(&config, "").unwrap(); + let options = super::CliOptions { + config_path: Some(config.display().to_string()), + ..Default::default() + }; + (dir, options) +} + +fn livekit_values(dir: &std::path::Path) -> super::BTreeMap { + let db = dir.join("accounts.db").display().to_string(); + values(&[ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("CODETRIAL_DB_PATH", &db), + ]) +} + +#[test] +fn prepare_builds_the_server_config_and_opens_the_account_database() { + let (dir, options) = launch_fixture("prepare"); + let prepared = super::prepare(&livekit_values(&dir), &options).unwrap(); + assert_eq!(prepared.config.pool.providers.len(), 1); + assert!(prepared.max_concurrent > 0); + assert!(dir.join("accounts.db").is_file(), "the database is opened"); + assert!( + prepared.config.tasks.is_none(), + "task mode waits for the bind" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn prepare_refuses_a_launch_without_livekit_credentials() { + let (dir, options) = launch_fixture("prepare-missing"); + let error = super::prepare(&values(&[]), &options).err().unwrap(); + assert!( + error.contains("missing required LiveKit credentials"), + "{error}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn finish_refuses_the_published_secret_on_a_public_address() { + let (dir, options) = launch_fixture("finish-public"); + let values = livekit_values(&dir); + let prepared = super::prepare(&values, &options).unwrap(); + let refusal = super::finish( + prepared.config, + &values, + &options, + addr("203.0.113.9:3000"), + super::LaunchedBy::Operator, + ) + .err() + .unwrap(); + assert!(!refusal.is_empty()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn finish_offers_a_launchers_child_no_setup_page_and_no_task_mode() { + let (dir, options) = launch_fixture("finish-child"); + let mut values = livekit_values(&dir); + values.insert( + "SESSION_SECRET".into(), + "a-real-secret-for-this-test-only-0123".into(), + ); + let prepared = super::prepare(&values, &options).unwrap(); + let (config, setup) = super::finish( + prepared.config, + &values, + &options, + addr("127.0.0.1:3000"), + super::LaunchedBy::Parent, + ) + .unwrap(); + assert!(setup.is_none(), "a child has no terminal anyone reads"); + assert!(config.tasks.is_none(), "task mode was not asked for"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_local_interviewer_needs_a_gemini_key_and_a_valid_config() { + let (dir, _) = launch_fixture("agent"); + let mut values = livekit_values(&dir); + assert!(super::local_agent_config(values.clone()).unwrap().is_none()); + values.insert("GOOGLE_API_KEY".into(), "google-key".into()); + assert!(super::local_agent_config(values.clone()).unwrap().is_some()); + values.insert("GEMINI_CONTEXT_TRIGGER_TOKENS".into(), "1000".into()); + let error = super::local_agent_config(values).err().unwrap(); + assert!(error.contains("GEMINI_CONTEXT"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A dispatcher that says whether an interview is still running or writing its +/// report, notes that admission was closed, and starts nothing. +struct Busy { + busy: std::sync::atomic::AtomicBool, + closed: std::sync::atomic::AtomicBool, +} + +impl codetrial::web::RoomDispatcher for Busy { + fn ensure_agent( + &self, + _: &str, + _: &codetrial::config::Provider, + _: codetrial::web::AgentJob, + ) -> Result<(), codetrial::web::DispatchRefusal> { + Ok(()) + } + + fn is_idle(&self) -> bool { + !self.busy.load(std::sync::atomic::Ordering::SeqCst) + } + + fn close_admission(&self) { + self.closed.store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +/// Serves with a stop that has already happened and a dispatcher that is busy +/// for `busy_for` (`None`: forever), and returns how the run ended, how long it +/// took and whether admission had been closed. +fn run_stopped( + label: &str, + busy_for: Option, + settle: Option, +) -> (Result<(), String>, std::time::Duration, bool) { + let (dir, options) = launch_fixture(label); + let values = livekit_values(&dir); + let prepared = super::prepare(&values, &options).unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let bound = listener.local_addr().unwrap(); + let (config, setup) = super::finish( + prepared.config, + &values, + &options, + bound, + super::LaunchedBy::Parent, + ) + .unwrap(); + let dispatcher = std::sync::Arc::new(Busy { + busy: std::sync::atomic::AtomicBool::new(true), + closed: std::sync::atomic::AtomicBool::new(false), + }); + if let Some(after) = busy_for { + let dispatcher = std::sync::Arc::clone(&dispatcher); + std::thread::spawn(move || { + std::thread::sleep(after); + dispatcher + .busy + .store(false, std::sync::atomic::Ordering::SeqCst); + }); + } + let serving = std::sync::Arc::clone(&dispatcher); + let began = std::time::Instant::now(); + + // On its own thread with a deadline, so a bound that stops working fails + // this test instead of hanging the suite. + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(super::serve_with( + listener, + config, + setup, + Some(serving), + async {}, + super::Bounds { + drain: None, + settle, + }, + )); + }); + let result = finished + .recv_timeout(std::time::Duration::from_secs(30)) + .expect("serve_with returned within its bounds"); + let elapsed = began.elapsed(); + let _ = std::fs::remove_dir_all(&dir); + let closed = dispatcher.closed.load(std::sync::atomic::Ordering::SeqCst); + (result, elapsed, closed) +} + +#[test] +fn a_stopping_server_waits_for_its_rooms_and_reports_to_finish() { + let (result, elapsed, closed) = run_stopped( + "settle-waits", + Some(std::time::Duration::from_millis(600)), + Some(std::time::Duration::from_secs(20)), + ); + result.unwrap(); + assert!( + closed, + "nothing new is admitted once the shutdown has begun" + ); + assert!( + elapsed >= std::time::Duration::from_millis(500), + "{elapsed:?}" + ); +} + +#[test] +fn rooms_still_running_at_the_settle_bound_are_reported_not_hidden() { + let (result, elapsed, closed) = run_stopped( + "settle-bound", + None, + Some(std::time::Duration::from_millis(300)), + ); + assert!(closed); + let error = result.unwrap_err(); + assert!(error.contains("settle bound"), "{error}"); + // It waited out the bound, and no longer. + assert!( + elapsed >= std::time::Duration::from_millis(250), + "{elapsed:?}" + ); + assert!(elapsed < std::time::Duration::from_secs(10), "{elapsed:?}"); +} + +#[test] +fn without_a_settle_bound_the_product_stops_without_waiting() { + let (result, elapsed, closed) = run_stopped("settle-none", None, None); + assert!( + closed, + "the product closes admission as its shutdown begins too" + ); + result.unwrap(); + assert!(elapsed < std::time::Duration::from_secs(5), "{elapsed:?}"); } diff --git a/tests/unit/dispatch.rs b/tests/unit/dispatch.rs index 22c9ce3a0..e76c84bac 100644 --- a/tests/unit/dispatch.rs +++ b/tests/unit/dispatch.rs @@ -363,8 +363,8 @@ async fn a_concurrent_burst_never_overbooks_and_released_capacity_returns() { Reservation::GuestRoomInUse => { panic!("signed-in reservations must not hit the guest room limit") } - Reservation::Existing | Reservation::Finalizing => { - panic!("burst ids are unique") + Reservation::Existing | Reservation::Finalizing | Reservation::Closed => { + panic!("burst ids are unique and the dispatcher is open") } } })); @@ -943,3 +943,79 @@ async fn a_released_start_gives_its_consent_and_slot_back() { assert!(dispatcher.state.lock().unwrap().starts.is_empty()); drop(accounts); } + +/// Closing is part of the one admission decision, so after it no way of +/// starting a room gets in, while a room already admitted is still served and +/// still counts as live until it ends. +#[tokio::test] +async fn a_closed_dispatcher_admits_no_new_room_but_keeps_the_ones_it_has() { + let dispatcher = release_dispatcher(8); + let provider = Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let admitted = started(&dispatcher, "interview-before-close", false, "candidate"); + assert!(!dispatcher.is_idle()); + + dispatcher.close_admission(); + + assert!(matches!( + dispatcher.reserve_with_kind("interview-after-close", false), + Reservation::Closed + )); + assert!(matches!( + dispatcher.reserve_with_kind("guest-after-close", true), + Reservation::Closed + )); + assert_eq!( + dispatcher.ensure_agent("interview-via-ensure", &provider, AgentJob::Interview), + Err(DispatchRefusal::Stopping) + ); + assert_eq!( + dispatcher.ensure_guest_agent("guest-via-ensure", &provider, "candidate"), + Err(DispatchRefusal::Stopping) + ); + assert_eq!( + dispatcher.ensure_agent_with_no_show_release("no-show", &provider, "candidate", None), + Err(DispatchRefusal::Stopping) + ); + // The room that was admitted is still the dispatcher's, and still live. + assert!(matches!( + dispatcher.reserve_with_kind("interview-before-close", false), + Reservation::Existing + )); + assert!(!dispatcher.is_idle()); + drop(admitted); + assert!(dispatcher.is_idle()); + assert_eq!( + dispatcher.state.lock().unwrap().live.len(), + 0, + "refusals left no room behind" + ); +} + +#[test] +fn a_dispatcher_with_nothing_to_close_ignores_the_request() { + struct Plain; + impl RoomDispatcher for Plain { + fn ensure_agent(&self, _: &str, _: &Provider, _: AgentJob) -> Result<(), DispatchRefusal> { + Ok(()) + } + } + let provider = Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let plain = Plain; + plain.close_admission(); + assert_eq!( + plain.ensure_agent("room", &provider, AgentJob::Interview), + Ok(()) + ); +} diff --git a/tests/unit/gemini.rs b/tests/unit/gemini.rs index 8d120389b..4aa772c7a 100644 --- a/tests/unit/gemini.rs +++ b/tests/unit/gemini.rs @@ -1,7 +1,4 @@ -//! The `tests` module of `src/gemini.rs`, which declares this file by path. -//! Everything here reaches into `src/gemini.rs` through `super`, so it is a -//! unit -//! test and not an integration test: private items are in scope. +//! Unit tests declared by src/gemini.rs, with its private items in scope. use super::*; use crate::config::load_from_pairs; @@ -4999,3 +4996,5 @@ fn every_failing_path_is_named_before_a_second_error_on_one() { } include!("gemini_parts/tool_policy_and_reader_pressure.rs"); + +include!("gemini_parts/scoped_report_endpoint.rs"); diff --git a/tests/unit/gemini/socket_loopback.rs b/tests/unit/gemini/socket_loopback.rs new file mode 100644 index 000000000..5914cc7e8 --- /dev/null +++ b/tests/unit/gemini/socket_loopback.rs @@ -0,0 +1,61 @@ +use crate::test_isolation as isolation; + +use super::*; + +#[test] +fn loopback_registration_rejects_every_non_fixture_destination() { + for endpoint in [ + "wss://127.0.0.1:9/?key=sequence-test-key", + "ws://localhost:9/?key=sequence-test-key", + "ws://127.0.0.2:9/?key=sequence-test-key", + "ws://127.0.0.1:9/", + "ws://127.0.0.1:9?key=sequence-test-key", + "ws://127.0.0.1:9/?key=other", + "ws://127.0.0.1:9/?key=sequence-test-key&other=1", + "ws://user@127.0.0.1:9/?key=sequence-test-key", + "ws://127.0.0.1:9/other?key=sequence-test-key", + ] { + assert!(install(endpoint).is_err()); + } +} + +#[test] +fn loopback_registration_is_scoped_one_shot_and_drops_without_a_connection() { + if isolation::isolated!("CODETRIAL_SOCKET_FIXTURE_TEST") { + return; + } + let endpoint = "ws://127.0.0.1:9/?key=sequence-test-key"; + let first = crate::gemini::GeminiLiveSession::loopback_send_limit_for_test(endpoint).unwrap(); + assert!(install(endpoint).is_err()); + assert!( + claim("ws://127.0.0.1:8/?key=sequence-test-key") + .unwrap() + .is_none() + ); + assert!(claim(endpoint).unwrap().is_some()); + assert!(claim(endpoint).is_err()); + assert_eq!(first.sender_buffers(), None); + drop(first); + let second = install(endpoint).unwrap(); + assert!(claim(endpoint).unwrap().is_some()); + drop(second); + assert!(claim(endpoint).unwrap().is_none()); +} + +#[test] +fn loopback_buffer_admission_uses_actual_getters_not_requested_capacity() { + assert!(admitted(0).is_err()); + assert_eq!(admitted(4608).unwrap(), 4608); + assert_eq!(admitted(16384).unwrap(), 16384); + assert!(admitted(16385).is_err()); +} + +#[tokio::test(flavor = "current_thread")] +async fn loopback_connect_without_registration_passes_through_without_opening_a_socket() { + assert!( + connect("ws://127.0.0.1:9/?key=sequence-test-key") + .await + .unwrap() + .is_none() + ); +} diff --git a/tests/unit/gemini_parts/scoped_report_endpoint.rs b/tests/unit/gemini_parts/scoped_report_endpoint.rs new file mode 100644 index 000000000..d4a646fe1 --- /dev/null +++ b/tests/unit/gemini_parts/scoped_report_endpoint.rs @@ -0,0 +1,112 @@ +// Scoped routing and observation for tests that exercise the real report path. + +#[derive(Default)] +struct ScopedReportState { + endpoint: String, + entries: Mutex>, +} + +static SCOPED_REPORTS: Mutex)>> = Mutex::new(Vec::new()); + +pub(crate) struct ScopedReport { + scope: String, + state: Arc, +} + +impl ScopedReport { + pub(crate) fn register(scope: &str, endpoint: &str) -> Result { + let url = reqwest::Url::parse(endpoint).map_err(|error| error.to_string())?; + let local = url + .host_str() + .and_then(|host| { + host.trim_matches(['[', ']']) + .parse::() + .ok() + }) + .is_some_and(|address| address.is_loopback()); + if scope.is_empty() + || url.scheme() != "http" + || !local + || !url.username().is_empty() + || url.password().is_some() + { + return Err("report fixture requires a named scope and loopback HTTP endpoint".into()); + } + let mut scopes = SCOPED_REPORTS.lock().unwrap(); + if scopes.iter().any(|(name, _)| name == scope) { + return Err("report fixture scope is already registered".into()); + } + let state = Arc::new(ScopedReportState { + endpoint: endpoint.into(), + ..Default::default() + }); + scopes.push((scope.into(), Arc::clone(&state))); + Ok(Self { + scope: scope.into(), + state, + }) + } + + pub(crate) fn entries(&self) -> Vec<(i64, std::time::Instant)> { + self.state.entries.lock().unwrap().clone() + } +} + +impl Drop for ScopedReport { + fn drop(&mut self) { + SCOPED_REPORTS + .lock() + .unwrap() + .retain(|(name, state)| name != &self.scope || !Arc::ptr_eq(state, &self.state)); + } +} + +pub(crate) fn scoped_report_endpoint(scope: &str) -> Option { + SCOPED_REPORTS + .lock() + .unwrap() + .iter() + .find(|(name, _)| name == scope) + .map(|(_, state)| state.endpoint.clone()) +} + +pub(crate) fn note_bounded_report(scope: &str, seed: i64) { + if let Some((_, state)) = SCOPED_REPORTS + .lock() + .unwrap() + .iter() + .find(|(name, _)| name == scope) + { + state + .entries + .lock() + .unwrap() + .push((seed, std::time::Instant::now())); + } +} + +#[test] +fn scoped_report_routes_are_local_unique_and_removed_on_drop() { + assert!(ScopedReport::register("", "http://127.0.0.1:1/").is_err()); + for endpoint in [ + "https://127.0.0.1/", + "http://example.com/", + "http://user@127.0.0.1/", + ] { + assert!(ScopedReport::register("invalid-scope", endpoint).is_err()); + } + let first = ScopedReport::register("report-scope-first", "http://127.0.0.1:1234/").unwrap(); + let other = ScopedReport::register("report-scope-other", "http://127.0.0.1:4321/").unwrap(); + assert!(ScopedReport::register("report-scope-first", "http://127.0.0.1:1234/").is_err()); + assert_eq!( + scoped_report_endpoint("report-scope-first").as_deref(), + Some("http://127.0.0.1:1234/") + ); + assert!(scoped_report_endpoint("unknown-report-scope").is_none()); + note_bounded_report("report-scope-first", 17); + assert_eq!(first.entries().len(), 1); + assert!(other.entries().is_empty()); + drop(first); + assert!(scoped_report_endpoint("report-scope-first").is_none()); + assert!(scoped_report_endpoint("report-scope-other").is_some()); +} diff --git a/tests/unit/isolation.rs b/tests/unit/isolation.rs new file mode 100644 index 000000000..5f27ed02d --- /dev/null +++ b/tests/unit/isolation.rs @@ -0,0 +1,120 @@ +use std::time::{Duration, Instant}; + +/// Runs the calling test again in a child process, alone, under `marker`, and +/// is true in the parent once the child has passed: the test returns then. The +/// name is read from the path of an item declared inside the test, so it +/// cannot fall out of step with the test's own; one that selected no test +/// would fail `isolated_checked` rather than pass vacuously. +macro_rules! isolated { + ($marker:expr) => {{ + fn here() {} + let test = std::any::type_name_of_val(&here) + .strip_suffix("::here") + .map(|path| path.trim_end_matches("::{{closure}}")) + .and_then(|path| path.split_once("::")) + .map(|(_, test)| test) + .expect("an item inside a test has the test's path"); + $crate::test_isolation::isolated_checked(test, $marker, false, |_| {}) + }}; +} +pub(crate) use isolated; + +pub(crate) fn isolated_checked( + name: &str, + marker: &str, + loopback_http: bool, + check: impl FnOnce(&std::process::Output), +) -> bool { + if std::env::var(marker).as_deref() == Ok(name) { + return false; + } + let guard = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + guard.set_nonblocking(true).unwrap(); + let proxy = format!("http://{}", guard.local_addr().unwrap()); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", name, "--nocapture"]) + .env(marker, name) + .env("NO_PROXY", if loopback_http { "127.0.0.1" } else { "" }) + .env("no_proxy", if loopback_http { "127.0.0.1" } else { "" }) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + for key in [ + "HTTPS_PROXY", + "https_proxy", + "HTTP_PROXY", + "http_proxy", + "ALL_PROXY", + "all_proxy", + ] { + command.env(key, &proxy); + } + let mut child = command.spawn().unwrap(); + + // Drained while the child runs: a child that writes more than a pipe holds + // (64 KiB on Linux) blocks in its write, and polling only for its exit then + // reads that block as a hang. + let drain = |stream: Option>| { + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(mut stream) = stream { + let _ = stream.read_to_end(&mut bytes); + } + bytes + }) + }; + let stdout = drain( + child + .stdout + .take() + .map(|pipe| Box::new(pipe) as Box), + ); + let stderr = drain( + child + .stderr + .take() + .map(|pipe| Box::new(pipe) as Box), + ); + let started = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > Duration::from_secs(20) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!( + "isolated test child timed out: {}", + String::from_utf8_lossy(&stderr.join().unwrap()) + ); + } + std::thread::sleep(Duration::from_millis(10)); + }; + let output = std::process::Output { + status, + stdout: stdout.join().unwrap(), + stderr: stderr.join().unwrap(), + }; + assert_eq!( + guard.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock, + "an isolated test must not contact an external provider" + ); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.lines().any(|line| line == "running 1 test") + && stdout.lines().any(|line| { + line.starts_with("test result: ok. 1 passed; 0 failed; 0 ignored;") + }), + "isolated child must execute exactly one passing test: {name}\n{stdout}" + ); + check(&output); + true +} diff --git a/tests/unit/launch.rs b/tests/unit/launch.rs new file mode 100644 index 000000000..280091233 --- /dev/null +++ b/tests/unit/launch.rs @@ -0,0 +1,98 @@ +use super::*; + +#[test] +fn unfinished_rooms_are_reported_together_with_the_shutdowns_own_error() { + let alone = unfinished(Ok(())); + assert!( + alone.starts_with("rooms or reports were still running"), + "{alone}" + ); + assert_eq!( + unfinished(Err("web server failed: slow".into())), + "web server failed: slow; rooms or reports were still running when the settle bound passed" + ); +} + +/// A config directory holding one provider file per id, removed on drop. +struct Providers(std::path::PathBuf); + +impl Providers { + fn new(label: &str, ids: &[&str]) -> Self { + let dir = + std::env::temp_dir().join(format!("codetrial-launch-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + for id in ids { + std::fs::write( + dir.join(format!("codetrial.env.{id}")), + format!( + "LIVEKIT_URL=wss://{id}.example\nLIVEKIT_API_KEY=k-{id}\nLIVEKIT_API_SECRET=s-{id}\n" + ), + ) + .unwrap(); + } + Self(dir) + } +} + +impl Drop for Providers { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn the_pool_gains_the_discovered_providers_in_the_order_asked_for() { + let dir = Providers::new("pool", &["eu", "tokyo"]); + let mut pool = crate::config::ProviderPool::default(); + extend_pool(&mut pool, false, &dir.0, "tokyo"); + let ids: Vec<_> = pool.providers.iter().map(|p| p.id.as_str()).collect(); + assert_eq!(ids, ["tokyo", "eu"]); + let report = pool_report(&pool.providers); + assert_eq!(report.len(), 1, "{report:?}"); + assert!( + report[0].starts_with("provider pooling: 2 provider(s) over 2 project(s): tokyo="), + "{report:?}" + ); + assert!(pool_report(&[]).is_empty()); +} + +#[test] +fn a_project_listed_twice_is_reported_as_a_warning() { + let dir = Providers::new("twice", &["eu"]); + let mut pool = crate::config::ProviderPool::default(); + extend_pool(&mut pool, false, &dir.0, ""); + let twin = pool.providers[0].clone(); + pool.providers.push(crate::config::Provider { + id: "copy".into(), + ..twin + }); + let report = pool_report(&pool.providers); + assert_eq!(report.len(), 2, "{report:?}"); + assert!(report[1].starts_with("WARNING: eu and copy are the same LiveKit project")); +} + +#[test] +fn the_launch_reads_its_cap_length_and_compiler_switch_from_the_values() { + let values = |pairs: &[(&str, &str)]| -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + assert_eq!( + read_max_concurrent(&values(&[("CODETRIAL_MAX_CONCURRENT_INTERVIEWS", "5")])), + 5 + ); + assert_eq!( + interview_duration_min(&values(&[("CODETRIAL_DURATION_MIN", "45")])), + 45 + ); + assert_eq!(interview_duration_min(&values(&[])), DEFAULT_DURATION_MIN); + for (value, enabled) in [("true", true), ("false", false)] { + assert_eq!( + compiler_explorer_enabled(&values(&[("CODETRIAL_COMPILER_EXPLORER_ENABLED", value)])), + enabled + ); + } +} diff --git a/tests/unit/livekit.rs b/tests/unit/livekit.rs index b0d94f8bb..0d5c732b5 100644 --- a/tests/unit/livekit.rs +++ b/tests/unit/livekit.rs @@ -3537,7 +3537,7 @@ fn audio_chunk() -> GeminiEvent { /// The text a fake socket was sent: a cold briefing goes out as realtime /// input, a resumed one as client content. -fn sent_text(sent: &serde_json::Value) -> &str { +pub(super) fn sent_text(sent: &serde_json::Value) -> &str { sent["realtimeInput"]["text"] .as_str() .or_else(|| sent["clientContent"]["turns"][0]["parts"][0]["text"].as_str()) diff --git a/tests/unit/livekit/handoff.rs b/tests/unit/livekit/handoff.rs index f3ad74f6f..1cd2e461b 100644 --- a/tests/unit/livekit/handoff.rs +++ b/tests/unit/livekit/handoff.rs @@ -1,4 +1,5 @@ use super::*; +use crate::test_isolation as isolation; use futures_util::{SinkExt, StreamExt}; use tokio_tungstenite::tungstenite::Message; @@ -412,14 +413,30 @@ async fn a_tick_recovers_when_long_quiet_or_a_failed_request_leaves_the_watchdog #[tokio::test] async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tick() { + if isolation::isolated!("CODETRIAL_HANDOFF_LISTENING_NOTE_TEST") { + return; + } let mut fixture = TickFixture::new(1500, 10).await; let listener = tokio::net::TcpSocket::new_v4().unwrap(); listener.set_recv_buffer_size(1024).unwrap(); + let listener_recv = listener.recv_buffer_size().unwrap(); + assert!((1..=16_384).contains(&listener_recv), "{listener_recv}"); listener.bind("127.0.0.1:0".parse().unwrap()).unwrap(); let listener = listener.listen(1).unwrap(); fixture.endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let registration = GeminiLiveSession::loopback_send_limit_for_test(&format!( + "{}?key=sequence-test-key", + fixture.endpoint + )) + .unwrap(); let peer = tokio::spawn(async move { let (socket, _) = listener.accept().await.unwrap(); + let accepted_recv = { + let socket_ref = socket2::SockRef::from(&socket); + socket_ref.set_recv_buffer_size(1024).unwrap(); + socket_ref.recv_buffer_size().unwrap() + }; + assert!((1..=16_384).contains(&accepted_recv), "{accepted_recv}"); let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); let setup = socket.next().await.unwrap().unwrap(); assert!( @@ -439,10 +456,11 @@ async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tic fixture .replace_speech(session::SpeechHandoff { checkpoint_after: Instant::now(), - transcript_fragments: vec!["x".repeat(32 * 1024 * 1024)], + transcript_fragments: vec!["x".repeat(128 * 1024)], discarded: 0, }) .await; + let sender_buffers = registration.sender_buffers(); assert!(fixture.turn.state.needs_cold_brief); assert!(fixture.turn.activity.owes_prompt()); assert_eq!(fixture.turn.activity.prompt_text.as_deref(), Some(OWED)); @@ -456,6 +474,10 @@ async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tic .await .unwrap() .unwrap(); + let (before, connected) = sender_buffers.expect("the replacement must use the bounded socket"); + assert!((1..=16_384).contains(&before), "{before}"); + assert!((1..=16_384).contains(&connected), "{connected}"); + drop(registration); } #[test] diff --git a/tests/unit/livekit/packets.rs b/tests/unit/livekit/packets.rs index 7710049c0..7e65e71de 100644 --- a/tests/unit/livekit/packets.rs +++ b/tests/unit/livekit/packets.rs @@ -79,3 +79,58 @@ fn closing_transition_is_logged_once_for_received_and_server_controls() { assert!(records[0]["turn"].is_null()); } } + +#[test] +fn a_test_result_reads_its_reaction_clock_for_received_and_server_packets() { + for received in [false, true] { + for expired in [false, true] { + let now = Instant::now(); + + // The pure decision pins the threshold. A synthetic future instant + // gives zero elapsed without waiting at the cooldown edge; opposite + // ages expose a wrong clock source. + let old = now.checked_sub(Duration::from_secs(100)).unwrap(); + let future = now.checked_add(Duration::from_secs(3600)).unwrap(); + let mut state = RuntimeState::default(); + assert!(!state.ended && !state.thinking_hold.is_active()); + let mut activity = RuntimeActivity::for_interview(now, 0, 7); + activity.last_test_reaction = if expired { old } else { future }; + activity.last_code_change = if expired { future } else { old }; + activity.last_interjection = activity.last_code_change; + let result = reduce_data_packet( + &mut state, + &mut activity, + crate::runtime::TOPIC_TEST_RESULTS, + &serde_json::json!({"passed":1,"total":2}), + received, + 1000, + ); + assert_eq!(state.test_runs, 1); + assert_eq!(state.last_test_run.as_ref().unwrap()["passed"], 1); + assert_eq!(state.last_test_run.as_ref().unwrap()["total"], 2); + assert!(state.tested_code.is_none()); + assert!(state.framework_evidence.is_empty()); + assert!(state.runner_unavailable.is_none()); + assert!(!state.ended && !state.thinking_hold.is_active()); + assert_eq!( + state.evidence_ledger.metrics.raw_events_received, + u64::from(received) + ); + assert!(result.test_run.is_some()); + assert_eq!( + result.generate_reply.is_some(), + expired, + "{received}/{expired}" + ); + assert_eq!( + result.update_last_test_reaction, expired, + "{received}/{expired}" + ); + assert_eq!( + result.update_last_interjection, expired, + "{received}/{expired}" + ); + assert!(result.held_context.is_none()); + } + } +} diff --git a/tests/unit/livekit/probe/acquisition.rs b/tests/unit/livekit/probe/acquisition.rs new file mode 100644 index 000000000..6d6bddea8 --- /dev/null +++ b/tests/unit/livekit/probe/acquisition.rs @@ -0,0 +1,1055 @@ +use super::*; +use crate::livekit::latency::ReplyLatency; +use serde_json::json; +use std::time::Instant; + +fn limits() -> Limits { + Limits { + batches: 3, + calls: 4, + bytes: 8192, + } +} +fn identity() -> IdentityView<'static> { + IdentityView { + room: "fixture", + session: 7, + socket: 1, + trace_turn: None, + clock: json!({"clock_uncertainty_ms":null}), + } +} +fn call(id: &str) -> GeminiFunctionCall { + GeminiFunctionCall { + id: id.into(), + name: "read_editor".into(), + args: json!({"private":"not retained"}), + } +} +fn answers(id: &str, result: Value) -> Vec<(GeminiFunctionCall, Value)> { + vec![(call(id), result)] +} + +#[test] +fn assigned_scope_and_positive_limits_are_required() { + assert!(Capture::new("", 7, limits()).is_err()); + assert!(Capture::new("fixture", 0, limits()).is_err()); + for bad in [ + Limits { + batches: 0, + ..limits() + }, + Limits { + calls: 0, + ..limits() + }, + Limits { + bytes: 1, + ..limits() + }, + ] { + assert!(Capture::new("fixture", 7, bad).is_err()); + } +} + +#[test] +fn room_and_session_admit_but_socket_changes_are_retained() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &answers("a", json!("authored"))); + let mut next = identity(); + next.socket = 2; + let _ = capture.record(next, &answers("b", json!("second"))); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches[1].identity.socket, 2); + assert_eq!(snapshot.batches[1].sequence, 1); + assert_eq!(snapshot.retained_calls, 2); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); + assert!( + !serde_json::to_string(&snapshot.batches) + .unwrap() + .contains("not retained") + ); + for (room, session) in [("", 0), ("foreign", 7), ("fixture", 8)] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let mut view = identity(); + view.room = room; + view.session = session; + let _ = capture.record(view, &answers("a", json!(null))); + let _ = capture.record(identity(), &answers("b", json!(null))); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + assert!(capture.snapshot().batches.is_empty()); + } +} + +#[test] +fn byte_budget_counts_utf8_escaping_and_array_punctuation_before_cloning() { + let executed = answers("a", json!({"value":"é\n\\\""})); + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &executed); + let exact = capture.snapshot().retained_bytes; + for (bytes, accepted) in [(exact, true), (exact - 1, false)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + let _ = capture.record(identity(), &executed); + assert_eq!(capture.snapshot().batches.len(), usize::from(accepted)); + assert_eq!( + capture.snapshot().failure, + (!accepted).then_some(Failure::Limit) + ); + } + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: exact, + ..limits() + }, + ) + .unwrap(); + let _ = capture.record(identity(), &executed); + let _ = capture.record(identity(), &answers("b", json!("second"))); + assert_eq!(capture.snapshot().failure, Some(Failure::Limit)); + assert_eq!(capture.snapshot().retained_bytes, exact); +} + +#[test] +fn count_limits_and_ambiguous_ids_latch_failure() { + for caps in [ + Limits { + batches: 1, + ..limits() + }, + Limits { + calls: 1, + ..limits() + }, + ] { + let capture = Capture::new("fixture", 7, caps).unwrap(); + let _ = capture.record(identity(), &answers("a", json!(null))); + let _ = capture.record(identity(), &answers("b", json!(null))); + let _ = capture.record(identity(), &[]); + assert_eq!(capture.snapshot().failure, Some(Failure::Limit)); + assert_eq!(capture.snapshot().retained_calls, 1); + } + for ids in [vec![""], vec!["same", "same"]] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let batch = ids + .into_iter() + .flat_map(|id| answers(id, json!(null))) + .collect::>(); + let _ = capture.record(identity(), &batch); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + assert!(capture.snapshot().batches.is_empty()); + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &answers("same", json!(null))); + let _ = capture.record(identity(), &answers("same", json!(null))); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + assert_eq!(capture.snapshot().batches.len(), 1); +} + +#[test] +fn identity_view_preserves_trace_and_existing_paired_clock_policy() { + let now = Instant::now(); + let mut latency = ReplyLatency::default(); + assert_eq!(latency.acquisition_identity().session, 0); + latency.bind("fixture", 7, 1, 1500); + let before = latency.acquisition_identity(); + assert_eq!(before.trace_turn, None); + assert!(before.clock["emitted_ms"].is_null()); + assert!(before.clock["start_unix_ms"].is_null()); + assert!(before.clock["emitted_unix_ms"].as_u64().is_some()); + assert_eq!(before.clock["clock_uncertainty_ms"], Value::Null); + latency.anchor(now); + let after = latency.acquisition_identity(); + assert_eq!(after.trace_turn, None); + assert!(after.clock["start_unix_ms"].as_u64().is_some()); + assert!(after.clock["emitted_unix_ms"].as_u64().is_some()); + assert!(after.clock["emitted_ms"].as_i64().is_some()); + assert!(after.clock["clock_uncertainty_ms"].as_u64().is_some()); + latency.prompt(now); + let turn = latency.acquisition_identity().trace_turn; + assert!(turn.is_some()); + assert_eq!(latency.acquisition_identity().trace_turn, turn); +} + +#[test] +fn constructor_rejects_scope_metadata_before_cloning_it() { + assert!( + Capture::new( + "fixture", + 7, + Limits { + bytes: 2, + ..limits() + } + ) + .is_err() + ); + let escaped_room = "\"".repeat(1024); + assert!( + Capture::new( + &escaped_room, + 7, + Limits { + bytes: 2048, + ..limits() + } + ) + .is_err() + ); + let multibyte_room = "é".repeat(1024); + assert!( + Capture::new( + &multibyte_room, + 7, + Limits { + bytes: 1536, + ..limits() + } + ) + .is_err() + ); + let large_room = "x".repeat(8192); + assert!( + Capture::new( + &large_room, + 7, + Limits { + bytes: 256, + ..limits() + } + ) + .is_err() + ); +} + +fn admitted_ticket(capture: &Arc, id: &str) -> Ticket { + let sequence = capture + .record(identity(), &answers(id, json!("executed"))) + .unwrap(); + Ticket { + capture: Arc::clone(capture), + sequence, + } +} + +#[test] +fn completion_preserves_exact_array_bytes_and_original_object_clock() { + let prototype = Capture::new("fixture", 7, limits()).unwrap(); + let _ = admitted_ticket(&prototype, "a"); + let null_bytes = prototype.snapshot().retained_bytes; + for (sent, allowance, accepted) in [(true, 0, true), (false, 0, false), (false, 1, true)] { + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: null_bytes + allowance, + ..limits() + }, + ) + .unwrap(); + let ticket = admitted_ticket(&capture, "a"); + let before = capture.snapshot(); + assert!(before.batches[0].identity.clock.is_object()); + ticket.complete(sent); + let after = capture.snapshot(); + assert_eq!(after.batches[0].identity, before.batches[0].identity); + assert_eq!(after.batches[0].sequence, 0); + assert_eq!(after.batches[0].calls, before.batches[0].calls); + assert_eq!(after.retained_calls, 1); + assert_eq!( + after.batches[0].local_send_completed, + accepted.then_some(sent) + ); + assert_eq!(after.failure, (!accepted).then_some(Failure::Limit)); + assert_eq!( + after.retained_bytes, + before.retained_bytes + usize::from(accepted && !sent) + ); + assert_eq!( + after.retained_bytes, + serde_json::to_vec(&after.batches).unwrap().len() + ); + } +} + +#[test] +fn rejected_or_latched_admission_cannot_update_an_earlier_batch() { + for rejected in [Failure::Scope, Failure::Limit, Failure::AmbiguousId] { + let caps = Limits { + batches: if rejected == Failure::Limit { 1 } else { 3 }, + ..limits() + }; + let capture = Capture::new("fixture", 7, caps).unwrap(); + assert!(capture.record(identity(), &[]).is_none()); + assert_eq!(capture.snapshot().failure, None); + let ticket = admitted_ticket(&capture, "first"); + let mut view = identity(); + if rejected == Failure::Scope { + view.room = "foreign"; + } + let id = if rejected == Failure::AmbiguousId { + "first" + } else { + "second" + }; + assert!( + capture + .record(view, &answers(id, json!("rejected"))) + .is_none() + ); + assert!( + capture + .record(identity(), &answers("later", json!("latched"))) + .is_none() + ); + let before = capture.snapshot(); + assert_eq!(before.failure, Some(rejected)); + ticket.complete(false); + let after = capture.snapshot(); + assert_eq!( + after, before, + "a latched capture refuses completion of an admitted ticket too" + ); + } +} + +#[test] +fn sequence_mismatch_and_duplicate_completion_fail_closed() { + for sequence in [0, 1] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let ticket = admitted_ticket(&capture, "first"); + if sequence == 0 { + ticket.complete(true); + } else { + drop(ticket); + } + let before = capture.snapshot(); + Ticket { + capture: Arc::clone(&capture), + sequence, + } + .complete(false); + let after = capture.snapshot(); + assert_eq!(after.failure, Some(Failure::Scope)); + assert_eq!(after.batches, before.batches); + assert_eq!(after.retained_bytes, before.retained_bytes); + } +} + +#[test] +fn dropping_a_ticket_leaves_completion_unknown() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let ticket = admitted_ticket(&capture, "first"); + let before = capture.snapshot(); + drop(ticket); + assert_eq!(capture.snapshot(), before); + assert_eq!(capture.snapshot().batches[0].local_send_completed, None); +} + +#[test] +fn owner_gate_changes_are_scoped_to_the_post_sample_tool_send_boundary() { + use super::owner::{Gate, Stage, Transition}; + let capture = Capture::new("fixture", 7, limits()).unwrap(); + assert!(Gate::new(&capture, 0, Stage::ToolSend, None).is_err()); + for stage in [ + Stage::Acknowledgement, + Stage::BoardSettlement, + Stage::BoardResend, + ] { + assert!(Gate::new(&capture, 1, stage, Some(Transition::Hold(true))).is_err()); + } + assert!(Gate::new(&capture, 1, Stage::ToolSend, Some(Transition::Hold(true))).is_ok()); +} + +#[test] +fn completion_can_finish_admitted_sequences_out_of_order() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let first = admitted_ticket(&capture, "first"); + let second = admitted_ticket(&capture, "second"); + second.complete(false); + first.complete(true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches[0].calls[0].id, "first"); + assert_eq!(snapshot.batches[0].local_send_completed, Some(true)); + assert_eq!(snapshot.batches[1].calls[0].id, "second"); + assert_eq!(snapshot.batches[1].local_send_completed, Some(false)); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); +} + +fn bound_scope_records(capture: &Arc, room: &str, session: u64) -> bool { + let mut id = identity(); + id.room = room; + id.session = session; + capture + .record(id, &answers("a", json!("authored"))) + .is_some() +} + +#[test] +fn an_unarmed_observer_never_arms_itself_from_a_bind() { + let arming = Arming::detached(limits()); + arming.on_bind("fixture", 7); + assert!(arming.capture().is_none()); + assert_eq!(arming.failure(), Some(ArmFailure::BindBeforeAssignment)); + arming.on_assignment("fixture"); + assert!(arming.capture().is_none(), "a failed observer stays failed"); +} + +#[test] +fn an_assignment_alone_admits_nothing_until_its_session_binds() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + assert!(arming.capture().is_none()); + assert_eq!(arming.failure(), None); + arming.on_bind("fixture", 7); + let capture = arming + .capture() + .expect("the expected room's bind admits it"); + assert!(bound_scope_records(&capture, "fixture", 7)); + assert!(!bound_scope_records(&capture, "fixture", 8)); + assert!(!bound_scope_records(&capture, "other", 7)); +} + +#[test] +fn a_second_room_or_a_foreign_bind_is_a_fixture_failure() { + for foreign in ["assign", "bind"] { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + match foreign { + "assign" => arming.on_assignment("other"), + _ => arming.on_bind("other", 7), + } + assert_eq!(arming.failure(), Some(ArmFailure::UnrelatedRoom)); + arming.on_bind("fixture", 7); + assert!(arming.capture().is_none(), "{foreign}"); + } +} + +#[test] +fn the_first_bound_session_is_latched_and_zero_is_refused() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 0); + assert_eq!(arming.failure(), Some(ArmFailure::Session)); + + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 7); + arming.on_bind("fixture", 7); + assert!( + arming.capture().is_some(), + "repeats of the same scope are fine" + ); + assert_eq!(arming.failure(), None); + arming.on_bind("fixture", 8); + assert_eq!(arming.failure(), Some(ArmFailure::Session)); + assert!(arming.capture().is_none()); +} + +#[test] +fn the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope() { + use crate::config::{PRIMARY_PROVIDER_ID, Provider}; + use crate::dispatch::LocalDispatcher; + use crate::web::{AgentJob, RoomDispatcher}; + + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_TEST") { + return; + } + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + assert!(install_arming(&arming).is_err()); + + // A current-thread runtime that is never driven: the room task the + // dispatcher spawns is queued and dropped with it, so it can never reach + // for the network, however the scheduler happens to fall. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let dispatcher = LocalDispatcher { + config: crate::config::load_from_pairs([ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("GOOGLE_API_KEY", "primary-google"), + ]) + .unwrap(), + runtime: runtime.handle().clone(), + state: Arc::default(), + max_concurrent: 2, + }; + let provider = Provider { + id: PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let mut latency = ReplyLatency::default(); + + // Nothing is observed before the assignment and the bind. + assert!(observe(&latency, &answers("early", json!("x"))).is_none()); + + dispatcher + .ensure_agent("fixture-room", &provider, AgentJob::Interview) + .unwrap(); + assert_eq!( + arming.failure(), + None, + "the expected room's own bind comes first" + ); + latency.bind("fixture-room", 7, 1, 1500); + assert!(observe(&latency, &answers("a", json!("x"))).is_some()); + + // A second room reaching dispatch fails the fixture and stops capture. + dispatcher + .ensure_agent("other-room", &provider, AgentJob::Interview) + .unwrap(); + assert_eq!(arming.failure(), Some(ArmFailure::UnrelatedRoom)); + assert!(observe(&latency, &answers("b", json!("x"))).is_none()); + + drop(installed); + drop(runtime); +} + +#[test] +fn a_blocked_or_removed_observer_leaves_no_capture_installed() { + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_BLOCKED_TEST") { + return; + } + + // Another capture holds the one slot: the fixture must see a failure rather + // than believe its scope went live. + let foreign = Capture::new("foreign", 1, limits()).unwrap(); + let held = install(&foreign).unwrap(); + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + arming.on_assignment("fixture"); + bound("fixture", 7); + assert_eq!(arming.failure(), Some(ArmFailure::Install)); + drop(installed); + drop(held); + + // Removing the observer takes its capture out, and an event that was + // already in flight afterwards installs nothing, because that capture would + // have no owner left to remove it. + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + arming.on_assignment("fixture"); + bound("fixture", 7); + assert!(arming.capture().is_some()); + drop(installed); + arming.on_bind("fixture", 7); + arming.on_assignment("fixture"); + let probe = Capture::new("probe", 1, limits()).unwrap(); + assert!(install(&probe).is_ok(), "a removed observer left a capture"); +} + +#[test] +fn a_teardown_clears_only_its_own_dispatch_hook() { + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_HOOK_TEST") { + return; + } + let first = Arming::new(limits()); + let installed = install_arming(&first).unwrap(); + // A successor that slipped in while this observer was being removed. + let successor: crate::dispatch::AssignmentHook = Arc::new(|_| {}); + crate::dispatch::set_assignment_hook(Some(Arc::clone(&successor))); + drop(installed); + assert!(crate::dispatch::assignment_hook_is(&successor)); + crate::dispatch::clear_assignment_hook(&successor); + assert!(!crate::dispatch::assignment_hook_is(&successor)); + + // And a plain reinstall after a removal is armed from the first event. + let second = Arming::new(limits()); + let guard = install_arming(&second).unwrap(); + crate::dispatch::notify_assigned("room"); + bound("room", 7); + assert_eq!(second.failure(), None); + assert!(second.capture().is_some()); + drop(guard); +} + +fn calls(ids: &[&str]) -> Vec { + ids.iter().map(|id| call(id)).collect() +} + +#[test] +fn a_received_call_is_recorded_before_it_executes() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a", "b"]), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 0); + assert_eq!(snapshot.receipts.len(), 2); + let first = &snapshot.receipts[0]; + assert_eq!( + (first.id.as_str(), first.name.as_str()), + ("a", "read_editor") + ); + assert_eq!((first.identity.session, first.identity.socket), (7, 1)); + assert!(!first.discarded); + assert!( + snapshot + .receipts + .iter() + .all(|receipt| receipt.batch.is_none()) + ); + // The arguments are never retained, only what identifies the call. + assert!( + !serde_json::to_string(&snapshot.receipts) + .unwrap() + .contains("not retained") + ); +} + +#[test] +fn executing_a_batch_links_only_the_receipts_it_answers() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a", "b"]), false); + capture + .record(identity(), &answers("a", json!("result"))) + .unwrap(); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.receipts[0].batch, Some(0)); + assert_eq!(snapshot.receipts[1].batch, None, "received, never executed"); +} + +#[test] +fn a_call_received_during_a_discard_is_kept_and_marked() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["late"]), true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert!(snapshot.receipts[0].discarded); +} + +#[test] +fn receipts_fail_closed_on_scope_reuse_and_limits() { + let scoped = Capture::new("fixture", 7, limits()).unwrap(); + let mut elsewhere = identity(); + elsewhere.session = 8; + scoped.receive(elsewhere, &calls(&["a"]), false); + assert_eq!(scoped.snapshot().failure, Some(Failure::Scope)); + assert!(scoped.snapshot().receipts.is_empty()); + + let reused = Capture::new("fixture", 7, limits()).unwrap(); + reused.receive(identity(), &calls(&["a"]), false); + reused.receive(identity(), &calls(&["a"]), false); + assert_eq!(reused.snapshot().failure, Some(Failure::AmbiguousId)); + + let twice = Capture::new("fixture", 7, limits()).unwrap(); + twice.receive(identity(), &calls(&["a", "a"]), false); + assert_eq!(twice.snapshot().failure, Some(Failure::AmbiguousId)); + + let blank = Capture::new("fixture", 7, limits()).unwrap(); + blank.receive(identity(), &calls(&[""]), false); + assert_eq!(blank.snapshot().failure, Some(Failure::AmbiguousId)); + + let many = Capture::new("fixture", 7, limits()).unwrap(); + many.receive(identity(), &calls(&["1", "2", "3", "4", "5"]), false); + assert_eq!(many.snapshot().failure, Some(Failure::Limit)); + assert!( + many.snapshot().receipts.is_empty(), + "nothing partial is kept" + ); + + let tiny = Capture::new( + "fixture", + 7, + Limits { + bytes: 150, + ..limits() + }, + ) + .unwrap(); + tiny.receive( + identity(), + &calls(&["a-long-call-id-that-will-not-fit-in-the-byte-budget"]), + false, + ); + assert_eq!(tiny.snapshot().failure, Some(Failure::Limit)); +} + +#[test] +fn a_latched_failure_stops_receipts_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.receive(identity(), &calls(&["a"]), false); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + capture.receive(identity(), &calls(&["fresh"]), false); + assert_eq!(capture.snapshot().receipts.len(), 1); +} + +fn receipt_bytes(capture: &Capture) -> (usize, usize) { + let snapshot = capture.snapshot(); + ( + snapshot.retained_receipt_bytes, + serde_json::to_vec(&snapshot.receipts).unwrap().len(), + ) +} + +#[test] +fn receipt_bytes_are_the_serialized_array_before_and_after_linking() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + assert_eq!( + receipt_bytes(&capture), + (2, 2), + "an empty array is two bytes" + ); + capture.receive(identity(), &calls(&["a"]), false); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual); + capture.receive(identity(), &calls(&["b", "c"]), false); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual, "commas between receipts are counted"); + capture + .record(identity(), &answers("b", json!("result"))) + .unwrap(); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual, "null becomes a sequence number"); + assert_eq!(capture.snapshot().failure, None); +} + +#[test] +fn the_receipt_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.receive(identity(), &calls(&["a"]), false); + let one = receipt_bytes(&probe).1; + for (bytes, fits) in [(one, true), (one - 1, false)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + assert_eq!(capture.snapshot().failure.is_none(), fits, "limit {bytes}"); + } + // A second receipt that does not fit leaves the first alone. + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: one, + ..limits() + }, + ) + .unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + let before = receipt_bytes(&capture); + capture.receive(identity(), &calls(&["b"]), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.receipts.len(), 1); + assert_eq!(receipt_bytes(&capture), before); +} + +#[test] +fn a_batch_answers_only_the_receipt_from_its_own_socket() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + let mut later = identity(); + later.socket = 2; + capture + .record(later, &answers("a", json!("result"))) + .unwrap(); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!( + snapshot.receipts[0].batch, None, + "received on socket 1, executed on 2" + ); + assert_eq!( + snapshot.batches.len(), + 1, + "the execution itself is still recorded" + ); +} + +#[test] +fn a_reply_keeps_its_newest_view_and_where_it_first_appeared() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.reply(identity(), "interviewer-0", "Hel", false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hello", false); + capture.receive(identity(), &calls(&["b"]), false); + let mut later = identity(); + later.trace_turn = Some(9); + later.clock = json!({"clock_uncertainty_ms":5}); + capture.reply(later, "interviewer-0", "Hello there", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + let [reply] = snapshot.replies.as_slice() else { + panic!("one segment, one reply: {:?}", snapshot.replies); + }; + assert_eq!(reply.text, "Hello there"); + assert!(reply.final_segment && !reply.revised); + assert_eq!(reply.views, 3); + // First seen after the receipt and before the batch that answered it. + assert_eq!((reply.receipts_before, reply.batches_before), (1, 0)); + // The turn went on across a second call and an answer. + assert_eq!((reply.receipts_last, reply.batches_last), (2, 1)); + assert_eq!(reply.identity, Identity::from_view(&identity())); +} + +#[test] +fn a_rewritten_view_is_marked_and_a_reopened_final_one_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.reply(identity(), "interviewer-0", "Hello there", false); + capture.reply(identity(), "interviewer-0", "Hello", false); + capture.reply(identity(), "interviewer-1", "Fine", true); + capture.reply(identity(), "interviewer-1", "Fine", false); + capture.reply(identity(), "interviewer-2", "Fine", true); + capture.reply(identity(), "interviewer-2", "Fine and", true); + // Ordinary growth afterwards does not make the page's view trustworthy. + capture.reply(identity(), "interviewer-0", "Hello there, again", true); + capture.reply(identity(), "interviewer-1", "Fine, thanks", true); + let revised: Vec<_> = capture + .snapshot() + .replies + .iter() + .map(|reply| reply.revised) + .collect(); + assert_eq!(revised, [true, true, false]); +} + +#[test] +fn a_segment_id_on_another_socket_is_another_reply() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.reply(identity(), "interviewer-0", "one", true); + let mut next = identity(); + next.socket = 2; + capture.reply(next, "interviewer-0", "two", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.replies.len(), 2); + assert_eq!(snapshot.replies[0].text, "one"); + assert_eq!(snapshot.replies[1].identity.socket, 2); +} + +#[test] +fn replies_fail_closed_on_scope_count_and_bytes() { + for (room, session) in [("foreign", 7), ("fixture", 8)] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let mut view = identity(); + view.room = room; + view.session = session; + capture.reply(view, "interviewer-0", "x", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Scope)); + assert!(snapshot.replies.is_empty()); + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + for index in 0..5 { + capture.reply(identity(), &format!("interviewer-{index}"), "x", true); + } + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.replies.len(), 4); + + // A view that no longer fits is dropped whole, and the earlier one stays. + let small = Limits { + bytes: 400, + ..limits() + }; + let capture = Capture::new("fixture", 7, small).unwrap(); + capture.reply(identity(), "interviewer-0", "short", false); + capture.reply(identity(), "interviewer-0", &"long".repeat(200), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.replies[0].text, "short"); +} + +#[test] +fn the_reply_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.reply(identity(), "interviewer-0", "exact", true); + let exact = serde_json::to_vec(&probe.snapshot().replies).unwrap().len(); + for (bytes, failure) in [(exact, None), (exact - 1, Some(Failure::Limit))] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.reply(identity(), "interviewer-0", "exact", true); + assert_eq!(capture.snapshot().failure, failure); + } +} + +#[test] +fn the_running_byte_counts_are_the_serialized_lengths() { + // Each view is charged for its own change, so the running count has to stay + // what serializing the whole array would give, through revisions that grow, + // shrink and reopen a reply. + let roomy = Limits { + calls: 8, + ..limits() + }; + let capture = Capture::new("fixture", 7, roomy).unwrap(); + for (id, text, last) in [ + ("interviewer-0", "Hel", false), + ("interviewer-0", "Hello there", true), + ("interviewer-1", "Fine", true), + ("interviewer-0", "Hi", false), + ("interviewer-2", "\"quoted\" \u{3b1}", true), + ] { + capture.reply(identity(), id, text, last); + capture.boundary(identity(), Ending::Complete, last); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!( + snapshot.retained_reply_bytes, + serde_json::to_vec(&snapshot.replies).unwrap().len() + ); + assert_eq!(snapshot.reply_bytes.len(), snapshot.replies.len()); + assert_eq!( + snapshot.retained_boundary_bytes, + serde_json::to_vec(&snapshot.boundaries).unwrap().len() + ); + } +} + +#[test] +fn the_boundary_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.boundary(identity(), Ending::Complete, true); + probe.boundary(identity(), Ending::Interrupted, false); + let exact = serde_json::to_vec(&probe.snapshot().boundaries) + .unwrap() + .len(); + for (bytes, failure, kept) in [(exact, None, 2), (exact - 1, Some(Failure::Limit), 1)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + capture.boundary(identity(), Ending::Interrupted, false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, failure); + assert_eq!(snapshot.boundaries.len(), kept); + } +} + +#[test] +fn a_latched_failure_stops_replies_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&[""]), false); + capture.reply(identity(), "interviewer-0", "x", true); + assert!(capture.snapshot().replies.is_empty()); +} + +#[test] +fn a_boundary_keeps_arrival_order_and_what_preceded_it() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + capture.receive(identity(), &calls(&["a"]), false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hi", false); + let mut next = identity(); + next.socket = 2; + capture.boundary(next, Ending::Interrupted, false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + let seen: Vec<_> = snapshot + .boundaries + .iter() + .map(|boundary| { + ( + boundary.ending, + boundary.accepted, + boundary.identity.socket, + boundary.receipts_before, + boundary.batches_before, + boundary.replies_before, + ) + }) + .collect(); + assert_eq!( + seen, + [ + (Ending::Complete, true, 1, 0, 0, 0), + (Ending::Interrupted, false, 2, 1, 1, 1) + ] + ); +} + +#[test] +fn boundaries_fail_closed_on_scope_count_and_bytes() { + let mut view = identity(); + view.session = 8; + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.boundary(view, Ending::Complete, true); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + + let capture = Capture::new("fixture", 7, limits()).unwrap(); + for _ in 0..5 { + capture.boundary(identity(), Ending::Complete, true); + } + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.boundaries.len(), 4); + + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.boundary(identity(), Ending::Complete, true); + let exact = serde_json::to_vec(&probe.snapshot().boundaries) + .unwrap() + .len(); + for (bytes, failure) in [(exact, None), (exact - 1, Some(Failure::Limit))] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + assert_eq!(capture.snapshot().failure, failure); + } +} + +#[test] +fn only_turn_endings_are_boundaries() { + use crate::gemini::GeminiEvent; + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_BOUNDARY_TEST") { + return; + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _installed = install(&capture).unwrap(); + let mut latency = ReplyLatency::default(); + latency.bind("fixture", 7, 1, 1500); + boundary(&latency, &GeminiEvent::OutputTranscript("x".into()), true); + boundary(&latency, &GeminiEvent::TurnComplete, true); + boundary(&latency, &GeminiEvent::Interrupted, false); + let endings: Vec<_> = capture + .snapshot() + .boundaries + .iter() + .map(|boundary| (boundary.ending, boundary.accepted)) + .collect(); + assert_eq!( + endings, + [(Ending::Complete, true), (Ending::Interrupted, false)] + ); +} + +#[test] +fn a_single_view_reply_starts_with_matching_counts() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hi", true); + let reply = &capture.snapshot().replies[0]; + assert_eq!((reply.receipts_before, reply.batches_before), (1, 1)); + assert_eq!((reply.receipts_last, reply.batches_last), (1, 1)); +} + +#[test] +fn the_report_carries_replies_and_boundaries() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 7); + let capture = arming.capture().unwrap(); + capture.reply(identity(), "interviewer-0", "Hi", true); + capture.boundary(identity(), Ending::Interrupted, false); + let report = arming.report(); + assert_eq!(report["capture"]["replies"][0]["text"], "Hi"); + let boundary = &report["capture"]["boundaries"][0]; + assert_eq!(boundary["ending"], "Interrupted"); + assert_eq!(boundary["accepted"], false); + assert_eq!(boundary["replies_before"], 1); +} diff --git a/tests/unit/livekit/probe/serving.rs b/tests/unit/livekit/probe/serving.rs new file mode 100644 index 000000000..83e2f0c95 --- /dev/null +++ b/tests/unit/livekit/probe/serving.rs @@ -0,0 +1,614 @@ +//! Serves the real web service and the local interviewer in this process, built +//! by the same startup phases and the same `launch::serve` that `codetrial web` +//! runs, with the arming observer installed before the first request, so a +//! browser run can leave behind the tool-call evidence of the interview it +//! drove. +//! +//! `serve_web_with_arming` is the ignored entry a launcher runs; the ordinary +//! tests beside it drive `serve` with a throwaway config so the fixture itself +//! is checked without credentials or a browser. +//! +//! Unix only (`probe.rs` gates the module): a TERM is a stop request, which +//! needs Unix signals, and the launcher keeps it in a session of its own. +//! +//! Where it differs from `codetrial web`, on purpose: it is always a launcher's +//! child (no Setup page, as with the desktop app's server), it does not clear +//! the operator's web error log, task mode is off, and it reads the process +//! environment exactly as the product does, so an ambient `NODE_ENV` or +//! `CODETRIAL_*` setting reaches it too. It waits up to its settle bound for +//! admitted rooms and reports to finish before the snapshot, and a room still +//! running at the bound fails the run. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde_json::json; + +use super::acquisition::{Arming, Limits, install_arming}; +use crate::launch::{ + Bounds, CliOptions, LaunchedBy, finish, load_values, local_agent_config, prepare, + serve as serve_web, +}; +use crate::livekit::latency::ReplyLatency; + +/// A launcher gives a config file and a directory the fixture owns; the +/// account database, the ready file, the stop file and the report all live in +/// that directory under fixed names. The account database therefore has no +/// default to fall back on: the default is the operator's own `codetrial.db`. +struct Settings { + config: PathBuf, + dir: PathBuf, + addr: String, + deadline: Duration, + /// How long open connections may hold the shutdown up before the run + /// reports without them. + drain: Duration, + /// How long admitted rooms and their reports may take to finish after the + /// last connection has gone, before the run reports them as unfinished. + settle: Duration, + /// Skips the startup probe of the provider's quota. Only the + /// credential-free tests set it; the entry a launcher runs keeps the + /// product's behavior. + offline: bool, +} + +impl Settings { + fn from_lookup(get: &dyn Fn(&str) -> Option) -> Result { + let required = |name: &str| { + get(name) + .filter(|value| !value.trim().is_empty()) + .map(PathBuf::from) + .ok_or_else(|| format!("{name} must be set")) + }; + let seconds = |name: &str, default: u64| { + get(name) + .map(|text| text.parse().map_err(|_| format!("{name} is not seconds"))) + .transpose() + .map(|value| Duration::from_secs(value.unwrap_or(default))) + }; + Ok(Self { + config: required("CODETRIAL_FIXTURE_CONFIG")?, + dir: required("CODETRIAL_FIXTURE_DIR")?, + addr: get("CODETRIAL_FIXTURE_ADDR").unwrap_or_else(|| "127.0.0.1:0".into()), + deadline: seconds("CODETRIAL_FIXTURE_DEADLINE_S", 7200)?, + + // From the launcher, which sizes its stop wait from this and the + // settle; the default is what it passes. + drain: seconds("CODETRIAL_FIXTURE_DRAIN_S", 10)?, + settle: seconds("CODETRIAL_FIXTURE_SETTLE_S", 120)?, + offline: false, + }) + } + + fn db(&self) -> PathBuf { + self.dir.join("accounts.db") + } + + fn ready(&self) -> PathBuf { + self.dir.join("ready") + } + + fn stop(&self) -> PathBuf { + self.dir.join("stop") + } + + fn out(&self) -> PathBuf { + self.dir.join("out.json") + } +} + +/// Serves until `stop` resolves, then writes what the observer saw and how the +/// server ended to `out.json`. The ready file holds the bound address; it is +/// written once the listener is bound, so a request that arrives before the +/// accept loop has run waits in the socket's backlog rather than being refused. +fn serve( + settings: &Settings, + stop: impl Future + Send + 'static, +) -> Result<(), String> { + clear_previous(settings)?; + + // Loopback only: it runs with a throwaway session secret and an account + // database nobody else should reach. + let addr: std::net::SocketAddr = settings + .addr + .parse() + .map_err(|error| format!("{} is not an address: {error}", settings.addr))?; + + // Canonical first, as `launch::reachable_from_elsewhere` does: a dual-stack + // socket names 127.0.0.1 as ::ffff:127.0.0.1. + if !addr.ip().to_canonical().is_loopback() { + return Err(format!("{addr} is not a loopback address")); + } + let options = CliOptions { + config_path: Some(settings.config.display().to_string()), + ..Default::default() + }; + let mut values = load_values(&options)?; + values.insert( + "CODETRIAL_DB_PATH".into(), + settings.db().display().to_string(), + ); + + let prepared = prepare(&values, &options)?; + let listener = std::net::TcpListener::bind(addr) + .map_err(|error| format!("cannot bind {addr}: {error}"))?; + listener + .set_nonblocking(true) + .map_err(|error| error.to_string())?; + let bound = listener.local_addr().map_err(|error| error.to_string())?; + let (mut config, setup) = finish( + prepared.config, + &values, + &options, + bound, + LaunchedBy::Parent, + )?; + config.probe_provider_quota &= !settings.offline; + let agent = local_agent_config(values)? + .ok_or("the fixture hosts the interviewer and needs a Gemini key")?; + + let arming = Arming::new(Limits { + batches: 512, + calls: 2048, + bytes: 4 << 20, + }); + let installed = install_arming(&arming)?; + publish(&settings.ready(), &bound.to_string())?; + let outcome = serve_web( + listener, + config, + setup, + Some(agent), + prepared.max_concurrent, + stop, + Bounds { + drain: Some(settings.drain), + settle: Some(settings.settle), + }, + ); + + let mut report = arming.report(); + report["bound"] = json!(bound.to_string()); + report["outcome"] = json!(outcome.clone().err().unwrap_or_else(|| "ok".into())); + drop(installed); + ended(outcome, publish(&settings.out(), &format!("{report:#}\n"))) +} + +/// How a run ended, once its report was or was not written: when both failed, +/// both are said, the serving failure first. +fn ended(outcome: Result<(), String>, written: Result<(), String>) -> Result<(), String> { + match (outcome, written) { + (Err(served), Err(unwritten)) => Err(format!("{served}; and {unwritten}")), + (outcome, written) => outcome.and(written), + } +} + +/// Written beside and renamed into place, so a reader never sees half a file. +fn publish(path: &Path, text: &str) -> Result<(), String> { + let partial = path.with_extension("partial"); + std::fs::write(&partial, text) + .and_then(|()| std::fs::rename(&partial, path)) + .map_err(|error| format!("cannot write {}: {error}", path.display())) +} + +/// Set once a TERM can be received, so a test sends one only then. +static TERM_REGISTERED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Resolves once the stop file exists or the deadline passes. +async fn stop_file(path: PathBuf, deadline: Duration) { + let began = Instant::now(); + while !path.exists() && began.elapsed() < deadline { + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// The stop file, or a TERM, which a launcher that cannot reach the file sends +/// to ask the same thing: drain, settle and write the report. Registered when +/// first polled, inside the fixture's runtime, which is after the ready file is +/// written: a TERM in that first moment still finds the default action and ends +/// the fixture without a report, so a launcher asks with the stop file and +/// keeps the fixture out of group-wide signals. Fails loudly if it cannot be +/// registered, since the fixture would otherwise carry on without its contract. +async fn stop_requested(path: PathBuf, deadline: Duration) { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("the TERM handler registers"); + TERM_REGISTERED.store(true, std::sync::atomic::Ordering::SeqCst); + + // Once this returns the handler stays installed and a later TERM does + // nothing, so a launcher's escalation after the stop file is the KILL. + tokio::select! { + () = stop_file(path, deadline) => {} + _ = term.recv() => {} + } +} + +/// Before anything can fail: a run that never reports must not leave an +/// earlier run's report to be read as its own. +fn clear_previous(settings: &Settings) -> Result<(), String> { + for stale in [settings.ready(), settings.out()] { + match std::fs::remove_file(&stale) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("cannot clear {}: {error}", stale.display())); + } + _ => {} + } + } + Ok(()) +} + +fn serve_until_stop_file(settings: &Settings) -> Result<(), String> { + clear_previous(settings)?; + if settings.stop().exists() { + return Err(format!( + "{} already exists and would end the run before it began", + settings.stop().display() + )); + } + serve(settings, stop_requested(settings.stop(), settings.deadline)) +} + +#[test] +#[ignore = "serves until stopped; an interview run through it spends provider quota"] +fn serve_web_with_arming() { + // The observer's hooks are process-wide, so another test's room or bind in + // this process would fail it for good. The launcher runs this one alone. + assert!( + std::env::args().any(|arg| arg == "--exact"), + "run this with --exact so no other test shares the process" + ); + let settings = Settings::from_lookup(&|name| std::env::var(name).ok()).unwrap(); + serve_until_stop_file(&settings).unwrap(); +} + +fn scratch(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "codetrial-serving-{label}-{}-{}", + std::process::id(), + crate::current_epoch_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Settings for a throwaway directory: a fake config, loopback port zero, and +/// no quota probe. +fn throwaway(dir: &Path) -> Settings { + let config = dir.join("codetrial.env.local"); + std::fs::write( + &config, + "LIVEKIT_URL=wss://primary.example\nLIVEKIT_API_KEY=primary-key\n\ + LIVEKIT_API_SECRET=primary-secret\nGOOGLE_API_KEY=google-key\n", + ) + .unwrap(); + Settings { + config, + dir: dir.to_path_buf(), + addr: "127.0.0.1:0".into(), + deadline: Duration::from_secs(60), + drain: Duration::from_secs(10), + settle: Duration::from_secs(10), + offline: true, + } +} + +fn get(addr: &str, path: &str) -> String { + use std::io::{Read, Write}; + let mut stream = std::net::TcpStream::connect(addr).unwrap(); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response +} + +fn wait_for(path: &Path) -> String { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(30) { + if let Ok(text) = std::fs::read_to_string(path) + && !text.is_empty() + { + return text; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("{} never appeared", path.display()); +} + +fn lookup( + config: Option<&str>, + dir: Option<&str>, + deadline: Option<&str>, +) -> impl Fn(&str) -> Option { + let (config, dir, deadline) = ( + config.map(String::from), + dir.map(String::from), + deadline.map(String::from), + ); + move |name| match name { + "CODETRIAL_FIXTURE_CONFIG" => config.clone(), + "CODETRIAL_FIXTURE_DIR" => dir.clone(), + "CODETRIAL_FIXTURE_DEADLINE_S" => deadline.clone(), + _ => None, + } +} + +#[test] +fn the_fixture_owns_one_directory_and_never_defaults_the_database() { + let settings = Settings::from_lookup(&lookup(Some("/c"), Some("/d"), None)).unwrap(); + assert_eq!(settings.db(), Path::new("/d/accounts.db")); + for (name, path) in [ + ("ready", settings.ready()), + ("stop", settings.stop()), + ("out.json", settings.out()), + ] { + assert_eq!(path, Path::new("/d").join(name)); + } + + // A blank directory counts as missing: a blank database path is read as + // absent by the launch code and would open the operator's own database. + for blank in [None, Some(""), Some(" "), Some("\t")] { + for (config, dir, missing) in [ + (blank, Some("/d"), "CODETRIAL_FIXTURE_CONFIG"), + (Some("/c"), blank, "CODETRIAL_FIXTURE_DIR"), + ] { + assert_eq!( + Settings::from_lookup(&lookup(config, dir, None)) + .err() + .unwrap(), + format!("{missing} must be set") + ); + } + } + assert!(Settings::from_lookup(&lookup(Some("/c"), Some("/d"), Some("soon"))).is_err()); + assert_eq!(settings.settle, Duration::from_secs(120)); + assert_eq!(settings.drain, Duration::from_secs(10)); + for (key, value, read) in [ + ("CODETRIAL_FIXTURE_SETTLE_S", "later", None), + ("CODETRIAL_FIXTURE_DRAIN_S", "soon", None), + ("CODETRIAL_FIXTURE_DRAIN_S", "3", Some(3)), + ] { + let lookup = |name: &str| match name { + "CODETRIAL_FIXTURE_CONFIG" => Some("/c".to_string()), + "CODETRIAL_FIXTURE_DIR" => Some("/d".to_string()), + name if name == key => Some(value.to_string()), + _ => None, + }; + let result = Settings::from_lookup(&lookup); + match read { + None => assert_eq!(result.err().unwrap(), format!("{key} is not seconds")), + Some(seconds) => assert_eq!(result.unwrap().drain, Duration::from_secs(seconds)), + } + } +} + +#[test] +fn a_stop_file_left_from_an_earlier_run_refuses_the_start() { + let dir = scratch("stale-stop"); + let settings = throwaway(&dir); + std::fs::write(settings.stop(), "").unwrap(); + // An earlier run's report goes too, so it cannot be kept as this one's. + std::fs::write(settings.out(), "{\"outcome\":\"ok\"}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + let error = serve_until_stop_file(&settings).unwrap_err(); + assert!( + error.contains("would end the run before it began"), + "{error}" + ); + assert!(!settings.db().exists() && !settings.out().exists()); + assert!(!settings.ready().exists()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_run_that_fails_and_cannot_report_says_both() { + let fail = |why: &str| Err::<(), String>(why.into()); + assert_eq!(ended(Ok(()), Ok(())), Ok(())); + assert_eq!(ended(fail("served"), Ok(())), fail("served")); + assert_eq!(ended(Ok(()), fail("unwritten")), fail("unwritten")); + assert_eq!( + ended(fail("served"), fail("unwritten")), + fail("served; and unwritten") + ); +} + +#[test] +fn an_address_off_loopback_refuses_the_start() { + let dir = scratch("public-addr"); + for (addr, refusal) in [ + ("0.0.0.0:0", "not a loopback address"), + ("192.0.2.1:0", "not a loopback address"), + ("localhost:0", "not an address"), + ] { + let mut settings = throwaway(&dir); + settings.addr = addr.into(); + let error = serve_until_stop_file(&settings).unwrap_err(); + assert!(error.contains(refusal), "{addr}: {error}"); + assert!(!settings.db().exists() && !settings.out().exists()); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_fixture_serves_the_real_service_and_reports_what_the_observer_saw() { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_TEST") { + return; + } + let dir = scratch("serve"); + let settings = throwaway(&dir); + let (ready, out, db) = (settings.ready(), settings.out(), settings.db()); + let stop = std::sync::Arc::new(tokio::sync::Notify::new()); + let stopping = std::sync::Arc::clone(&stop); + let server = + std::thread::spawn(move || serve(&settings, async move { stopping.notified().await })); + + let addr = wait_for(&ready); + assert!(get(&addr, "/healthz").starts_with("HTTP/1.1 200"), "{addr}"); + assert!( + get(&addr, "/setup").starts_with("HTTP/1.1 404"), + "a child has no Setup" + ); + assert!(db.is_file(), "the named database is the one opened"); + + // The product's own boundaries: the dispatcher's assignment, the latency + // bind, the executed tool answers and the local send outcome. + crate::dispatch::notify_assigned("fixture-room"); + let mut latency = ReplyLatency::default(); + latency.bind("fixture-room", 7, 1, 1500); + super::acquisition::observe(&latency, &fixture_answers("call-1")) + .expect("the armed scope records the batch") + .complete(true); + stop.notify_one(); + server.join().unwrap().unwrap(); + + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert_eq!(report["outcome"], "ok", "{report}"); + assert_eq!(report["bound"], addr, "{report}"); + // Present and null, not merely absent: a missing key must not pass. + assert_eq!(report.get("arming_failure"), Some(&serde_json::Value::Null)); + let capture = &report["capture"]; + assert_eq!( + capture.get("failure"), + Some(&serde_json::Value::Null), + "{report}" + ); + assert!(capture["retained_bytes"].as_u64().unwrap() > 0, "{report}"); + assert!( + !report.to_string().contains("not retained"), + "arguments are not kept" + ); + assert_eq!(capture["retained_calls"], 1, "{report}"); + assert_eq!(capture["batches"].as_array().unwrap().len(), 1, "{report}"); + let batch = &capture["batches"][0]; + assert_eq!(batch["identity"]["room"], "fixture-room", "{report}"); + assert_eq!(batch["identity"]["session"], 7, "{report}"); + assert_eq!(batch["calls"][0]["id"], "call-1", "{report}"); + assert_eq!(batch["calls"][0]["name"], "read_editor", "{report}"); + assert_eq!(batch["calls"][0]["result"], "authored result", "{report}"); + assert_eq!(batch["local_send_completed"], true, "{report}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_term_is_a_stop_request_and_the_run_still_reports() { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_TEST") { + return; + } + let dir = scratch("term"); + let settings = throwaway(&dir); + let (ready, out) = (settings.ready(), settings.out()); + let server = std::thread::spawn(move || serve_until_stop_file(&settings)); + wait_for(&ready); + + // The handler is registered when the stop future is first polled, which the + // bound address does not wait for. + let began = Instant::now(); + while !TERM_REGISTERED.load(std::sync::atomic::Ordering::SeqCst) { + assert!(began.elapsed() < Duration::from_secs(30), "no TERM handler"); + std::thread::sleep(Duration::from_millis(20)); + } + let status = std::process::Command::new("kill") + .args(["-TERM", &std::process::id().to_string()]) + .status() + .unwrap(); + assert!(status.success()); + server.join().unwrap().unwrap(); + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert_eq!(report["outcome"], "ok", "{report}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting() { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_DRAIN_TEST") { + return; + } + let dir = scratch("drain"); + let mut settings = throwaway(&dir); + settings.drain = Duration::from_secs(1); + let (ready, out) = (settings.ready(), settings.out()); + let stop = std::sync::Arc::new(tokio::sync::Notify::new()); + let stopping = std::sync::Arc::clone(&stop); + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let result = serve(&settings, async move { stopping.notified().await }); + let _ = done.send(result); + }); + + let addr = wait_for(&ready); + // A login whose body never finishes arriving, held open past the stop. + use std::io::Write; + let mut held = std::net::TcpStream::connect(&addr).unwrap(); + write!( + held, + "POST /api/login HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/json\r\n\ + Content-Length: 1000\r\n\r\n{{\"partial\":" + ) + .unwrap(); + std::thread::sleep(Duration::from_millis(300)); + stop.notify_one(); + let result = finished + .recv_timeout(Duration::from_secs(20)) + .expect("the run reported within its drain bound"); + let error = result.unwrap_err(); + assert!(error.contains("shutdown bound"), "{error}"); + + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert!( + report["outcome"] + .as_str() + .unwrap() + .contains("shutdown bound"), + "{report}" + ); + drop(held); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_run_replaces_the_artifacts_an_earlier_run_left() { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_STALE_TEST") { + return; + } + let dir = scratch("stale-artifacts"); + let settings = throwaway(&dir); + std::fs::write(settings.out(), "{\"outcome\":\"ok\",\"stale\":true}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + // A run that fails to start leaves neither file behind. + let mut broken = throwaway(&dir); + broken.config = dir.join("missing.env.local"); + assert!(serve(&broken, async {}).is_err()); + assert!(!settings.out().exists() && !settings.ready().exists()); + std::fs::write(settings.out(), "{\"outcome\":\"ok\",\"stale\":true}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + + // A stop that has already happened: the run starts, serves nothing, + // reports. + serve(&settings, async {}).unwrap(); + let report: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(settings.out()).unwrap()).unwrap(); + assert!(report.get("stale").is_none(), "{report}"); + assert_eq!(report["outcome"], "ok", "{report}"); + let ready = std::fs::read_to_string(settings.ready()).unwrap(); + assert_ne!(ready, "127.0.0.1:1"); + assert_eq!(report["bound"], ready.as_str(), "{report}"); + assert!( + !settings.dir.join("out.partial").exists(), + "published by rename" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +fn fixture_answers(id: &str) -> Vec<(crate::gemini::GeminiFunctionCall, serde_json::Value)> { + vec![( + crate::gemini::GeminiFunctionCall { + id: id.into(), + name: "read_editor".into(), + args: json!({"private": "not retained"}), + }, + json!("authored result"), + )] +} diff --git a/tests/unit/livekit/recovery.rs b/tests/unit/livekit/recovery.rs index c0d46a2e9..7ec841030 100644 --- a/tests/unit/livekit/recovery.rs +++ b/tests/unit/livekit/recovery.rs @@ -1,3 +1,5 @@ +use crate::test_isolation as isolation; + use super::super::tests::{fake_recording_socket, hold_turn, test_output_audio}; use super::*; use crate::config::load_from_pairs; @@ -19,98 +21,7 @@ fn isolated_checked_with_loopback( check: impl FnOnce(&std::process::Output), ) -> bool { let name = format!("livekit::recovery::tests::{name}"); - if std::env::var("CODETRIAL_RECOVERY_TEST").as_deref() == Ok(&name) { - return false; - } - let guard = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - guard.set_nonblocking(true).unwrap(); - let proxy = format!("http://{}", guard.local_addr().unwrap()); - let mut command = std::process::Command::new(std::env::current_exe().unwrap()); - command - .args(["--exact", &name, "--nocapture"]) - .env("CODETRIAL_RECOVERY_TEST", &name) - .env("NO_PROXY", if loopback_http { "127.0.0.1" } else { "" }) - .env("no_proxy", if loopback_http { "127.0.0.1" } else { "" }) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); - for key in [ - "HTTPS_PROXY", - "https_proxy", - "HTTP_PROXY", - "http_proxy", - "ALL_PROXY", - "all_proxy", - ] { - command.env(key, &proxy); - } - let mut child = command.spawn().unwrap(); - - // Drained while the child runs: a child that writes more than a pipe holds - // (64 KiB on Linux) blocks in its write, and polling only for its exit then - // reads that block as a hang. - let drain = |stream: Option>| { - std::thread::spawn(move || { - let mut bytes = Vec::new(); - if let Some(mut stream) = stream { - let _ = stream.read_to_end(&mut bytes); - } - bytes - }) - }; - let stdout = drain( - child - .stdout - .take() - .map(|pipe| Box::new(pipe) as Box), - ); - let stderr = drain( - child - .stderr - .take() - .map(|pipe| Box::new(pipe) as Box), - ); - let started = Instant::now(); - let status = loop { - if let Some(status) = child.try_wait().unwrap() { - break status; - } - if started.elapsed() > Duration::from_secs(20) { - child.kill().unwrap(); - child.wait().unwrap(); - panic!( - "recovery child timed out: {}", - String::from_utf8_lossy(&stderr.join().unwrap()) - ); - } - std::thread::sleep(Duration::from_millis(10)); - }; - let output = std::process::Output { - status, - stdout: stdout.join().unwrap(), - stderr: stderr.join().unwrap(), - }; - assert_eq!( - guard.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock, - "the recovery test must not contact an external provider" - ); - assert!( - output.status.success(), - "{}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.lines().any(|line| line == "running 1 test") - && stdout.lines().any(|line| { - line.starts_with("test result: ok. 1 passed; 0 failed; 0 ignored;") - }), - "isolated child must execute exactly one passing test: {name}\n{stdout}" - ); - check(&output); - true + isolation::isolated_checked(&name, "CODETRIAL_RECOVERY_TEST", loopback_http, check) } #[test] @@ -278,7 +189,14 @@ impl Rig { } async fn run(&mut self) -> ControlFlow<()> { - let boot = crate::runtime::bootstrap(&self.config, "recovery-test", Some("two-sum"), 45); + tokio::time::timeout(Duration::from_secs(10), self.run_result()) + .await + .expect("the real room recovery must be bounded in the test") + .unwrap() + } + + async fn run_result(&mut self) -> Result, Error> { + let boot = crate::runtime::bootstrap(&self.config, self.room.name(), Some("two-sum"), 45); let interview = InterviewContext { config: &self.config, keys: &self.keys, @@ -291,7 +209,7 @@ impl Rig { }; let prefix = self.prefix; let speech = self.speech.take(); - tokio::time::timeout(Duration::from_secs(10), async { + async { let mut context = self.turn.context( &mut self.output, &mut self.gemini, @@ -314,22 +232,22 @@ impl Rig { } else { on_gemini_event(&self.room, &mut context, None, &mut self.loops, interview).await } - }) + } .await - .expect("the real room recovery must be bounded in the test") - .unwrap() } - async fn finish(mut self) { + /// Joins the old socket's peer and returns every frame it received. + async fn old_frames(&mut self) -> Vec { self.gemini.shutdown().await.unwrap(); - assert_eq!( - tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) - .await - .unwrap() - .unwrap() - .len(), - self.expected_old_messages - ); + tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) + .await + .unwrap() + .unwrap() + } + + async fn finish(mut self) { + let frames = self.old_frames().await; + assert_eq!(frames.len(), self.expected_old_messages); } } @@ -567,3 +485,7 @@ fn a_stream_end_or_a_new_stream_breaks_the_gap_and_replayed_frames_are_not_deque assert!(after.contains("mic_gap_max_ms=20 "), "{after}"); assert!(after.contains("mic_gap_over_1s=0 "), "{after}"); } + +include!("recovery_parts/report_generation_cardinality.rs"); + +include!("recovery_parts/acquisition_tool_sends.rs"); diff --git a/tests/unit/livekit/recovery_live.rs b/tests/unit/livekit/recovery_live.rs index cdfe4e278..ea34d4cc6 100644 --- a/tests/unit/livekit/recovery_live.rs +++ b/tests/unit/livekit/recovery_live.rs @@ -1128,3 +1128,5 @@ include!("recovery_live_parts/native_worker_and_room_disconnect.rs"); // room loop, one clip arm per invocation. #[path = "recovery_continuity.rs"] mod continuity; + +include!("recovery_live_parts/report_generation_cardinality.rs"); diff --git a/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs b/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs new file mode 100644 index 000000000..bc797fae6 --- /dev/null +++ b/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs @@ -0,0 +1,828 @@ +// Real SDK delivery with a hand-built driver clock and local model endpoints. + +#[derive(Clone, Copy, Debug, PartialEq)] +enum G2Case { + EndThenDeadline, + DeadlineFirst, + RoomClosed, + AllReady, +} + +impl G2Case { + fn selected() -> Self { + match std::env::var("CODETRIAL_G2_CASE").as_deref() { + Ok("end") => Self::EndThenDeadline, + Ok("deadline") => Self::DeadlineFirst, + Ok("closed") => Self::RoomClosed, + Ok("all") => Self::AllReady, + _ => panic!("CODETRIAL_G2_CASE is end, deadline, closed or all"), + } + } + + fn name(self) -> &'static str { + match self { + Self::EndThenDeadline => "g2-end-then-deadline", + Self::DeadlineFirst => "g2-deadline-first", + Self::RoomClosed => "g2-room-closed", + Self::AllReady => "g2-all-ready", + } + } + + fn actions(self) -> Vec<&'static str> { + let mut actions = vec!["candidate_join", "agent_join"]; + if matches!(self, Self::RoomClosed | Self::AllReady) { + actions.push("room_delete"); + } + actions.push("controlled_driver_start"); + if !matches!(self, Self::RoomClosed | Self::AllReady) { + actions.push("room_delete"); + } + actions + } + + fn initial_count(self) -> usize { + usize::from(self != Self::RoomClosed) + } + + fn terminal(self) -> &'static str { + match self { + Self::EndThenDeadline => "End", + Self::DeadlineFirst | Self::AllReady => "Deadline", + Self::RoomClosed => "RoomClosed", + } + } +} + +struct G2TerminalRegistration(String); + +impl Drop for G2TerminalRegistration { + fn drop(&mut self) { + super::super::super::LIVE_TERMINAL_PROBES + .lock() + .unwrap() + .retain(|(name, _)| name != &self.0); + } +} + +fn g2_native_config() -> crate::config::AgentConfig { + let mut pairs = crate::config::read_config_file(Path::new( + &std::env::var("CODETRIAL_LIVE_CONFIG").expect("isolated native config path"), + )) + .expect("read isolated native config"); + pairs.retain(|(name, _)| name != "GOOGLE_API_KEY" && name != "GOOGLE_API_KEYS"); + pairs.push(("GOOGLE_API_KEY".into(), "g2-fixture-only-key".into())); + let config = crate::config::load_from_pairs(pairs).expect("valid isolated native config"); + register_secrets([ + config.livekit_api_key.clone(), + config.livekit_api_secret.clone(), + ]); + config +} + +async fn g2_delete(config: &crate::config::AgentConfig, room: &str) -> Result<(), String> { + tokio::time::timeout( + Duration::from_secs(10), + super::super::super::room_service::delete_room( + &config.livekit_url, + &config.livekit_api_key, + &config.livekit_api_secret, + room, + crate::current_epoch_seconds(), + ), + ) + .await + .map_err(|_| "native room delete timed out".to_owned())? + .map_err(|error| redact(&error.to_string())) +} + +async fn g2_queue_until( + source: &mut tokio::sync::mpsc::UnboundedReceiver, + target: &tokio::sync::mpsc::UnboundedSender, + closed: bool, +) -> Result<(), String> { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + let event = source.recv().await.ok_or_else(|| "native agent event stream ended before required input".to_owned())?; + let matched = if closed { + matches!(&event, RoomEvent::Disconnected { .. }) + } else { + matches!(&event, RoomEvent::DataReceived { topic, payload, .. } + if topic.as_deref() == Some(crate::runtime::TOPIC_CONTROL) + && serde_json::from_slice::(payload).is_ok_and(|body| body["type"] == "end_interview")) + }; + target.send(event).map_err(|_| "driver event receiver ended".to_owned())?; + if matched { return Ok(()); } + } + }).await.map_err(|_| "native input arrival was not observed in 20 seconds".to_owned())? +} + +fn g2_seed_report_state(state: &mut crate::agent::RuntimeState) { + state.code = SEED_SOLUTION.into(); + state.language = "python".into(); + crate::livekit::tests::receive_test_run(state); + for phase in ["test", "optimizations"] { + crate::agent::record_framework_evidence( + state, + &json!({ + "phase": phase, "source": crate::livekit::tests::observed_source(phase), + "kind": "observed", "confidence": 90, "summary": "Synthetic G2 fixture evidence." + }), + ) + .unwrap(); + } + state.behavioral_round_started = true; + state.evidence_ledger.conversation.candidate_turns = 1; + let rounds = crate::agent::ReportRounds::of(state); + assert!( + rounds.behavioral_opened && rounds.follow_ups_released && rounds.communication_assessed() + ); +} + +impl super::Rig { + async fn g2_repeat_deadline(&mut self) -> Result, String> { + let boot = crate::runtime::bootstrap(&self.config, self.room.name(), Some("two-sum"), 45); + let interview = super::super::super::InterviewContext { + config: &self.config, + keys: &self.keys, + boot: &boot, + live_endpoint: Some(&self.endpoint), + started_at: std::time::Instant::now(), + candidate_identity: "candidate", + events: &self.events, + slot: None, + }; + let mut context = self.turn.context( + &mut self.output, + &mut self.gemini, + &mut self.board, + &mut self.media, + ); + super::super::super::on_hard_deadline(&self.room, &mut context, &mut self.loops, interview) + .await + .map_err(|error| redact(&error.to_string())) + } +} + +type G2DriverOutcome = (super::Rig, Vec, bool, bool, bool); + +struct G2NativeRun { + config: crate::config::AgentConfig, + room_name: String, + agent_identity: String, + rig: Option, + candidate: Option, + bridge: Option>, + driver: Option>, + shared: Arc, + fixture: super::G2ReportFixture, + terminals: tokio::sync::mpsc::UnboundedReceiver<(&'static str, std::time::Instant)>, + _terminal_scope: G2TerminalRegistration, + attempted_join: bool, + attempted_delete: bool, + joined_driver: bool, + first_break: bool, + guard_reentered: bool, + guard_ok: bool, + outcomes: Vec, +} + +impl G2NativeRun { + async fn new(config: crate::config::AgentConfig, room_name: String) -> Self { + let agent_identity = crate::runtime::agent_identity(&room_name); + let mut rig = super::Rig::new("ws://127.0.0.1:9/".into()).await; + rig.config = config.clone(); + rig.loops.recovery_terminal = None; + rig.loops.recovery_inputs = None; + g2_seed_report_state(&mut rig.turn.state); + let fixture = super::G2ReportFixture::start(&room_name).await; + let (terminal, terminals) = tokio::sync::mpsc::unbounded_channel(); + super::super::super::LIVE_TERMINAL_PROBES + .lock() + .unwrap() + .push((room_name.clone(), terminal)); + Self { + config, + room_name: room_name.clone(), + agent_identity, + rig: Some(rig), + candidate: None, + bridge: None, + driver: None, + shared: Shared::new(), + fixture, + terminals, + _terminal_scope: G2TerminalRegistration(room_name), + attempted_join: false, + attempted_delete: false, + joined_driver: false, + first_break: false, + guard_reentered: false, + guard_ok: false, + outcomes: Vec::new(), + } + } + + async fn delete(&mut self, permits: &mut PermitClient) -> Result<(), String> { + self.attempted_delete = true; + permits + .guarded("room_delete", g2_delete(&self.config, &self.room_name)) + .await + .map_err(|error| error.to_string())? + } + + async fn exercise( + &mut self, + permits: &mut PermitClient, + case: G2Case, + mutant: bool, + ) -> Result<(), String> { + if permits.stop_requested() { + return Err("launcher stopped before joins".into()); + } + self.attempted_join = true; + let token = crate::web::token_response( + &crate::web::TokenConfig { + api_key: &self.config.livekit_api_key, + api_secret: &self.config.livekit_api_secret, + server_url: &self.config.livekit_url, + recording_max_min: None, + }, + br#"{"problemId":"two-sum","durationMin":10}"#, + &self.room_name, + "candidate", + crate::current_epoch_seconds(), + ) + .map_err(|error| format!("candidate token: {error}"))? + .token; + if permits.stop_requested() { + return Err("launcher stopped before candidate join".into()); + } + self.candidate = Some( + permits + .guarded( + "candidate_join", + connect_candidate( + &self.config.livekit_url, + &token, + &self.shared, + &self.agent_identity, + ), + ) + .await + .map_err(|error| error.to_string())??, + ); + if permits.stop_requested() { + return Err("launcher stopped before agent join".into()); + } + let (agent, mut native_events) = permits + .guarded("agent_join", async { + tokio::time::timeout( + Duration::from_secs(30), + super::super::super::join_room( + &self.config, + &self.room_name, + &self.agent_identity, + crate::current_epoch_seconds(), + ), + ) + .await + .map_err(|_| "agent connect timed out".to_owned())? + .map_err(|error| redact(&error.to_string())) + }) + .await + .map_err(|error| error.to_string())??; + self.rig.as_mut().unwrap().room = super::super::super::room_link::RoomLink::live(agent); + let (forward, received) = tokio::sync::mpsc::unbounded_channel(); + self.rig.as_mut().unwrap().events = tokio::sync::Mutex::new(received); + if case != G2Case::RoomClosed { + if permits.stop_requested() { + return Err("launcher stopped before End".into()); + } + permits + .guarded( + FINAL_ACTION_ID, + self.candidate + .as_ref() + .unwrap() + .room + .local_participant() + .publish_data(DataPacket { + payload: serde_json::to_vec( + &json!({"type":"end_interview", "reason":"candidate_ended"}), + ) + .unwrap(), + topic: Some(crate::runtime::TOPIC_CONTROL.into()), + reliable: true, + ..Default::default() + }), + ) + .await + .map_err(|error| error.to_string())? + .map_err(|error| redact(&error.to_string()))?; + g2_queue_until(&mut native_events, &forward, false).await?; + } + if matches!(case, G2Case::RoomClosed | G2Case::AllReady) { + if permits.stop_requested() { + return Err("launcher stopped before delete".into()); + } + self.delete(permits).await?; + g2_queue_until(&mut native_events, &forward, true).await?; + } + self.bridge = Some(tokio::spawn(async move { + while let Some(event) = native_events.recv().await { + if forward.send(event).is_err() { + break; + } + } + })); + if permits.stop_requested() { + return Err("launcher stopped before driver".into()); + } + permits + .permit("controlled_driver_start") + .await + .map_err(|error| error.to_string())?; + let mut rig = self.rig.take().unwrap(); + rig.loops.deadline_at = if matches!(case, G2Case::DeadlineFirst | G2Case::AllReady) { + tokio::time::Instant::now() + } else { + tokio::time::Instant::now() + Duration::from_secs(540) + }; + self.driver = Some(tokio::spawn(async move { + let first = rig + .run_result() + .await + .map_err(|error| redact(&error.to_string())); + let first_break = matches!(&first, Ok(std::ops::ControlFlow::Break(()))); + let mut outcomes = vec![format!("{first:?}")]; + let mut guard_reentered = false; + let mut guard_ok = false; + if case == G2Case::EndThenDeadline && first_break { + let repeated = rig.g2_repeat_deadline().await; + guard_reentered = true; + guard_ok = matches!(&repeated, Ok(std::ops::ControlFlow::Break(()))); + outcomes.push(format!("{repeated:?}")); + } + (rig, outcomes, first_break, guard_reentered, guard_ok) + })); + let stop_file = permits.stop_file.clone(); + let joined = tokio::select! { + biased; + () = async { + loop { + if stop_file.as_deref().is_some_and(Path::exists) { break; } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } => return Err("launcher stopped during driver; cleanup follows".into()), + result = tokio::time::timeout( + Duration::from_secs(if mutant { 1020 } else { 540 }), + self.driver.as_mut().unwrap(), + ) => result.map_err(|_| "driver join timed out".to_owned())? + .map_err(|error| format!("driver join failed: {error}"))?, + }; + self.driver.take(); + self.rig = Some(joined.0); + self.outcomes = joined.1; + self.first_break = joined.2; + self.guard_reentered = joined.3; + self.guard_ok = joined.4; + self.joined_driver = true; + Ok(()) + } + + async fn cleanup(&mut self, permits: &mut PermitClient) -> Vec { + let mut failures = Vec::new(); + if let Some(driver) = self.driver.take() { + driver.abort(); + let _ = tokio::time::timeout(Duration::from_secs(2), driver).await; + failures.push("driver aborted: cardinality acceptance is inconclusive".into()); + } + if let Some(rig) = &mut self.rig { + if !g2_agent_closed( + tokio::time::timeout(Duration::from_secs(10), rig.room.close()).await, + ) { + failures.push("agent Room close did not complete".into()); + } + if !matches!( + tokio::time::timeout(Duration::from_secs(1), rig.gemini.shutdown()).await, + Ok(Ok(())) + ) { + failures.push("local Gemini close did not complete".into()); + } + if let Some(mut old) = rig.old.take() + && !matches!( + tokio::time::timeout(Duration::from_secs(2), &mut old).await, + Ok(Ok(_)) + ) + { + old.abort(); + let _ = old.await; + failures.push("local Gemini peer did not join".into()); + } + } + if self.attempted_join + && !self.attempted_delete + && let Err(error) = self.delete(permits).await + { + failures.push(format!("cleanup room delete: {error}")); + } + if let Some(candidate) = self.candidate.take() { + if !matches!( + tokio::time::timeout(Duration::from_secs(10), candidate.room.close()).await, + Ok(Ok(())) | Ok(Err(::livekit::RoomError::AlreadyClosed)) + ) { + failures.push("candidate Room close did not complete".into()); + } + let mut observer = candidate.observer; + if !matches!( + tokio::time::timeout(Duration::from_secs(10), &mut observer).await, + Ok(Ok(())) + ) { + observer.abort(); + let _ = observer.await; + failures.push("candidate observer did not join".into()); + } + } + if let Some(mut bridge) = self.bridge.take() + && !matches!( + tokio::time::timeout(Duration::from_secs(10), &mut bridge).await, + Ok(Ok(())) + ) + { + bridge.abort(); + let _ = bridge.await; + failures.push("SDK event bridge did not join".into()); + } + failures + } +} + +/// NOT RUN: local-ready scheduling with real SDK packets/receipts, not the +/// production room-loop clock or proof of every native arrival ordering. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "NOT RUN: real LiveKit Room; requires a prospective G2 manifest, launcher permits and isolated native config; Gemini endpoints stay local"] +async fn g2_native_report_generation_and_guarded_deadline_reentry() { + let case = G2Case::selected(); + let mutant = match std::env::var("CODETRIAL_G2_GUARD_MUTANT").as_deref() { + Err(_) => false, + Ok("1") => { + assert_eq!(case, G2Case::EndThenDeadline); + true + } + _ => panic!("CODETRIAL_G2_GUARD_MUTANT is absent or 1"), + }; + let cap = if mutant { 1200 } else { 720 }; + let mut permits = PermitClient::from_env() + .await + .expect("prospective G2 launcher budget"); + let run_id = format!( + "{}{}", + case.name(), + if mutant { "-guard-mutant" } else { "" } + ); + assert_eq!(permits.manifest["run_id"], run_id); + assert_eq!(permits.manifest["actions"], json!(case.actions())); + assert_eq!(permits.manifest["action_ceiling"], case.actions().len() + 1); + assert_eq!(permits.manifest["outer_wall_cap_s"], cap); + assert_eq!( + permits.manifest["observed_budgets"]["external_gemini_http"], + 0 + ); + assert_eq!( + permits.manifest["observed_budgets"]["gemini_live_connections"], + 0 + ); + assert_eq!( + permits.manifest["observed_budgets"]["native_room_connections"], + 2 + ); + assert_eq!( + permits.manifest["observed_budgets"]["local_report_http_attempts"], + if mutant { 10 } else { 5 } + ); + let settings = g2_settings(&permits.manifest); + assert_eq!(settings["case"], case.name()); + assert_eq!(settings["guard_mutant"], mutant); + let room_name = settings["room_name"].as_str().unwrap().to_owned(); + assert_eq!( + permits.manifest["named_resources"], + json!([{"kind":"room", "name":room_name}]) + ); + let config = g2_native_config(); + g2_require_deny_proxy(&config, &settings); + let mut run = G2NativeRun::new(config, room_name).await; + let result = run.exercise(&mut permits, case, mutant).await; + let cleanup = run.cleanup(&mut permits).await; + assert!( + result.is_ok() && cleanup.is_empty() && run.joined_driver, + "INCONCLUSIVE: native driver/setup/cleanup incomplete: {result:?} {cleanup:?}" + ); + if case == G2Case::EndThenDeadline { + assert!( + run.first_break && run.guard_reentered && run.outcomes.len() == 2, + "INCONCLUSIVE: real End must finish and the guarded deadline handler must run" + ); + if !mutant { + assert!( + run.guard_ok, + "INCONCLUSIVE: baseline deadline re-entry must return Ok(Break), not an unrelated error" + ); + } + } else if case != G2Case::AllReady { + assert!( + run.first_break, + "INCONCLUSIVE: terminal handler did not leave normally" + ); + } + let mut expected_permits = case.actions(); + if case != G2Case::RoomClosed { + expected_permits.push(FINAL_ACTION_ID); + } + let mut granted = permits + .granted + .iter() + .map(String::as_str) + .collect::>(); + granted.sort(); + expected_permits.sort(); + assert_eq!( + granted, expected_permits, + "one reserved End and only declared nonfinal actions" + ); + let entries = run.fixture.scope.entries(); + let inputs = run.fixture.bodies.lock().unwrap().len(); + let terminal = run + .terminals + .try_recv() + .expect("real driver terminal observed") + .0; + let observed = run.shared.observed.lock().unwrap(); + emit( + "g2_counts", + json!({"case":case.name(),"bounded_entries":entries.len(),"local_http_attempts":inputs,"final_copies":observed.tally.final_copies,"final_distinct":observed.tally.final_ids.len(),"terminal":terminal,"outcomes":run.outcomes,"driver_joined":run.joined_driver,"first_break":run.first_break,"guard_reentered":run.guard_reentered,"guard_ok":run.guard_ok}), + ); + if mutant { + assert_eq!( + entries.len(), + 2, + "mutant precondition: exactly two actual bounded entries" + ); + assert_eq!( + inputs, 2, + "mutant precondition: exactly two actual local report HTTP requests" + ); + eprintln!("G2_GUARD_MUTANT_REACHED_TWO_REAL_GENERATIONS"); + } + assert_eq!( + entries.len(), + case.initial_count(), + "initial bounded generation cardinality; accepted regeneration was never requested" + ); + assert_eq!( + inputs, + case.initial_count(), + "valid local report fixture uses one HTTP attempt per generation" + ); + assert_eq!(terminal, case.terminal()); + assert!( + run.terminals.try_recv().is_err(), + "one real recovery terminal" + ); + assert_eq!(observed.tally.provisional, 0); + if matches!(case, G2Case::EndThenDeadline | G2Case::DeadlineFirst) { + assert_eq!(observed.tally.final_ids.len(), 1); + assert!( + observed.tally.final_copies >= 1, + "actual SDK final packet received" + ); + assert_eq!(observed.ack_failures, 0); + } else { + assert_eq!( + observed.tally.final_copies, 0, + "closed native room delivers no final packet" + ); + } +} + +#[test] +fn g2_native_rows_declare_finite_zero_gemini_budgets() { + for case in [ + G2Case::EndThenDeadline, + G2Case::DeadlineFirst, + G2Case::RoomClosed, + G2Case::AllReady, + ] { + assert_eq!(case.actions().len() + 1, 5); + assert!(!case.actions().contains(&"retry_report")); + assert!(!case.actions().contains(&FINAL_ACTION_ID)); + } + assert_eq!(4 * 720 + 1200, 68 * 60); +} + +fn g2_agent_closed( + result: Result< + Result<(), Box>, + tokio::time::error::Elapsed, + >, +) -> bool { + match result { + Ok(Ok(())) => true, + Ok(Err(error)) => matches!( + error.downcast_ref::<::livekit::RoomError>(), + Some(::livekit::RoomError::AlreadyClosed) + ), + Err(_) => false, + } +} + +fn g2_require_deny_proxy(config: &crate::config::AgentConfig, settings: &Value) { + let proxy = std::env::var("CODETRIAL_G2_DENY_PROXY") + .expect("run through the G2 non-forwarding proxy wrapper"); + let parsed = reqwest::Url::parse(&proxy).expect("local denial proxy URL"); + assert_eq!(parsed.scheme(), "http"); + assert!( + parsed + .host_str() + .and_then(|host| host + .trim_matches(['[', ']']) + .parse::() + .ok()) + .is_some_and(|address| address.is_loopback()) + ); + assert!(parsed.username().is_empty() && parsed.password().is_none()); + for key in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert_eq!( + std::env::var(key).as_deref(), + Ok(proxy.as_str()), + "mandatory outside-HTTP denial" + ); + } + let bypass = std::env::var("NO_PROXY").expect("explicit native host bypasses"); + assert_eq!(std::env::var("no_proxy").as_deref(), Ok(bypass.as_str())); + let host = reqwest::Url::parse(&config.livekit_url) + .unwrap() + .host_str() + .unwrap() + .to_owned(); + let mut native = Vec::new(); + for entry in bypass.split(',') { + assert!( + !entry.is_empty() + && !entry.contains('*') + && !entry.to_ascii_lowercase().contains("googleapis.com") + ); + if entry != "127.0.0.1" && entry != "::1" { + native.push(entry.to_owned()); + } + } + assert!( + bypass.split(',').any(|entry| entry == host), + "the configured native host is explicitly allowed" + ); + let configured = reqwest::Url::parse(&config.livekit_url).unwrap(); + let configured_port = configured.port_or_known_default().unwrap(); + let endpoints = settings["native_endpoints"].as_array().unwrap(); + assert!( + endpoints + .iter() + .any(|endpoint| endpoint["host"] == host && endpoint["port"] == configured_port), + "the configured SDK host:port must be prospectively allowed for CONNECT" + ); + let mut endpoint_hosts = endpoints + .iter() + .map(|endpoint| { + let endpoint_host = endpoint["host"].as_str().unwrap(); + let port = endpoint["port"].as_u64().unwrap(); + assert!( + (1..=65535).contains(&port) + && !endpoint_host.contains('*') + && !endpoint_host + .to_ascii_lowercase() + .contains("googleapis.com") + ); + endpoint_host.to_owned() + }) + .collect::>(); + endpoint_hosts.sort(); + endpoint_hosts.dedup(); + let actual_allow: Vec = serde_json::from_str( + &std::env::var("CODETRIAL_G2_PROXY_ALLOW") + .expect("proxy exports its actual exact CONNECT allowlist"), + ) + .unwrap(); + let as_pairs = |values: &[Value]| { + let mut pairs = values + .iter() + .map(|endpoint| { + ( + endpoint["host"].as_str().unwrap().to_owned(), + endpoint["port"].as_u64().unwrap(), + ) + }) + .collect::>(); + pairs.sort(); + pairs + }; + assert_eq!( + as_pairs(&actual_allow), + as_pairs(endpoints), + "actual proxy CONNECT policy equals prospectively hashed settings" + ); + let mut declared = settings["native_hosts"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_owned()) + .collect::>(); + native.sort(); + declared.sort(); + assert_eq!( + endpoint_hosts, declared, + "HTTP bypasses are only the declared SDK endpoint hosts" + ); + declared.retain(|entry| entry != "127.0.0.1" && entry != "::1"); + assert_eq!( + native, declared, + "only prospectively named native hosts bypass the restricted proxy" + ); +} + +fn g2_settings(manifest: &Value) -> Value { + let bytes = std::fs::read( + std::env::var("CODETRIAL_G2_SETTINGS").expect("prospective G2 settings path"), + ) + .unwrap(); + assert_eq!( + crate::sha256_hex(&[&bytes]), + manifest["settings_sha256"].as_str().unwrap() + ); + let settings: Value = serde_json::from_slice(&bytes).unwrap(); + let room = settings["room_name"].as_str().unwrap(); + assert!( + !room.is_empty() + && room.len() <= 63 + && room + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) + ); + assert!( + room.starts_with(manifest["run_id"].as_str().unwrap()), + "cleanup owns only the prospectively named run room" + ); + settings +} + +/// Controller-owned idempotent cleanup of the one prospectively named Room. +#[tokio::test] +#[ignore = "NOT RUN: controller-owned G2 named-resource cleanup; requires the frozen manifest/settings and native config"] +async fn g2_cleanup_named_native_room() { + let bytes = + std::fs::read(std::env::var("CODETRIAL_G2_MANIFEST").expect("prospective manifest path")) + .unwrap(); + let manifest: Value = serde_json::from_slice(&bytes).unwrap(); + let settings = g2_settings(&manifest); + let requested = + std::env::var("CODETRIAL_G2_CLEAN_ROOM").expect("controller supplied exact named room"); + assert_eq!(settings["room_name"], requested); + assert_eq!( + manifest["named_resources"], + json!([{"kind":"room", "name":requested}]) + ); + let config = g2_native_config(); + g2_require_deny_proxy(&config, &settings); + g2_delete(&config, &requested) + .await + .expect("bounded owned-room cleanup"); +} + +/// Offline config metadata only: writes host/port without calling a provider. +#[test] +#[ignore = "offline config metadata operation; requires selected config and new private output; no provider or Room call"] +fn g2_write_native_endpoint_settings() { + let out = std::env::var("CODETRIAL_G2_ENDPOINTS_OUT") + .expect("new private endpoint metadata output path"); + let config = g2_native_config(); + let url = reqwest::Url::parse(&config.livekit_url).expect("selected native URL parses"); + assert!(matches!(url.scheme(), "ws" | "wss")); + let host = url.host_str().expect("selected native URL has a host"); + let port = url + .port_or_known_default() + .expect("selected native URL has a port"); + let bytes = serde_json::to_vec_pretty(&json!({ + "native_hosts":[host], "native_endpoints":[{"host":host,"port":port}] + })) + .unwrap(); + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(out) + .expect("endpoint metadata output must be new"); + std::io::Write::write_all(&mut output, &bytes).expect("write endpoint metadata only"); +} diff --git a/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs new file mode 100644 index 000000000..db9342a0f --- /dev/null +++ b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs @@ -0,0 +1,676 @@ +async fn acquisition_receive_tool(rig: &mut Rig, id: &str, tool: &str) { + let mut context = rig.turn.context( + &mut rig.output, + &mut rig.gemini, + &mut rig.board, + &mut rig.media, + ); + session::handle_gemini_event( + &rig.room, + &mut context, + crate::gemini::GeminiEvent::ToolCall(vec![crate::gemini::GeminiFunctionCall { + id: id.into(), + name: tool.into(), + args: serde_json::json!({"fromLine":1}), + }]), + Interruptible::Yes, + ) + .await + .unwrap(); +} + +async fn acquisition_receive_editor(rig: &mut Rig, id: &str) { + acquisition_receive_tool(rig, id, crate::runtime::TOOL_READ_EDITOR).await; +} + +async fn acquisition_actual_tool_send(failed: bool) { + use crate::livekit::probe::acquisition::{Capture, Limits, install}; + for (held, discarded) in [(false, false), (true, false), (false, true)] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + rig.turn.state.code = "def solve(): return 'send-result-sentinel'".into(); + if held { + rig.turn.state.thinking_hold = crate::agent::ThinkingHold::Held { + since: Instant::now(), + }; + } + rig.turn.activity.socket.discarding_output = discarded; + let now = Instant::now(); + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + rig.turn.activity.latency.anchor(now); + rig.turn.activity.latency.prompt(now); + let capture = Capture::new( + "recovery-test", + 7, + Limits { + batches: 1, + calls: 1, + bytes: 65536, + }, + ) + .unwrap(); + let installed = install(&capture).unwrap(); + if failed { + rig.gemini.shutdown().await.unwrap(); + } + acquisition_receive_editor(&mut rig, "actual-tool-send").await; + // Join the real peer before any assertion can end this fixture. + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 1); + let batch = &snapshot.batches[0]; + assert_eq!(batch.calls.len(), 1); + assert_eq!(batch.calls[0].id, "actual-tool-send"); + + // Every executed call was received first, and a call that arrived under + // a discard says so. + assert_eq!(snapshot.receipts.len(), 1); + assert_eq!(snapshot.receipts[0].id, "actual-tool-send"); + assert_eq!(snapshot.receipts[0].batch, Some(0)); + + // A hold silences the interviewer as a discard does: the call is still + // answered, and neither kind is to be tied to a later reply. + assert_eq!(snapshot.receipts[0].discarded, held || discarded); + assert_eq!(batch.calls[0].name, crate::runtime::TOOL_READ_EDITOR); + assert!( + batch.calls[0].result["result"] + .as_str() + .unwrap() + .contains("send-result-sentinel") + ); + assert_eq!( + batch.local_send_completed, + Some(!failed), + "actual owner await result, independent of held/discarded timing gates" + ); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); + let responses = frames + .iter() + .filter_map(|frame| frame.pointer("/toolResponse/functionResponses")) + .filter_map(serde_json::Value::as_array) + .flatten() + .collect::>(); + assert_eq!(responses.len(), usize::from(!failed)); + if let Some(response) = responses.first() { + assert_eq!(response["id"], "actual-tool-send"); + assert!( + response["response"] == batch.calls[0].result, + "actual wire result matches captured execution" + ); + } + } +} + +#[tokio::test] +async fn acquisition_tool_send_ok_records_real_completion() { + if isolated("acquisition_tool_send_ok_records_real_completion") { + return; + } + acquisition_actual_tool_send(false).await; +} + +#[tokio::test] +async fn acquisition_tool_send_error_records_returned_failure() { + if isolated("acquisition_tool_send_error_records_returned_failure") { + return; + } + acquisition_actual_tool_send(true).await; +} + +async fn acquisition_reply_rig() -> ( + Rig, + std::sync::Arc, +) { + use crate::livekit::probe::acquisition::{Capture, Limits}; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + let capture = Capture::new( + "recovery-test", + 7, + Limits { + batches: 2, + calls: 4, + bytes: 65536, + }, + ) + .unwrap(); + (rig, capture) +} + +fn acquisition_gate_capture( + rig: &mut Rig, +) -> std::sync::Arc { + use crate::livekit::probe::acquisition::{Capture, Limits}; + let now = Instant::now(); + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + rig.turn.activity.latency.anchor(now); + rig.turn.activity.latency.prompt(now); + Capture::new( + "recovery-test", + 7, + Limits { + batches: 2, + calls: 2, + bytes: 65536, + }, + ) + .unwrap() +} + +#[tokio::test] +async fn acquisition_controlled_owner_await_cancellation_preserves_execution_boundary() { + if isolated("acquisition_controlled_owner_await_cancellation_preserves_execution_boundary") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + for stage in [ + Stage::Acknowledgement, + Stage::BoardSettlement, + Stage::BoardResend, + Stage::ToolSend, + ] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let tool = match stage { + Stage::BoardSettlement => { + rig.turn.state.example_drawing_enabled = true; + crate::runtime::TOOL_READ_BOARD + } + Stage::BoardResend => { + rig.turn.state.interview_mode = crate::agent::InterviewMode::Whiteboard; + rig.board.seed_recovery_image(vec![0x5a; 64]); + crate::runtime::TOOL_READ_BOARD + } + _ => crate::runtime::TOOL_READ_EDITOR, + }; + let capture = acquisition_gate_capture(&mut rig); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, stage, None).unwrap(); + let held = owner::install(&gate).unwrap(); + let mut call = Box::pin(acquisition_receive_tool(&mut rig, "cancelled-owner", tool)); + tokio::select! { + entered = gate.entered() => { entered.unwrap(); }, + () = &mut call => panic!("owner must reach its selected controlled await"), + () = tokio::time::sleep(Duration::from_secs(2)) => panic!("controlled owner gate did not enter"), + } + drop(call); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + let executed = matches!(stage, Stage::BoardResend | Stage::ToolSend); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), usize::from(executed), "{stage:?}"); + + // Received before the acknowledgement gate, so cancellation at any gate + // leaves the call on record, executed only if the batch got that far. + assert_eq!(snapshot.receipts.len(), 1, "{stage:?}"); + assert_eq!(snapshot.receipts[0].id, "cancelled-owner"); + assert_eq!( + snapshot.receipts[0].batch, + executed.then_some(0), + "{stage:?}" + ); + assert!(!snapshot.receipts[0].discarded); + if let Some(batch) = snapshot.batches.first() { + assert_eq!(batch.calls[0].id, "cancelled-owner"); + assert_eq!( + batch.local_send_completed, None, + "controlled cancellation is unknown, not Err" + ); + } + assert!( + frames.is_empty(), + "controlled gate is before underlying I/O polling" + ); + } +} + +#[tokio::test] +async fn acquisition_admitted_ticket_completes_original_capture_after_reinstallation() { + if isolated("acquisition_admitted_ticket_completes_original_capture_after_reinstallation") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let original = acquisition_gate_capture(&mut rig); + let replacement = acquisition_gate_capture(&mut rig); + let installed = install(&original).unwrap(); + let gate = Gate::new(&original, 1, Stage::ToolSend, None).unwrap(); + let held = owner::install(&gate).unwrap(); + let actor = async { + gate.entered().await.unwrap(); + assert_eq!(original.snapshot().batches.len(), 1); + drop(installed); + let replacement_installed = install(&replacement).unwrap(); + gate.release(); + replacement_installed + }; + let ((), replacement_installed) = tokio::join!( + acquisition_receive_editor(&mut rig, "original-ticket"), + actor + ); + let frames = rig.old_frames().await; + assert_eq!(tool_response_ids(&frames), ["original-ticket"]); + let first = original.snapshot(); + let second = replacement.snapshot(); + drop(replacement_installed); + drop(held); + assert_eq!(first.failure, None); + assert_eq!(first.batches[0].local_send_completed, Some(true)); + assert_eq!(first.batches[0].calls[0].id, "original-ticket"); + assert_eq!(second.failure, None); + assert!( + second.batches.is_empty(), + "completion must never resolve the current ACTIVE capture" + ); +} + +async fn acquisition_timing_gate_divergence( + initial: bool, + transition: crate::livekit::probe::acquisition::owner::Transition, +) { + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage, Transition}, + }; + for failed in [false, true] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + match transition { + Transition::Hold(_) => { + if !initial { + rig.turn.state.thinking_hold = crate::agent::ThinkingHold::Held { + since: Instant::now(), + }; + } + } + Transition::Discard(_) => rig.turn.activity.socket.discarding_output = !initial, + } + let capture = acquisition_gate_capture(&mut rig); + rig.turn + .activity + .turn_timing + .note_transcript(Instant::now()); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, Stage::ToolSend, Some(transition)).unwrap(); + let held = owner::install(&gate).unwrap(); + if failed { + rig.gemini.shutdown().await.unwrap(); + } + let actor = async { + gate.entered().await.unwrap(); + gate.release(); + }; + let ((), ()) = tokio::join!( + acquisition_receive_editor(&mut rig, "divergent-gates"), + actor + ); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + let later = + !rig.turn.state.interviewer_silenced() && !rig.turn.activity.socket.discarding_output; + assert_eq!( + later, !initial, + "the synthetic transition must actually diverge the two gates" + ); + let timing = rig + .turn + .activity + .turn_timing + .finish( + Instant::now(), + crate::livekit::timing::TimingOutcome::NoAudio, + ) + .unwrap(); + assert_eq!( + timing.tool_calls, + u64::from(initial), + "initial timed sample independently controls execution accounting" + ); + assert_eq!(timing.tool_response_errors, u64::from(initial && failed)); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 1); + assert_eq!(snapshot.batches[0].local_send_completed, Some(!failed)); + assert_eq!(tool_response_ids(&frames).len(), usize::from(!failed)); + } +} + +fn acquisition_check_later_log(output: &std::process::Output, later: bool) { + let stderr = String::from_utf8_lossy(&output.stderr); + let logged = stderr + .lines() + .filter_map(|line| line.strip_prefix("codetrial latency ")) + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|record| record["stage"] == "tool_response") + .count(); + assert_eq!( + logged, + if later { 2 } else { 0 }, + "later log gate observed separately from the initial timed sample" + ); +} + +#[tokio::test] +async fn acquisition_tool_send_hold_turns_timing_off_after_initial_sample() { + if isolated_checked( + "acquisition_tool_send_hold_turns_timing_off_after_initial_sample", + |out| acquisition_check_later_log(out, false), + ) { + return; + } + acquisition_timing_gate_divergence( + true, + crate::livekit::probe::acquisition::owner::Transition::Hold(true), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_release_turns_logging_on_after_untimed_execution() { + if isolated_checked( + "acquisition_tool_send_release_turns_logging_on_after_untimed_execution", + |out| acquisition_check_later_log(out, true), + ) { + return; + } + acquisition_timing_gate_divergence( + false, + crate::livekit::probe::acquisition::owner::Transition::Hold(false), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_discard_turns_logging_off_after_timed_execution() { + if isolated_checked( + "acquisition_tool_send_discard_turns_logging_off_after_timed_execution", + |out| acquisition_check_later_log(out, false), + ) { + return; + } + acquisition_timing_gate_divergence( + true, + crate::livekit::probe::acquisition::owner::Transition::Discard(true), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_discard_end_turns_logging_on_after_untimed_execution() { + if isolated_checked( + "acquisition_tool_send_discard_end_turns_logging_on_after_untimed_execution", + |out| acquisition_check_later_log(out, true), + ) { + return; + } + acquisition_timing_gate_divergence( + false, + crate::livekit::probe::acquisition::owner::Transition::Discard(false), + ) + .await; +} + +#[tokio::test] +async fn acquisition_consumed_gate_does_not_hold_a_later_tool_batch() { + if isolated("acquisition_consumed_gate_does_not_hold_a_later_tool_batch") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let capture = acquisition_gate_capture(&mut rig); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, Stage::ToolSend, None).unwrap(); + let held = owner::install(&gate).unwrap(); + assert!(owner::install(&gate).is_err()); + let actor = async { + gate.entered().await.unwrap(); + gate.release(); + }; + let ((), ()) = tokio::join!(acquisition_receive_editor(&mut rig, "first-gated"), actor); + tokio::time::timeout( + Duration::from_secs(2), + acquisition_receive_editor(&mut rig, "later-ungated"), + ) + .await + .unwrap(); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + assert_eq!(tool_response_ids(&frames), ["first-gated", "later-ungated"]); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 2); + assert!( + snapshot + .batches + .iter() + .all(|batch| batch.local_send_completed == Some(true)) + ); +} + +async fn acquisition_event(rig: &mut Rig, event: crate::gemini::GeminiEvent) { + let mut context = rig.turn.context( + &mut rig.output, + &mut rig.gemini, + &mut rig.board, + &mut rig.media, + ); + session::handle_gemini_event(&rig.room, &mut context, event, Interruptible::Yes) + .await + .unwrap(); +} + +#[tokio::test] +async fn acquisition_keeps_the_interviewer_segments_the_page_was_sent() { + if isolated("acquisition_keeps_the_interviewer_segments_the_page_was_sent") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_receive_editor(&mut rig, "before-reply").await; + acquisition_event( + &mut rig, + GeminiEvent::OutputTranscript("The answer ".into()), + ) + .await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("is 42.".into())).await; + // The candidate's own speech closes with the same turn and is not a reply. + rig.media.identity = Some("candidate".into()); + acquisition_event(&mut rig, GeminiEvent::InputTranscript("my words".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Next.".into())).await; + let _ = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + let replies: Vec<_> = snapshot + .replies + .iter() + .map(|reply| (reply.text.as_str(), reply.final_segment, reply.views)) + .collect(); + + // The first turn's newest view, sealed by the turn's end; the next turn's + // segment is still open and is its own reply. + assert_eq!( + replies, + [("The answer is 42.", true, 3), ("Next.", false, 1)] + ); + assert_ne!(snapshot.replies[0].id, snapshot.replies[1].id); + assert!(snapshot.replies.iter().all(|reply| !reply.revised)); + // The tool call preceded both, and was answered before either. + assert!( + snapshot + .replies + .iter() + .all(|reply| (reply.receipts_before, reply.batches_before) == (1, 1)) + ); +} + +#[tokio::test] +async fn acquisition_seals_the_line_a_cut_marked_before_the_next_words() { + if isolated("acquisition_seals_the_line_a_cut_marked_before_the_next_words") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Cut words.".into())).await; + rig.turn + .state + .interviewer_delivery + .cancel(crate::agent::DeliveryCut::Generation, Instant::now()); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("New words.".into())).await; + let snapshot = capture.snapshot(); + drop(installed); + let replies: Vec<_> = snapshot + .replies + .iter() + .map(|reply| (reply.text.as_str(), reply.final_segment)) + .collect(); + assert_eq!(replies, [("Cut words.", true), ("New words.", false)]); +} + +#[tokio::test] +async fn acquisition_marks_an_ending_the_guard_swallowed() { + if isolated("acquisition_marks_an_ending_the_guard_swallowed") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::{Ending, install}; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Said.".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("More.".into())).await; + acquisition_event(&mut rig, GeminiEvent::Interrupted).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let snapshot = capture.snapshot(); + drop(installed); + let seen: Vec<_> = snapshot + .boundaries + .iter() + .map(|boundary| (boundary.ending, boundary.accepted, boundary.replies_before)) + .collect(); + + // The ending that follows an interruption closes nothing: the provider + // sends it for the generation the interruption already cut. + assert_eq!( + seen, + [ + (Ending::Complete, true, 1), + (Ending::Interrupted, true, 2), + (Ending::Complete, false, 2) + ] + ); +} + +#[tokio::test] +async fn acquisition_does_not_take_a_departed_candidates_words_for_a_reply() { + if isolated("acquisition_does_not_take_a_departed_candidates_words_for_a_reply") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + rig.media.identity = Some("candidate".into()); + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::InputTranscript("my words".into())).await; + + // The candidate leaves before the turn closes, so the close publishes their + // final segment without an identity, as the agent. + rig.media.identity = None; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + assert!(snapshot.replies.is_empty(), "{:?}", snapshot.replies); +} + +/// The report the fixture leaves behind, from a script that exercises every +/// shape the JS consumer reads: a reply that cites an executed call, a plain +/// completed reply, and an interrupted one whose late ending is swallowed. +/// Wall-clock readings and the byte counts that include them are replaced with +/// zero, because a fixture cannot hold the time the test ran; the consumer's +/// tests put times back. +#[tokio::test] +async fn acquisition_report_is_the_wire_fixture_the_consumer_reads() { + if isolated("acquisition_report_is_the_wire_fixture_the_consumer_reads") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::{Arming, Limits, install}; + let (mut rig, _) = acquisition_reply_rig().await; + let arming = Arming::detached(Limits { + batches: 8, + calls: 8, + bytes: 65536, + }); + arming.on_assignment("recovery-test"); + arming.on_bind("recovery-test", 7); + let capture = arming.capture().unwrap(); + let installed = install(&capture).unwrap(); + acquisition_receive_editor(&mut rig, "call-1").await; + for text in ["REPLY-A one. ", "two."] { + acquisition_event(&mut rig, GeminiEvent::OutputTranscript(text.into())).await; + } + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("REPLY-B.".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event( + &mut rig, + GeminiEvent::OutputTranscript("REPLY-C cut".into()), + ) + .await; + acquisition_event(&mut rig, GeminiEvent::Interrupted).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let _ = rig.old_frames().await; + let mut report = arming.report(); + drop(installed); + fn zero_clocks(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(map) => { + for (key, inner) in map.iter_mut() { + // Byte counts embed the digits of the clock they carry; the + // call count does not, and is kept. + if key == "emitted_unix_ms" + || (key.starts_with("retained_") && key.ends_with("_bytes")) + { + *inner = 0.into(); + } else { + zero_clocks(inner); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(zero_clocks), + _ => {} + } + } + zero_clocks(&mut report); + let path = "tests/fixtures/acquisition-report.json"; + let rendered = format!("{}\n", serde_json::to_string_pretty(&report).unwrap()); + if std::env::var_os("CODETRIAL_UPDATE_FIXTURES").is_some() { + std::fs::write(path, &rendered).unwrap(); + } + assert_eq!( + std::fs::read_to_string(path).unwrap(), + rendered, + "regenerate with CODETRIAL_UPDATE_FIXTURES=1" + ); +} diff --git a/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs b/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs index 071b9f6d1..c8b383503 100644 --- a/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs +++ b/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs @@ -981,7 +981,7 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { let kept = |driver: &Driver<'_, '_, '_>| { ( driver.held_contexts.len(), - driver.held_bytes, + driver.held_contexts.iter().map(String::len).sum::(), driver.held_omitted, driver.held_omitted_total, ) @@ -1001,11 +1001,7 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { // Two that together reach the bound exactly are both kept; one more byte // drops the first and only the first. driver.held_contexts.clear(); - ( - driver.held_bytes, - driver.held_omitted, - driver.held_omitted_total, - ) = (0, 0, 0); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); driver.retain_held_context("a".repeat(40_000)); driver.retain_held_context("b".repeat(MAX_HELD_BYTES - 40_000)); assert_eq!(kept(&driver), (2, MAX_HELD_BYTES, 0, 0)); @@ -1013,13 +1009,20 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { assert_eq!(kept(&driver), (2, MAX_HELD_BYTES - 40_000 + 1, 1, 1)); assert!(driver.held_contexts[0].starts_with('b')); + // Room for one can take more than one eviction, each subtracted from the + // running total: dropping only the first would leave the bound exceeded. + driver.held_contexts.clear(); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); + for (fill, len) in [('a', 30_000), ('b', 30_000), ('c', 4_000)] { + driver.retain_held_context(fill.to_string().repeat(len)); + } + driver.retain_held_context("d".repeat(40_000)); + assert_eq!(kept(&driver), (2, 44_000, 2, 2)); + assert!(driver.held_contexts[0].starts_with('c')); + // The count bound drops the oldest, one at a time. driver.held_contexts.clear(); - ( - driver.held_bytes, - driver.held_omitted, - driver.held_omitted_total, - ) = (0, 0, 0); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); for n in 0..MAX_HELD_CONTEXTS { driver.retain_held_context(format!("{n}")); } @@ -1029,10 +1032,6 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { assert_eq!(driver.held_contexts.len(), MAX_HELD_CONTEXTS); assert_eq!((driver.held_omitted, driver.held_omitted_total), (1, 1)); assert_eq!(driver.held_contexts[0], "1"); - assert_eq!( - driver.held_bytes, - driver.held_contexts.iter().map(String::len).sum::() - ); drop(driver); rig.finish().await; } @@ -1163,7 +1162,6 @@ async fn reference_debt_survives_noop_packets_and_action_supersession() { } assert!(driver.context.state.earlier_steps_named.is_empty()); driver.held_contexts.push_back("unsent held summary".into()); - driver.held_bytes = "unsent held summary".len(); driver.round = Some("skipped"); driver.loops.pending_round_notice = Some("started"); assert!(!driver.drain(&mut closed).await.unwrap()); diff --git a/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs b/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs new file mode 100644 index 000000000..16023a0cd --- /dev/null +++ b/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs @@ -0,0 +1,172 @@ +// Initial bounded generation is distinct from provider attempts and receipts. + +#[tokio::test] +async fn g2_bounded_report_observes_only_polled_scoped_invocations() { + if isolated_checked_with_loopback( + "g2_bounded_report_observes_only_polled_scoped_invocations", + true, + |_| {}, + ) { + return; + } + let config = g2_fixture_config(); + let boot = crate::runtime::bootstrap(&config, "g2-polled-entry", Some("two-sum"), 45); + let keys = crate::gemini::GeminiKeys::single("g2-fixture-key"); + let refused = std::sync::atomic::AtomicBool::new(false); + let scope = crate::gemini::tests::ScopedReport::register(boot.room_name, "http://127.0.0.1:9/") + .unwrap(); + let future = super::super::report::generate_report_bounded( + &boot, + "fixture evidence", + &[], + ( + crate::agent::ReportRounds { + behavioral_opened: true, + follow_ups_released: true, + candidate_turns: 1, + }, + false, + ), + &keys, + 31, + &refused, + ); + tokio::pin!(future); + assert!( + scope.entries().is_empty(), + "constructing a lazy generation does not start it" + ); + + // The refused endpoint keeps the generation retrying, so the first poll + // leaves it pending and the second one is a real second poll. + let finished = std::future::poll_fn(|cx| { + std::task::Poll::Ready(std::future::Future::poll(future.as_mut(), cx).is_ready()) + }) + .await; + assert!(!finished, "the first poll leaves the generation pending"); + assert_eq!(scope.entries().len(), 1, "actual bounded body entered once"); + let _ = std::future::poll_fn(|cx| { + std::task::Poll::Ready(std::future::Future::poll(future.as_mut(), cx).is_ready()) + }) + .await; + assert_eq!( + scope.entries().len(), + 1, + "polling the same future does not recount" + ); + assert_eq!(scope.entries()[0].0, 31); +} + +#[tokio::test] +async fn g2_scoped_report_uses_actual_generator_material_and_wire() { + if isolated_checked_with_loopback( + "g2_scoped_report_uses_actual_generator_material_and_wire", + true, + |_| {}, + ) { + return; + } + let config = g2_fixture_config(); + let boot = crate::runtime::bootstrap(&config, "g2-generator-wire", Some("two-sum"), 45); + let keys = crate::gemini::GeminiKeys::single("g2-fixture-key"); + let refused = std::sync::atomic::AtomicBool::new(false); + let fixture = G2ReportFixture::start(boot.room_name).await; + let result = tokio::time::timeout( + Duration::from_secs(2), + super::super::report::generate_report_bounded( + &boot, + "unique g2 assessment material", + &[], + ( + crate::agent::ReportRounds { + behavioral_opened: true, + follow_ups_released: true, + candidate_turns: 1, + }, + false, + ), + &keys, + 37, + &refused, + ), + ) + .await; + result + .expect("registered report route reaches its local HTTP fixture") + .unwrap() + .unwrap(); + assert_eq!(fixture.scope.entries().len(), 1); + let bodies = fixture.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 1); + assert!( + bodies[0] + .to_string() + .contains("unique g2 assessment material") + ); + assert_eq!(bodies[0]["generationConfig"]["seed"], 37); + assert_eq!( + bodies[0]["generationConfig"]["responseSchema"], + crate::agent::report_response_schema(true) + ); +} + +#[test] +fn g2_repeated_server_end_cannot_create_a_second_finish_intent() { + let mut state = RuntimeState::default(); + let payload = serde_json::json!({"type":"end_interview", "reason":"candidate_ended"}); + let first = crate::agent::apply_server_event_at(&mut state, TOPIC_CONTROL, &payload, 0.0, 1); + assert!(first.finish_interview.is_some()); + let deadline = serde_json::json!({"type":"end_interview", "reason":"time_up"}); + let second = crate::agent::apply_server_event_at(&mut state, TOPIC_CONTROL, &deadline, 0.0, 2); + assert!( + second.finish_interview.is_none(), + "deadline re-entry cannot finish an ended assessment twice" + ); +} + +fn g2_fixture_config() -> crate::config::AgentConfig { + load_from_pairs([ + ("LIVEKIT_URL", "wss://example.livekit.cloud"), + ("LIVEKIT_API_KEY", "devkey"), + ("LIVEKIT_API_SECRET", "devsecret"), + ("GOOGLE_API_KEY", "g2-fixture-key"), + ]) + .unwrap() +} + +struct G2ReportFixture { + scope: crate::gemini::tests::ScopedReport, + bodies: std::sync::Arc>>, + server: tokio::task::JoinHandle<()>, +} + +impl G2ReportFixture { + async fn start(scope: &str) -> Self { + let bodies = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = std::sync::Arc::clone(&bodies); + let answer = crate::gemini::tests::valid_report(); + let app = axum::Router::new().route("/", axum::routing::post(move |axum::Json(body): axum::Json| { + let seen = std::sync::Arc::clone(&seen); + let answer = answer.clone(); + async move { + seen.lock().unwrap().push(body); + axum::Json(serde_json::json!({"candidates":[{"content":{"parts":[{"text":answer.to_string()}]}}]})) + } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let scope = crate::gemini::tests::ScopedReport::register(scope, &endpoint).unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + Self { + scope, + bodies, + server, + } + } +} + +impl Drop for G2ReportFixture { + fn drop(&mut self) { + self.server.abort(); + } +} diff --git a/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs b/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs index ee0863547..ae1d4716d 100644 --- a/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs +++ b/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs @@ -1126,14 +1126,25 @@ impl WriteSite { } } -fn write_site_usage(output: &std::process::Output, drain: &str, rows: usize) { +fn write_site_usage( + output: &std::process::Output, + drain: &str, + rows: usize, + reference: Option<&str>, +) { let stderr = String::from_utf8_lossy(&output.stderr); let usage = stderr .lines() .filter(|line| line.starts_with("codetrial live_turn_usage ")) .collect::>(); - assert_eq!(usage.len(), rows * 2); - for pair in usage.as_chunks::<2>().0 { + let total_rows = rows + usize::from(reference.is_some()); + assert_eq!(usage.len(), total_rows * 2); + for (index, pair) in usage.as_chunks::<2>().0.iter().enumerate() { + let drain = if index == 0 { + reference.unwrap_or(drain) + } else { + drain + }; for (line, socket, cause, prompt, event) in [(pair[0], 1, "tool", 11, 1), (pair[1], 2, drain, 29, 2)] { @@ -1155,6 +1166,6 @@ fn write_site_usage(output: &std::process::Output, drain: &str, rows: usize) { .lines() .filter(|line| line.starts_with("recovery ingress: ")) .count(), - rows + total_rows ); } diff --git a/tests/unit/livekit/recovery_parts/terminal_matrices.rs b/tests/unit/livekit/recovery_parts/terminal_matrices.rs index ecee98aae..e87e8c5eb 100644 --- a/tests/unit/livekit/recovery_parts/terminal_matrices.rs +++ b/tests/unit/livekit/recovery_parts/terminal_matrices.rs @@ -1067,6 +1067,8 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() ) { return; } + const LISTENING_CONTEXT: &str = + "[SYSTEM EVENT] The candidate continues speaking. Keep listening; no reply is requested."; let mut rows = 0; for phase in PREFIX_PHASES { for cause in PREFIX_STOPS { @@ -1126,7 +1128,7 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() let note: serde_json::Value = serde_json::from_str(note.to_text().unwrap()).unwrap(); assert_eq!(note["clientContent"]["turnComplete"], false); - assert!(note.to_string().contains("Keep listening")); + assert_eq!(super::super::tests::sent_text(¬e), LISTENING_CONTEXT); messages.push(note); } } @@ -1283,6 +1285,15 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() rig.turn.state.evidence_ledger.metrics.turn_prompt_count, u32::from(phase.transcribed()) ); + assert_eq!( + rig.turn.state.evidence_ledger.metrics.turn_prompt_bytes, + if phase.transcribed() { + u64::try_from(LISTENING_CONTEXT.len()).unwrap() + } else { + 0 + }, + "{phase:?} {cause:?}" + ); assert_eq!( rig.turn .state diff --git a/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs b/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs index 39b05dcc6..03323049f 100644 --- a/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs +++ b/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs @@ -1277,7 +1277,6 @@ async fn omitted_held_contexts_are_counted_once_and_reach_the_wire_once() { assert!(driver.pending()); assert!(!driver.drain(&mut fresh).await.unwrap()); assert!(driver.held_contexts.is_empty()); - assert_eq!(driver.held_bytes, 0); assert_eq!((driver.held_omitted, driver.held_omitted_total), (0, 8)); assert!(!driver.pending()); feed(&mut driver, 33); diff --git a/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs b/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs index 4219e87b4..7552b7efc 100644 --- a/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs +++ b/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs @@ -21,8 +21,14 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool } else { &PREFIX_STOPS }; + let needs_reference = !physical && !after && site != WriteSite::BoardImage; + let reference = needs_reference.then_some((true, OpenStop::End)); + let cases = reference + .into_iter() + .chain(causes.iter().map(|&cause| (after, cause))); + let mut reference_bytes = None; let mut rows = 0; - for &cause in causes { + for (index, (after, cause)) in cases.enumerate() { let resumed = site == WriteSite::SilentEditor; let mut rig = Rig::with_old_checkpoint("ws://127.0.0.1:9/".into(), Some(11), resumed).await; rig.gemini.input_cause = Some("tool"); @@ -332,6 +338,10 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool .is_err() ); assert!(rig.loops.recovery_writes.is_empty()); + let constructed = ( + rig.turn.state.evidence_ledger.metrics.turn_prompt_count, + rig.turn.state.evidence_ledger.metrics.turn_prompt_bytes, + ); rig.finish().await; finish.send(()).unwrap(); let (messages, received) = server.await.unwrap(); @@ -343,9 +353,41 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool expected_before, "no write after the targeted one may reach the wire: {site:?} {cause:?}" ); + let expected: Option = match site { + WriteSite::Brief | WriteSite::ListeningBrief => Some(1), + WriteSite::HeldContext | WriteSite::SilentEditor => Some(2), + WriteSite::HeldAction => Some(3), + WriteSite::BoardImage => None, + }; + if let Some(expected) = expected { + assert_eq!(constructed.0, expected, "{site:?} {cause:?} after={after}"); + let delivered: u64 = messages + .iter() + .map(|message| { + u64::try_from(super::super::tests::sent_text(message).len()).unwrap() + }) + .sum(); + + // The reference shares this fixture and sums both clock and body. + // Cancellation must retain the same full charge, not just more + // bytes than the earlier writes delivered. + if after { + assert_eq!(constructed.1, delivered, "{site:?} {cause:?}"); + if needs_reference && index == 0 { + reference_bytes = Some(delivered); + } + } else { + assert_eq!( + constructed.1, + reference_bytes.expect("the matching completed write ran first"), + "{site:?} {cause:?} retains the full constructed input" + ); + assert!(constructed.1 > delivered, "{site:?} {cause:?}"); + } + } rows += 1; } - assert_eq!(rows, causes.len()); + assert_eq!(rows, causes.len() + usize::from(needs_reference)); } macro_rules! write_site_test { @@ -367,7 +409,11 @@ macro_rules! write_site_test { } else { $site.drain_cause() }; - if isolated_checked(stringify!($name), |out| write_site_usage(out, drain, rows)) { + let reference = (!$physical && !$after && $site != WriteSite::BoardImage) + .then_some($site.after_cause()); + if isolated_checked(stringify!($name), |out| { + write_site_usage(out, drain, rows, reference); + }) { return; } write_site_terminal_matrix($site, $physical, $after).await; @@ -1244,15 +1290,6 @@ impl Rig { .await .unwrap(); } - - /// Joins the old socket's peer and returns every frame it received. - async fn old_frames(&mut self) -> Vec { - self.gemini.shutdown().await.unwrap(); - tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) - .await - .unwrap() - .unwrap() - } } /// Like `scripted_fresh_gemini`, but a ready socket keeps what it is sent. diff --git a/tests/unit/livekit/report.rs b/tests/unit/livekit/report.rs index 99d7cb92a..32bff2a8c 100644 --- a/tests/unit/livekit/report.rs +++ b/tests/unit/livekit/report.rs @@ -1621,11 +1621,12 @@ fn recovery_offer_matches_browser_limits_and_generation_deadline() { limits["quotaRetryAfterSeconds"] ); - // The regenerated report goes through the same receipt wait, so a page that - // stops waiting before every attempt could land drops one in flight. + // Generation can finish while the candidate is away; a return within the + // grace must still leave every receipt attempt time to land. let delivery = DELIVERY_WAIT * DELIVERY_ATTEMPTS as u32; assert!( - limits["retryWaitSeconds"].as_u64().unwrap() > (REPORT_TIMEOUT + delivery).as_secs() + 3 + limits["retryWaitSeconds"].as_u64().unwrap() + > (REPORT_TIMEOUT + REJOIN_GRACE + delivery).as_secs() + 3 ); } @@ -2011,47 +2012,18 @@ fn only_the_candidate_coming_and_going_moves_a_recovery_wait() { } } -/// Drives a recovery wait from a script of events, each sent after its delay. -async fn scripted_recovery( - script: Vec<(u64, RecoveryEvent)>, - generation: impl std::future::Future, - readiness: Readiness, -) -> (Option, Vec) { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut fixture = RecoveryFixture::new(rx); - let sender = async move { - for (delay, event) in script { - tokio::time::sleep(std::time::Duration::from_secs(delay)).await; - let _ = tx.send(event); - } - tx - }; - let (result, _tx) = tokio::join!( - recover_report( - &mut fixture, - None, - generation, - REPORT_RECOVERY_WINDOW, - std::time::Duration::ZERO, - tokio::time::Instant::now(), - move || readiness, - ), - sender, - ); - (result, fixture.1) -} +include!("report/scripted_recovery.rs"); /// A full LiveKit rejoin looks like the candidate leaving and coming back. An /// accepted regeneration keeps running through it, and the report it returns /// while they are away waits for them instead of going into an empty room. #[tokio::test(start_paused = true)] async fn a_rejoin_within_the_grace_keeps_an_accepted_regeneration() { - let started = tokio::time::Instant::now(); let generation = async { tokio::time::sleep(std::time::Duration::from_secs(5)).await; Ok(Ok(serde_json::json!({"result": "complete"}))) }; - let (result, notices) = scripted_recovery( + let (result, notices, returned_at) = timed_scripted_recovery( vec![ (0, RecoveryEvent::Retry), (1, RecoveryEvent::Away), @@ -2063,7 +2035,7 @@ async fn a_rejoin_within_the_grace_keeps_an_accepted_regeneration() { .await; assert_eq!(result.unwrap().unwrap().unwrap()["result"], "complete"); assert_eq!(notices, [RecoveryNotice::Accepted]); - assert!(started.elapsed() >= std::time::Duration::from_secs(20)); + assert_eq!(returned_at, std::time::Duration::from_secs(20)); } /// Gone past the grace is gone, before a retry or during the regeneration. diff --git a/tests/unit/livekit/report/scripted_recovery.rs b/tests/unit/livekit/report/scripted_recovery.rs new file mode 100644 index 000000000..fc4737d1d --- /dev/null +++ b/tests/unit/livekit/report/scripted_recovery.rs @@ -0,0 +1,73 @@ +/// Drives a recovery wait from a script of events, each sent after its delay. +async fn scripted_recovery( + script: Vec<(u64, RecoveryEvent)>, + generation: impl std::future::Future, + readiness: Readiness, +) -> (Option, Vec) { + let (result, notices, _) = timed_scripted_recovery(script, generation, readiness).await; + (result, notices) +} + +async fn timed_scripted_recovery( + script: Vec<(u64, RecoveryEvent)>, + generation: impl std::future::Future, + readiness: Readiness, +) -> (Option, Vec, Duration) { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let mut fixture = RecoveryFixture::new(rx); + let started = tokio::time::Instant::now(); + let sender = async move { + for (delay, event) in script { + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + let _ = tx.send(event); + } + tx + }; + // The sender can finish later than the holder and hide an early return. + let ((result, returned_at), _tx) = tokio::join!( + async { + let result = recover_report( + &mut fixture, + None, + generation, + REPORT_RECOVERY_WINDOW, + std::time::Duration::ZERO, + tokio::time::Instant::now(), + move || readiness, + ) + .await; + (result, started.elapsed()) + }, + sender, + ); + (result, fixture.1, returned_at) +} + +#[tokio::test(start_paused = true)] +async fn a_late_rejoin_fits_the_browser_report_wait() { + let generation = async { + tokio::time::sleep(Duration::from_secs(124)).await; + Ok(Ok(serde_json::json!({"result": "complete"}))) + }; + let (result, notices, returned_at) = timed_scripted_recovery( + vec![ + (0, RecoveryEvent::Retry), + (123, RecoveryEvent::Away), + (29, RecoveryEvent::Back), + ], + generation, + Readiness::Ready, + ) + .await; + assert_eq!(result.unwrap().unwrap().unwrap()["result"], "complete"); + assert_eq!(notices, [RecoveryNotice::Accepted]); + assert_eq!(returned_at, Duration::from_secs(152)); + let limits: serde_json::Value = + serde_json::from_str(include_str!("../../../fixtures/report-recovery.json")).unwrap(); + let browser_wait = Duration::from_secs(limits["retryWaitSeconds"].as_u64().unwrap()); + let delivery = DELIVERY_WAIT * u32::try_from(DELIVERY_ATTEMPTS).unwrap(); + assert!( + browser_wait > returned_at + delivery, + "browser wait {browser_wait:?} must cover holder return {returned_at:?} and delivery {delivery:?}" + ); +} diff --git a/tests/unit/livekit/session.rs b/tests/unit/livekit/session.rs index 435e168b4..af24e53d5 100644 --- a/tests/unit/livekit/session.rs +++ b/tests/unit/livekit/session.rs @@ -362,63 +362,7 @@ fn continuity_reminders_leave_opening_default_and_inactive_inputs_alone() { } } -#[test] -fn real_tool_dispatch_records_only_tools_before_reply_audio() { - let start = Instant::now(); - let mut activity = RuntimeActivity::new(start); - let mut state = RuntimeState { - code: "return 1;".into(), - ..RuntimeState::default() - }; - let call = GeminiFunctionCall { - id: "read-1".into(), - name: TOOL_READ_EDITOR.into(), - args: serde_json::json!({}), - }; - activity.note_candidate_finished(start, false); - let response = execute_timed_tool_batch( - &mut state, - &mut activity, - vec![call.clone(), call.clone()], - 0, - true, - ); - assert!( - response[0].1["result"] - .as_str() - .is_some_and(|code| code.contains("return 1;")) - ); - let first = activity - .turn_timing - .note_audio(start + Duration::from_secs(1), true) - .unwrap(); - assert_eq!(first.tool_calls, 2); - assert_eq!(first.tool_wait, Some(first.tool_exec)); - assert!(first.pre_tool.is_some()); - execute_timed_tool_batch( - &mut state, - &mut activity, - vec![call.clone(), call.clone()], - 0, - true, - ); - assert!( - activity - .turn_timing - .finish(start + Duration::from_secs(2), TimingOutcome::NoAudio) - .is_none() - ); - activity.note_candidate_finished(start + Duration::from_secs(3), false); - execute_timed_tool_batch(&mut state, &mut activity, vec![call], 0, false); - let next = activity - .turn_timing - .note_audio(start + Duration::from_secs(4), true) - .unwrap(); - assert_eq!( - next.tool_calls, 0, - "discarded/held tool responses must not be carried forward" - ); -} +include!("session_parts/tool_execution.rs"); #[test] fn timing_transcripts_follow_the_existing_late_fragment_guard() { diff --git a/tests/unit/livekit/session/observations.rs b/tests/unit/livekit/session/observations.rs index cc3b80d46..f38a1d2c7 100644 --- a/tests/unit/livekit/session/observations.rs +++ b/tests/unit/livekit/session/observations.rs @@ -1,3 +1,5 @@ +use crate::test_isolation as isolation; + use super::*; use crate::livekit::session::{execute_tool_batch_observed, execute_tool_call}; use serde_json::json; @@ -147,6 +149,7 @@ fn accepted_close_request_is_observed_before_tool_batch_response_timing() { ], 0, true, + &mut None, ); assert_eq!(answers.len(), 2); assert!( @@ -216,3 +219,112 @@ fn accepted_close_request_is_observed_before_tool_batch_response_timing() { ); } } + +#[test] +fn executed_tool_capture_precedes_send_and_telemetry_gates() { + if isolation::isolated!("CODETRIAL_ACQUISITION_CAPTURE_TEST") { + return; + } + use crate::livekit::probe::acquisition::{Capture, Failure, Limits, install}; + let capture = Capture::new( + "acquisition-fixture", + 7, + Limits { + batches: 3, + calls: 4, + bytes: 16384, + }, + ) + .unwrap(); + let installed = install(&capture).unwrap(); + assert!(install(&capture).is_err()); + let mut state = RuntimeState { + code: "def solve(items):\n return 'authored-result-sentinel'\n".into(), + ..RuntimeState::default() + }; + state.evidence_ledger.code.parser_observation = Some(crate::agent::CodeObservation::Parsed); + state.evidence_ledger.code.substantive_revision = 1; + let now = Instant::now(); + let mut activity = RuntimeActivity::for_interview(now, 0, 7); + activity.latency.bind("acquisition-fixture", 7, 1, 1500); + activity.latency.anchor(now); + activity.latency.prompt(now); + let call = |id: &str| GeminiFunctionCall { + id: id.into(), + name: TOOL_READ_EDITOR.into(), + args: json!({"fromLine":1}), + }; + let answers = execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("actual-received-call-id"), call("second-id")], + 0, + false, + &mut None, + ); + let snapshot = capture.snapshot(); + assert_eq!( + snapshot.batches.len(), + 1, + "executed answers must survive without any later send/timing observation" + ); + let batch = &snapshot.batches[0]; + for (captured, (executed, result)) in batch.calls.iter().zip(&answers) { + assert_eq!(captured.id, executed.id); + assert_eq!(captured.name, executed.name); + assert_eq!(&captured.result, result); + } + assert_eq!(batch.calls.len(), 2); + assert!( + batch.calls[0].result["result"] + .as_str() + .unwrap() + .contains("authored-result-sentinel") + ); + assert!(batch.calls[0].result.get("editorReview").is_none()); + assert!(batch.calls[1].result["editorReview"].as_str().is_some()); + assert_eq!(batch.local_send_completed, None); + assert_eq!(batch.identity.socket, 1); + assert!(batch.identity.trace_turn.is_some()); + let record = tool_timing( + &mut activity, + &state, + &answers, + HintCounters::capture(&state), + timing(false), + ); + assert!(!record.to_string().contains("actual-received-call-id")); + assert!(!record.to_string().contains("authored-result-sentinel")); + activity.latency.set_socket(2); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("replacement-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().batches[1].identity.socket, 2); + activity.latency.bind("foreign-room", 8, 3, 1500); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("foreign-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + assert_eq!(capture.snapshot().batches.len(), 2); + drop(installed); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("uninstalled-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().batches.len(), 2); + let _reinstalled = install(&capture).unwrap(); +} diff --git a/tests/unit/livekit/session_parts/tool_execution.rs b/tests/unit/livekit/session_parts/tool_execution.rs new file mode 100644 index 000000000..68bd41005 --- /dev/null +++ b/tests/unit/livekit/session_parts/tool_execution.rs @@ -0,0 +1,61 @@ +// Included in the session test module so existing test paths stay unchanged. + +#[test] +fn real_tool_dispatch_records_only_tools_before_reply_audio() { + let start = Instant::now(); + let mut activity = RuntimeActivity::new(start); + let mut state = RuntimeState { + code: "return 1;".into(), + ..RuntimeState::default() + }; + let call = GeminiFunctionCall { + id: "read-1".into(), + name: TOOL_READ_EDITOR.into(), + args: serde_json::json!({}), + }; + activity.note_candidate_finished(start, false); + let response = execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call.clone(), call.clone()], + 0, + true, + &mut None, + ); + assert!( + response[0].1["result"] + .as_str() + .is_some_and(|code| code.contains("return 1;")) + ); + let first = activity + .turn_timing + .note_audio(start + Duration::from_secs(1), true) + .unwrap(); + assert_eq!(first.tool_calls, 2); + assert_eq!(first.tool_wait, Some(first.tool_exec)); + assert!(first.pre_tool.is_some()); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call.clone(), call.clone()], + 0, + true, + &mut None, + ); + assert!( + activity + .turn_timing + .finish(start + Duration::from_secs(2), TimingOutcome::NoAudio) + .is_none() + ); + activity.note_candidate_finished(start + Duration::from_secs(3), false); + execute_timed_tool_batch(&mut state, &mut activity, vec![call], 0, false, &mut None); + let next = activity + .turn_timing + .note_audio(start + Duration::from_secs(4), true) + .unwrap(); + assert_eq!( + next.tool_calls, 0, + "discarded/held tool responses must not be carried forward" + ); +} diff --git a/web/report-recovery.js b/web/report-recovery.js index 7ec3dfed7..807636681 100644 --- a/web/report-recovery.js +++ b/web/report-recovery.js @@ -2,7 +2,7 @@ export const reportRecoveryLimits = Object.freeze({ expiresInSeconds: 300, retryAfterSeconds: 30, quotaRetryAfterSeconds: 60, - retryWaitSeconds: 145, + retryWaitSeconds: 175, // The agent announces the end of its window. This page's clock starts when // the offer arrives, later than the agent's, so it waits a little past it // for that word and falls back on its own only when the word never comes. @@ -61,8 +61,8 @@ export function createReportRecovery({ report = null; ready = false; } - // The agent gives regeneration its own 125-second deadline, and the report - // it produces may take every delivery attempt to arrive. + // The agent gives regeneration its own 125-second deadline. A ready report + // can wait through the rejoin grace and still need every delivery attempt. function waitForReport() { if (wait) timers.clearTimeout(wait); wait = timers.setTimeout(