From 1c55318036533da0465d7570ba3249243d75a6ae Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Thu, 5 Nov 2026 11:13:24 +0800 Subject: [PATCH 1/7] Derive Rust file ceilings from Git history Keep file-size limits in the style checker instead of a manually updated JSON table. Bound existing large paths by HEAD, the comparison baseline and their committed history, so shrinking automatically lowers limits. Keep new paths within the default ceiling and retire deleted exemptions. Fail on incomplete baseline history instead of silently skipping checks. Remove the obsolete ratchet command and document the Git requirements. The hotspot ranking reads the limits the checker derives from HEAD. Change-Id: Ia54c92ff56d7bb87401c134b154810f49150c2fe --- .claude/skills/codetrial-improve/SKILL.md | 2 +- .claude/skills/codetrial-improve/rust.md | 40 ++-- docs/development.md | 19 +- scripts/ceiling-hotspots.py | 55 ++--- scripts/check-rust-style.py | 170 +++++++++------- scripts/rust-style-ceilings.json | 29 --- tests/test_ceiling_hotspots.py | 65 +++--- tests/test_check_rust_style.py | 237 ++++++++++++++++------ 8 files changed, 378 insertions(+), 239 deletions(-) delete mode 100644 scripts/rust-style-ceilings.json diff --git a/.claude/skills/codetrial-improve/SKILL.md b/.claude/skills/codetrial-improve/SKILL.md index ccafe12eb..5176440c0 100644 --- a/.claude/skills/codetrial-improve/SKILL.md +++ b/.claude/skills/codetrial-improve/SKILL.md @@ -188,7 +188,7 @@ None of this is a gate; each piece names what it can and cannot see. | `scripts/agent-failure-loop.py` (PostToolUse) | Prompts for the shared premise on the second identical gate failure | Know the premise, or block | | `scripts/test.sh` receipts | Records lane, verdict, skipped count and the fingerprint of the tree as the run started, under the git directory | Prove the run was complete, or stop an agent with a shell from writing one; CI recomputes the verdict | | `scripts/agent-closeout-check.py` (Stop) | Tells the user once per tree when it has changes the gate has not passed on in every lane (`all`, or rust, portable and browser) | Block, or see a gate run elsewhere | -| `scripts/ceiling-hotspots.py` | Ranks the files recorded as exceptions to the size limit by commits in a window times lines | Say a file is worth splitting, or whether its tests cover it; it is one input | +| `scripts/ceiling-hotspots.py` | Ranks the Rust files over the default size limit by commits in a window times lines | Say a file is worth splitting, or whether its tests cover it; it is one input | | pre-push note | Notes `src/` changes with no `tests/` change | Judge whether a test was needed | ## Local evidence at closeout diff --git a/.claude/skills/codetrial-improve/rust.md b/.claude/skills/codetrial-improve/rust.md index fcac3f86a..9882190a5 100644 --- a/.claude/skills/codetrial-improve/rust.md +++ b/.claude/skills/codetrial-improve/rust.md @@ -54,17 +54,27 @@ through `clippy.toml`. ## File size -New Rust files stay within 1500 lines. Existing larger files are recorded in -`scripts/rust-style-ceilings.json` and may only shrink: after splitting one by -responsibility, run `scripts/check-rust-style.py --ratchet` to lower its entry. -`scripts/ceiling-hotspots.py` ranks them by churn times size, as one input to -choosing which to split first. Count a file's mutants before splitting it with -`cargo mutants --list -f | wc -l`; `make mutants` downloads a compiler -first. Never raise an entry to land a change; a new module is the answer. The -gate passes `--baseline origin/main`, so a raised entry or a new exemption fails -against what main records, and under CI a baseline ref that was not fetched -fails too, so a shallow checkout cannot skip it. Test bodies live under -`tests/unit/` and answer to the same ceiling. +New Rust files stay within 1500 lines. By default, the checker derives each +limit from the committed file at the same path in HEAD, with a floor of 1500; +working files may shrink freely but larger files may not grow. Without Git +every file gets 1500 and the gate's baseline cannot be read, so a source +archive fails the gate; run it from a clone. +`scripts/ceiling-hotspots.py` ranks the larger files by churn times size, as +one input to choosing which to split first. Count a file's mutants before +splitting it with `cargo mutants --list -f | wc -l`; `make mutants` +downloads a compiler first. + +The gate passes `--baseline origin/main`. For a larger file present in both +that baseline and HEAD, its limit is the smallest size across those two trees +and every reachable committed version at that path in `origin/main..HEAD`, +with a floor of 1500. All reachable commits count, including merged history. +A committed shrink automatically ratchets the limit; a new or renamed path +gets 1500, and deletion retires the allowance. Split a large file by +responsibility rather than grow it. An explicit baseline requires resolvable +HEAD and baseline refs, common history and a complete, non-shallow +repository; an error fails locally and in CI. Diverged refs are supported, +but the branch must still satisfy any smaller baseline limit. Test bodies +live under `tests/unit/` and answer to the same ceiling. ### Splitting a file @@ -106,9 +116,9 @@ What splitting `src/agent/prompts.rs` (3944 lines, six modules under fn name`) fails with "is still defined here" in `cargo test --lib` and not in clippy, and a source digest that hashes the old file (the report evaluation's identity) stops covering the code that left it. -7. **Run `scripts/check-rust-style.py --ratchet`** so the old entry leaves - `scripts/rust-style-ceilings.json`, then clippy at `-D warnings`, then - `./scripts/test.sh rust`. +7. **Commit the split, then run clippy** at `-D warnings` and + `./scripts/test.sh rust`. The old file's limit drops to its committed size + on its own; there is no table to update. A comment added to a file that sits exactly at its ceiling fails the checker, so a note for a ceilinged file goes here or in the split, not in the file. @@ -188,7 +198,7 @@ in [recipes/rust.md](recipes/rust.md). cargo fmt cargo clippy --locked --workspace --all-targets cargo doc --locked --workspace --no-deps -python3 scripts/check-rust-style.py +python3 scripts/check-rust-style.py --baseline origin/main python3 scripts/check-rust-style.py --workspace cargo test --locked ``` diff --git a/docs/development.md b/docs/development.md index 8c2f59ad4..6e889b1b6 100644 --- a/docs/development.md +++ b/docs/development.md @@ -195,10 +195,21 @@ locks held across await and `mem::forget` are explicit denials too. `unsafe` count at zero — `forbid` rather than `deny`, so a local `#[allow]` cannot reintroduce it. `scripts/check-rust-style.py` holds what a lint cannot ask: that this table stays strict, that no `allow` or bare `#[ignore]` appears, -that no marker or CJK text lands in Rust or `web/`, and that no file passes the -line ceiling in `scripts/rust-style-ceilings.json`, which `--ratchet` only -lowers. The rules and the lints left off on purpose are in the Rust profile of -codetrial-improve, `.claude/skills/codetrial-improve/rust.md`. +that no marker or CJK text lands in Rust or `web/`, and that no file passes its +Git-derived line ceiling. By default, a file may not exceed 1500 lines or its +committed size at the same path in HEAD, whichever is larger. Without Git +every file gets the 1500-line limit and the explicit baseline below cannot be +read, so a source archive fails this gate; run it from a clone. The gate passes +`--baseline origin/main`: larger files present in both the baseline and HEAD +are limited by their smallest size across those two trees and every reachable +commit in `origin/main..HEAD`, with a floor of 1500. Committed shrinking +automatically ratchets the limit; new or renamed paths get 1500, and deletion +retires an allowance. An explicit baseline requires resolvable HEAD and +baseline refs, common history and a complete, non-shallow repository; missing +history fails locally and in CI. Diverged refs are supported, but the branch +must still satisfy any smaller baseline limit. The rules and the lints left +off on purpose are in the Rust profile of codetrial-improve, +`.claude/skills/codetrial-improve/rust.md`. The compiler those lints run on is pinned in `rust-toolchain.toml`, so a new Rust release cannot fail an unchanged tree: Dependabot proposes each bump as diff --git a/scripts/ceiling-hotspots.py b/scripts/ceiling-hotspots.py index b7902c970..1fe06e17d 100644 --- a/scripts/ceiling-hotspots.py +++ b/scripts/ceiling-hotspots.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """Rank oversized files by how often they change times how large they are. -`scripts/rust-style-ceilings.json` says which Rust files are over the size -ceiling; it does not say which to split first. A big file nobody touches costs -nothing, and a big file every change goes through costs on every change. This -prints, per top-level directory, each file with its lines, its recorded ceiling, -the commits that touched it in a window, and score = commits x lines. - - default the files named in the ceilings JSON +`scripts/check-rust-style.py` says which Rust files are over the size ceiling; +it does not say which to split first. A big file nobody touches costs nothing, +and a big file every change goes through costs on every change. This prints, +per top-level directory, each file with its lines, its ceiling, the commits +that touched it in a window, and score = commits x lines. + + default every Rust file the style checker sees above its default + ceiling, with the limit it derives from HEAD --under DIR instead, every tracked file under DIR (a directory such as `web`) with at least --min-lines lines; there is no ceiling column to fill, so it shows `-`. `--suffix .js` keeps only @@ -25,16 +26,25 @@ """ import argparse -import json +import importlib.util import subprocess import sys from collections import Counter from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -CEILINGS = "scripts/rust-style-ceilings.json" DEFAULT_SINCE = "90.days" +# The gate's own line counts and the limits it derives from HEAD. The gate run +# with `--baseline` can be stricter: it also takes the smallest size since that +# ref, so a file that shrank on a branch and grew back shows a larger ceiling +# here than the one the gate enforces. +_SPEC = importlib.util.spec_from_file_location( + "check_rust_style", Path(__file__).with_name("check-rust-style.py") +) +STYLE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(STYLE) + class GitError(Exception): """git exited non-zero; the message is its standard error.""" @@ -81,20 +91,17 @@ def commit_counts(repo, since, paths): return Counter(name for name in out.split("\0") if name) -def ceiling_rows(repo, ceilings): - """{path: ceiling} for each recorded file that still exists.""" - if not isinstance(ceilings, dict) or not all( - isinstance(name, str) and type(ceiling) is int - for name, ceiling in ceilings.items() - ): - raise ValueError(f"{CEILINGS} is not a map of path to line ceiling") - rows = {} - for name in sorted(ceilings): - if (repo / name).is_file(): - rows[name] = ceilings[name] - else: - print(f"skipped {name}: no such file", file=sys.stderr) - return rows +def ceiling_rows(repo): + """{path: ceiling} for each Rust file over the default ceiling. A file with + no committed version, or one at or under the default in HEAD, gets the + default, as the gate gives it.""" + current = STYLE.sizes(repo) + limits = STYLE.git_ceilings(repo, current) + return { + name: limits.get(name, STYLE.DEFAULT_CEILING) + for name, lines in sorted(current.items()) + if lines > STYLE.DEFAULT_CEILING + } def under_rows(repo, directory, min_lines, suffixes=(), excludes=()): @@ -176,7 +183,7 @@ def main(argv=None): repo, args.under, args.min_lines, args.suffix, args.exclude ) else: - rows = ceiling_rows(repo, json.loads((repo / CEILINGS).read_text())) + rows = ceiling_rows(repo) ranked = rank(repo, rows, args.since) except (GitError, OSError, ValueError) as error: print(f"ceiling-hotspots: {error}", file=sys.stderr) diff --git a/scripts/check-rust-style.py b/scripts/check-rust-style.py index bd211cb6a..a66e68f5b 100644 --- a/scripts/check-rust-style.py +++ b/scripts/check-rust-style.py @@ -17,12 +17,11 @@ no comment naming the untracked working documents. language No CJK in Rust or the browser half; fixtures that need multi-byte text use Greek or emoji. - ceiling A file does not grow past the size recorded for it in - scripts/rust-style-ceilings.json, or past DEFAULT_CEILING when - it has no entry. `--ratchet` lowers entries to what the files - are now and never raises one. `--baseline REF` also refuses an - entry above, or absent from, the one REF records; under CI an unfetched REF - is itself a finding. + ceiling New files stay within DEFAULT_CEILING. Existing larger files + cannot grow beyond their committed size in HEAD. --baseline REF + also uses the smallest committed size since REF; deletion retires + an exemption. Limits come from Git, without a generated table. + An explicit baseline needs complete history and never skips errors. `--workspace` audits Cargo's own membership for lint inheritance and needs Cargo; the default run needs none. Exit status 1 when anything is reported. @@ -31,7 +30,6 @@ import argparse import functools import json -import os import re import subprocess import sys @@ -39,7 +37,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -CEILINGS = "scripts/rust-style-ceilings.json" DEFAULT_CEILING = 1500 RUST_ROOTS = ("src", "tests", "scripts/hook-check") @@ -394,14 +391,6 @@ def check_ceilings(current, recorded): f"{rel}: ceiling: {lines} lines, over {ceiling}; " "split it by responsibility, do not raise the ceiling" ) - for rel, ceiling in sorted(recorded.items()): - if rel not in current: - yield f"{CEILINGS}: {rel} no longer exists; drop its entry" - elif current[rel] < ceiling: - yield ( - f"{CEILINGS}: {rel} is {current[rel]} lines, " - f"under its {ceiling}; run --ratchet so it cannot grow back" - ) def git_ok(root, *args): @@ -418,46 +407,94 @@ def git_ok(root, *args): return result.stdout if result.returncode == 0 else None -def baseline_findings(root, ref, recorded): - """Ceiling findings against `ref`. +def git_ceilings(root, current, baseline=None): + """Derive allowances from committed paths, including every branch version.""" + head = git_ok(root, "rev-parse", "--verify", "-q", "HEAD^{commit}") + if head is None and baseline is None: + return {} - A ref that does not resolve is a finding under CI, where a shallow checkout - would otherwise skip the ratchet without a word, and a note elsewhere. A ref - that has no ceilings file yet, the change that introduces it, is skipped. - """ - if git_ok(root, "rev-parse", "--verify", "-q", f"{ref}^{{commit}}") is None: - if os.environ.get("CI") == "true": - return [f"baseline {ref} is not fetched; fetch it so the ratchet can run"] - print(f"baseline {ref} not found; skipped", file=sys.stderr) - return [] - text = git_ok(root, "show", f"{ref}:{CEILINGS}") - if text is None: - print(f"baseline {ref} has no ceilings file; skipped", file=sys.stderr) - return [] - return list(check_baseline(recorded, json.loads(text))) - - -def check_baseline(recorded, base): - """A ceiling never rises, and no file gains an exemption, since `base`.""" - for rel, ceiling in sorted(recorded.items()): - if rel not in base: - yield f"{CEILINGS}: {rel} is a new exemption; split the file instead" - elif ceiling > base[rel]: - yield f"{CEILINGS}: {rel} rose from {base[rel]} to {ceiling}; split the file instead" - - -def ratchet(current, recorded): - """Lower every entry to the file's size now; add none, raise none.""" - return { - rel: min(ceiling, current[rel]) - for rel, ceiling in recorded.items() - if rel in current and current[rel] > DEFAULT_CEILING + def required(*args): + value = git_ok(root, *args) + if value is None: + raise ValueError( + "cannot read Git size baseline; fetch its complete history" + ) + return value + + if baseline is not None: + ref = git_ok( + root, + "rev-parse", + "--verify", + "-q", + "--end-of-options", + f"{baseline}^{{commit}}", + ) + if ref is None: + raise ValueError( + f"baseline {baseline} is not fetched; fetch its complete history" + ) + if ( + head is None + or required("rev-parse", "--is-shallow-repository").strip() != "false" + ): + raise ValueError( + "Git size baseline needs complete history; fetch --unshallow" + ) + required("merge-base", "--all", ref.strip(), head.strip()) + history = required( + "rev-list", "--parents", "--full-history", f"{ref.strip()}..{head.strip()}" + ) + else: + ref, history = head, "" + + paths = sorted(path for path, size in current.items() if size > DEFAULT_CEILING) + if not paths: + return {} + trees, counts = {}, {} + + def tree(commit): + if commit not in trees: + rows = required( + "--literal-pathspecs", + "ls-tree", + "-rz", + "--full-tree", + commit, + "--", + *paths, + ) + trees[commit] = { + path: meta.split()[2] + for row in rows.split("\0") + if row + for meta, path in [row.split("\t", 1)] + if path in current and path.endswith(".rs") + } + return trees[commit] + + def lines(oid): + if oid not in counts: + counts[oid] = len(required("cat-file", "blob", oid).splitlines()) + return counts[oid] + + base, committed = tree(ref.strip()), tree(head.strip()) + limits = { + path: max(DEFAULT_CEILING, min(lines(base[path]), lines(committed[path]))) + for path, size in current.items() + if size > DEFAULT_CEILING and path in base and path in committed } - - -def load_ceilings(root): - path = root / CEILINGS - return json.loads(path.read_text()) if path.exists() else {} + for row in history.splitlines(): + commit, *parents = row.split() + version = tree(commit) + for path in limits: + if path in version: + limits[path] = min( + limits[path], max(DEFAULT_CEILING, lines(version[path])) + ) + elif any(path in tree(parent) for parent in parents): + limits[path] = DEFAULT_CEILING + return limits def run(root, baseline=None): @@ -467,22 +504,17 @@ def run(root, baseline=None): findings.extend(check_rust(root, path)) for path in web_files(root): findings.extend(check_web(root, path)) - recorded = load_ceilings(root) - findings.extend(check_ceilings(sizes(root), recorded)) - if baseline: - findings.extend(baseline_findings(root, baseline, recorded)) + current = sizes(root) + try: + findings.extend(check_ceilings(current, git_ceilings(root, current, baseline))) + except ValueError as error: + findings.append(f"ceiling: {error}") return findings def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) - modes = parser.add_mutually_exclusive_group() - modes.add_argument( - "--ratchet", - action="store_true", - help="lower recorded ceilings to current sizes", - ) - modes.add_argument( + parser.add_argument( "--workspace", action="store_true", help="audit Cargo workspace lint inheritance (requires Cargo and cached dependencies)", @@ -490,16 +522,10 @@ def main(): parser.add_argument( "--baseline", metavar="REF", - help="also refuse ceilings above those REF records; skipped when REF has none", + help="derive size limits from REF and all committed versions since it (needs complete Git history)", ) parser.add_argument("--root", type=Path, default=ROOT, help=argparse.SUPPRESS) args = parser.parse_args() - if args.ratchet: - kept = ratchet(sizes(args.root), load_ceilings(args.root)) - (args.root / CEILINGS).write_text( - json.dumps(dict(sorted(kept.items())), indent=2) + "\n" - ) - return 0 findings = ( list(check_lint_table(args.root)) + list(check_workspace(args.root)) if args.workspace diff --git a/scripts/rust-style-ceilings.json b/scripts/rust-style-ceilings.json deleted file mode 100644 index ec7689e5f..000000000 --- a/scripts/rust-style-ceilings.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "src/agent.rs": 2004, - "src/agent/evidence.rs": 2135, - "src/agent/problem_variants.rs": 1819, - "src/agent/report.rs": 1730, - "src/gemini.rs": 1870, - "src/livekit.rs": 1935, - "src/livekit/session.rs": 2946, - "src/livekit/turn.rs": 2333, - "tests/agent/framework.rs": 4640, - "tests/agent/prompts.rs": 2380, - "tests/agent/runtime.rs": 2605, - "tests/cli.rs": 3183, - "tests/interview_behavior.rs": 1706, - "tests/recording.rs": 3864, - "tests/unit/accounts/schema.migration_tests.rs": 1505, - "tests/unit/agent/evidence.rs": 3761, - "tests/unit/gemini.report_eval.rs": 2691, - "tests/unit/gemini.rs": 5001, - "tests/unit/livekit.rs": 6946, - "tests/unit/livekit/cost.rs": 2179, - "tests/unit/livekit/report.rs": 2821, - "tests/unit/livekit/session.rs": 6023, - "tests/unit/livekit/turn.rs": 3405, - "tests/unit/tasks/session.rs": 1861, - "tests/web/accounts.rs": 1808, - "tests/web/recording.rs": 2496, - "tests/web/token.rs": 2151 -} diff --git a/tests/test_ceiling_hotspots.py b/tests/test_ceiling_hotspots.py index 5e446577d..0504a4e7c 100644 --- a/tests/test_ceiling_hotspots.py +++ b/tests/test_ceiling_hotspots.py @@ -1,7 +1,6 @@ """The hotspot ranking orders files by commits times lines, inside the window.""" import importlib.util -import json import os import subprocess import sys @@ -16,6 +15,9 @@ # A commit far outside any window the tests ask for. LONG_AGO = "2001-01-01T00:00:00 +0000" +# The style checker's default ceiling; only files above it are ranked by default. +OVER = 1501 + def run(*args): return subprocess.run( @@ -76,11 +78,6 @@ def commit(self, files, date=None): self.git("add", "-A") self.git("commit", "-q", "-m", "change", date=date) - def ceilings(self, entries): - path = self.repo / "scripts/rust-style-ceilings.json" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(entries)) - def test_score_is_commits_times_lines_and_orders_the_table(self): self.commit({"src/big.rs": 10, "src/busy.rs": 5, "src/idle.rs": 100}) self.commit({"src/busy.rs": 6}) @@ -162,29 +159,31 @@ def test_under_filters_by_size_suffix_and_prefix(self): self.assertEqual(rows, {"web/big.js": None}) def test_cli_prints_one_table_per_directory_with_the_revision(self): - self.commit({"src/a.rs": 3, "tests/t.rs": 2}) - self.ceilings({"src/a.rs": 3, "tests/t.rs": 2}) + self.commit({"src/a.rs": OVER, "tests/t.rs": OVER + 1}) done = run("--repo", str(self.repo)) self.assertEqual(done.returncode, 0, done.stderr) self.assertIn("# history at ", done.stdout) self.assertIn("## src/ (1 files)", done.stdout) self.assertIn("## tests/ (1 files)", done.stdout) - def test_a_ceiling_for_a_missing_file_is_reported_and_skipped(self): - self.commit({"src/a.rs": 3}) - self.ceilings({"src/a.rs": 3, "src/gone.rs": 9}) + def test_a_file_at_the_default_ceiling_is_not_ranked(self): + self.commit({"src/a.rs": OVER, "src/small.rs": OVER - 1, "web/a.js": OVER}) done = run("--repo", str(self.repo)) self.assertEqual(done.returncode, 0, done.stderr) - self.assertIn("skipped src/gone.rs", done.stderr) - self.assertNotIn("src/gone.rs", done.stdout) - - def test_a_ceilings_file_of_the_wrong_shape_exits_two(self): - self.commit({"src/a.rs": 3}) - for shape in ([1, 2], {"src/a.rs": "3"}, {"src/a.rs": True}): - self.ceilings(shape) - done = run("--repo", str(self.repo)) - self.assertEqual(done.returncode, 2, (shape, done.stderr)) - self.assertIn("not a map of path to line ceiling", done.stderr) + self.assertIn("src/a.rs", done.stdout) + self.assertNotIn("src/small.rs", done.stdout) + self.assertNotIn("web/a.js", done.stdout) + + def test_the_ceiling_is_the_committed_size_not_the_working_one(self): + self.commit({"src/a.rs": OVER + 9}) + (self.repo / "src/a.rs").write_text("x\n" * (OVER + 20)) + (self.repo / "src/new.rs").write_text("x\n" * OVER) + done = run("--repo", str(self.repo)) + self.assertEqual(done.returncode, 0, done.stderr) + self.assertEqual( + row_of(done.stdout, "src/a.rs")[1:3], [str(OVER + 20), str(OVER + 9)] + ) + self.assertEqual(row_of(done.stdout, "src/new.rs")[1:3], [str(OVER), "1500"]) def test_cli_under_shows_a_dash_where_no_ceiling_exists(self): self.commit({"web/big.js": 20}) @@ -195,13 +194,13 @@ def test_cli_under_shows_a_dash_where_no_ceiling_exists(self): ) def test_a_file_with_no_commit_in_the_window_scores_zero(self): - self.commit({"src/a.rs": 4}, date=LONG_AGO) - self.ceilings({"src/a.rs": 4}) + self.commit({"src/a.rs": OVER}, date=LONG_AGO) done = run("--repo", str(self.repo), "--since", "30.days") self.assertEqual(done.returncode, 0, done.stderr) self.assertIn("since 30.days", done.stdout) self.assertEqual( - row_of(done.stdout, "src/a.rs"), ["src/a.rs", "4", "4", "0", "0"] + row_of(done.stdout, "src/a.rs"), + ["src/a.rs", str(OVER), str(OVER), "0", "0"], ) def test_under_with_nothing_to_list_exits_two(self): @@ -216,23 +215,19 @@ def test_outside_a_repository_exits_two(self): self.assertEqual(done.returncode, 2) self.assertIn("ceiling-hotspots:", done.stderr) - def test_the_real_tree_lists_every_recorded_ceiling_that_exists(self): - recorded = json.loads((ROOT / "scripts/rust-style-ceilings.json").read_text()) - present = [name for name in recorded if (ROOT / name).is_file()] + def test_the_real_tree_lists_every_file_over_the_default_ceiling(self): + mod = self.mod.STYLE + over = [ + n for n, lines in mod.sizes(ROOT).items() if lines > mod.DEFAULT_CEILING + ] done = run("--repo", str(ROOT)) self.assertEqual(done.returncode, 0, done.stderr) listed = [ line.split()[0] for line in done.stdout.splitlines() - if line.split() and line.split()[0] in recorded + if line.split() and line.split()[0] in over ] - self.assertEqual(sorted(listed), sorted(present)) - # The gate fails any file above its entry, so a row over its ceiling - # would mean this script counts lines differently from the gate. - for line in done.stdout.splitlines(): - cells = line.split() - if cells and cells[0] in recorded: - self.assertLessEqual(int(cells[1]), int(cells[2]), cells[0]) + self.assertEqual(sorted(listed), sorted(over)) if __name__ == "__main__": diff --git a/tests/test_check_rust_style.py b/tests/test_check_rust_style.py index 0007aa90b..43242859e 100644 --- a/tests/test_check_rust_style.py +++ b/tests/test_check_rust_style.py @@ -9,7 +9,6 @@ import unittest from pathlib import Path from types import SimpleNamespace -from unittest import mock from unittest.mock import Mock SCRIPT = Path(__file__).resolve().parents[1] / "scripts/check-rust-style.py" @@ -257,39 +256,167 @@ def test_cjk_is_refused_in_rust_and_the_browser_half(self): self.findings("src/lib.rs", f'const S: &str = "{greek}";\n'), [] ) - def test_a_file_cannot_pass_its_ceiling(self): - body = "fn f() {}\n" * 1501 - self.assertFires("ceiling", "src/big.rs", body) - self.write("scripts/rust-style-ceilings.json", json.dumps({"src/big.rs": 1501})) - self.assertEqual(run(self.root).stdout, "") - self.write("src/big.rs", body + "fn g() {}\n") - grown = run(self.root) - self.assertIn("src/big.rs: ceiling", grown.stdout) - self.assertEqual(grown.returncode, 1) - - def test_a_ceiling_above_the_file_is_refused_until_ratcheted(self): - self.write("src/big.rs", "fn f() {}\n" * 1600) - self.write("scripts/rust-style-ceilings.json", json.dumps({"src/big.rs": 1700})) - self.assertIn("run --ratchet", run(self.root).stdout) - run(self.root, "--ratchet") - self.assertEqual(run(self.root).stdout, "") - - def test_a_ceiling_for_a_deleted_file_is_refused(self): - self.write( - "scripts/rust-style-ceilings.json", json.dumps({"src/gone.rs": 2000}) + def test_a_new_file_cannot_exceed_the_default_ceiling(self): + self.assertEqual(self.findings("src/big.rs", "// line\n" * 1500), []) + self.assertFires("ceiling", "src/big.rs", "// line\n" * 1501) + + def start_history(self): + self.git("init", "-q") + self.git("symbolic-ref", "HEAD", "refs/heads/main") + self.commit_tree("base") + self.git("tag", "base") + + def commit_tree(self, message): + self.git("add", "-A") + self.git("commit", "-qm", message) + + def test_git_history_provides_ceilings_without_a_table(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.assertFalse((self.root / "scripts/rust-style-ceilings.json").exists()) + self.assertEqual(self.git("status", "--porcelain").stdout, b"") + self.write("src/big.rs", "// line\n" * 1601) + self.assertTrue(any("over 1600" in x for x in self.style.run(self.root))) + + def test_committed_shrink_cannot_grow_back(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.write("src/big.rs", "// line\n" * 1550) + self.assertEqual(self.style.run(self.root, "base"), []) + self.commit_tree("shrink") + self.write("src/big.rs", "// line\n" * 1560) + self.commit_tree("regrow") + self.assertEqual( + self.style.run(self.root), [] + ) # HEAD checks uncommitted growth. + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "base")) ) - self.assertIn("src/gone.rs no longer exists", run(self.root).stdout) - def test_ratchet_lowers_and_never_raises(self): - self.write("src/big.rs", "fn f() {}\n" * 1600) - self.write( - "scripts/rust-style-ceilings.json", - json.dumps({"src/big.rs": 1700, "src/small.rs": 1800}), + def test_new_committed_and_renamed_paths_get_no_exemption(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("mv", "src/big.rs", "src/renamed.rs") + self.write("src/new.rs", "// line\n" * 1600) + self.commit_tree("new paths") + findings = self.style.run(self.root, "base") + for name in ["renamed.rs", "new.rs"]: + self.assertTrue(any(name in x and "over 1500" in x for x in findings)) + + def test_deletion_retires_the_allowance(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + (self.root / "src/big.rs").unlink() + self.assertEqual(self.style.run(self.root, "base"), []) + self.commit_tree("delete") + self.write("src/big.rs", "// line\n" * 1600) + self.commit_tree("recreate") + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "base")) + ) + + def test_shrinking_below_default_keeps_the_general_default(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.write("src/big.rs", "// line\n" * 1400) + self.commit_tree("shrink") + self.write("src/big.rs", "// line\n" * 1500) + self.assertEqual(self.style.run(self.root, "base"), []) + # One past the default is checked against the default, rather than + # against the 1600 the base allowed. + self.write("src/big.rs", "// line\n" * 1501) + findings = self.style.run(self.root, "base") + self.assertTrue(any("over 1500" in x for x in findings), findings) + + def test_merge_history_cannot_hide_a_smaller_version(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/big.rs", "// line\n" * 1550) + self.commit_tree("shrink on side") + self.git("checkout", "main") + self.git("merge", "--no-ff", "-s", "ours", "side", "-m", "merge") + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "base")) + ) + + def test_a_merge_deletion_retires_the_allowance(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/other.rs", "// other\n") + self.commit_tree("side changes") + self.git("checkout", "main") + self.git("merge", "--no-ff", "--no-commit", "side") + self.git("rm", "src/big.rs") + self.commit_tree("merge deletes") + self.write("src/big.rs", "// line\n" * 1600) + self.commit_tree("recreate") + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "base")) + ) + + def test_unrelated_history_cannot_supply_an_allowance(self): + self.start_history() + self.git("checkout", "--orphan", "unrelated") + self.commit_tree("other root") + self.assertTrue( + any( + "cannot read Git size baseline" in x + for x in self.style.run(self.root, "base") + ) + ) + + def test_diverged_baseline_keeps_branch_history(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "side") + self.write("src/big.rs", "// line\n" * 1550) + self.commit_tree("shrink") + self.git("checkout", "main") + self.write("src/other.rs", "// other\n") + self.commit_tree("main advances") + self.git("checkout", "side") + self.write("src/big.rs", "// line\n" * 1560) + self.assertTrue( + any("over 1550" in x for x in self.style.run(self.root, "main")) + ) + + def test_git_and_worktree_count_newlines_the_same_way(self): + name = "src/large name " + chr(0x3B1) + ".rs" + self.write(name, "// line\r\n" * 1500 + "// last") + self.start_history() + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.write(name, "// line\n" * 1500 + "// last\n") + self.assertEqual(self.style.run(self.root, "HEAD"), []) + self.write(name, "// line\n" * 1500 + "// last\n// added") + self.assertTrue( + any("over 1501" in x for x in self.style.run(self.root, "HEAD")) + ) + + def test_a_descendant_baseline_cannot_relax_head(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + self.git("checkout", "-qb", "ahead") + self.write("src/big.rs", "// line\n" * 2000) + self.commit_tree("ahead grows") + self.git("checkout", "main") + self.write("src/big.rs", "// line\n" * 1900) + self.assertTrue( + any("over 1600" in x for x in self.style.run(self.root, "ahead")) + ) + + def test_a_baseline_only_path_is_still_new_in_the_worktree(self): + self.start_history() + self.git("checkout", "-qb", "ahead") + self.write("src/new.rs", "// line\n" * 1600) + self.commit_tree("ahead adds") + self.git("checkout", "main") + self.write("src/new.rs", "// line\n" * 1600) + self.assertTrue( + any("over 1500" in x for x in self.style.run(self.root, "ahead")) ) - self.write("src/small.rs", "fn f() {}\n") - run(self.root, "--ratchet") - kept = json.loads((self.root / "scripts/rust-style-ceilings.json").read_text()) - self.assertEqual(kept, {"src/big.rs": 1600}) def test_every_required_lint_refuses_weakening_or_removal(self): for group, lints in self.style.REQUIRED_LINTS.items(): @@ -445,36 +572,28 @@ def git(self, *args): check=True, ) - def test_a_ceiling_cannot_rise_or_appear_since_the_baseline(self): - body = "fn f() {}\n" * 1600 - ceilings = "scripts/rust-style-ceilings.json" - self.write("src/big.rs", body) - self.write(ceilings, json.dumps({"src/big.rs": 1600})) - self.git("init", "-q") - self.git("add", "-A") - self.git("commit", "-qm", "base") - self.assertEqual(self.style.run(self.root, "HEAD"), []) - self.write("src/big.rs", body + "fn g() {}\n" * 100) - self.write(ceilings, json.dumps({"src/big.rs": 1700})) + def test_explicit_missing_baseline_fails(self): + # The checker reads no CI variable, so there is one answer. The test + # does not set one: the runner executes cases in threads, and a changed + # os.environ under a thread that is spawning git makes the spawn fail + # with EFAULT. A repository with history, so the refusal is of the + # absent ref and not of the directory. + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() self.assertTrue( - any( - "rose from 1600 to 1700" in x for x in self.style.run(self.root, "HEAD") - ) + any("not fetched" in x for x in self.style.run(self.root, "no-such-ref")) ) - self.write("src/big.rs", body) - self.write("src/new.rs", body) - self.write(ceilings, json.dumps({"src/big.rs": 1600, "src/new.rs": 1600})) - self.assertTrue( - any("new exemption" in x for x in self.style.run(self.root, "HEAD")) - ) - with mock.patch.dict(os.environ, {"CI": ""}): - self.assertEqual(self.style.run(self.root, "no-such-ref"), []) - with mock.patch.dict(os.environ, {"CI": "true"}): - self.assertTrue( - any( - "not fetched" in x for x in self.style.run(self.root, "no-such-ref") - ) - ) + + def test_shallow_history_cannot_claim_a_ratchet(self): + self.write("src/big.rs", "// line\n" * 1600) + self.start_history() + with tempfile.TemporaryDirectory() as clone: + self.git("clone", "-q", "--depth", "1", self.root.as_uri(), clone) + findings = self.style.run(Path(clone), "HEAD") + self.assertTrue(any("complete history" in x for x in findings)) + + def test_legacy_ratchet_command_is_removed(self): + self.assertEqual(run(self.root, "--ratchet").returncode, 2) if __name__ == "__main__": From 6ece8b61d5080b974ef4e4c02fd15fe2cc737837 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 4 Nov 2026 15:12:40 +0800 Subject: [PATCH 2/7] Tighten recovery and report handling and tests A regenerated report could finish while its candidate was away within the rejoin grace, and the browser fallback finalized the provisional failure before that result was ready. Include the rejoin grace in the browser retry wait and its wire fixture, and derive the recovery driver's retained context bytes from its queue instead of a cached total kept in step by hand. Several recovery properties were asserted loosely or not at all, and two browser suites were unreliable. Compare canceled writes with their constructed input, observe report generation by room scope through native rooms, pin the reaction clock, share the isolated-process checks and the rig socket join, bound socket buffers in loopback fixtures, narrow the recovery driver's visibility, honor a custom Cargo target directory and join the workbench writers before removing a bundle. Change-Id: Id92733aef590fc694affb07d6cd3e2dbcad62709 --- Cargo.lock | 1 + Cargo.toml | 2 + docs/provider-cost-and-degradation.md | 13 +- src/gemini/lifecycle.rs | 8 + src/gemini/report_calls.rs | 7 +- src/gemini/socket.rs | 9 + src/gemini/socket/loopback.rs | 118 +++ src/lib.rs | 4 + src/livekit/recovery.rs | 45 +- src/livekit/report.rs | 2 + tests/browser/face-detector.test.js | 25 +- tests/browser/report-recovery.test.js | 21 + tests/browser/review-workbench.test.js | 70 +- tests/fixtures/report-recovery.json | 2 +- tests/unit/gemini.rs | 7 +- tests/unit/gemini/socket_loopback.rs | 65 ++ .../gemini_parts/scoped_report_endpoint.rs | 112 +++ tests/unit/isolation.rs | 101 +++ tests/unit/livekit.rs | 2 +- tests/unit/livekit/handoff.rs | 33 +- tests/unit/livekit/packets.rs | 55 ++ tests/unit/livekit/recovery.rs | 132 +-- tests/unit/livekit/recovery_live.rs | 2 + .../report_generation_cardinality.rs | 828 ++++++++++++++++++ ...entry_cancellation_and_driver_decisions.rs | 30 +- .../report_generation_cardinality.rs | 172 ++++ .../recovery_parts/retry_and_room_waits.rs | 19 +- .../recovery_parts/terminal_matrices.rs | 13 +- .../tool_work_and_final_sends.rs | 1 - ...te_sites_and_replacement_classification.rs | 61 +- tests/unit/livekit/report.rs | 42 +- .../unit/livekit/report/scripted_recovery.rs | 73 ++ web/report-recovery.js | 6 +- 33 files changed, 1821 insertions(+), 260 deletions(-) create mode 100644 src/gemini/socket/loopback.rs create mode 100644 tests/unit/gemini/socket_loopback.rs create mode 100644 tests/unit/gemini_parts/scoped_report_endpoint.rs create mode 100644 tests/unit/isolation.rs create mode 100644 tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs create mode 100644 tests/unit/livekit/recovery_parts/report_generation_cardinality.rs create mode 100644 tests/unit/livekit/report/scripted_recovery.rs diff --git a/Cargo.lock b/Cargo.lock index 4b805a9a4..63895d55d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,6 +324,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "socket2", "tokio", "tokio-tungstenite", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index 9d7841885..f4866e30d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,8 @@ windows-permissions = "0.2.4" # Independent implementations verify production signatures and digest fixtures. hmac = "0.13.0" sha2 = "0.11.1" +# Borrowed socket inspection for real loopback transport fixtures. +socket2 = "0.6.5" livekit-protocol = "0.8.2" # LiveKit's transport, reached directly to check the vendored proxy path. livekit-net = "0.1.4" diff --git a/docs/provider-cost-and-degradation.md b/docs/provider-cost-and-degradation.md index bcf84dcac..a1652b4db 100644 --- a/docs/provider-cost-and-degradation.md +++ b/docs/provider-cost-and-degradation.md @@ -225,12 +225,13 @@ answers each request on the control topic, `report_retry` with status `accepted` or `early` (with the seconds still to wait, which does not spend the regeneration), and announces `closed` at expiry or when no key can ever answer; a duplicate request during -regeneration gets no answer. The browser waits at most 145 seconds from its -request or the acceptance, whichever came last, so neither a reconnect nor a -lost answer cuts off a regeneration still in progress. The terminal failure -determines eligibility, so a transient burst followed by a schema failure -regenerates as a schema failure. Reloads and process restarts cannot recover -the inputs. +regeneration gets no answer. The browser waits at most 175 seconds from its +request or the acceptance, whichever came last. This covers the configured +generation deadline, one final rejoin grace and every delivery attempt. This +browser bound does not bound arbitrary SDK publish or host scheduling stalls. +The terminal failure determines eligibility, so a transient burst followed by +a schema failure regenerates as a schema failure. Reloads and process +restarts cannot recover the inputs. Every report packet, provisional, regenerated or final, is delivered the same way. The agent publishes those same bytes at most three times, each with a diff --git a/src/gemini/lifecycle.rs b/src/gemini/lifecycle.rs index d2ff7e3c7..713c7f541 100644 --- a/src/gemini/lifecycle.rs +++ b/src/gemini/lifecycle.rs @@ -66,6 +66,14 @@ impl LiveReaderInspection { } impl GeminiLiveSession { + /// Limits one authored loopback connection without replacing its transport. + #[cfg(test)] + pub(crate) fn loopback_send_limit_for_test( + endpoint: &str, + ) -> std::io::Result { + super::socket::loopback::install(endpoint) + } + /// Leaves a fail-fast session in the room while its owner retires the /// transport. Metadata and unconsumed usage stay with the returned socket. pub(crate) fn retire(&mut self) -> Self { diff --git a/src/gemini/report_calls.rs b/src/gemini/report_calls.rs index 3fdba0dc1..2e6f2df4b 100644 --- a/src/gemini/report_calls.rs +++ b/src/gemini/report_calls.rs @@ -25,10 +25,15 @@ pub(crate) async fn generate_report_with_keys( rounds: crate::agent::ReportRounds, run: ReportRun<'_>, ) -> Result> { + #[cfg(test)] + let url = tests::scoped_report_endpoint(run.scope) + .unwrap_or_else(|| gemini_generate_content_url(model)); + #[cfg(not(test))] + let url = gemini_generate_content_url(model); generate_report_with_keys_at( ReportCalls { keys, - url: gemini_generate_content_url(model), + url, material, communication_assessed: rounds.communication_assessed(), budget: ReportCallBudget::new(), diff --git a/src/gemini/socket.rs b/src/gemini/socket.rs index 6a8a96114..8c302fe91 100644 --- a/src/gemini/socket.rs +++ b/src/gemini/socket.rs @@ -28,6 +28,11 @@ pub(super) async fn connect(url: &str) -> Result { let connector = Some(Connector::Rustls( std::sync::Arc::clone(crate::tls_config()), )); + #[cfg(test)] + if let Some(stream) = loopback::connect(url).await? { + let (socket, _) = client_async_tls_with_config(request, stream, None, connector).await?; + return Ok(socket); + } let Some(proxy) = proxy_for(request.uri(), &Matcher::from_env()) else { let (socket, _) = connect_async_tls_with_config(request, None, false, connector).await?; return Ok(socket); @@ -73,3 +78,7 @@ async fn tunnel(proxy: &Intercept, target: &Uri) -> Result #[cfg(test)] #[path = "../../tests/unit/gemini/socket.rs"] mod tests; + +#[cfg(test)] +#[path = "socket/loopback.rs"] +pub(super) mod loopback; diff --git a/src/gemini/socket/loopback.rs b/src/gemini/socket/loopback.rs new file mode 100644 index 000000000..6e74b41e7 --- /dev/null +++ b/src/gemini/socket/loopback.rs @@ -0,0 +1,118 @@ +//! A scoped socket-option fixture, retaining the real TCP and WebSocket path. + +use std::io; +use std::net::{Ipv4Addr, SocketAddrV4}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use tokio::net::{TcpSocket, TcpStream}; + +static ACTIVE: Mutex>> = Mutex::new(None); +const REQUESTED: u32 = 1024; +const MAX_BUFFER: u32 = 16384; + +struct State { + endpoint: String, + address: SocketAddrV4, + claimed: AtomicBool, + buffers: Mutex>, +} + +/// Owned by one isolated fixture until its real session and peer are joined. +pub(crate) struct Registration { + state: Arc, +} + +pub(in crate::gemini) fn install(endpoint: &str) -> io::Result { + let parsed = reqwest::Url::parse(endpoint).map_err(io::Error::other)?; + if parsed.scheme() != "ws" + || parsed.host_str() != Some("127.0.0.1") + || !parsed.username().is_empty() + || parsed.password().is_some() + || parsed.path() != "/" + || parsed.query() != Some("key=sequence-test-key") + || parsed.fragment().is_some() + || parsed.as_str() != endpoint + || parsed.port().is_none_or(|port| port == 0) + { + return Err(io::Error::other( + "socket fixture requires its exact authored loopback URL", + )); + } + let state = Arc::new(State { + endpoint: parsed.to_string(), + address: SocketAddrV4::new(Ipv4Addr::LOCALHOST, parsed.port().unwrap_or(0)), + claimed: AtomicBool::new(false), + buffers: Mutex::new(None), + }); + let mut active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + if active.is_some() { + return Err(io::Error::other("a socket fixture is already installed")); + } + *active = Some(Arc::clone(&state)); + Ok(Registration { state }) +} + +fn claim(endpoint: &str) -> io::Result>> { + let active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + let Some(state) = active.as_ref().filter(|state| state.endpoint == endpoint) else { + return Ok(None); + }; + if state.claimed.swap(true, Ordering::SeqCst) { + return Err(io::Error::other("the socket fixture was already consumed")); + } + Ok(Some(Arc::clone(state))) +} + +fn admitted(bytes: u32) -> io::Result { + if (1..=MAX_BUFFER).contains(&bytes) { + Ok(bytes) + } else { + Err(io::Error::other( + "socket buffer exceeds the declared fixture bound", + )) + } +} + +pub(super) async fn connect(endpoint: &str) -> io::Result> { + let Some(state) = claim(endpoint)? else { + return Ok(None); + }; + let socket = TcpSocket::new_v4()?; + socket.set_send_buffer_size(REQUESTED)?; + let before = admitted(socket.send_buffer_size()?)?; + let stream = socket.connect(state.address.into()).await?; + let connected = admitted( + u32::try_from(socket2::SockRef::from(&stream).send_buffer_size()?) + .map_err(io::Error::other)?, + )?; + *state + .buffers + .lock() + .unwrap_or_else(|error| error.into_inner()) = Some((before, connected)); + Ok(Some(stream)) +} + +impl Registration { + pub(crate) fn sender_buffers(&self) -> Option<(u32, u32)> { + *self + .state + .buffers + .lock() + .unwrap_or_else(|error| error.into_inner()) + } +} + +impl Drop for Registration { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(|error| error.into_inner()); + if active + .as_ref() + .is_some_and(|state| Arc::ptr_eq(state, &self.state)) + { + *active = None; + } + } +} + +#[path = "../../../tests/unit/gemini/socket_loopback.rs"] +mod tests; diff --git a/src/lib.rs b/src/lib.rs index 917987c19..4bae31362 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -12,6 +12,10 @@ pub mod tasks; pub mod token; pub mod web; +#[cfg(test)] +#[path = "../tests/unit/isolation.rs"] +mod test_isolation; + /// Lives here rather than in one of the modules because `accounts` stamps rows /// with it and `web` stamps tokens with it, and neither should have to depend /// on the other for a clock. diff --git a/src/livekit/recovery.rs b/src/livekit/recovery.rs index 3f268fa3b..d0a08bf41 100644 --- a/src/livekit/recovery.rs +++ b/src/livekit/recovery.rs @@ -30,7 +30,7 @@ pub(super) enum Terminal { #[cfg(test)] impl Terminal { - pub(super) fn name(&self) -> &'static str { + fn name(&self) -> &'static str { match self { Self::End { .. } => "End", Self::Deadline => "Deadline", @@ -44,9 +44,9 @@ impl Terminal { pub(super) struct Cancelled; pub(super) struct Packet { - pub(super) topic: Option, - pub(super) sender: Option, - pub(super) payload: Vec, + topic: Option, + sender: Option, + payload: Vec, } /// How many unsent context summaries a recovery keeps, and their total size. @@ -181,29 +181,28 @@ struct Action { /// A test-only barrier after input preparation and before transport polling. #[cfg(test)] pub(super) struct WriteBarrier { - pub(super) phase: &'static str, - pub(super) entered: std::sync::Arc, - pub(super) release: std::sync::Arc, + phase: &'static str, + entered: std::sync::Arc, + release: std::sync::Arc, } /// A test-only action run in the poll that completes a labelled write, before /// the driver can observe any later input. #[cfg(test)] pub(super) struct AfterWrite { - pub(super) phase: &'static str, - pub(super) action: Box, + phase: &'static str, + action: Box, } pub(super) struct Driver<'r, 'c, 'i> { - pub(super) room: &'r RoomLink, - pub(super) context: &'r mut GeminiEventContext<'c>, - pub(super) loops: &'r mut RoomLoop, - pub(super) interview: InterviewContext<'i>, - pub(super) ingress: Ingress, - pub(super) revision: u64, - pub(super) terminal: Option, + room: &'r RoomLink, + context: &'r mut GeminiEventContext<'c>, + loops: &'r mut RoomLoop, + interview: InterviewContext<'i>, + ingress: Ingress, + revision: u64, + terminal: Option, held_contexts: VecDeque, - held_bytes: usize, held_omitted: usize, held_omitted_total: usize, held_steps: Vec<&'static str>, @@ -236,7 +235,6 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { revision: 0, terminal: None, held_contexts: VecDeque::new(), - held_bytes: 0, held_omitted: 0, held_omitted_total: 0, held_steps: Vec::new(), @@ -343,17 +341,17 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { self.held_omitted_total += 1; return; } + let mut held_bytes = self.held_contexts.iter().map(String::len).sum::(); while self.held_contexts.len() >= MAX_HELD_CONTEXTS - || self.held_bytes + context.len() > MAX_HELD_BYTES + || held_bytes + context.len() > MAX_HELD_BYTES { let Some(old) = self.held_contexts.pop_front() else { break; }; - self.held_bytes -= old.len(); + held_bytes -= old.len(); self.held_omitted += 1; self.held_omitted_total += 1; } - self.held_bytes += context.len(); self.held_contexts.push_back(context); } @@ -505,7 +503,7 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { } } - pub(super) async fn wait(&mut self, op: impl Future) -> Result { + async fn wait(&mut self, op: impl Future) -> Result { tokio::pin!(op); loop { if self.terminal.is_some() { @@ -635,7 +633,7 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { } } - pub(super) async fn flush_notices(&mut self) -> Result<(), Cancelled> { + async fn flush_notices(&mut self) -> Result<(), Cancelled> { if let Some(round) = self.round { let message = serde_json::json!({"type":"round_state", "round":"behavioral", "status":round}); @@ -703,7 +701,6 @@ impl<'r, 'c, 'i> Driver<'r, 'c, 'i> { async fn flush_contexts(&mut self, session: &mut GeminiLiveSession) -> Result { while !self.held_contexts.is_empty() || self.held_omitted != 0 { let contexts = std::mem::take(&mut self.held_contexts); - self.held_bytes = 0; let steps = std::mem::take(&mut self.held_steps); let omitted = std::mem::take(&mut self.held_omitted); let mut body = contexts.into_iter().collect::>().join("\n"); diff --git a/src/livekit/report.rs b/src/livekit/report.rs index 78e597de3..04710cc01 100644 --- a/src/livekit/report.rs +++ b/src/livekit/report.rs @@ -125,6 +125,8 @@ pub(super) async fn generate_report_bounded( seed: i64, refused: &std::sync::atomic::AtomicBool, ) -> GeneratedReport { + #[cfg(test)] + crate::gemini::tests::note_bounded_report(boot.room_name, seed); tokio::time::timeout( REPORT_TIMEOUT, generate_report_with_keys( diff --git a/tests/browser/face-detector.test.js b/tests/browser/face-detector.test.js index 1e8b41896..b36e1e464 100644 --- a/tests/browser/face-detector.test.js +++ b/tests/browser/face-detector.test.js @@ -12,13 +12,7 @@ import { test, before, after } from "node:test"; import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { createServer } from "node:net"; -import { - existsSync, - mkdirSync, - mkdtempSync, - rmSync, - writeFileSync, -} from "node:fs"; +import { existsSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { launchChromium, root } from "./source.js"; @@ -42,7 +36,10 @@ async function freePort() { let PORT = 0; let BASE = ""; -const binary = resolve(root, "target/debug/codetrial"); +const targetDir = resolve( + process.env.CARGO_TARGET_DIR || resolve(root, "target"), +); +const binary = join(targetDir, "debug/codetrial"); let server = null; let browser = null; @@ -79,12 +76,7 @@ before(async () => { server = null; return; } - // `target/` is missing on a clean checkout, and `mkdtemp` needs its parent. - // Creating it keeps a missing build a skipped suite, which is what the - // chromium guard above already does, rather than an ENOENT thrown out of - // `before`. - mkdirSync(resolve(root, "target"), { recursive: true }); - workDir = mkdtempSync(join(root, "target", "face-detector-")); + workDir = mkdtempSync(join(targetDir, "face-detector-")); const configPath = join(workDir, "codetrial.env.local"); writeFileSync( configPath, @@ -322,7 +314,10 @@ test("a VideoFrame is not a frame this detector can read", async (t) => { /// and the check reads "unavailable" as ready: no face is ever required. /// Detection has to go through `face-worker.js`, the one response granted eval. test("the setup face check detects a face under the page's own policy", async (t) => { - if (!browser) return t.skip("playwright chromium unavailable"); + if (!browser || !server) { + t.skip("playwright browser or built server binary unavailable"); + return; + } const page = await browser.newPage(); try { await page.goto(BASE, { waitUntil: "domcontentloaded" }); diff --git a/tests/browser/report-recovery.test.js b/tests/browser/report-recovery.test.js index b1d72fff6..ae5aa0468 100644 --- a/tests/browser/report-recovery.test.js +++ b/tests/browser/report-recovery.test.js @@ -396,6 +396,27 @@ test("page finalizes a regeneration once, cancelling provisional timers", async ); }); +test("page keeps the room for a report held through the rejoin grace", async () => { + const page = pageHarness(); + await page.receiveReport(page.room, packet(raw)); + page.advance(30_000); + assert.equal(page.reportRecovery.retry(), true); + page.receiveControl(control(accepted)); + // A 124-second generation can be held until a rejoin at 152 seconds. + page.advance(152_000); + await settle(); + assert.equal(page.scope.state.phase, "report_recovery"); + assert.equal(page.scope.state.room, page.room); + assert.ok(!page.events.includes("disconnect")); + await page.receiveReport( + page.room, + packet({ incomplete: true, summary: "retained regeneration" }), + ); + await settle(); + assert.equal(page.scope.state.phase, "report"); + assert.equal(page.saves.at(-1).summary, "retained regeneration"); +}); + test("the agent's closing notice finalizes the page through its control topic", async () => { const page = pageHarness(); await page.receiveReport(page.room, packet(raw)); diff --git a/tests/browser/review-workbench.test.js b/tests/browser/review-workbench.test.js index 33316162d..27de6733c 100644 --- a/tests/browser/review-workbench.test.js +++ b/tests/browser/review-workbench.test.js @@ -33,7 +33,26 @@ async function start( manifestFiles = null, ) { const bundle = mkdtempSync(join(tmpdir(), "codetrial-workbench-test-")); - t.after(() => rmSync(bundle, { recursive: true, force: true })); + const writers = []; + const own = (child) => { + const closed = new Promise((resolve) => child.once("close", resolve)); + writers.push({ child, closed }); + return child; + }; + let page; + t.after(async () => { + try { + await page?.close(); + } finally { + for (const { child, closed } of writers.reverse()) { + if (child.pid && child.exitCode === null && child.signalCode === null) + child.kill(); + await closed; + } + // The reviewer can write presence files until these processes close. + rmSync(bundle, { recursive: true, force: true }); + } + }); const files = ["src/example.rs", "tests/example.rs"]; writeFileSync( join(bundle, "manifest.json"), @@ -85,30 +104,24 @@ Path(sys.argv[sys.argv.index('--output-last-message')+1]).write_text(json.dumps( env.PATH = `${bin}:${env.PATH}`; args.push("--reviewer", "codex"); } - const server = spawn(python, [helper, "serve", "--port", "0", ...args], { - cwd: root, - env, - stdio: ["ignore", "pipe", "pipe"], - }); - t.after(async () => { - if (server.exitCode === null) { - const exited = once(server, "exit"); - server.kill(); - await exited; - } - }); + const server = own( + spawn(python, [helper, "serve", "--port", "0", ...args], { + cwd: root, + env, + stdio: ["ignore", "pipe", "pipe"], + }), + ); const lines = createInterface({ input: server.stdout }); const [url] = await once(lines, "line", { signal: t.signal }); lines.close(); - const page = await browser.newPage(); - t.after(() => page.close()); + page = await browser.newPage(); await page.goto(url); await page.locator("#workspace").waitFor({ state: "visible" }); const inspect = () => JSON.parse( execFileSync(python, [helper, "inspect", ...args], { encoding: "utf8" }), ); - return { bundle, args, page, url, inspect, files, server }; + return { bundle, args, page, url, inspect, files, server, own }; } test( @@ -116,7 +129,7 @@ test( { timeout: 30000 }, async (t) => { if (!browser) return t.skip("playwright chromium unavailable"); - const { page, args, bundle, inspect } = await start(t); + const { page, args, bundle, inspect, own } = await start(t); await page.getByText(/Commit threshold reached .*at least 200/).waitFor(); assert.equal(inspect().total, 4); assert.equal(await page.locator("#answers-form textarea").count(), 1); @@ -134,18 +147,17 @@ test( await page.getByText("Saved on this computer", { exact: true }).waitFor(); const saved = inspect(); assert.equal(saved.status, "ready_to_send"); - const waiting = spawn(python, [ - helper, - "wait", - ...args, - "--after", - String(saved.revision), - "--timeout", - "10", - ]); - t.after(() => { - if (waiting.exitCode === null) waiting.kill(); - }); + const waiting = own( + spawn(python, [ + helper, + "wait", + ...args, + "--after", + String(saved.revision), + "--timeout", + "10", + ]), + ); let received = ""; waiting.stdout.on("data", (chunk) => { received += chunk; diff --git a/tests/fixtures/report-recovery.json b/tests/fixtures/report-recovery.json index 9265471ac..5df7992b7 100644 --- a/tests/fixtures/report-recovery.json +++ b/tests/fixtures/report-recovery.json @@ -2,7 +2,7 @@ "expiresInSeconds": 300, "retryAfterSeconds": 30, "quotaRetryAfterSeconds": 60, - "retryWaitSeconds": 145, + "retryWaitSeconds": 175, "closeGraceSeconds": 15, "retryStatuses": [ "accepted", diff --git a/tests/unit/gemini.rs b/tests/unit/gemini.rs index 8d120389b..4aa772c7a 100644 --- a/tests/unit/gemini.rs +++ b/tests/unit/gemini.rs @@ -1,7 +1,4 @@ -//! The `tests` module of `src/gemini.rs`, which declares this file by path. -//! Everything here reaches into `src/gemini.rs` through `super`, so it is a -//! unit -//! test and not an integration test: private items are in scope. +//! Unit tests declared by src/gemini.rs, with its private items in scope. use super::*; use crate::config::load_from_pairs; @@ -4999,3 +4996,5 @@ fn every_failing_path_is_named_before_a_second_error_on_one() { } include!("gemini_parts/tool_policy_and_reader_pressure.rs"); + +include!("gemini_parts/scoped_report_endpoint.rs"); diff --git a/tests/unit/gemini/socket_loopback.rs b/tests/unit/gemini/socket_loopback.rs new file mode 100644 index 000000000..2eb98ca55 --- /dev/null +++ b/tests/unit/gemini/socket_loopback.rs @@ -0,0 +1,65 @@ +use crate::test_isolation as isolation; + +use super::*; + +#[test] +fn loopback_registration_rejects_every_non_fixture_destination() { + for endpoint in [ + "wss://127.0.0.1:9/?key=sequence-test-key", + "ws://localhost:9/?key=sequence-test-key", + "ws://127.0.0.2:9/?key=sequence-test-key", + "ws://127.0.0.1:9/", + "ws://127.0.0.1:9?key=sequence-test-key", + "ws://127.0.0.1:9/?key=other", + "ws://127.0.0.1:9/?key=sequence-test-key&other=1", + "ws://user@127.0.0.1:9/?key=sequence-test-key", + "ws://127.0.0.1:9/other?key=sequence-test-key", + ] { + assert!(install(endpoint).is_err()); + } +} + +#[test] +fn loopback_registration_is_scoped_one_shot_and_drops_without_a_connection() { + let name = format!( + "{}::loopback_registration_is_scoped_one_shot_and_drops_without_a_connection", + module_path!().split_once("::").unwrap().1, + ); + if isolation::isolated_checked(&name, "CODETRIAL_SOCKET_FIXTURE_TEST", false, |_| {}) { + return; + } + let endpoint = "ws://127.0.0.1:9/?key=sequence-test-key"; + let first = crate::gemini::GeminiLiveSession::loopback_send_limit_for_test(endpoint).unwrap(); + assert!(install(endpoint).is_err()); + assert!( + claim("ws://127.0.0.1:8/?key=sequence-test-key") + .unwrap() + .is_none() + ); + assert!(claim(endpoint).unwrap().is_some()); + assert!(claim(endpoint).is_err()); + assert_eq!(first.sender_buffers(), None); + drop(first); + let second = install(endpoint).unwrap(); + assert!(claim(endpoint).unwrap().is_some()); + drop(second); + assert!(claim(endpoint).unwrap().is_none()); +} + +#[test] +fn loopback_buffer_admission_uses_actual_getters_not_requested_capacity() { + assert!(admitted(0).is_err()); + assert_eq!(admitted(4608).unwrap(), 4608); + assert_eq!(admitted(16384).unwrap(), 16384); + assert!(admitted(16385).is_err()); +} + +#[tokio::test(flavor = "current_thread")] +async fn loopback_connect_without_registration_passes_through_without_opening_a_socket() { + assert!( + connect("ws://127.0.0.1:9/?key=sequence-test-key") + .await + .unwrap() + .is_none() + ); +} diff --git a/tests/unit/gemini_parts/scoped_report_endpoint.rs b/tests/unit/gemini_parts/scoped_report_endpoint.rs new file mode 100644 index 000000000..d4a646fe1 --- /dev/null +++ b/tests/unit/gemini_parts/scoped_report_endpoint.rs @@ -0,0 +1,112 @@ +// Scoped routing and observation for tests that exercise the real report path. + +#[derive(Default)] +struct ScopedReportState { + endpoint: String, + entries: Mutex>, +} + +static SCOPED_REPORTS: Mutex)>> = Mutex::new(Vec::new()); + +pub(crate) struct ScopedReport { + scope: String, + state: Arc, +} + +impl ScopedReport { + pub(crate) fn register(scope: &str, endpoint: &str) -> Result { + let url = reqwest::Url::parse(endpoint).map_err(|error| error.to_string())?; + let local = url + .host_str() + .and_then(|host| { + host.trim_matches(['[', ']']) + .parse::() + .ok() + }) + .is_some_and(|address| address.is_loopback()); + if scope.is_empty() + || url.scheme() != "http" + || !local + || !url.username().is_empty() + || url.password().is_some() + { + return Err("report fixture requires a named scope and loopback HTTP endpoint".into()); + } + let mut scopes = SCOPED_REPORTS.lock().unwrap(); + if scopes.iter().any(|(name, _)| name == scope) { + return Err("report fixture scope is already registered".into()); + } + let state = Arc::new(ScopedReportState { + endpoint: endpoint.into(), + ..Default::default() + }); + scopes.push((scope.into(), Arc::clone(&state))); + Ok(Self { + scope: scope.into(), + state, + }) + } + + pub(crate) fn entries(&self) -> Vec<(i64, std::time::Instant)> { + self.state.entries.lock().unwrap().clone() + } +} + +impl Drop for ScopedReport { + fn drop(&mut self) { + SCOPED_REPORTS + .lock() + .unwrap() + .retain(|(name, state)| name != &self.scope || !Arc::ptr_eq(state, &self.state)); + } +} + +pub(crate) fn scoped_report_endpoint(scope: &str) -> Option { + SCOPED_REPORTS + .lock() + .unwrap() + .iter() + .find(|(name, _)| name == scope) + .map(|(_, state)| state.endpoint.clone()) +} + +pub(crate) fn note_bounded_report(scope: &str, seed: i64) { + if let Some((_, state)) = SCOPED_REPORTS + .lock() + .unwrap() + .iter() + .find(|(name, _)| name == scope) + { + state + .entries + .lock() + .unwrap() + .push((seed, std::time::Instant::now())); + } +} + +#[test] +fn scoped_report_routes_are_local_unique_and_removed_on_drop() { + assert!(ScopedReport::register("", "http://127.0.0.1:1/").is_err()); + for endpoint in [ + "https://127.0.0.1/", + "http://example.com/", + "http://user@127.0.0.1/", + ] { + assert!(ScopedReport::register("invalid-scope", endpoint).is_err()); + } + let first = ScopedReport::register("report-scope-first", "http://127.0.0.1:1234/").unwrap(); + let other = ScopedReport::register("report-scope-other", "http://127.0.0.1:4321/").unwrap(); + assert!(ScopedReport::register("report-scope-first", "http://127.0.0.1:1234/").is_err()); + assert_eq!( + scoped_report_endpoint("report-scope-first").as_deref(), + Some("http://127.0.0.1:1234/") + ); + assert!(scoped_report_endpoint("unknown-report-scope").is_none()); + note_bounded_report("report-scope-first", 17); + assert_eq!(first.entries().len(), 1); + assert!(other.entries().is_empty()); + drop(first); + assert!(scoped_report_endpoint("report-scope-first").is_none()); + assert!(scoped_report_endpoint("report-scope-other").is_some()); +} diff --git a/tests/unit/isolation.rs b/tests/unit/isolation.rs new file mode 100644 index 000000000..24f8e78d4 --- /dev/null +++ b/tests/unit/isolation.rs @@ -0,0 +1,101 @@ +use std::time::{Duration, Instant}; + +pub(crate) fn isolated_checked( + name: &str, + marker: &str, + loopback_http: bool, + check: impl FnOnce(&std::process::Output), +) -> bool { + if std::env::var(marker).as_deref() == Ok(name) { + return false; + } + let guard = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + guard.set_nonblocking(true).unwrap(); + let proxy = format!("http://{}", guard.local_addr().unwrap()); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", name, "--nocapture"]) + .env(marker, name) + .env("NO_PROXY", if loopback_http { "127.0.0.1" } else { "" }) + .env("no_proxy", if loopback_http { "127.0.0.1" } else { "" }) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + for key in [ + "HTTPS_PROXY", + "https_proxy", + "HTTP_PROXY", + "http_proxy", + "ALL_PROXY", + "all_proxy", + ] { + command.env(key, &proxy); + } + let mut child = command.spawn().unwrap(); + + // Drained while the child runs: a child that writes more than a pipe holds + // (64 KiB on Linux) blocks in its write, and polling only for its exit then + // reads that block as a hang. + let drain = |stream: Option>| { + std::thread::spawn(move || { + let mut bytes = Vec::new(); + if let Some(mut stream) = stream { + let _ = stream.read_to_end(&mut bytes); + } + bytes + }) + }; + let stdout = drain( + child + .stdout + .take() + .map(|pipe| Box::new(pipe) as Box), + ); + let stderr = drain( + child + .stderr + .take() + .map(|pipe| Box::new(pipe) as Box), + ); + let started = Instant::now(); + let status = loop { + if let Some(status) = child.try_wait().unwrap() { + break status; + } + if started.elapsed() > Duration::from_secs(20) { + child.kill().unwrap(); + child.wait().unwrap(); + panic!( + "isolated test child timed out: {}", + String::from_utf8_lossy(&stderr.join().unwrap()) + ); + } + std::thread::sleep(Duration::from_millis(10)); + }; + let output = std::process::Output { + status, + stdout: stdout.join().unwrap(), + stderr: stderr.join().unwrap(), + }; + assert_eq!( + guard.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock, + "an isolated test must not contact an external provider" + ); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.lines().any(|line| line == "running 1 test") + && stdout.lines().any(|line| { + line.starts_with("test result: ok. 1 passed; 0 failed; 0 ignored;") + }), + "isolated child must execute exactly one passing test: {name}\n{stdout}" + ); + check(&output); + true +} diff --git a/tests/unit/livekit.rs b/tests/unit/livekit.rs index b0d94f8bb..0d5c732b5 100644 --- a/tests/unit/livekit.rs +++ b/tests/unit/livekit.rs @@ -3537,7 +3537,7 @@ fn audio_chunk() -> GeminiEvent { /// The text a fake socket was sent: a cold briefing goes out as realtime /// input, a resumed one as client content. -fn sent_text(sent: &serde_json::Value) -> &str { +pub(super) fn sent_text(sent: &serde_json::Value) -> &str { sent["realtimeInput"]["text"] .as_str() .or_else(|| sent["clientContent"]["turns"][0]["parts"][0]["text"].as_str()) diff --git a/tests/unit/livekit/handoff.rs b/tests/unit/livekit/handoff.rs index f3ad74f6f..ed492fc8b 100644 --- a/tests/unit/livekit/handoff.rs +++ b/tests/unit/livekit/handoff.rs @@ -1,4 +1,5 @@ use super::*; +use crate::test_isolation as isolation; use futures_util::{SinkExt, StreamExt}; use tokio_tungstenite::tungstenite::Message; @@ -412,14 +413,39 @@ async fn a_tick_recovers_when_long_quiet_or_a_failed_request_leaves_the_watchdog #[tokio::test] async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tick() { + let name = format!( + "{}::a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tick", + module_path!().split_once("::").unwrap().1 + ); + if isolation::isolated_checked( + &name, + "CODETRIAL_HANDOFF_LISTENING_NOTE_TEST", + false, + |_| {}, + ) { + return; + } let mut fixture = TickFixture::new(1500, 10).await; let listener = tokio::net::TcpSocket::new_v4().unwrap(); listener.set_recv_buffer_size(1024).unwrap(); + let listener_recv = listener.recv_buffer_size().unwrap(); + assert!((1..=16_384).contains(&listener_recv), "{listener_recv}"); listener.bind("127.0.0.1:0".parse().unwrap()).unwrap(); let listener = listener.listen(1).unwrap(); fixture.endpoint = format!("ws://{}/", listener.local_addr().unwrap()); + let registration = GeminiLiveSession::loopback_send_limit_for_test(&format!( + "{}?key=sequence-test-key", + fixture.endpoint + )) + .unwrap(); let peer = tokio::spawn(async move { let (socket, _) = listener.accept().await.unwrap(); + let accepted_recv = { + let socket_ref = socket2::SockRef::from(&socket); + socket_ref.set_recv_buffer_size(1024).unwrap(); + socket_ref.recv_buffer_size().unwrap() + }; + assert!((1..=16_384).contains(&accepted_recv), "{accepted_recv}"); let mut socket = tokio_tungstenite::accept_async(socket).await.unwrap(); let setup = socket.next().await.unwrap().unwrap(); assert!( @@ -439,10 +465,11 @@ async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tic fixture .replace_speech(session::SpeechHandoff { checkpoint_after: Instant::now(), - transcript_fragments: vec!["x".repeat(32 * 1024 * 1024)], + transcript_fragments: vec!["x".repeat(128 * 1024)], discarded: 0, }) .await; + let sender_buffers = registration.sender_buffers(); assert!(fixture.turn.state.needs_cold_brief); assert!(fixture.turn.activity.owes_prompt()); assert_eq!(fixture.turn.activity.prompt_text.as_deref(), Some(OWED)); @@ -456,6 +483,10 @@ async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tic .await .unwrap() .unwrap(); + let (before, connected) = sender_buffers.expect("the replacement must use the bounded socket"); + assert!((1..=16_384).contains(&before), "{before}"); + assert!((1..=16_384).contains(&connected), "{connected}"); + drop(registration); } #[test] diff --git a/tests/unit/livekit/packets.rs b/tests/unit/livekit/packets.rs index 7710049c0..7e65e71de 100644 --- a/tests/unit/livekit/packets.rs +++ b/tests/unit/livekit/packets.rs @@ -79,3 +79,58 @@ fn closing_transition_is_logged_once_for_received_and_server_controls() { assert!(records[0]["turn"].is_null()); } } + +#[test] +fn a_test_result_reads_its_reaction_clock_for_received_and_server_packets() { + for received in [false, true] { + for expired in [false, true] { + let now = Instant::now(); + + // The pure decision pins the threshold. A synthetic future instant + // gives zero elapsed without waiting at the cooldown edge; opposite + // ages expose a wrong clock source. + let old = now.checked_sub(Duration::from_secs(100)).unwrap(); + let future = now.checked_add(Duration::from_secs(3600)).unwrap(); + let mut state = RuntimeState::default(); + assert!(!state.ended && !state.thinking_hold.is_active()); + let mut activity = RuntimeActivity::for_interview(now, 0, 7); + activity.last_test_reaction = if expired { old } else { future }; + activity.last_code_change = if expired { future } else { old }; + activity.last_interjection = activity.last_code_change; + let result = reduce_data_packet( + &mut state, + &mut activity, + crate::runtime::TOPIC_TEST_RESULTS, + &serde_json::json!({"passed":1,"total":2}), + received, + 1000, + ); + assert_eq!(state.test_runs, 1); + assert_eq!(state.last_test_run.as_ref().unwrap()["passed"], 1); + assert_eq!(state.last_test_run.as_ref().unwrap()["total"], 2); + assert!(state.tested_code.is_none()); + assert!(state.framework_evidence.is_empty()); + assert!(state.runner_unavailable.is_none()); + assert!(!state.ended && !state.thinking_hold.is_active()); + assert_eq!( + state.evidence_ledger.metrics.raw_events_received, + u64::from(received) + ); + assert!(result.test_run.is_some()); + assert_eq!( + result.generate_reply.is_some(), + expired, + "{received}/{expired}" + ); + assert_eq!( + result.update_last_test_reaction, expired, + "{received}/{expired}" + ); + assert_eq!( + result.update_last_interjection, expired, + "{received}/{expired}" + ); + assert!(result.held_context.is_none()); + } + } +} diff --git a/tests/unit/livekit/recovery.rs b/tests/unit/livekit/recovery.rs index c0d46a2e9..06ac8afc9 100644 --- a/tests/unit/livekit/recovery.rs +++ b/tests/unit/livekit/recovery.rs @@ -1,3 +1,5 @@ +use crate::test_isolation as isolation; + use super::super::tests::{fake_recording_socket, hold_turn, test_output_audio}; use super::*; use crate::config::load_from_pairs; @@ -19,98 +21,7 @@ fn isolated_checked_with_loopback( check: impl FnOnce(&std::process::Output), ) -> bool { let name = format!("livekit::recovery::tests::{name}"); - if std::env::var("CODETRIAL_RECOVERY_TEST").as_deref() == Ok(&name) { - return false; - } - let guard = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - guard.set_nonblocking(true).unwrap(); - let proxy = format!("http://{}", guard.local_addr().unwrap()); - let mut command = std::process::Command::new(std::env::current_exe().unwrap()); - command - .args(["--exact", &name, "--nocapture"]) - .env("CODETRIAL_RECOVERY_TEST", &name) - .env("NO_PROXY", if loopback_http { "127.0.0.1" } else { "" }) - .env("no_proxy", if loopback_http { "127.0.0.1" } else { "" }) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()); - for key in [ - "HTTPS_PROXY", - "https_proxy", - "HTTP_PROXY", - "http_proxy", - "ALL_PROXY", - "all_proxy", - ] { - command.env(key, &proxy); - } - let mut child = command.spawn().unwrap(); - - // Drained while the child runs: a child that writes more than a pipe holds - // (64 KiB on Linux) blocks in its write, and polling only for its exit then - // reads that block as a hang. - let drain = |stream: Option>| { - std::thread::spawn(move || { - let mut bytes = Vec::new(); - if let Some(mut stream) = stream { - let _ = stream.read_to_end(&mut bytes); - } - bytes - }) - }; - let stdout = drain( - child - .stdout - .take() - .map(|pipe| Box::new(pipe) as Box), - ); - let stderr = drain( - child - .stderr - .take() - .map(|pipe| Box::new(pipe) as Box), - ); - let started = Instant::now(); - let status = loop { - if let Some(status) = child.try_wait().unwrap() { - break status; - } - if started.elapsed() > Duration::from_secs(20) { - child.kill().unwrap(); - child.wait().unwrap(); - panic!( - "recovery child timed out: {}", - String::from_utf8_lossy(&stderr.join().unwrap()) - ); - } - std::thread::sleep(Duration::from_millis(10)); - }; - let output = std::process::Output { - status, - stdout: stdout.join().unwrap(), - stderr: stderr.join().unwrap(), - }; - assert_eq!( - guard.accept().unwrap_err().kind(), - std::io::ErrorKind::WouldBlock, - "the recovery test must not contact an external provider" - ); - assert!( - output.status.success(), - "{}\n{}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - let stdout = String::from_utf8_lossy(&output.stdout); - assert!( - stdout.lines().any(|line| line == "running 1 test") - && stdout.lines().any(|line| { - line.starts_with("test result: ok. 1 passed; 0 failed; 0 ignored;") - }), - "isolated child must execute exactly one passing test: {name}\n{stdout}" - ); - check(&output); - true + isolation::isolated_checked(&name, "CODETRIAL_RECOVERY_TEST", loopback_http, check) } #[test] @@ -278,7 +189,14 @@ impl Rig { } async fn run(&mut self) -> ControlFlow<()> { - let boot = crate::runtime::bootstrap(&self.config, "recovery-test", Some("two-sum"), 45); + tokio::time::timeout(Duration::from_secs(10), self.run_result()) + .await + .expect("the real room recovery must be bounded in the test") + .unwrap() + } + + async fn run_result(&mut self) -> Result, Error> { + let boot = crate::runtime::bootstrap(&self.config, self.room.name(), Some("two-sum"), 45); let interview = InterviewContext { config: &self.config, keys: &self.keys, @@ -291,7 +209,7 @@ impl Rig { }; let prefix = self.prefix; let speech = self.speech.take(); - tokio::time::timeout(Duration::from_secs(10), async { + async { let mut context = self.turn.context( &mut self.output, &mut self.gemini, @@ -314,22 +232,22 @@ impl Rig { } else { on_gemini_event(&self.room, &mut context, None, &mut self.loops, interview).await } - }) + } .await - .expect("the real room recovery must be bounded in the test") - .unwrap() } - async fn finish(mut self) { + /// Joins the old socket's peer and returns every frame it received. + async fn old_frames(&mut self) -> Vec { self.gemini.shutdown().await.unwrap(); - assert_eq!( - tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) - .await - .unwrap() - .unwrap() - .len(), - self.expected_old_messages - ); + tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) + .await + .unwrap() + .unwrap() + } + + async fn finish(mut self) { + let frames = self.old_frames().await; + assert_eq!(frames.len(), self.expected_old_messages); } } @@ -567,3 +485,5 @@ fn a_stream_end_or_a_new_stream_breaks_the_gap_and_replayed_frames_are_not_deque assert!(after.contains("mic_gap_max_ms=20 "), "{after}"); assert!(after.contains("mic_gap_over_1s=0 "), "{after}"); } + +include!("recovery_parts/report_generation_cardinality.rs"); diff --git a/tests/unit/livekit/recovery_live.rs b/tests/unit/livekit/recovery_live.rs index cdfe4e278..ea34d4cc6 100644 --- a/tests/unit/livekit/recovery_live.rs +++ b/tests/unit/livekit/recovery_live.rs @@ -1128,3 +1128,5 @@ include!("recovery_live_parts/native_worker_and_room_disconnect.rs"); // room loop, one clip arm per invocation. #[path = "recovery_continuity.rs"] mod continuity; + +include!("recovery_live_parts/report_generation_cardinality.rs"); diff --git a/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs b/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs new file mode 100644 index 000000000..bc797fae6 --- /dev/null +++ b/tests/unit/livekit/recovery_live_parts/report_generation_cardinality.rs @@ -0,0 +1,828 @@ +// Real SDK delivery with a hand-built driver clock and local model endpoints. + +#[derive(Clone, Copy, Debug, PartialEq)] +enum G2Case { + EndThenDeadline, + DeadlineFirst, + RoomClosed, + AllReady, +} + +impl G2Case { + fn selected() -> Self { + match std::env::var("CODETRIAL_G2_CASE").as_deref() { + Ok("end") => Self::EndThenDeadline, + Ok("deadline") => Self::DeadlineFirst, + Ok("closed") => Self::RoomClosed, + Ok("all") => Self::AllReady, + _ => panic!("CODETRIAL_G2_CASE is end, deadline, closed or all"), + } + } + + fn name(self) -> &'static str { + match self { + Self::EndThenDeadline => "g2-end-then-deadline", + Self::DeadlineFirst => "g2-deadline-first", + Self::RoomClosed => "g2-room-closed", + Self::AllReady => "g2-all-ready", + } + } + + fn actions(self) -> Vec<&'static str> { + let mut actions = vec!["candidate_join", "agent_join"]; + if matches!(self, Self::RoomClosed | Self::AllReady) { + actions.push("room_delete"); + } + actions.push("controlled_driver_start"); + if !matches!(self, Self::RoomClosed | Self::AllReady) { + actions.push("room_delete"); + } + actions + } + + fn initial_count(self) -> usize { + usize::from(self != Self::RoomClosed) + } + + fn terminal(self) -> &'static str { + match self { + Self::EndThenDeadline => "End", + Self::DeadlineFirst | Self::AllReady => "Deadline", + Self::RoomClosed => "RoomClosed", + } + } +} + +struct G2TerminalRegistration(String); + +impl Drop for G2TerminalRegistration { + fn drop(&mut self) { + super::super::super::LIVE_TERMINAL_PROBES + .lock() + .unwrap() + .retain(|(name, _)| name != &self.0); + } +} + +fn g2_native_config() -> crate::config::AgentConfig { + let mut pairs = crate::config::read_config_file(Path::new( + &std::env::var("CODETRIAL_LIVE_CONFIG").expect("isolated native config path"), + )) + .expect("read isolated native config"); + pairs.retain(|(name, _)| name != "GOOGLE_API_KEY" && name != "GOOGLE_API_KEYS"); + pairs.push(("GOOGLE_API_KEY".into(), "g2-fixture-only-key".into())); + let config = crate::config::load_from_pairs(pairs).expect("valid isolated native config"); + register_secrets([ + config.livekit_api_key.clone(), + config.livekit_api_secret.clone(), + ]); + config +} + +async fn g2_delete(config: &crate::config::AgentConfig, room: &str) -> Result<(), String> { + tokio::time::timeout( + Duration::from_secs(10), + super::super::super::room_service::delete_room( + &config.livekit_url, + &config.livekit_api_key, + &config.livekit_api_secret, + room, + crate::current_epoch_seconds(), + ), + ) + .await + .map_err(|_| "native room delete timed out".to_owned())? + .map_err(|error| redact(&error.to_string())) +} + +async fn g2_queue_until( + source: &mut tokio::sync::mpsc::UnboundedReceiver, + target: &tokio::sync::mpsc::UnboundedSender, + closed: bool, +) -> Result<(), String> { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + let event = source.recv().await.ok_or_else(|| "native agent event stream ended before required input".to_owned())?; + let matched = if closed { + matches!(&event, RoomEvent::Disconnected { .. }) + } else { + matches!(&event, RoomEvent::DataReceived { topic, payload, .. } + if topic.as_deref() == Some(crate::runtime::TOPIC_CONTROL) + && serde_json::from_slice::(payload).is_ok_and(|body| body["type"] == "end_interview")) + }; + target.send(event).map_err(|_| "driver event receiver ended".to_owned())?; + if matched { return Ok(()); } + } + }).await.map_err(|_| "native input arrival was not observed in 20 seconds".to_owned())? +} + +fn g2_seed_report_state(state: &mut crate::agent::RuntimeState) { + state.code = SEED_SOLUTION.into(); + state.language = "python".into(); + crate::livekit::tests::receive_test_run(state); + for phase in ["test", "optimizations"] { + crate::agent::record_framework_evidence( + state, + &json!({ + "phase": phase, "source": crate::livekit::tests::observed_source(phase), + "kind": "observed", "confidence": 90, "summary": "Synthetic G2 fixture evidence." + }), + ) + .unwrap(); + } + state.behavioral_round_started = true; + state.evidence_ledger.conversation.candidate_turns = 1; + let rounds = crate::agent::ReportRounds::of(state); + assert!( + rounds.behavioral_opened && rounds.follow_ups_released && rounds.communication_assessed() + ); +} + +impl super::Rig { + async fn g2_repeat_deadline(&mut self) -> Result, String> { + let boot = crate::runtime::bootstrap(&self.config, self.room.name(), Some("two-sum"), 45); + let interview = super::super::super::InterviewContext { + config: &self.config, + keys: &self.keys, + boot: &boot, + live_endpoint: Some(&self.endpoint), + started_at: std::time::Instant::now(), + candidate_identity: "candidate", + events: &self.events, + slot: None, + }; + let mut context = self.turn.context( + &mut self.output, + &mut self.gemini, + &mut self.board, + &mut self.media, + ); + super::super::super::on_hard_deadline(&self.room, &mut context, &mut self.loops, interview) + .await + .map_err(|error| redact(&error.to_string())) + } +} + +type G2DriverOutcome = (super::Rig, Vec, bool, bool, bool); + +struct G2NativeRun { + config: crate::config::AgentConfig, + room_name: String, + agent_identity: String, + rig: Option, + candidate: Option, + bridge: Option>, + driver: Option>, + shared: Arc, + fixture: super::G2ReportFixture, + terminals: tokio::sync::mpsc::UnboundedReceiver<(&'static str, std::time::Instant)>, + _terminal_scope: G2TerminalRegistration, + attempted_join: bool, + attempted_delete: bool, + joined_driver: bool, + first_break: bool, + guard_reentered: bool, + guard_ok: bool, + outcomes: Vec, +} + +impl G2NativeRun { + async fn new(config: crate::config::AgentConfig, room_name: String) -> Self { + let agent_identity = crate::runtime::agent_identity(&room_name); + let mut rig = super::Rig::new("ws://127.0.0.1:9/".into()).await; + rig.config = config.clone(); + rig.loops.recovery_terminal = None; + rig.loops.recovery_inputs = None; + g2_seed_report_state(&mut rig.turn.state); + let fixture = super::G2ReportFixture::start(&room_name).await; + let (terminal, terminals) = tokio::sync::mpsc::unbounded_channel(); + super::super::super::LIVE_TERMINAL_PROBES + .lock() + .unwrap() + .push((room_name.clone(), terminal)); + Self { + config, + room_name: room_name.clone(), + agent_identity, + rig: Some(rig), + candidate: None, + bridge: None, + driver: None, + shared: Shared::new(), + fixture, + terminals, + _terminal_scope: G2TerminalRegistration(room_name), + attempted_join: false, + attempted_delete: false, + joined_driver: false, + first_break: false, + guard_reentered: false, + guard_ok: false, + outcomes: Vec::new(), + } + } + + async fn delete(&mut self, permits: &mut PermitClient) -> Result<(), String> { + self.attempted_delete = true; + permits + .guarded("room_delete", g2_delete(&self.config, &self.room_name)) + .await + .map_err(|error| error.to_string())? + } + + async fn exercise( + &mut self, + permits: &mut PermitClient, + case: G2Case, + mutant: bool, + ) -> Result<(), String> { + if permits.stop_requested() { + return Err("launcher stopped before joins".into()); + } + self.attempted_join = true; + let token = crate::web::token_response( + &crate::web::TokenConfig { + api_key: &self.config.livekit_api_key, + api_secret: &self.config.livekit_api_secret, + server_url: &self.config.livekit_url, + recording_max_min: None, + }, + br#"{"problemId":"two-sum","durationMin":10}"#, + &self.room_name, + "candidate", + crate::current_epoch_seconds(), + ) + .map_err(|error| format!("candidate token: {error}"))? + .token; + if permits.stop_requested() { + return Err("launcher stopped before candidate join".into()); + } + self.candidate = Some( + permits + .guarded( + "candidate_join", + connect_candidate( + &self.config.livekit_url, + &token, + &self.shared, + &self.agent_identity, + ), + ) + .await + .map_err(|error| error.to_string())??, + ); + if permits.stop_requested() { + return Err("launcher stopped before agent join".into()); + } + let (agent, mut native_events) = permits + .guarded("agent_join", async { + tokio::time::timeout( + Duration::from_secs(30), + super::super::super::join_room( + &self.config, + &self.room_name, + &self.agent_identity, + crate::current_epoch_seconds(), + ), + ) + .await + .map_err(|_| "agent connect timed out".to_owned())? + .map_err(|error| redact(&error.to_string())) + }) + .await + .map_err(|error| error.to_string())??; + self.rig.as_mut().unwrap().room = super::super::super::room_link::RoomLink::live(agent); + let (forward, received) = tokio::sync::mpsc::unbounded_channel(); + self.rig.as_mut().unwrap().events = tokio::sync::Mutex::new(received); + if case != G2Case::RoomClosed { + if permits.stop_requested() { + return Err("launcher stopped before End".into()); + } + permits + .guarded( + FINAL_ACTION_ID, + self.candidate + .as_ref() + .unwrap() + .room + .local_participant() + .publish_data(DataPacket { + payload: serde_json::to_vec( + &json!({"type":"end_interview", "reason":"candidate_ended"}), + ) + .unwrap(), + topic: Some(crate::runtime::TOPIC_CONTROL.into()), + reliable: true, + ..Default::default() + }), + ) + .await + .map_err(|error| error.to_string())? + .map_err(|error| redact(&error.to_string()))?; + g2_queue_until(&mut native_events, &forward, false).await?; + } + if matches!(case, G2Case::RoomClosed | G2Case::AllReady) { + if permits.stop_requested() { + return Err("launcher stopped before delete".into()); + } + self.delete(permits).await?; + g2_queue_until(&mut native_events, &forward, true).await?; + } + self.bridge = Some(tokio::spawn(async move { + while let Some(event) = native_events.recv().await { + if forward.send(event).is_err() { + break; + } + } + })); + if permits.stop_requested() { + return Err("launcher stopped before driver".into()); + } + permits + .permit("controlled_driver_start") + .await + .map_err(|error| error.to_string())?; + let mut rig = self.rig.take().unwrap(); + rig.loops.deadline_at = if matches!(case, G2Case::DeadlineFirst | G2Case::AllReady) { + tokio::time::Instant::now() + } else { + tokio::time::Instant::now() + Duration::from_secs(540) + }; + self.driver = Some(tokio::spawn(async move { + let first = rig + .run_result() + .await + .map_err(|error| redact(&error.to_string())); + let first_break = matches!(&first, Ok(std::ops::ControlFlow::Break(()))); + let mut outcomes = vec![format!("{first:?}")]; + let mut guard_reentered = false; + let mut guard_ok = false; + if case == G2Case::EndThenDeadline && first_break { + let repeated = rig.g2_repeat_deadline().await; + guard_reentered = true; + guard_ok = matches!(&repeated, Ok(std::ops::ControlFlow::Break(()))); + outcomes.push(format!("{repeated:?}")); + } + (rig, outcomes, first_break, guard_reentered, guard_ok) + })); + let stop_file = permits.stop_file.clone(); + let joined = tokio::select! { + biased; + () = async { + loop { + if stop_file.as_deref().is_some_and(Path::exists) { break; } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } => return Err("launcher stopped during driver; cleanup follows".into()), + result = tokio::time::timeout( + Duration::from_secs(if mutant { 1020 } else { 540 }), + self.driver.as_mut().unwrap(), + ) => result.map_err(|_| "driver join timed out".to_owned())? + .map_err(|error| format!("driver join failed: {error}"))?, + }; + self.driver.take(); + self.rig = Some(joined.0); + self.outcomes = joined.1; + self.first_break = joined.2; + self.guard_reentered = joined.3; + self.guard_ok = joined.4; + self.joined_driver = true; + Ok(()) + } + + async fn cleanup(&mut self, permits: &mut PermitClient) -> Vec { + let mut failures = Vec::new(); + if let Some(driver) = self.driver.take() { + driver.abort(); + let _ = tokio::time::timeout(Duration::from_secs(2), driver).await; + failures.push("driver aborted: cardinality acceptance is inconclusive".into()); + } + if let Some(rig) = &mut self.rig { + if !g2_agent_closed( + tokio::time::timeout(Duration::from_secs(10), rig.room.close()).await, + ) { + failures.push("agent Room close did not complete".into()); + } + if !matches!( + tokio::time::timeout(Duration::from_secs(1), rig.gemini.shutdown()).await, + Ok(Ok(())) + ) { + failures.push("local Gemini close did not complete".into()); + } + if let Some(mut old) = rig.old.take() + && !matches!( + tokio::time::timeout(Duration::from_secs(2), &mut old).await, + Ok(Ok(_)) + ) + { + old.abort(); + let _ = old.await; + failures.push("local Gemini peer did not join".into()); + } + } + if self.attempted_join + && !self.attempted_delete + && let Err(error) = self.delete(permits).await + { + failures.push(format!("cleanup room delete: {error}")); + } + if let Some(candidate) = self.candidate.take() { + if !matches!( + tokio::time::timeout(Duration::from_secs(10), candidate.room.close()).await, + Ok(Ok(())) | Ok(Err(::livekit::RoomError::AlreadyClosed)) + ) { + failures.push("candidate Room close did not complete".into()); + } + let mut observer = candidate.observer; + if !matches!( + tokio::time::timeout(Duration::from_secs(10), &mut observer).await, + Ok(Ok(())) + ) { + observer.abort(); + let _ = observer.await; + failures.push("candidate observer did not join".into()); + } + } + if let Some(mut bridge) = self.bridge.take() + && !matches!( + tokio::time::timeout(Duration::from_secs(10), &mut bridge).await, + Ok(Ok(())) + ) + { + bridge.abort(); + let _ = bridge.await; + failures.push("SDK event bridge did not join".into()); + } + failures + } +} + +/// NOT RUN: local-ready scheduling with real SDK packets/receipts, not the +/// production room-loop clock or proof of every native arrival ordering. +#[tokio::test(flavor = "multi_thread")] +#[ignore = "NOT RUN: real LiveKit Room; requires a prospective G2 manifest, launcher permits and isolated native config; Gemini endpoints stay local"] +async fn g2_native_report_generation_and_guarded_deadline_reentry() { + let case = G2Case::selected(); + let mutant = match std::env::var("CODETRIAL_G2_GUARD_MUTANT").as_deref() { + Err(_) => false, + Ok("1") => { + assert_eq!(case, G2Case::EndThenDeadline); + true + } + _ => panic!("CODETRIAL_G2_GUARD_MUTANT is absent or 1"), + }; + let cap = if mutant { 1200 } else { 720 }; + let mut permits = PermitClient::from_env() + .await + .expect("prospective G2 launcher budget"); + let run_id = format!( + "{}{}", + case.name(), + if mutant { "-guard-mutant" } else { "" } + ); + assert_eq!(permits.manifest["run_id"], run_id); + assert_eq!(permits.manifest["actions"], json!(case.actions())); + assert_eq!(permits.manifest["action_ceiling"], case.actions().len() + 1); + assert_eq!(permits.manifest["outer_wall_cap_s"], cap); + assert_eq!( + permits.manifest["observed_budgets"]["external_gemini_http"], + 0 + ); + assert_eq!( + permits.manifest["observed_budgets"]["gemini_live_connections"], + 0 + ); + assert_eq!( + permits.manifest["observed_budgets"]["native_room_connections"], + 2 + ); + assert_eq!( + permits.manifest["observed_budgets"]["local_report_http_attempts"], + if mutant { 10 } else { 5 } + ); + let settings = g2_settings(&permits.manifest); + assert_eq!(settings["case"], case.name()); + assert_eq!(settings["guard_mutant"], mutant); + let room_name = settings["room_name"].as_str().unwrap().to_owned(); + assert_eq!( + permits.manifest["named_resources"], + json!([{"kind":"room", "name":room_name}]) + ); + let config = g2_native_config(); + g2_require_deny_proxy(&config, &settings); + let mut run = G2NativeRun::new(config, room_name).await; + let result = run.exercise(&mut permits, case, mutant).await; + let cleanup = run.cleanup(&mut permits).await; + assert!( + result.is_ok() && cleanup.is_empty() && run.joined_driver, + "INCONCLUSIVE: native driver/setup/cleanup incomplete: {result:?} {cleanup:?}" + ); + if case == G2Case::EndThenDeadline { + assert!( + run.first_break && run.guard_reentered && run.outcomes.len() == 2, + "INCONCLUSIVE: real End must finish and the guarded deadline handler must run" + ); + if !mutant { + assert!( + run.guard_ok, + "INCONCLUSIVE: baseline deadline re-entry must return Ok(Break), not an unrelated error" + ); + } + } else if case != G2Case::AllReady { + assert!( + run.first_break, + "INCONCLUSIVE: terminal handler did not leave normally" + ); + } + let mut expected_permits = case.actions(); + if case != G2Case::RoomClosed { + expected_permits.push(FINAL_ACTION_ID); + } + let mut granted = permits + .granted + .iter() + .map(String::as_str) + .collect::>(); + granted.sort(); + expected_permits.sort(); + assert_eq!( + granted, expected_permits, + "one reserved End and only declared nonfinal actions" + ); + let entries = run.fixture.scope.entries(); + let inputs = run.fixture.bodies.lock().unwrap().len(); + let terminal = run + .terminals + .try_recv() + .expect("real driver terminal observed") + .0; + let observed = run.shared.observed.lock().unwrap(); + emit( + "g2_counts", + json!({"case":case.name(),"bounded_entries":entries.len(),"local_http_attempts":inputs,"final_copies":observed.tally.final_copies,"final_distinct":observed.tally.final_ids.len(),"terminal":terminal,"outcomes":run.outcomes,"driver_joined":run.joined_driver,"first_break":run.first_break,"guard_reentered":run.guard_reentered,"guard_ok":run.guard_ok}), + ); + if mutant { + assert_eq!( + entries.len(), + 2, + "mutant precondition: exactly two actual bounded entries" + ); + assert_eq!( + inputs, 2, + "mutant precondition: exactly two actual local report HTTP requests" + ); + eprintln!("G2_GUARD_MUTANT_REACHED_TWO_REAL_GENERATIONS"); + } + assert_eq!( + entries.len(), + case.initial_count(), + "initial bounded generation cardinality; accepted regeneration was never requested" + ); + assert_eq!( + inputs, + case.initial_count(), + "valid local report fixture uses one HTTP attempt per generation" + ); + assert_eq!(terminal, case.terminal()); + assert!( + run.terminals.try_recv().is_err(), + "one real recovery terminal" + ); + assert_eq!(observed.tally.provisional, 0); + if matches!(case, G2Case::EndThenDeadline | G2Case::DeadlineFirst) { + assert_eq!(observed.tally.final_ids.len(), 1); + assert!( + observed.tally.final_copies >= 1, + "actual SDK final packet received" + ); + assert_eq!(observed.ack_failures, 0); + } else { + assert_eq!( + observed.tally.final_copies, 0, + "closed native room delivers no final packet" + ); + } +} + +#[test] +fn g2_native_rows_declare_finite_zero_gemini_budgets() { + for case in [ + G2Case::EndThenDeadline, + G2Case::DeadlineFirst, + G2Case::RoomClosed, + G2Case::AllReady, + ] { + assert_eq!(case.actions().len() + 1, 5); + assert!(!case.actions().contains(&"retry_report")); + assert!(!case.actions().contains(&FINAL_ACTION_ID)); + } + assert_eq!(4 * 720 + 1200, 68 * 60); +} + +fn g2_agent_closed( + result: Result< + Result<(), Box>, + tokio::time::error::Elapsed, + >, +) -> bool { + match result { + Ok(Ok(())) => true, + Ok(Err(error)) => matches!( + error.downcast_ref::<::livekit::RoomError>(), + Some(::livekit::RoomError::AlreadyClosed) + ), + Err(_) => false, + } +} + +fn g2_require_deny_proxy(config: &crate::config::AgentConfig, settings: &Value) { + let proxy = std::env::var("CODETRIAL_G2_DENY_PROXY") + .expect("run through the G2 non-forwarding proxy wrapper"); + let parsed = reqwest::Url::parse(&proxy).expect("local denial proxy URL"); + assert_eq!(parsed.scheme(), "http"); + assert!( + parsed + .host_str() + .and_then(|host| host + .trim_matches(['[', ']']) + .parse::() + .ok()) + .is_some_and(|address| address.is_loopback()) + ); + assert!(parsed.username().is_empty() && parsed.password().is_none()); + for key in [ + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "http_proxy", + "https_proxy", + "all_proxy", + ] { + assert_eq!( + std::env::var(key).as_deref(), + Ok(proxy.as_str()), + "mandatory outside-HTTP denial" + ); + } + let bypass = std::env::var("NO_PROXY").expect("explicit native host bypasses"); + assert_eq!(std::env::var("no_proxy").as_deref(), Ok(bypass.as_str())); + let host = reqwest::Url::parse(&config.livekit_url) + .unwrap() + .host_str() + .unwrap() + .to_owned(); + let mut native = Vec::new(); + for entry in bypass.split(',') { + assert!( + !entry.is_empty() + && !entry.contains('*') + && !entry.to_ascii_lowercase().contains("googleapis.com") + ); + if entry != "127.0.0.1" && entry != "::1" { + native.push(entry.to_owned()); + } + } + assert!( + bypass.split(',').any(|entry| entry == host), + "the configured native host is explicitly allowed" + ); + let configured = reqwest::Url::parse(&config.livekit_url).unwrap(); + let configured_port = configured.port_or_known_default().unwrap(); + let endpoints = settings["native_endpoints"].as_array().unwrap(); + assert!( + endpoints + .iter() + .any(|endpoint| endpoint["host"] == host && endpoint["port"] == configured_port), + "the configured SDK host:port must be prospectively allowed for CONNECT" + ); + let mut endpoint_hosts = endpoints + .iter() + .map(|endpoint| { + let endpoint_host = endpoint["host"].as_str().unwrap(); + let port = endpoint["port"].as_u64().unwrap(); + assert!( + (1..=65535).contains(&port) + && !endpoint_host.contains('*') + && !endpoint_host + .to_ascii_lowercase() + .contains("googleapis.com") + ); + endpoint_host.to_owned() + }) + .collect::>(); + endpoint_hosts.sort(); + endpoint_hosts.dedup(); + let actual_allow: Vec = serde_json::from_str( + &std::env::var("CODETRIAL_G2_PROXY_ALLOW") + .expect("proxy exports its actual exact CONNECT allowlist"), + ) + .unwrap(); + let as_pairs = |values: &[Value]| { + let mut pairs = values + .iter() + .map(|endpoint| { + ( + endpoint["host"].as_str().unwrap().to_owned(), + endpoint["port"].as_u64().unwrap(), + ) + }) + .collect::>(); + pairs.sort(); + pairs + }; + assert_eq!( + as_pairs(&actual_allow), + as_pairs(endpoints), + "actual proxy CONNECT policy equals prospectively hashed settings" + ); + let mut declared = settings["native_hosts"] + .as_array() + .unwrap() + .iter() + .map(|value| value.as_str().unwrap().to_owned()) + .collect::>(); + native.sort(); + declared.sort(); + assert_eq!( + endpoint_hosts, declared, + "HTTP bypasses are only the declared SDK endpoint hosts" + ); + declared.retain(|entry| entry != "127.0.0.1" && entry != "::1"); + assert_eq!( + native, declared, + "only prospectively named native hosts bypass the restricted proxy" + ); +} + +fn g2_settings(manifest: &Value) -> Value { + let bytes = std::fs::read( + std::env::var("CODETRIAL_G2_SETTINGS").expect("prospective G2 settings path"), + ) + .unwrap(); + assert_eq!( + crate::sha256_hex(&[&bytes]), + manifest["settings_sha256"].as_str().unwrap() + ); + let settings: Value = serde_json::from_slice(&bytes).unwrap(); + let room = settings["room_name"].as_str().unwrap(); + assert!( + !room.is_empty() + && room.len() <= 63 + && room + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) + ); + assert!( + room.starts_with(manifest["run_id"].as_str().unwrap()), + "cleanup owns only the prospectively named run room" + ); + settings +} + +/// Controller-owned idempotent cleanup of the one prospectively named Room. +#[tokio::test] +#[ignore = "NOT RUN: controller-owned G2 named-resource cleanup; requires the frozen manifest/settings and native config"] +async fn g2_cleanup_named_native_room() { + let bytes = + std::fs::read(std::env::var("CODETRIAL_G2_MANIFEST").expect("prospective manifest path")) + .unwrap(); + let manifest: Value = serde_json::from_slice(&bytes).unwrap(); + let settings = g2_settings(&manifest); + let requested = + std::env::var("CODETRIAL_G2_CLEAN_ROOM").expect("controller supplied exact named room"); + assert_eq!(settings["room_name"], requested); + assert_eq!( + manifest["named_resources"], + json!([{"kind":"room", "name":requested}]) + ); + let config = g2_native_config(); + g2_require_deny_proxy(&config, &settings); + g2_delete(&config, &requested) + .await + .expect("bounded owned-room cleanup"); +} + +/// Offline config metadata only: writes host/port without calling a provider. +#[test] +#[ignore = "offline config metadata operation; requires selected config and new private output; no provider or Room call"] +fn g2_write_native_endpoint_settings() { + let out = std::env::var("CODETRIAL_G2_ENDPOINTS_OUT") + .expect("new private endpoint metadata output path"); + let config = g2_native_config(); + let url = reqwest::Url::parse(&config.livekit_url).expect("selected native URL parses"); + assert!(matches!(url.scheme(), "ws" | "wss")); + let host = url.host_str().expect("selected native URL has a host"); + let port = url + .port_or_known_default() + .expect("selected native URL has a port"); + let bytes = serde_json::to_vec_pretty(&json!({ + "native_hosts":[host], "native_endpoints":[{"host":host,"port":port}] + })) + .unwrap(); + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(out) + .expect("endpoint metadata output must be new"); + std::io::Write::write_all(&mut output, &bytes).expect("write endpoint metadata only"); +} diff --git a/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs b/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs index 071b9f6d1..c8b383503 100644 --- a/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs +++ b/tests/unit/livekit/recovery_parts/entry_cancellation_and_driver_decisions.rs @@ -981,7 +981,7 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { let kept = |driver: &Driver<'_, '_, '_>| { ( driver.held_contexts.len(), - driver.held_bytes, + driver.held_contexts.iter().map(String::len).sum::(), driver.held_omitted, driver.held_omitted_total, ) @@ -1001,11 +1001,7 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { // Two that together reach the bound exactly are both kept; one more byte // drops the first and only the first. driver.held_contexts.clear(); - ( - driver.held_bytes, - driver.held_omitted, - driver.held_omitted_total, - ) = (0, 0, 0); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); driver.retain_held_context("a".repeat(40_000)); driver.retain_held_context("b".repeat(MAX_HELD_BYTES - 40_000)); assert_eq!(kept(&driver), (2, MAX_HELD_BYTES, 0, 0)); @@ -1013,13 +1009,20 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { assert_eq!(kept(&driver), (2, MAX_HELD_BYTES - 40_000 + 1, 1, 1)); assert!(driver.held_contexts[0].starts_with('b')); + // Room for one can take more than one eviction, each subtracted from the + // running total: dropping only the first would leave the bound exceeded. + driver.held_contexts.clear(); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); + for (fill, len) in [('a', 30_000), ('b', 30_000), ('c', 4_000)] { + driver.retain_held_context(fill.to_string().repeat(len)); + } + driver.retain_held_context("d".repeat(40_000)); + assert_eq!(kept(&driver), (2, 44_000, 2, 2)); + assert!(driver.held_contexts[0].starts_with('c')); + // The count bound drops the oldest, one at a time. driver.held_contexts.clear(); - ( - driver.held_bytes, - driver.held_omitted, - driver.held_omitted_total, - ) = (0, 0, 0); + (driver.held_omitted, driver.held_omitted_total) = (0, 0); for n in 0..MAX_HELD_CONTEXTS { driver.retain_held_context(format!("{n}")); } @@ -1029,10 +1032,6 @@ async fn held_contexts_are_kept_within_a_count_and_a_byte_bound() { assert_eq!(driver.held_contexts.len(), MAX_HELD_CONTEXTS); assert_eq!((driver.held_omitted, driver.held_omitted_total), (1, 1)); assert_eq!(driver.held_contexts[0], "1"); - assert_eq!( - driver.held_bytes, - driver.held_contexts.iter().map(String::len).sum::() - ); drop(driver); rig.finish().await; } @@ -1163,7 +1162,6 @@ async fn reference_debt_survives_noop_packets_and_action_supersession() { } assert!(driver.context.state.earlier_steps_named.is_empty()); driver.held_contexts.push_back("unsent held summary".into()); - driver.held_bytes = "unsent held summary".len(); driver.round = Some("skipped"); driver.loops.pending_round_notice = Some("started"); assert!(!driver.drain(&mut closed).await.unwrap()); diff --git a/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs b/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs new file mode 100644 index 000000000..16023a0cd --- /dev/null +++ b/tests/unit/livekit/recovery_parts/report_generation_cardinality.rs @@ -0,0 +1,172 @@ +// Initial bounded generation is distinct from provider attempts and receipts. + +#[tokio::test] +async fn g2_bounded_report_observes_only_polled_scoped_invocations() { + if isolated_checked_with_loopback( + "g2_bounded_report_observes_only_polled_scoped_invocations", + true, + |_| {}, + ) { + return; + } + let config = g2_fixture_config(); + let boot = crate::runtime::bootstrap(&config, "g2-polled-entry", Some("two-sum"), 45); + let keys = crate::gemini::GeminiKeys::single("g2-fixture-key"); + let refused = std::sync::atomic::AtomicBool::new(false); + let scope = crate::gemini::tests::ScopedReport::register(boot.room_name, "http://127.0.0.1:9/") + .unwrap(); + let future = super::super::report::generate_report_bounded( + &boot, + "fixture evidence", + &[], + ( + crate::agent::ReportRounds { + behavioral_opened: true, + follow_ups_released: true, + candidate_turns: 1, + }, + false, + ), + &keys, + 31, + &refused, + ); + tokio::pin!(future); + assert!( + scope.entries().is_empty(), + "constructing a lazy generation does not start it" + ); + + // The refused endpoint keeps the generation retrying, so the first poll + // leaves it pending and the second one is a real second poll. + let finished = std::future::poll_fn(|cx| { + std::task::Poll::Ready(std::future::Future::poll(future.as_mut(), cx).is_ready()) + }) + .await; + assert!(!finished, "the first poll leaves the generation pending"); + assert_eq!(scope.entries().len(), 1, "actual bounded body entered once"); + let _ = std::future::poll_fn(|cx| { + std::task::Poll::Ready(std::future::Future::poll(future.as_mut(), cx).is_ready()) + }) + .await; + assert_eq!( + scope.entries().len(), + 1, + "polling the same future does not recount" + ); + assert_eq!(scope.entries()[0].0, 31); +} + +#[tokio::test] +async fn g2_scoped_report_uses_actual_generator_material_and_wire() { + if isolated_checked_with_loopback( + "g2_scoped_report_uses_actual_generator_material_and_wire", + true, + |_| {}, + ) { + return; + } + let config = g2_fixture_config(); + let boot = crate::runtime::bootstrap(&config, "g2-generator-wire", Some("two-sum"), 45); + let keys = crate::gemini::GeminiKeys::single("g2-fixture-key"); + let refused = std::sync::atomic::AtomicBool::new(false); + let fixture = G2ReportFixture::start(boot.room_name).await; + let result = tokio::time::timeout( + Duration::from_secs(2), + super::super::report::generate_report_bounded( + &boot, + "unique g2 assessment material", + &[], + ( + crate::agent::ReportRounds { + behavioral_opened: true, + follow_ups_released: true, + candidate_turns: 1, + }, + false, + ), + &keys, + 37, + &refused, + ), + ) + .await; + result + .expect("registered report route reaches its local HTTP fixture") + .unwrap() + .unwrap(); + assert_eq!(fixture.scope.entries().len(), 1); + let bodies = fixture.bodies.lock().unwrap(); + assert_eq!(bodies.len(), 1); + assert!( + bodies[0] + .to_string() + .contains("unique g2 assessment material") + ); + assert_eq!(bodies[0]["generationConfig"]["seed"], 37); + assert_eq!( + bodies[0]["generationConfig"]["responseSchema"], + crate::agent::report_response_schema(true) + ); +} + +#[test] +fn g2_repeated_server_end_cannot_create_a_second_finish_intent() { + let mut state = RuntimeState::default(); + let payload = serde_json::json!({"type":"end_interview", "reason":"candidate_ended"}); + let first = crate::agent::apply_server_event_at(&mut state, TOPIC_CONTROL, &payload, 0.0, 1); + assert!(first.finish_interview.is_some()); + let deadline = serde_json::json!({"type":"end_interview", "reason":"time_up"}); + let second = crate::agent::apply_server_event_at(&mut state, TOPIC_CONTROL, &deadline, 0.0, 2); + assert!( + second.finish_interview.is_none(), + "deadline re-entry cannot finish an ended assessment twice" + ); +} + +fn g2_fixture_config() -> crate::config::AgentConfig { + load_from_pairs([ + ("LIVEKIT_URL", "wss://example.livekit.cloud"), + ("LIVEKIT_API_KEY", "devkey"), + ("LIVEKIT_API_SECRET", "devsecret"), + ("GOOGLE_API_KEY", "g2-fixture-key"), + ]) + .unwrap() +} + +struct G2ReportFixture { + scope: crate::gemini::tests::ScopedReport, + bodies: std::sync::Arc>>, + server: tokio::task::JoinHandle<()>, +} + +impl G2ReportFixture { + async fn start(scope: &str) -> Self { + let bodies = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let seen = std::sync::Arc::clone(&bodies); + let answer = crate::gemini::tests::valid_report(); + let app = axum::Router::new().route("/", axum::routing::post(move |axum::Json(body): axum::Json| { + let seen = std::sync::Arc::clone(&seen); + let answer = answer.clone(); + async move { + seen.lock().unwrap().push(body); + axum::Json(serde_json::json!({"candidates":[{"content":{"parts":[{"text":answer.to_string()}]}}]})) + } + })); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/", listener.local_addr().unwrap()); + let scope = crate::gemini::tests::ScopedReport::register(scope, &endpoint).unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + Self { + scope, + bodies, + server, + } + } +} + +impl Drop for G2ReportFixture { + fn drop(&mut self) { + self.server.abort(); + } +} diff --git a/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs b/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs index ee0863547..ae1d4716d 100644 --- a/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs +++ b/tests/unit/livekit/recovery_parts/retry_and_room_waits.rs @@ -1126,14 +1126,25 @@ impl WriteSite { } } -fn write_site_usage(output: &std::process::Output, drain: &str, rows: usize) { +fn write_site_usage( + output: &std::process::Output, + drain: &str, + rows: usize, + reference: Option<&str>, +) { let stderr = String::from_utf8_lossy(&output.stderr); let usage = stderr .lines() .filter(|line| line.starts_with("codetrial live_turn_usage ")) .collect::>(); - assert_eq!(usage.len(), rows * 2); - for pair in usage.as_chunks::<2>().0 { + let total_rows = rows + usize::from(reference.is_some()); + assert_eq!(usage.len(), total_rows * 2); + for (index, pair) in usage.as_chunks::<2>().0.iter().enumerate() { + let drain = if index == 0 { + reference.unwrap_or(drain) + } else { + drain + }; for (line, socket, cause, prompt, event) in [(pair[0], 1, "tool", 11, 1), (pair[1], 2, drain, 29, 2)] { @@ -1155,6 +1166,6 @@ fn write_site_usage(output: &std::process::Output, drain: &str, rows: usize) { .lines() .filter(|line| line.starts_with("recovery ingress: ")) .count(), - rows + total_rows ); } diff --git a/tests/unit/livekit/recovery_parts/terminal_matrices.rs b/tests/unit/livekit/recovery_parts/terminal_matrices.rs index ecee98aae..e87e8c5eb 100644 --- a/tests/unit/livekit/recovery_parts/terminal_matrices.rs +++ b/tests/unit/livekit/recovery_parts/terminal_matrices.rs @@ -1067,6 +1067,8 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() ) { return; } + const LISTENING_CONTEXT: &str = + "[SYSTEM EVENT] The candidate continues speaking. Keep listening; no reply is requested."; let mut rows = 0; for phase in PREFIX_PHASES { for cause in PREFIX_STOPS { @@ -1126,7 +1128,7 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() let note: serde_json::Value = serde_json::from_str(note.to_text().unwrap()).unwrap(); assert_eq!(note["clientContent"]["turnComplete"], false); - assert!(note.to_string().contains("Keep listening")); + assert_eq!(super::super::tests::sent_text(¬e), LISTENING_CONTEXT); messages.push(note); } } @@ -1283,6 +1285,15 @@ async fn prefix_terminal_matrix_preserves_received_evidence_before_any_restart() rig.turn.state.evidence_ledger.metrics.turn_prompt_count, u32::from(phase.transcribed()) ); + assert_eq!( + rig.turn.state.evidence_ledger.metrics.turn_prompt_bytes, + if phase.transcribed() { + u64::try_from(LISTENING_CONTEXT.len()).unwrap() + } else { + 0 + }, + "{phase:?} {cause:?}" + ); assert_eq!( rig.turn .state diff --git a/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs b/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs index 39b05dcc6..03323049f 100644 --- a/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs +++ b/tests/unit/livekit/recovery_parts/tool_work_and_final_sends.rs @@ -1277,7 +1277,6 @@ async fn omitted_held_contexts_are_counted_once_and_reach_the_wire_once() { assert!(driver.pending()); assert!(!driver.drain(&mut fresh).await.unwrap()); assert!(driver.held_contexts.is_empty()); - assert_eq!(driver.held_bytes, 0); assert_eq!((driver.held_omitted, driver.held_omitted_total), (0, 8)); assert!(!driver.pending()); feed(&mut driver, 33); diff --git a/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs b/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs index 4219e87b4..7552b7efc 100644 --- a/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs +++ b/tests/unit/livekit/recovery_parts/write_sites_and_replacement_classification.rs @@ -21,8 +21,14 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool } else { &PREFIX_STOPS }; + let needs_reference = !physical && !after && site != WriteSite::BoardImage; + let reference = needs_reference.then_some((true, OpenStop::End)); + let cases = reference + .into_iter() + .chain(causes.iter().map(|&cause| (after, cause))); + let mut reference_bytes = None; let mut rows = 0; - for &cause in causes { + for (index, (after, cause)) in cases.enumerate() { let resumed = site == WriteSite::SilentEditor; let mut rig = Rig::with_old_checkpoint("ws://127.0.0.1:9/".into(), Some(11), resumed).await; rig.gemini.input_cause = Some("tool"); @@ -332,6 +338,10 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool .is_err() ); assert!(rig.loops.recovery_writes.is_empty()); + let constructed = ( + rig.turn.state.evidence_ledger.metrics.turn_prompt_count, + rig.turn.state.evidence_ledger.metrics.turn_prompt_bytes, + ); rig.finish().await; finish.send(()).unwrap(); let (messages, received) = server.await.unwrap(); @@ -343,9 +353,41 @@ async fn write_site_terminal_matrix(site: WriteSite, physical: bool, after: bool expected_before, "no write after the targeted one may reach the wire: {site:?} {cause:?}" ); + let expected: Option = match site { + WriteSite::Brief | WriteSite::ListeningBrief => Some(1), + WriteSite::HeldContext | WriteSite::SilentEditor => Some(2), + WriteSite::HeldAction => Some(3), + WriteSite::BoardImage => None, + }; + if let Some(expected) = expected { + assert_eq!(constructed.0, expected, "{site:?} {cause:?} after={after}"); + let delivered: u64 = messages + .iter() + .map(|message| { + u64::try_from(super::super::tests::sent_text(message).len()).unwrap() + }) + .sum(); + + // The reference shares this fixture and sums both clock and body. + // Cancellation must retain the same full charge, not just more + // bytes than the earlier writes delivered. + if after { + assert_eq!(constructed.1, delivered, "{site:?} {cause:?}"); + if needs_reference && index == 0 { + reference_bytes = Some(delivered); + } + } else { + assert_eq!( + constructed.1, + reference_bytes.expect("the matching completed write ran first"), + "{site:?} {cause:?} retains the full constructed input" + ); + assert!(constructed.1 > delivered, "{site:?} {cause:?}"); + } + } rows += 1; } - assert_eq!(rows, causes.len()); + assert_eq!(rows, causes.len() + usize::from(needs_reference)); } macro_rules! write_site_test { @@ -367,7 +409,11 @@ macro_rules! write_site_test { } else { $site.drain_cause() }; - if isolated_checked(stringify!($name), |out| write_site_usage(out, drain, rows)) { + let reference = (!$physical && !$after && $site != WriteSite::BoardImage) + .then_some($site.after_cause()); + if isolated_checked(stringify!($name), |out| { + write_site_usage(out, drain, rows, reference); + }) { return; } write_site_terminal_matrix($site, $physical, $after).await; @@ -1244,15 +1290,6 @@ impl Rig { .await .unwrap(); } - - /// Joins the old socket's peer and returns every frame it received. - async fn old_frames(&mut self) -> Vec { - self.gemini.shutdown().await.unwrap(); - tokio::time::timeout(Duration::from_secs(2), self.old.take().unwrap()) - .await - .unwrap() - .unwrap() - } } /// Like `scripted_fresh_gemini`, but a ready socket keeps what it is sent. diff --git a/tests/unit/livekit/report.rs b/tests/unit/livekit/report.rs index 99d7cb92a..32bff2a8c 100644 --- a/tests/unit/livekit/report.rs +++ b/tests/unit/livekit/report.rs @@ -1621,11 +1621,12 @@ fn recovery_offer_matches_browser_limits_and_generation_deadline() { limits["quotaRetryAfterSeconds"] ); - // The regenerated report goes through the same receipt wait, so a page that - // stops waiting before every attempt could land drops one in flight. + // Generation can finish while the candidate is away; a return within the + // grace must still leave every receipt attempt time to land. let delivery = DELIVERY_WAIT * DELIVERY_ATTEMPTS as u32; assert!( - limits["retryWaitSeconds"].as_u64().unwrap() > (REPORT_TIMEOUT + delivery).as_secs() + 3 + limits["retryWaitSeconds"].as_u64().unwrap() + > (REPORT_TIMEOUT + REJOIN_GRACE + delivery).as_secs() + 3 ); } @@ -2011,47 +2012,18 @@ fn only_the_candidate_coming_and_going_moves_a_recovery_wait() { } } -/// Drives a recovery wait from a script of events, each sent after its delay. -async fn scripted_recovery( - script: Vec<(u64, RecoveryEvent)>, - generation: impl std::future::Future, - readiness: Readiness, -) -> (Option, Vec) { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let mut fixture = RecoveryFixture::new(rx); - let sender = async move { - for (delay, event) in script { - tokio::time::sleep(std::time::Duration::from_secs(delay)).await; - let _ = tx.send(event); - } - tx - }; - let (result, _tx) = tokio::join!( - recover_report( - &mut fixture, - None, - generation, - REPORT_RECOVERY_WINDOW, - std::time::Duration::ZERO, - tokio::time::Instant::now(), - move || readiness, - ), - sender, - ); - (result, fixture.1) -} +include!("report/scripted_recovery.rs"); /// A full LiveKit rejoin looks like the candidate leaving and coming back. An /// accepted regeneration keeps running through it, and the report it returns /// while they are away waits for them instead of going into an empty room. #[tokio::test(start_paused = true)] async fn a_rejoin_within_the_grace_keeps_an_accepted_regeneration() { - let started = tokio::time::Instant::now(); let generation = async { tokio::time::sleep(std::time::Duration::from_secs(5)).await; Ok(Ok(serde_json::json!({"result": "complete"}))) }; - let (result, notices) = scripted_recovery( + let (result, notices, returned_at) = timed_scripted_recovery( vec![ (0, RecoveryEvent::Retry), (1, RecoveryEvent::Away), @@ -2063,7 +2035,7 @@ async fn a_rejoin_within_the_grace_keeps_an_accepted_regeneration() { .await; assert_eq!(result.unwrap().unwrap().unwrap()["result"], "complete"); assert_eq!(notices, [RecoveryNotice::Accepted]); - assert!(started.elapsed() >= std::time::Duration::from_secs(20)); + assert_eq!(returned_at, std::time::Duration::from_secs(20)); } /// Gone past the grace is gone, before a retry or during the regeneration. diff --git a/tests/unit/livekit/report/scripted_recovery.rs b/tests/unit/livekit/report/scripted_recovery.rs new file mode 100644 index 000000000..fc4737d1d --- /dev/null +++ b/tests/unit/livekit/report/scripted_recovery.rs @@ -0,0 +1,73 @@ +/// Drives a recovery wait from a script of events, each sent after its delay. +async fn scripted_recovery( + script: Vec<(u64, RecoveryEvent)>, + generation: impl std::future::Future, + readiness: Readiness, +) -> (Option, Vec) { + let (result, notices, _) = timed_scripted_recovery(script, generation, readiness).await; + (result, notices) +} + +async fn timed_scripted_recovery( + script: Vec<(u64, RecoveryEvent)>, + generation: impl std::future::Future, + readiness: Readiness, +) -> (Option, Vec, Duration) { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let mut fixture = RecoveryFixture::new(rx); + let started = tokio::time::Instant::now(); + let sender = async move { + for (delay, event) in script { + tokio::time::sleep(std::time::Duration::from_secs(delay)).await; + let _ = tx.send(event); + } + tx + }; + // The sender can finish later than the holder and hide an early return. + let ((result, returned_at), _tx) = tokio::join!( + async { + let result = recover_report( + &mut fixture, + None, + generation, + REPORT_RECOVERY_WINDOW, + std::time::Duration::ZERO, + tokio::time::Instant::now(), + move || readiness, + ) + .await; + (result, started.elapsed()) + }, + sender, + ); + (result, fixture.1, returned_at) +} + +#[tokio::test(start_paused = true)] +async fn a_late_rejoin_fits_the_browser_report_wait() { + let generation = async { + tokio::time::sleep(Duration::from_secs(124)).await; + Ok(Ok(serde_json::json!({"result": "complete"}))) + }; + let (result, notices, returned_at) = timed_scripted_recovery( + vec![ + (0, RecoveryEvent::Retry), + (123, RecoveryEvent::Away), + (29, RecoveryEvent::Back), + ], + generation, + Readiness::Ready, + ) + .await; + assert_eq!(result.unwrap().unwrap().unwrap()["result"], "complete"); + assert_eq!(notices, [RecoveryNotice::Accepted]); + assert_eq!(returned_at, Duration::from_secs(152)); + let limits: serde_json::Value = + serde_json::from_str(include_str!("../../../fixtures/report-recovery.json")).unwrap(); + let browser_wait = Duration::from_secs(limits["retryWaitSeconds"].as_u64().unwrap()); + let delivery = DELIVERY_WAIT * u32::try_from(DELIVERY_ATTEMPTS).unwrap(); + assert!( + browser_wait > returned_at + delivery, + "browser wait {browser_wait:?} must cover holder return {returned_at:?} and delivery {delivery:?}" + ); +} diff --git a/web/report-recovery.js b/web/report-recovery.js index 7ec3dfed7..807636681 100644 --- a/web/report-recovery.js +++ b/web/report-recovery.js @@ -2,7 +2,7 @@ export const reportRecoveryLimits = Object.freeze({ expiresInSeconds: 300, retryAfterSeconds: 30, quotaRetryAfterSeconds: 60, - retryWaitSeconds: 145, + retryWaitSeconds: 175, // The agent announces the end of its window. This page's clock starts when // the offer arrives, later than the agent's, so it waits a little past it // for that word and falls back on its own only when the word never comes. @@ -61,8 +61,8 @@ export function createReportRecovery({ report = null; ready = false; } - // The agent gives regeneration its own 125-second deadline, and the report - // it produces may take every delivery attempt to arrive. + // The agent gives regeneration its own 125-second deadline. A ready report + // can wait through the rejoin grace and still need every delivery attempt. function waitForReport() { if (wait) timers.clearTimeout(wait); wait = timers.setTimeout( From d3c1e8db9ab86f869c954312f08559501d43732e Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 6 Nov 2026 09:49:52 +0800 Subject: [PATCH 3/7] Judge decoded swap audio in long interviews A long interview reported its decoded signal as not assessed even when a replacement swap had been installed, because only the soak flow ran the media probe. Install the probe for the long flow, mark each installed swap once, and carry the judged result into the report as a separate status that never changes the exit code or the qualification. Each judged row keeps the samples behind it, cut to the fields the judgment read and at most four per swap with the decisive one kept, and a covered row carries the sample that covers it. The swap marker and the settle step move to swap-audio-owner.cjs, shared with the soak. Change-Id: Ieb285e4e87208246e92368dc61c3e78a35bd2c48 --- docs/latency-observability.md | 7 +- scripts/browser-check.cjs | 164 +++++++------ scripts/long-interview-driver.cjs | 47 +++- scripts/soak-judge.cjs | 103 +++++++- scripts/swap-audio-owner.cjs | 89 +++++++ tests/browser/long-interview-driver.test.js | 166 +++++++++++++ tests/browser/soak-judge.test.js | 113 +++++++++ tests/browser/swap-audio-owner.test.js | 258 ++++++++++++++++++++ 8 files changed, 865 insertions(+), 82 deletions(-) create mode 100644 scripts/swap-audio-owner.cjs create mode 100644 tests/browser/swap-audio-owner.test.js diff --git a/docs/latency-observability.md b/docs/latency-observability.md index 17c632b5c..61bc3e51c 100644 --- a/docs/latency-observability.md +++ b/docs/latency-observability.md @@ -225,9 +225,10 @@ node --test tests/browser/soak-judge.test.js tests/browser/long-interview-driver A long-interview report's `assessment_scope` is `structural`, even when `run_kind` declares a qualification workload. Its `qualification` remains -`not_evaluated`: checkpoint `reply_content` and post-swap `decoded_signal` -are `not_assessed` by this driver. Separate evidence is required before -qualifying a build. `exposure_s` ends at the harness's observation boundary; +`not_evaluated`: checkpoint `reply_content` is `not_assessed` by this driver, +and post-swap `decoded_signal` (`verified`, `not_required`, `failed` or +`not_assessed`) is reported beside it and changes neither the qualification +nor the exit code. Separate evidence is required before qualifying a build. `exposure_s` ends at the harness's observation boundary; the analyzer's exposure can extend through the terminal socket's end and must not be substituted for it. `end_permit_granted` records permission, while `end_clicked` records the page action; the countdown can win after the permit diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index bf30ae363..a66411666 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -59,15 +59,17 @@ const { planFromEnv, validateEnvironment, } = require("./long-interview-plan.cjs"); +const { + judgeSettledSwapAudio, + startSwapMarker, +} = require("./swap-audio-owner.cjs"); const { clockEvidenceLines, completeLines, episodeEvidenceLines, - installedSwapKeys, isRotation, judgeEpisodeReplies, judgeSoak, - judgeSwapAudio, parseClock, parseEpisodes, parseObligations, @@ -510,6 +512,10 @@ const CONNECTION_CAP_SECONDS = 600; /// `GoAway`'s notice plus a stalled reply's twenty seconds. const RESUME_GRACE_MS = 45_000; +// What the shared settle needs from this process: the grace the page is given +// and where its per-episode lines go. +const SETTLE = { graceMs: RESUME_GRACE_MS, log: console.log }; + /// The two runtime lines that are not part of a replacement: the advisory and /// its expiry. Matched literally, printed by src/livekit.rs. What a replacement /// was is read from the fields of its `replacement:` line by @@ -786,6 +792,7 @@ async function longInterview( tokenRequestsMs, joinGuarded, tokenExpiredNow, + probing = false, ) { const plan = longPlan; const countdownStartS = parseCountdown( @@ -849,21 +856,34 @@ async function longInterview( return pageSide.awaitServerDeadline(); }, }; + const swapMarker = probing + ? startSwapMarker({ + page, + roomName, + agentOutput, + newLines: serverLogTail(), + }) + : null; // The schedule's clock starts here: connected, greeted, the join already // dispatched through the budget by the media gate. - const run = await runLongInterview({ - plan, - schedule: prepared.schedule, - playback: prepared.playback, - clipSource: prepared.clipSource, - budget: runBudget, - controller, - joinedAtMs: Date.now(), - interviewStartMs, - joinDispatched: joinGuarded, - observe, - failure: failure() ?? new Promise(() => {}), - }); + let run; + try { + run = await runLongInterview({ + plan, + schedule: prepared.schedule, + playback: prepared.playback, + clipSource: prepared.clipSource, + budget: runBudget, + controller, + joinedAtMs: Date.now(), + interviewStartMs, + joinDispatched: joinGuarded, + observe, + failure: failure() ?? new Promise(() => {}), + }); + } finally { + await swapMarker?.halt(); + } ended = true; refresh(); const clock = parseClock(`${[...evidence.keys()].join("\n")}\n`, roomName); @@ -881,6 +901,33 @@ async function longInterview( ), graceStimuli: 0, }); + const episodeText = `${[...episodeLines.keys()].join("\n")}\n`; + const episodes = parseEpisodes(episodeText, roomName); + // A probe that could not mark or settle leaves the signal not assessed + // rather than failing the run. + let audio = null; + let replies = null; + if (swapMarker) { + try { + await swapMarker.stop(); + replies = judgeEpisodeReplies( + episodes, + parseObligations(episodeText, roomName), + true, + ); + audio = await judgeSettledSwapAudio( + page, + replies, + "long-interview", + SETTLE, + ); + } catch (error) { + console.error(`long-interview: decoded audio not assessed: ${error}`); + } + await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } const log = `${agentOutput.join("")}\n${readServerLog()}`; const serverLog = process.env.SERVER_LOG; const logsComplete = @@ -897,10 +944,9 @@ async function longInterview( observation, clock, tokenRequestsMs, - episodes: parseEpisodes( - `${[...episodeLines.keys()].join("\n")}\n`, - roomName, - ).episodes, + episodes: episodes.episodes, + audio, + replies, logs: { truncated: false, evicted: false, complete: logsComplete }, browserDisconnected, tokenExpired: tokenExpiredNow(), @@ -987,8 +1033,6 @@ async function soakInterview( // as it goes past for the same reason: the agent's output is a rolling // window. const evidence = new Map(); - // The installed swaps already handed to the page to sample again. - const marked = new Set(); // One read per call rather than one per signal: this is the whole server // log, and the loop runs for minutes. @@ -1016,24 +1060,6 @@ async function soakInterview( parseObligations(evidenceText(), roomName), final, ); - // Each installed swap is marked in the page once, as soon as its end line is - // read, so the sample that answers it can only open on a Speaking transition - // after this point. Read late, never early: a transition between the install - // and this call is missed and the next one is used instead, which can fail - // a swap whose reply was quicker than a poll but never passes one on audio - // from before it. - const newLines = serverLogTail(); - const markNewSwaps = async () => { - if (!probing) return; - for (const key of installedSwapKeys( - `${agentOutput.join("")}\n${newLines()}`, - roomName, - )) { - if (marked.has(key)) continue; - marked.add(key); - await page.evaluate((swap) => globalThis.__markJimSwap?.(swap), key); - } - }; const failure = (reasons) => new Error(`${reasons.join("; ")} during ${soakSeconds}s soak`); const judge = (final) => @@ -1055,29 +1081,15 @@ async function soakInterview( return soakSummary; }; - // Marked on its own clock, not the loops' below: those wait on log parsing - // and room-service calls, and a swap read late can miss the transition that - // answered it. - let marking = true; - let markError = null; - const marker = (async () => { - while (marking) { - await sleep(250); - if (!marking) break; - try { - await markNewSwaps(); - } catch (error) { - markError = error; - marking = false; - } - } - })(); - const stopMarking = async () => { - marking = false; - await marker; - if (markError) throw markError; - await markNewSwaps(); - }; + const swapMarker = probing + ? startSwapMarker({ + page, + roomName, + agentOutput, + newLines: serverLogTail(), + }) + : null; + const stopMarking = async () => swapMarker?.stop(); try { let nextRefresh = 0; @@ -1169,16 +1181,7 @@ async function soakInterview( // cancels what still waits, and an unsampled swap counts against the soak. let audio = null; if (probing) { - const rearms = await page.evaluate( - (waitMs) => globalThis.__settleJimSwapAudio(waitMs), - RESUME_GRACE_MS, - ); - audio = judgeSwapAudio(replies, rearms); - for (const e of audio.episodes) { - console.log( - `soak episode ${e.episode} audio: ${e.status} ${e.reasons.join("; ")}`.trimEnd(), - ); - } + audio = await judgeSettledSwapAudio(page, replies, "soak", SETTLE); } const result = record(final, replies, audio); @@ -1195,8 +1198,7 @@ async function soakInterview( throw failure(detail.length > 0 ? detail : final.reasons); } } finally { - marking = false; - await marker; + await swapMarker?.halt(); } } @@ -1360,6 +1362,7 @@ async function isolateRustAgent( let bargeReplay = null; let jimAudio = null; let probingAudio = false; + let probingSwapAudio = false; if (process.env.BROWSER_CHECK_FLOW === "barge") { await page.addInitScript( installBargeReplay, @@ -3025,7 +3028,11 @@ PatternLexicon.prototype.search = function(word) { ), ); probingAudio = credentialed && ["voice", "barge", "soak"].includes(flow); - if (probingAudio) { + // The long interview judges the decoded audio of its own swaps inside + // longInterview; the generic end-of-flow assertion on Jim's whole track + // would fail a tone fixture that never makes Jim speak. + probingSwapAudio = credentialed && flow === "long-interview"; + if (probingAudio || probingSwapAudio) { await page.evaluate(mediaProbeSource()); } let joinGuarded = false; @@ -3157,6 +3164,7 @@ PatternLexicon.prototype.search = function(word) { consoleErrors.some((line) => /token.*(expired|invalid)|expired.*token/i.test(line), ), + probingSwapAudio, ); } let testResultText = null; @@ -3326,6 +3334,12 @@ PatternLexicon.prototype.search = function(word) { .evaluate(() => globalThis.__finishJimAudioProbe(true)) .catch(() => null); } + if (probingSwapAudio) { + // Stops the probe's sampling on any failure, whichever step threw. + await page + .evaluate(() => globalThis.__finishJimAudioProbe(true)) + .catch(() => null); + } if ( process.env.BROWSER_CHECK_CAPTURE && (floorLogPath || probingAudio) && diff --git a/scripts/long-interview-driver.cjs b/scripts/long-interview-driver.cjs index 5ec6a3753..c1579b062 100644 --- a/scripts/long-interview-driver.cjs +++ b/scripts/long-interview-driver.cjs @@ -30,6 +30,7 @@ const path = require("path"); const sched = require("./long-interview-schedule.cjs"); const timing_ = require("./long-interview-timing.cjs"); const { BudgetDenied } = require("./run-budget-guard.cjs"); +const { summarizeSwapAudio } = require("./soak-judge.cjs"); const CLIP_DIR_ENV = "BROWSER_CHECK_LONG_CLIP_DIR"; const TONE_FIXTURE_ENV = "BROWSER_CHECK_LONG_TONE_FIXTURE"; @@ -435,9 +436,40 @@ function judgeDispatch(schedule, playback, run) { return errors; } +// A swap row is a verdict and at most four small samples, so the rows cost a +// fixed amount per swap; the cap keeps a runaway page from growing the report, +// and what it cuts is counted. +const DECODED_ROWS_KEPT = 512; + +/// The judged swap rows with the samples behind them, for a reader that has +/// only the report. By default the capture file sits beside the artifact +/// directory, not in it, so these do not draw on the archive bound; a capture +/// placed inside that directory counts against it like any other file. +function decodedSignalRows(audio) { + const rows = (Array.isArray(audio?.episodes) ? audio.episodes : []).filter( + (row) => row && typeof row === "object", + ); + return { + decoded_signal_rows: rows + .slice(0, DECODED_ROWS_KEPT) + .map(({ swap, episode, status, reasons, ...detail }) => ({ + swap, + episode, + status, + reasons, + samples: detail.samples ?? [], + samples_dropped: detail.samples_dropped ?? 0, + ...(detail.covered_by ? { covered_by: detail.covered_by } : {}), + })), + decoded_signal_rows_dropped: Math.max(0, rows.length - DECODED_ROWS_KEPT), + }; +} + /// The run's verdict, its failure classes and the exit code the process /// should leave with. `observation` is judgeObservation's result and -/// `episodes` the parsed replacement episodes of the room. +/// `episodes` the parsed replacement episodes of the room. `audio` is +/// judgeSwapAudio's result and `replies` the reply judge's, when the page ran +/// the decoded-audio probe. function finalizeLongRun(o) { const { plan, schedule, playback, run, clipSource, observation } = o; const timing = timingForPlan(plan); @@ -517,6 +549,7 @@ function finalizeLongRun(o) { ? "failed" : resolved.status; + const signal = summarizeSwapAudio(o.audio, o.replies); const speechClips = clipSource.kind === CLIP_SOURCE.speech; const covered = !run.aborted && dispatchErrors.length === 0 && run.uncovered.length === 0; @@ -538,7 +571,11 @@ function finalizeLongRun(o) { assessment_scope: "structural", qualification: "not_evaluated", reply_content: "not_assessed", - decoded_signal: "not_assessed", + decoded_signal: signal.status, + ...decodedSignalRows(o.audio), + ...(signal.status === "failed" + ? { decoded_signal_reasons: signal.reasons } + : {}), end_mode: plan.endMode, duration_min: plan.durationMin, observation_target_s: plan.observationTargetS, @@ -575,7 +612,11 @@ function finalizeLongRun(o) { failure_classes: resolved.artifacts.map((a) => a.class), not_exercised: [ "reply content: checkpoint answers are not judged by this driver", - "decoded signal: native capture records do not verify a post-swap decoded waveform", + ...(signal.status === "not_assessed" + ? [ + "decoded signal: native capture records do not verify a post-swap decoded waveform", + ] + : []), ...(cleanupAssessed ? [] : ["cleanup: nothing active after teardown was checked by the worker"]), diff --git a/scripts/soak-judge.cjs b/scripts/soak-judge.cjs index 77877b854..1d98c47f8 100644 --- a/scripts/soak-judge.cjs +++ b/scripts/soak-judge.cjs @@ -793,6 +793,52 @@ function isFreshDecodedSample(r) { const SWAP_AUDIO_OK = ["not_required", "verified", "covered_by_later_swap"]; +/// What a judged row keeps of each sample behind it: the fields the judgment +/// read, and nothing the page might add. A sample is a handful of numbers, so +/// a row costs a fixed, small amount however long the interview runs. +const SAMPLE_FIELDS = [ + "seq", + "status", + "reason", + "edgesAtMark", + "windowEdge", + "counterPairs", + "bytesReceived", + "packetsReceived", + "peakRms", +]; +const MAX_SAMPLES_PER_SWAP = 4; + +const MAX_SAMPLE_TEXT = 200; + +/// A page-supplied reason is cut, so a long one cannot grow a row. +const keptValue = (value) => + typeof value === "string" ? value.slice(0, MAX_SAMPLE_TEXT) : value; + +const keptSample = (sample) => + Object.fromEntries( + SAMPLE_FIELDS.filter((field) => field in sample).map((field) => [ + field, + keptValue(sample[field]), + ]), + ); + +/// The samples a row keeps: the one that decided it (the first fresh one, +/// else the first that was ok) always, then the rest in the order taken, up +/// to the bound. A cap that cut by position alone could keep four stale +/// samples and drop the fresh one the verdict rests on. +function keptSamples(own) { + const decisive = + own.find(isFreshDecodedSample) ?? own.find((r) => r.status === "ok"); + const kept = own.slice(0, MAX_SAMPLES_PER_SWAP); + // Past the bound, the decisive sample takes the last slot, which keeps the + // rest in the order they were taken. + if (decisive && !kept.includes(decisive)) { + kept[MAX_SAMPLES_PER_SWAP - 1] = decisive; + } + return kept.map(keptSample); +} + /// Whether the audio the browser decoded after each swap that answered a reply /// is on record. `replies` is the result of judgeEpisodeReplies and `rearms` /// the per-swap samples the page took after marking each installed swap @@ -807,13 +853,25 @@ const SWAP_AUDIO_OK = ["not_required", "verified", "covered_by_later_swap"]; /// nothing about physical audibility or the delay before it began. function judgeSwapAudio(replies, rearms = []) { const rows = []; + // Not Map.groupBy, which needs Node 21; the checks run on Node 18. + const bySwap = new Map(); + for (const r of rearms) { + if (!bySwap.has(r.swap)) bySwap.set(r.swap, []); + bySwap.get(r.swap).push(r); + } for (const e of replies.episodes) { const swap = `${e.pid}:${e.session}:${e.episode}`; + const own = bySwap.get(swap) ?? []; + // Kept with the verdict it supports, so the rows can be audited later + // without the page; dropped ones are counted, never silently lost. + const kept = keptSamples(own); const row = { swap, episode: e.episode, status: "not_required", reasons: [], + samples: kept, + samples_dropped: own.length - kept.length, }; rows.push(row); if (e.status !== "pass" || e.class !== "answered") continue; @@ -821,7 +879,6 @@ function judgeSwapAudio(replies, rearms = []) { row.status = status; row.reasons.push(reason); }; - const own = rearms.filter((r) => r.swap === swap); if (own.length === 0) { fail( "missing_sample", @@ -843,6 +900,13 @@ function judgeSwapAudio(replies, rearms = []) { const silent = own.find((r) => /silent/.test(r.status)); if (superseded && covering) { row.status = "covered_by_later_swap"; + // The sample that covers it belongs to another swap's row, whose cap + // can drop it, so it is carried here as well. + row.covered_by = { + swap: covering.swap, + seq: covering.seq, + sample: keptSample(covering), + }; } else if (silent) { fail( "silent", @@ -864,6 +928,41 @@ function judgeSwapAudio(replies, rearms = []) { }; } +/// The decoded-signal status of a judgeSwapAudio result, read from its episode +/// rows alone: `failed` with their reasons when any row is not an accepted +/// status, `verified` when a sample was owed and taken, `not_required` when +/// none was owed, and `not_assessed` when the rows are not usable, never a +/// pass. +/// +/// `replies` is the reply judge's result for the same episodes. The audio +/// judge skips an episode the reply judge could not place (no reply record, an +/// unfinished line), so those read as `not_required`, and one verified +/// neighbor would hide the gap: when the replies are not all placed, a +/// would-be `verified` or `not_required` is `not_assessed` instead. A signal +/// that was demonstrated to fail stays `failed`. +function summarizeSwapAudio(audio, replies) { + const rows = audio?.episodes; + const usable = + Array.isArray(rows) && + rows.every( + (e) => + e && + typeof e.status === "string" && + Array.isArray(e.reasons) && + e.reasons.every((r) => typeof r === "string"), + ); + if (!usable) return { status: "not_assessed", reasons: [] }; + const failed = rows.filter((e) => !SWAP_AUDIO_OK.includes(e.status)); + if (failed.length) { + return { status: "failed", reasons: failed.flatMap((e) => e.reasons) }; + } + if (replies && replies.ok === false) { + return { status: "not_assessed", reasons: [] }; + } + const owed = rows.some((e) => e.status !== "not_required"); + return { status: owed ? "verified" : "not_required", reasons: [] }; +} + /// What a soak writes about itself. Every field is the verdict of a part, so a /// reader cannot mistake a pilot that completed for a qualification: in pilot /// mode `qualification` is `not_evaluated` whatever `verdict` says. @@ -927,6 +1026,8 @@ module.exports = { judgeEpisodes, judgeSoak, judgeSwapAudio, + MAX_SAMPLES_PER_SWAP, + summarizeSwapAudio, judgeEpisodeReplies, parseClock, parseVerdictMode, diff --git a/scripts/swap-audio-owner.cjs b/scripts/swap-audio-owner.cjs new file mode 100644 index 000000000..6fdccddd9 --- /dev/null +++ b/scripts/swap-audio-owner.cjs @@ -0,0 +1,89 @@ +"use strict"; + +// The one owner of the decoded-audio marker and its settle, shared by the soak, +// voice and long-interview flows. It lives apart from browser-check.cjs so its +// ordering, cancellation and page-failure behavior can be driven with a page +// that is only an object with `evaluate`. + +const { installedSwapKeys, judgeSwapAudio } = require("./soak-judge.cjs"); + +const POLL_MS = 250; + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +/// Marks each installed swap in the page once, as soon as its end line is +/// read, so the sample that answers it can only open on a Speaking transition +/// after that point. Read late, never early: a transition between the install +/// and this call is missed and the next one is used instead, which can fail a +/// swap whose reply was quicker than a poll but never passes one on audio from +/// before it. It runs on its own clock, not the callers' loops: those wait on +/// log parsing and room-service calls, and a swap read late can miss the +/// transition that answered it. `halt` ends the loop on a failure path; +/// `stop` ends it, rethrows a marking failure and marks what is left. +/// +/// `newLines` returns the log gained since its last call and `agentOutput` is +/// the agent's retained output; the swaps are read from both. `keysOf` and +/// `wait` are the real parser and timer unless a caller says otherwise. +function startSwapMarker({ + page, + roomName, + agentOutput, + newLines, + keysOf = (text) => installedSwapKeys(text, roomName), + wait = sleep, + pollMs = POLL_MS, +}) { + const marked = new Set(); + let marking = true; + let markError = null; + const markNewSwaps = async () => { + for (const key of keysOf(`${agentOutput.join("")}\n${newLines()}`)) { + if (marked.has(key)) continue; + marked.add(key); + await page.evaluate((swap) => globalThis.__markJimSwap?.(swap), key); + } + }; + const loop = (async () => { + while (marking) { + await wait(pollMs); + if (!marking) break; + try { + await markNewSwaps(); + } catch (error) { + markError = error; + marking = false; + } + } + })(); + const halt = async () => { + marking = false; + await loop; + }; + return { + halt, + async stop() { + await halt(); + if (markError) throw markError; + await markNewSwaps(); + }, + }; +} + +/// The decoded audio each answered swap owes: the page is given the rest of +/// the grace for the first Speaking transition after the last swap, then +/// cancels what still waits, and the result is judged against the episodes. +async function judgeSettledSwapAudio(page, replies, label, { graceMs, log }) { + const rearms = await page.evaluate( + (waitMs) => globalThis.__settleJimSwapAudio(waitMs), + graceMs, + ); + const audio = judgeSwapAudio(replies, rearms); + for (const e of audio.episodes) { + log( + `${label} episode ${e.episode} audio: ${e.status} ${e.reasons.join("; ")}`.trimEnd(), + ); + } + return audio; +} + +module.exports = { judgeSettledSwapAudio, startSwapMarker }; diff --git a/tests/browser/long-interview-driver.test.js b/tests/browser/long-interview-driver.test.js index 2a5d4870f..16a6d8810 100644 --- a/tests/browser/long-interview-driver.test.js +++ b/tests/browser/long-interview-driver.test.js @@ -1084,6 +1084,33 @@ test("a run the page's countdown ended is reported as such, and its capture is k assert.match(cjs, /longCaptureWritten = true;\s*fs\.writeFileSync/); }); +test("the long interview shares the soak's swap marker and keeps the generic Jim audio assertion off", () => { + const cjs = fs.readFileSync( + path.join(ROOT, "scripts", "browser-check.cjs"), + "utf8", + ); + // The probe is installed for the long flow, but only the soak, voice and + // barge flows assert Jim's whole track. + assert.match( + cjs, + /probingSwapAudio = credentialed && flow === "long-interview"/, + ); + assert.match(cjs, /\["voice", "barge", "soak"\]\.includes\(flow\)/); + // One marker and one settle-and-judge owner serve both flows. + // It lives in scripts/swap-audio-owner.cjs (tested in swap-audio-owner.test.js); + // the two flows call it and neither keeps a copy of its own. + assert.match(cjs, /require\("\.\/swap-audio-owner\.cjs"\)/); + assert.match(cjs, /startSwapMarker\(\{/); + assert.match(cjs, /judgeSettledSwapAudio\(/); + assert.equal(cjs.match(/__markJimSwap|__settleJimSwapAudio/g), null); + // The verdict reaches the report, and any failure still stops the probe. + assert.match(cjs, /\n\s+audio,\n\s+replies,\n\s+logs: \{ truncated: false/); + assert.match( + cjs, + /if \(probingSwapAudio\) \{[^}]*__finishJimAudioProbe\(true\)/, + ); +}); + test("a completed long interview reports structural scope without claiming qualification", async () => { const finished = finalize(await play(SMOKE), { cleanup: undefined }); assert.equal(finished.exitCode, EXIT.pass); @@ -1101,6 +1128,145 @@ test("a completed long interview reports structural scope without claiming quali ); }); +const swapRow = (status, reasons = []) => ({ + swap: "1:2:1", + episode: 1, + status, + reasons, +}); + +test("a decoded-signal verdict from the media probe is reported apart from the structural result", async () => { + const played = await play(SMOKE); + const cases = [ + [{ ok: true, episodes: [swapRow("verified")], reasons: [] }, "verified"], + [ + { ok: true, episodes: [swapRow("covered_by_later_swap")], reasons: [] }, + "verified", + ], + [ + { ok: true, episodes: [swapRow("not_required")], reasons: [] }, + "not_required", + ], + [{ ok: true, episodes: [], reasons: [] }, "not_required"], + ]; + for (const [audio, expected] of cases) { + const finished = finalize(played, { audio }); + assert.equal(finished.report.decoded_signal, expected); + assert.equal(finished.exitCode, EXIT.pass); + assert.equal(finished.report.qualification, "not_evaluated"); + assert.equal(finished.report.reply_content, "not_assessed"); + assert.ok( + !finished.report.not_exercised.some((n) => /^decoded signal:/.test(n)), + ); + } +}); + +test("the judged swap rows and their samples are kept in the report, bounded", async () => { + const played = await play(SMOKE); + const sample = { seq: 1, status: "ok", peakRms: 0.2 }; + const rows = Array.from({ length: 600 }, (_, i) => ({ + ...swapRow("verified"), + episode: i + 1, + samples: [sample], + samples_dropped: 3, + extra: "not copied", + })); + const kept = finalize(played, { + audio: { ok: true, episodes: rows, reasons: [] }, + }).report; + assert.equal(kept.decoded_signal_rows.length, 512); + assert.equal(kept.decoded_signal_rows_dropped, 88); + assert.deepEqual(kept.decoded_signal_rows[0], { + swap: "1:2:1", + episode: 1, + status: "verified", + reasons: [], + samples: [sample], + samples_dropped: 3, + }); + const covered = finalize(played, { + audio: { + ok: true, + episodes: [ + { + ...swapRow("covered_by_later_swap"), + covered_by: { swap: "1:2:2", seq: 2 }, + }, + ], + reasons: [], + }, + }).report; + assert.deepEqual(covered.decoded_signal_rows[0].covered_by, { + swap: "1:2:2", + seq: 2, + }); + const none = finalize(played, { audio: undefined }).report; + assert.deepEqual(none.decoded_signal_rows, []); + assert.equal(none.decoded_signal_rows_dropped, 0); +}); + +test("a missing or failed decoded sample is a failed signal, never a pass", async () => { + const played = await play(SMOKE); + const reasons = ["no post-swap audio sample was taken for episode 1"]; + const finished = finalize(played, { + audio: { + ok: false, + episodes: [swapRow("missing_sample", reasons)], + reasons, + }, + }); + assert.equal(finished.report.decoded_signal, "failed"); + assert.deepEqual(finished.report.decoded_signal_reasons, reasons); + assert.equal(finished.report.qualification, "not_evaluated"); + assert.equal( + finished.report.full_interview, + finalize(played).report.full_interview, + ); +}); + +test("an unplaced reply keeps a clean-looking signal from being reported", async () => { + const played = await play(SMOKE); + const audio = { episodes: [swapRow("verified")] }; + for (const [replies, expected] of [ + [{ ok: false, episodes: [] }, "not_assessed"], + [{ ok: true, episodes: [] }, "verified"], + [null, "verified"], + ]) { + const finished = finalize(played, { audio, replies }); + assert.equal(finished.report.decoded_signal, expected); + } +}); + +test("a status the judge does not accept is a failed signal", async () => { + const played = await play(SMOKE); + const finished = finalize(played, { + audio: { episodes: [swapRow("bogus", ["odd"])] }, + }); + assert.equal(finished.report.decoded_signal, "failed"); + assert.deepEqual(finished.report.decoded_signal_reasons, ["odd"]); +}); + +test("an unusable decoded-signal verdict leaves the signal not assessed", async () => { + const played = await play(SMOKE); + for (const audio of [ + null, + {}, + { ok: true }, + { ok: true, episodes: "x" }, + { ok: true, episodes: [{}], reasons: [] }, + { ok: true, episodes: [null], reasons: [] }, + { ok: true, episodes: [{ status: 1, reasons: [] }] }, + { ok: true, episodes: [{ status: "verified", reasons: [7] }] }, + { ok: true, episodes: [{ status: "verified" }] }, + ]) { + const finished = finalize(played, { audio }); + assert.equal(finished.report.decoded_signal, "not_assessed"); + assert.ok( + finished.report.not_exercised.some((n) => /^decoded signal:/.test(n)), + ); + } +}); + for (const plan of [ SMOKE, { ...SMOKE, runKind: "pilot", durationMin: 60, observationTargetS: 3600 }, diff --git a/tests/browser/soak-judge.test.js b/tests/browser/soak-judge.test.js index a3750599f..669779005 100644 --- a/tests/browser/soak-judge.test.js +++ b/tests/browser/soak-judge.test.js @@ -874,6 +874,42 @@ test("a swap that owed no audio is not made to speak, whatever its sample says", assert.deepEqual(statuses(held), [[1, "not_required"]]); }); +test("a covering sample its own row's cap drops is still carried by the row it covers", () => { + const log = owedSwap(1) + owedSwap(2); + const superseded = rearm(1, { + seq: 1, + status: "superseded", + reason: "superseded-by-later-swap", + windowEdge: null, + counterPairs: 0, + bytesReceived: null, + packetsReceived: null, + peakRms: null, + }); + // Swap 2's own verdict rests on its first sample (seq 0, before swap 1's); + // the one that covers swap 1 is its fifth, past the cap of four. + const stale = { windowEdge: 4 }; + const second = [ + rearm(2, { seq: 0 }), + rearm(2, { seq: 2, ...stale }), + rearm(2, { seq: 3, ...stale }), + rearm(2, { seq: 4, ...stale }), + rearm(2, { seq: 5, pageSecret: "not kept", reason: "r".repeat(300) }), + ]; + const audio = audioOf(log, [superseded, ...second]); + const [first, other] = audio.episodes; + assert.equal(first.status, "covered_by_later_swap"); + assert.equal(first.covered_by.seq, 5); + assert.equal(first.covered_by.sample.seq, 5); + // Carried through the same whitelist and cut as a row's own samples. + assert.ok(!("pageSecret" in first.covered_by.sample)); + assert.equal(first.covered_by.sample.reason.length, 200); + assert.ok( + !other.samples.some((s) => s.seq === 5), + "the cap dropped it there", + ); +}); + test("a swap superseded by a later swap is covered only by that swap's fresh sample", () => { const log = owedSwap(1) + owedSwap(2); const superseded = rearm(1, { @@ -887,6 +923,13 @@ test("a swap superseded by a later swap is covered only by that swap's fresh sam }); const covered = audioOf(log, [superseded, rearm(2)]); assert.equal(covered.ok, true); + // The row names the other swap's sample that covers it, and carries it. + assert.deepEqual(covered.episodes[0].covered_by, { + swap: "7:1:2", + seq: 2, + sample: covered.episodes[1].samples[0], + }); + assert.equal(covered.episodes[1].covered_by, undefined); assert.deepEqual(statuses(covered), [ [1, "covered_by_later_swap"], [2, "verified"], @@ -1166,6 +1209,50 @@ test("the carried reply in the series rejects audio accepted on the old socket", ); }); +test("a judged row keeps the samples behind it, whitelisted and bounded", () => { + const replies = seriesReplies(); + const samples = [1, 2, 3, 4, 5, 6].map((n) => + rearm(2, { seq: n, pageSecret: "not kept", windowEdge: 5 + n }), + ); + const row = judge + .judgeSwapAudio(replies, samples) + .episodes.find((e) => e.episode === 2); + assert.equal(row.status, "verified"); + assert.equal(row.samples.length, judge.MAX_SAMPLES_PER_SWAP); + assert.equal(row.samples_dropped, 2); + assert.deepEqual( + row.samples.map((sample) => sample.seq), + [1, 2, 3, 4], + ); + assert.ok(row.samples.every((sample) => !("pageSecret" in sample))); + assert.equal(row.samples[0].peakRms, 0.2); + // The sample a verdict rests on survives the cap even when it comes last. + const stale = [1, 2, 3, 4].map((n) => + rearm(2, { seq: n, windowEdge: 4, status: "ok" }), + ); + const fifth = judge + .judgeSwapAudio(replies, [...stale, rearm(2, { seq: 5 })]) + .episodes.find((e) => e.episode === 2); + assert.equal(fifth.status, "verified"); + assert.ok(fifth.samples.some((sample) => sample.seq === 5)); + assert.equal(fifth.samples.length, judge.MAX_SAMPLES_PER_SWAP); + assert.deepEqual( + fifth.samples.map((sample) => sample.seq), + [1, 2, 3, 5], + ); + // A page-supplied reason is cut rather than carried whole. + const long = judge + .judgeSwapAudio(replies, [rearm(2, { reason: "x".repeat(5000) })]) + .episodes.find((e) => e.episode === 2); + assert.equal(long.samples[0].reason.length, 200); + // A swap that owed nothing keeps what was sampled for it too, and one with no + // sample keeps none rather than inventing them. + const none = judge + .judgeSwapAudio(replies, []) + .episodes.find((e) => e.episode === 2); + assert.deepEqual([none.samples, none.samples_dropped], [[], 0]); +}); + test("native capture records in the series do not prove fresh decoded audio", () => { const replies = seriesReplies(); assert.equal(replies.ok, true); @@ -1187,3 +1274,29 @@ test("native capture records in the series do not prove fresh decoded audio", () 5, ); }); + +test("a swap whose reply left no record is not assessed, never read as no sample owed", () => { + const gap = + fixtures.start(1, { obligation: 3, origin: "prompt" }) + + fixtures.end(1, "installed", { obligation: 3 }); + const replies = replyTo(gap + owedSwap(2)); + assert.equal(replies.ok, false); + // The audio judge skips what the reply judge could not place, so on its own + // it reports the gap as nothing owed and a verified neighbor hides it. + const audio = judge.judgeSwapAudio(replies, [rearm(2)]); + assert.deepEqual(statuses(audio), [ + [1, "not_required"], + [2, "verified"], + ]); + assert.equal(judge.summarizeSwapAudio(audio).status, "verified"); + assert.equal(judge.summarizeSwapAudio(audio, replies).status, "not_assessed"); + // A demonstrated signal failure is still a failure, whatever else is missing. + const unsampled = judge.judgeSwapAudio(replies, []); + assert.equal(judge.summarizeSwapAudio(unsampled, replies).status, "failed"); + // With every reply placed the replies argument changes nothing. + const clean = replyTo(owedSwap(1)); + assert.equal( + judge.summarizeSwapAudio(audioOf(owedSwap(1), [rearm(1)]), clean).status, + "verified", + ); +}); diff --git a/tests/browser/swap-audio-owner.test.js b/tests/browser/swap-audio-owner.test.js new file mode 100644 index 000000000..5758c6d88 --- /dev/null +++ b/tests/browser/swap-audio-owner.test.js @@ -0,0 +1,258 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import owner from "../../scripts/swap-audio-owner.cjs"; +import * as fixtures from "./soak-judge-fixtures.js"; + +const { judgeSettledSwapAudio, startSwapMarker } = owner; + +/// A page that is only `evaluate`: it runs the function it is given against a +/// stand-in global, so what the marker and the settle ask of the page is +/// observable without a browser. +function fakePage({ failOn = null } = {}) { + const calls = []; + const page = { + calls, + async evaluate(fn, arg) { + const marks = []; + const stand = { + __markJimSwap: (key) => { + if (key === failOn) throw new Error(`page refused ${key}`); + marks.push(key); + }, + __settleJimSwapAudio: async (waitMs) => { + calls.push(["settle", waitMs]); + return page.rearms ?? []; + }, + }; + const real = globalThis.__markJimSwap; + const realSettle = globalThis.__settleJimSwapAudio; + Object.assign(globalThis, stand); + try { + const out = await fn(arg); + marks.forEach((key) => calls.push(["mark", key])); + return out; + } finally { + globalThis.__markJimSwap = real; + globalThis.__settleJimSwapAudio = realSettle; + } + }, + }; + return page; +} + +/// A clock the test steps by hand instead of waiting 250 ms for. +function manualClock() { + const waiting = []; + const delays = []; + return { + delays, + wait: (ms) => { + delays.push(ms); + return new Promise((resolve) => waiting.push(resolve)); + }, + async tick() { + const due = waiting.splice(0); + due.forEach((resolve) => resolve()); + // Let the loop run its body before the test looks. + await new Promise((resolve) => setImmediate(resolve)); + }, + }; +} + +// Swap keys are the words "swap:KEY" in the text; the parser under test is the +// soak judge's, covered by its own tests. +const keysOf = (text) => + [...text.matchAll(/swap:(\S+)/g)].map((match) => match[1]); + +/// Ends the loop the way production does: its pending wait runs out, which the +/// test's clock does by hand. +async function ending(clock, promise) { + await clock.tick(); + return promise; +} + +function marker(overrides = {}) { + const clock = manualClock(); + const agentOutput = []; + let pending = ""; + const page = overrides.page ?? fakePage(); + const handle = startSwapMarker({ + page, + roomName: "room", + agentOutput, + newLines: () => { + const text = pending; + pending = ""; + return text; + }, + keysOf, + wait: clock.wait, + ...overrides.options, + }); + return { + clock, + page, + handle, + agentOutput, + log: (text) => { + pending += text; + }, + }; +} + +test("each installed swap is marked once, in the order it was read", async () => { + const { clock, page, handle, log, agentOutput } = marker(); + agentOutput.push("swap:7:1:1\n"); + log("swap:7:1:2\nswap:7:1:1\n"); + await clock.tick(); + agentOutput.push("swap:7:1:3\n"); + await clock.tick(); + await clock.tick(); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [ + ["mark", "7:1:1"], + ["mark", "7:1:2"], + ["mark", "7:1:3"], + ]); +}); + +test("the loop polls on its own 250 ms clock", async () => { + const { clock, handle } = marker(); + await clock.tick(); + await ending(clock, handle.stop()); + assert.ok(clock.delays.length >= 1); + assert.ok( + clock.delays.every((ms) => ms === 250), + clock.delays.join(), + ); +}); + +test("with the real parser, an installed swap line in the output is marked", async () => { + const clock = manualClock(); + const page = fakePage(); + const handle = startSwapMarker({ + page, + roomName: fixtures.ROOM, + agentOutput: [fixtures.swapped(1, 3, "answered", 2)], + newLines: () => "", + wait: clock.wait, + }); + await clock.tick(); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [["mark", "7:1:1"]]); +}); + +test("stop marks what the loop had not yet read before it returns", async () => { + const { clock, page, handle, log } = marker(); + log("swap:7:1:9\n"); + await ending(clock, handle.stop()); + assert.deepEqual(page.calls, [["mark", "7:1:9"]]); +}); + +test("a page that fails to mark stops the loop and is rethrown by stop", async () => { + const page = fakePage({ failOn: "7:1:2" }); + const { clock, handle, log } = marker({ page }); + log("swap:7:1:1\nswap:7:1:2\nswap:7:1:3\n"); + await clock.tick(); + // The loop gave up at the failure and reads nothing more on its own. + log("swap:7:1:4\n"); + await clock.tick(); + await clock.tick(); + const rejected = assert.rejects(handle.stop(), /page refused 7:1:2/); + await clock.tick(); + await rejected; + // Nothing after the failure was marked, and the loop did not carry on. + assert.deepEqual(page.calls, [["mark", "7:1:1"]]); +}); + +test("halt after a failed mark neither rethrows nor marks again", async () => { + const page = fakePage({ failOn: "7:1:1" }); + const { clock, handle, log } = marker({ page }); + log("swap:7:1:1\n"); + await clock.tick(); + await ending(clock, handle.halt()); + assert.deepEqual(page.calls, []); + // A halted loop reads nothing more, however long the log grows. + log("swap:7:1:2\n"); + await clock.tick(); + assert.deepEqual(page.calls, []); +}); + +test("halt waits for the loop to finish, and a wake after it marks nothing", async () => { + const { clock, page, handle, log } = marker(); + log("swap:7:1:5\n"); + let finished = false; + const halting = handle.halt().then(() => { + finished = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(finished, false, "halt returned before the loop did"); + await clock.tick(); + await halting; + assert.equal(finished, true); + assert.deepEqual(page.calls, []); +}); + +test("the settle hands the page its grace, judges its samples and reports each episode", async () => { + const page = fakePage(); + // Episode 1 has its fresh sample; episode 2 has none. + page.rearms = [ + { + swap: "7:1:1", + seq: 1, + status: "ok", + reason: null, + edgesAtMark: 4, + windowEdge: 5, + counterPairs: 3, + bytesReceived: 900, + packetsReceived: 9, + peakRms: 0.2, + }, + ]; + const lines = []; + const answered = (episode) => ({ + pid: 7, + session: 1, + episode, + status: "pass", + class: "answered", + }); + const audio = await judgeSettledSwapAudio( + page, + { episodes: [answered(1), answered(2)] }, + "soak", + { graceMs: 1234, log: (line) => lines.push(line) }, + ); + assert.deepEqual(page.calls, [["settle", 1234]]); + assert.deepEqual( + audio.episodes.map((e) => [e.episode, e.status]), + [ + [1, "verified"], + [2, "missing_sample"], + ], + ); + assert.equal(lines.length, 2); + // A row with no reasons carries no trailing space. + assert.equal(lines[0], "soak episode 1 audio: verified"); + assert.match( + lines[1], + /^soak episode 2 audio: missing_sample no post-swap audio sample/, + ); +}); + +test("a page that fails to settle propagates, so the caller can still stop the probe", async () => { + const page = { + evaluate: async () => { + throw new Error("page closed"); + }, + }; + await assert.rejects( + judgeSettledSwapAudio(page, { episodes: [] }, "soak", { + graceMs: 1, + log: () => {}, + }), + /page closed/, + ); +}); From f84eeda58e4559f712963789c6808a609ca18138 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Sat, 7 Nov 2026 05:22:20 +0800 Subject: [PATCH 4/7] Hold long-interview replies to their questions The first paid pilot's content judge failed all 23 checkpoints, and reading the replies against the answer key showed the key at fault: it demanded facts the questions did not ask for, its rubrics missed plain ways of saying the same thing, and the whiteboard schedule wanted a revision marker that read_board never returns, since it gives a count and an image. Judge a revision where it can be read, in what the tool returned, and a board revision by its stroke count; widen the rubrics with negatives so a number that merely contains the token is not the bound. The catalogue changes, so its version, review ids and pins move on purpose. Change-Id: I7bed8e45c1ce5722c8a2140737f2c448308dfb67 --- docs/development.md | 14 ++ scripts/long-interview-page.cjs | 9 +- scripts/long-interview-schedule.cjs | 116 ++++++--- tests/agent/whiteboard.rs | 21 ++ tests/browser/long-interview-schedule.test.js | 231 ++++++++++++++++-- 5 files changed, 331 insertions(+), 60 deletions(-) diff --git a/docs/development.md b/docs/development.md index 6e889b1b6..9aaa5c760 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1132,6 +1132,20 @@ reserve is revalidated before launch against the resolved handover grace at or after the boundary is `late_idle` or `late_terminal_preempted` and never counts toward the two completed rotations a run needs. +What a reply is judged on is what its question asks and no more +(`scripts/long-interview-schedule.cjs`). An inspect question asks about the +duplicate case, so the revision is judged in what the tool read (the answer +key's `tool.result_fact_ids`), and a reply may state the current revision +without penalty but not an earlier one. A board has no marker a model can read +back: `read_board` gives it a count and an image, never text, so a whiteboard +revision is told apart by the stroke count the tool reports (four strokes are +drawn per revision and the board keeps them, so revision n reads `4n strokes`), +its questions do not ask for a revision, and `tests/agent/whiteboard.rs` pins +the wording the judge reads, which the judge's own test takes from +`tests/golden/prompts.json`. The rubrics are lexical and were checked against +the first pilot's real replies. Changing any of this changes the catalogue, so +`SCHEDULE_VERSION`, the pins and every clip manifest move with it. + `run` also samples the owned process tree (RSS, open descriptors and threads for every process, keyed by PID and start time so a reused PID is another process) every 30 seconds into `resource-samples.jsonl`, for either arm of a diff --git a/scripts/long-interview-page.cjs b/scripts/long-interview-page.cjs index e547d194c..1ccc6281e 100644 --- a/scripts/long-interview-page.cjs +++ b/scripts/long-interview-page.cjs @@ -172,15 +172,16 @@ function pageController(page, { selectLanguage, mode = "coding" }) { } /// Draws one closed shape per entry of the revision's strokes, in a row of its -/// own, so each revision leaves a board that differs from the last. The board -/// has no text tool, so the marker cannot be written on it: a revision is told -/// apart by position and shape count, not by a legible marker. +/// own, so each revision leaves a board that differs from the last. The marker +/// is not drawn: read_board gives the model a stroke count and an image, never +/// text, so a revision is told apart by the board's running stroke count, which +/// is four more after each revision (the judge reads it in the tool's text). async function drawBoardRevision(page, fixture, index) { const bounds = await page.locator("#board").boundingBox(); if (!bounds) throw new Error("the visible board had no drawing bounds"); const row = bounds.y + 30 + ((index - 1) % 6) * 70; let column = bounds.x + 30; - for (let i = 0; i < fixture.strokes.length; i++) { + for (let i = 0; i < fixture.strokes; i++) { await page.mouse.move(column, row); await page.mouse.down(); await page.mouse.move(column + 80, row, { steps: 4 }); diff --git a/scripts/long-interview-schedule.cjs b/scripts/long-interview-schedule.cjs index 2b2440172..171510271 100644 --- a/scripts/long-interview-schedule.cjs +++ b/scripts/long-interview-schedule.cjs @@ -29,7 +29,7 @@ const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); -const SCHEDULE_VERSION = 1; +const SCHEDULE_VERSION = 3; const CYCLE_S = 300; /// The end and report reservation; no stimulus is scheduled inside it. const STOP_BEFORE_END_S = 120; @@ -124,12 +124,21 @@ const AUTHORED = Object.freeze({ sentence: "My constraints are at most ten thousand numbers, exactly one valid pair, and never reuse the same element.", review: { - id: "constraints-v1", + id: "constraints-v2", reviewer: "schedule author", status: "reviewed", }, concepts: [ - ["ten thousand", "10000", "10,000", "10^4"], + [ + "ten thousand", + "ten-thousand", + "10 thousand", + "10k", + "10000", + "10,000", + "10 000", + "10^4", + ], ["exactly one", "one valid pair", "single valid pair", "one pair"], [ "same element", @@ -142,7 +151,7 @@ const AUTHORED = Object.freeze({ }, duplicate_case: { review: { - id: "duplicate-v1", + id: "duplicate-v2", reviewer: "schedule author", status: "reviewed", }, @@ -150,8 +159,15 @@ const AUTHORED = Object.freeze({ [ "different indices", "distinct indices", + "different index", + "distinct index", + "different position", + "distinct position", + "different positions", + "distinct positions", "two different", "two distinct", + "two separate", "separate indices", "not the same index", ], @@ -176,11 +192,21 @@ const AUTHORED = Object.freeze({ prime: "Could you compare the map approach with checking every pair?", barge: "Let me clarify the example first. I am still explaining my reasoning.", - state_question: - "What are the original constraints, our current stage, and the latest {thing}?", - final_question: - "Before we wrap up, what is the question I still have open, and what is the latest {thing}?", - things: { coding: "source revision", whiteboard: "diagram revision" }, + // What each question asks for is all its reply is held to. A board's + // revision cannot be asked for: read_board gives the model a count and an + // image, never text, and a marker drawn in the image is not something to + // judge a reply on. The whiteboard questions do not ask for a revision; it is + // judged in what read_board reports, the stroke count. + state_question: { + coding: + "What are the original constraints, our current stage, and the latest source revision?", + whiteboard: "What are the original constraints, and our current stage?", + }, + final_question: { + coding: + "Before we wrap up, what is the question I still have open, and what is the latest source revision?", + whiteboard: "Before we wrap up, what is the question I still have open?", + }, }); const canonicalize = (value) => { @@ -198,7 +224,7 @@ const hashOf = (value) => sha256(canonicalize(value)); /// Pinned. Edit the catalogue above, bump SCHEDULE_VERSION, and replace this. const AUTHORED_SHA256 = - "323220654cdf1c5ce18e4bd3a739c9b45557f891e94536a92720035c1deceb1f"; + "55bfd3f2315601dd598a42bb9ceb0c9e3f0d0acabcecb82d2aceeb7a20f3468b"; function checkAuthoredPin(expected = AUTHORED_SHA256) { const actual = hashOf(AUTHORED); @@ -226,6 +252,9 @@ function nonceFor(mode, checkpointId) { return `${first[digest[0] % first.length]} ${second[digest[1] % second.length]}`; } +/// Shapes the page controller draws per board revision, one stroke each. +const STROKES_PER_REVISION = 4; + function markerFor(mode, index) { return `rev${pad2(index)}-${sha256(`${SCHEDULE_VERSION}|${mode}|rev|${index}`).slice(0, 6)}`; } @@ -251,11 +280,12 @@ function revisionFixture(mode, index) { ].join("\n"); return { marker, kind: "editor", language: "javascript", source }; } + // The board keeps every revision's strokes, and its reading is the count. return { kind: "board", marker, - label: `complement map example ${marker}`, - strokes: ["box nums", "arrow need", "box seen", `label ${marker}`], + strokes: STROKES_PER_REVISION, + strokes_total: STROKES_PER_REVISION * index, }; } @@ -324,6 +354,7 @@ const rubricMatch = (spec) => ({ /// The schedule and its answer key for `mode`, derived from `plan` /// ({durationMin, observationTargetS}, as long-interview-plan.cjs validated). function generateSchedule(mode, plan) { + const coding = mode === "coding"; checkAuthoredPin(); if (!TOOL_BY_MODE[mode]) { throw new Error(`mode must be coding or whiteboard, not ${mode}`); @@ -334,7 +365,6 @@ function generateSchedule(mode, plan) { observationTargetS, ); const tool = TOOL_BY_MODE[mode]; - const thing = AUTHORED.things[mode]; const recalls = recallPlan(cycles); const steps = [{ action_id: JOIN_ACTION_ID, at_s: 0, kind: "join" }]; @@ -403,12 +433,22 @@ function generateSchedule(mode, plan) { `We are in the ${phase} stage.`, { mode: "folded", tokens: [phase] }, ); - const revisionFact = factOf( - `revision_${c}`, - base + 110, - `The latest ${thing} is ${revision.marker}.`, - { mode: "exact", tokens: [revision.marker] }, - ); + // A coding revision is a marker a reply states; a board's is the count + // read_board reports, judged in what the tool read and never in a reply, so + // its id is not revision_* (the prefix a reply may not state a stale one of). + const revisionFact = coding + ? factOf( + `revision_${c}`, + base + 110, + `The latest source revision is ${revision.marker}.`, + { mode: "exact", tokens: [revision.marker] }, + ) + : factOf( + `board_${c}`, + base + 110, + `The board holds ${revision.strokes_total} strokes.`, + { mode: "exact", tokens: [`${revision.strokes_total} strokes`] }, + ); facts.push(phaseFact, revisionFact); const planParts = [`Stage ${cycle}. We are now in the ${phase} stage.`]; @@ -506,7 +546,7 @@ function generateSchedule(mode, plan) { addClip(id("barge"), "barge", [AUTHORED.barge], base + 175); const recallId = recalls.get(cycle); - const stateQuestion = AUTHORED.state_question.replace("{thing}", thing); + const stateQuestion = AUTHORED.state_question[mode]; const recallSeed = recallId ? AUTHORED.seeds.find((s) => s.id === recallId) : null; @@ -536,6 +576,8 @@ function generateSchedule(mode, plan) { response_window_s: nextInput(inspectEnd, base + 175), clip_id: id("ask_inspect"), cycle, + // What was seeded for this question. The key below asks less of the + // reply: the revision is judged in what the tool read. source_fact_ids: ["duplicate_case", revisionFact.id], required_tool: { name: tool, call_id_required: true }, required_tools: [tool], @@ -543,7 +585,9 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: id("cp_inspect"), nonce: inspectNonce, - element_fact_ids: ["duplicate_case", revisionFact.id], + // Only what the question asks. The revision is checked in what the tool + // read, below, and a reply may state it without being faulted. + element_fact_ids: ["duplicate_case"], tool: { name: tool, result_fact_ids: [revisionFact.id], @@ -596,7 +640,9 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: id("cp_state"), nonce: stateNonce, - element_fact_ids: stateFacts, + element_fact_ids: stateFacts.filter( + (id) => coding || id !== revisionFact.id, + ), tools, }); } @@ -610,15 +656,14 @@ function generateSchedule(mode, plan) { ); facts.push(pendingFact); const finalNonce = nonceFor(mode, "cp_final"); - const finalText = `${AUTHORED.final_question.replace("{thing}", thing)} Please also say the code words ${finalNonce}.`; + const finalText = `${AUTHORED.final_question[mode]} Please also say the code words ${finalNonce}.`; const finalSpeech = addClip( "final_checkpoint", "checkpoint", [finalText], finalAt, ); - const lastRevisionFact = `revision_${pad2(cycles)}`; - const finalPhaseFact = `phase_${pad2(cycles)}`; + const lastRevisionFact = `${coding ? "revision" : "board"}_${pad2(cycles)}`; steps.push({ action_id: "final_checkpoint", at_s: finalAt, @@ -626,7 +671,11 @@ function generateSchedule(mode, plan) { clip_id: "final_checkpoint", checkpoint_id: "cp_final", }); - const finalFacts = [pendingFact.id, finalPhaseFact, lastRevisionFact]; + // What the final question asks for, and no more: the open question and, for + // a coding interview, the latest revision; never the stage. A board's + // revision is judged in what the tool read, so it is a source, not an element. + const finalFacts = [pendingFact.id, lastRevisionFact]; + const finalElements = coding ? finalFacts : [pendingFact.id]; checkpoints.push({ checkpoint_id: "cp_final", kind: "final", @@ -642,7 +691,7 @@ function generateSchedule(mode, plan) { keyCheckpoints.push({ checkpoint_id: "cp_final", nonce: finalNonce, - element_fact_ids: finalFacts, + element_fact_ids: finalElements, tools: [ { name: tool, @@ -909,13 +958,14 @@ function judgeCheckpoint(bundle, evidence, options = {}) { const misses = matchFact(fact.match, reply.text); if (misses.length) fail(`reply lacks ${id}: ${misses[0]}`); } - // A reply that names an earlier revision is not about the current one. + // A reply that names an earlier revision is not about the current one. The + // current one is whichever the checkpoint asks for or its tool must show. + const current = new Set([ + ...key.element_fact_ids, + ...(key.tools ?? [key.tool]).flatMap((t) => t.result_fact_ids), + ]); for (const fact of answerKey.facts) { - if ( - !/^revision_\d+$/.test(fact.id) || - key.element_fact_ids.includes(fact.id) - ) - continue; + if (!/^revision_\d+$/.test(fact.id) || current.has(fact.id)) continue; if (factIntegrity(fact) && matchFact(fact.match, reply.text).length === 0) { fail(`reply states the stale revision ${fact.id}`); } diff --git a/tests/agent/whiteboard.rs b/tests/agent/whiteboard.rs index bb48a623a..c2220eb32 100644 --- a/tests/agent/whiteboard.rs +++ b/tests/agent/whiteboard.rs @@ -206,6 +206,27 @@ fn a_board_reports_its_own_age_and_an_empty_one_says_so() { assert_eq!(board_age_seconds(&state), Some(0)); } +/// What the long-interview judge reads. The fixture draws no text, so a +/// revision is told apart by the stroke count `read_board` reports, and the +/// judge (`scripts/long-interview-schedule.cjs`) looks for " strokes" in +/// this text. Its test reads the frozen output in `tests/golden/prompts.json`, +/// so a rewording breaks it there too; this pins the phrase on this side. +#[test] +fn read_board_states_the_stroke_count_the_whiteboard_judge_matches() { + for strokes in [4, 20, 68] { + let text = read_board_text(strokes, 3, Some(12), 30); + assert!( + text.contains(&format!( + "image: {strokes} strokes, as the candidate last left it" + )), + "{text}" + ); + } + // An undrawn board says there is nothing to count, and so cannot match one. + let empty = read_board_text(0, 0, None, 30); + assert!(!empty.contains("strokes"), "{empty}"); +} + /// The reviewer of a whiteboard interview is pointed at the board and at /// nothing that does not exist. /// diff --git a/tests/browser/long-interview-schedule.test.js b/tests/browser/long-interview-schedule.test.js index 0b5ba55ee..085ce9867 100644 --- a/tests/browser/long-interview-schedule.test.js +++ b/tests/browser/long-interview-schedule.test.js @@ -53,9 +53,9 @@ const bundles = { // Pinned on purpose: a change to the authored questions, the timing or the // answer key moves these, and the change then has to be made on purpose. const PINS = { - coding: "1a926968c26adbf61aeea60a5d10dd5a461180fc12686d196d30e94f7b2313a1", + coding: "c8099dc2a945e42b147f431525557a481faba3373a9ee507307f9669f1b35cc9", whiteboard: - "d5851de32e3f63803d0adaab9a93e72ab732aa65a9d05f213f444bb6540c9b93", + "84e1a9fe28e0a0a3eced4ec2b8ba3b2551d98abc5a8291575bac4fc1e5952900", }; test("the authored catalogue is hash pinned", () => { @@ -216,8 +216,20 @@ test("every 90 minute schedule recalls facts seeded 30 minutes earlier", () => { test("the nonce is never the whole answer in the answer key", () => { for (const mode of ["coding", "whiteboard"]) { for (const k of bundles[mode].answer_key.checkpoints) { - assert.ok(k.element_fact_ids.length >= 2); + assert.ok(k.element_fact_ids.length >= 1); assert.ok((k.tools ?? [k.tool]).length >= 1); + // Nothing is required of the reply that its question did not ask for. + if (k.checkpoint_id === "cp_final") { + const last = bundles[mode].answer_key.facts + .map((f) => f.id) + .filter((id) => /^revision_\d+$/.test(id)) + .at(-1); + assert.deepEqual( + k.element_fact_ids, + mode === "coding" ? ["pending_question", last] : ["pending_question"], + mode, + ); + } } } }); @@ -310,6 +322,19 @@ test("interview-run-budget.py validate consumes the generated manifest fields", // ---------------------------------------------------------------- the judge +test("an inspect question asks about the duplicate case, not the revision", () => { + // The first paid pilot's c04 reply answered what was asked and said the code + // words, and failed because the key also wanted a marker the question never + // asked for. The revision is checked where it belongs: in what the tool read. + for (const mode of ["coding", "whiteboard"]) { + for (const k of bundles[mode].answer_key.checkpoints) { + if (!k.checkpoint_id.endsWith("_cp_inspect")) continue; + assert.deepEqual(k.element_fact_ids, ["duplicate_case"], k.checkpoint_id); + assert.match(k.tool.result_fact_ids.join(), /^(revision|board)_\d+$/); + } + } +}); + const CP_STATE = "c07_cp_state"; // a recall: two tool reads, five facts const CP_INSPECT = "c05_cp_inspect"; @@ -333,17 +358,30 @@ test("a faithful answer passes every checkpoint of both schedules", () => { } }); +const PASS = { verdict: "pass", reasons: [] }; +const reasonsOf = (mode, evidence) => + judgeCheckpoint(bundles[mode], evidence).reasons.join("\n"); const key = (mode, id) => bundles[mode].answer_key.checkpoints.find((k) => k.checkpoint_id === id); const factsOf = (mode) => bundles[mode].answer_key.facts; -const marker = (mode, n) => - factsOf(mode).find((f) => f.id === `revision_${String(n).padStart(2, "0")}`) +const factToken = (mode, prefix, n) => + factsOf(mode).find((f) => f.id === `${prefix}_${String(n).padStart(2, "0")}`) .match.tokens[0]; +const boardToken = (n) => factToken("whiteboard", "board", n); +const marker = (mode, n) => factToken(mode, "revision", n); // Each fixture names the reason it must fail. Every one of them keeps the // nonce, and all but the first few keep the expected tokens, so a judge that // stops at the nonce or at the words passes them. const NEGATIVES = [ + { + name: "an inspect reply that states an earlier revision", + cp: CP_INSPECT, + reason: /stale revision revision_04/, + edit: (e) => { + e.reply.text += ` ${marker("coding", 4)}`; + }, + }, { name: "nonce only, tool evidence valid", cp: CP_STATE, @@ -372,21 +410,21 @@ const NEGATIVES = [ }, { name: "wrong revision stated in the reply", - cp: CP_INSPECT, - reason: /lacks revision_05|stale revision/, + cp: CP_STATE, + reason: /lacks revision_07|stale revision/, edit: (e) => { e.reply.text = e.reply.text.replace( - marker("coding", 5), - marker("coding", 4), + marker("coding", 7), + marker("coding", 6), ); }, }, { name: "current and stale revision both stated", - cp: CP_INSPECT, - reason: /stale revision revision_04/, + cp: CP_STATE, + reason: /stale revision revision_06/, edit: (e) => { - e.reply.text += ` ${marker("coding", 4)}`; + e.reply.text += ` ${marker("coding", 6)}`; }, }, { @@ -533,7 +571,7 @@ const NEGATIVES = [ ]; test("the negative fixtures are a fixed set and every one fails", () => { - assert.equal(NEGATIVES.length, 22); + assert.equal(NEGATIVES.length, 23); for (const fixture of NEGATIVES) { const evidence = evidenceFor("coding", fixture.cp); // Each starts from a passing answer, so the edit alone is what fails it. @@ -597,15 +635,95 @@ test("a judge key that drops the schedule's tool requirement is refused", () => ); }); +// What read_board really answers with: the frozen output tests/agent/prompts.rs +// pins to the Rust that writes it, with its count swapped. The count is the +// part of a board a reply cannot read off the fixture's drawing with no text on +// it, so it is what tells one revision from the next. +const GOLDEN = JSON.parse( + fs.readFileSync(path.join(ROOT, "tests", "golden", "prompts.json"), "utf8"), +); +assert.match(GOLDEN.readBoard, /: 22 strokes,/); +const boardText = (strokes) => + GOLDEN.readBoard.replace(": 22 strokes,", `: ${strokes} strokes,`); + +test("a whiteboard revision is told apart by the stroke count read_board reports", () => { + // Four strokes are drawn per revision and the board keeps them, so revision n + // reads 4n strokes. No marker is drawn: read_board gives a count and an image. + for (const n of [1, 5, 11, 17]) { + assert.equal(boardToken(n), `${4 * n} strokes`); + } + const evidence = evidenceFor("whiteboard", CP_INSPECT); + evidence.tool_calls[0].result_text = boardText(20); + assert.deepEqual(judgeCheckpoint(bundles.whiteboard, evidence), PASS); + // The count of the revision before, a count that merely contains this one, + // and a board that was never drawn on are not the current board. + for (const stale of [ + boardText(16), + boardText(120), + boardText(2), + GOLDEN.readBoardEmpty, + ]) { + evidence.tool_calls[0].result_text = stale; + assert.match( + reasonsOf("whiteboard", evidence), + /result lacks board_05/, + stale, + ); + } +}); + +test("every whiteboard checkpoint passes on the real read_board text and is held to nothing more", () => { + for (const c of bundles.whiteboard.schedule.checkpoints) { + const k = key("whiteboard", c.checkpoint_id); + const need = (k.tools ?? [k.tool]).find((t) => t.name === "read_board"); + const board = need.result_fact_ids.find((id) => id.startsWith("board_")); + const count = factsOf("whiteboard").find((f) => f.id === board).match + .tokens[0]; + const evidence = evidenceFor("whiteboard", c.checkpoint_id); + const call = evidence.tool_calls.find((x) => x.name === "read_board"); + call.result_text = boardText(Number.parseInt(count, 10)); + assert.deepEqual( + judgeCheckpoint(bundles.whiteboard, evidence), + PASS, + c.checkpoint_id, + ); + // A reply may talk about counts without being faulted for a stale one. + evidence.reply.text += + " Each revision adds 4 strokes, so the board is much fuller now."; + assert.deepEqual( + judgeCheckpoint(bundles.whiteboard, evidence), + PASS, + c.checkpoint_id, + ); + } +}); + +test("a whiteboard question never asks for a revision no one can read", () => { + const { schedule, answer_key: answerKey } = bundles.whiteboard; + for (const clip of schedule.clips) { + assert.ok(!/revision/i.test(clip.source_text), clip.clip_id); + } + for (const k of answerKey.checkpoints) { + assert.ok( + !k.element_fact_ids.some((id) => /^(revision|board)_/.test(id)), + k.checkpoint_id, + ); + // It is still judged: in what the tool read. + assert.ok( + (k.tools ?? [k.tool]).some((t) => + t.result_fact_ids.some((id) => id.startsWith("board_")), + ), + k.checkpoint_id, + ); + } +}); + test("the same fixtures fail on the whiteboard schedule", () => { - // Board revisions carry the same markers through read_board. + // Board revisions are judged from the stroke count read_board reports. const evidence = evidenceFor("whiteboard", CP_INSPECT); assert.equal(evidence.tool_calls[0].name, "read_board"); - evidence.tool_calls[0].result_text = marker("whiteboard", 4); - assert.match( - judgeCheckpoint(bundles.whiteboard, evidence).reasons.join("\n"), - /result lacks revision_05/, - ); + evidence.tool_calls[0].result_text = boardToken(4); + assert.match(reasonsOf("whiteboard", evidence), /result lacks board_05/); const coding = evidenceFor("whiteboard", CP_INSPECT); coding.tool_calls[0].name = "read_editor"; assert.match( @@ -659,15 +777,82 @@ test("an unreviewed rubric cannot certify a reply", () => { assert.match(verdict.reasons.join("\n"), /rubric is not reviewed/); }); -test("exact tokens are case sensitive and folded tokens are not", () => { +test("an inspect reply may state the current revision without being faulted for it", () => { const evidence = evidenceFor("coding", CP_INSPECT); + evidence.reply.text += ` The latest source revision is ${marker("coding", 5)}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS); +}); + +test("the constraints rubric accepts the ways a model says ten thousand", () => { + for (const said of [ + "between 2 and 10 thousand transactions", + "ten-thousand numbers", + "up to 10k", + "at most 10\u00a0000 numbers", + ]) { + const evidence = evidenceFor("coding", CP_STATE); + const before = evidence.reply.text; + evidence.reply.text = before.replace(/ten thousand/i, said); + assert.notEqual(evidence.reply.text, before, "the answer states the bound"); + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS, said); + } + // A number that merely contains the token is not the bound. + for (const not of [ + "110k numbers", + "210 thousand numbers", + "model10kweights", + ]) { + const evidence = evidenceFor("coding", CP_STATE); + evidence.reply.text = evidence.reply.text.replace(/ten thousand/i, not); + assert.match( + reasonsOf("coding", evidence), + /rubric concept 1 is missing/, + not, + ); + } +}); + +test("the duplicate-case rubric accepts the plain ways of saying two different indices", () => { + for (const said of [ + "it uses a different index each time", + "the two positions are different positions", + "it takes two separate indices", + "each element has a distinct index", + // The pilot's own c01 reply, which the first rubric missed. + "you'll always be getting the index of a previous, different position", + "the second one is a distinct position", + ]) { + const evidence = evidenceFor("coding", CP_INSPECT); + evidence.reply.text = `${said}. ${key("coding", CP_INSPECT).nonce}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS, said); + } +}); + +test("the final question is answered by its pending question and revision alone", () => { + const k = key("coding", "cp_final"); + const last = k.element_fact_ids.at(-1); + const number = Number(last.replace("revision_", "")); + const evidence = evidenceFor("coding", "cp_final"); + const pending = factsOf("coding").find((f) => f.id === "pending_question"); + evidence.reply.text = `${k.nonce}. The open question is whether to return them in ${pending.match.tokens[0]}, and the latest source revision is ${marker("coding", number)}.`; + assert.deepEqual(judgeCheckpoint(bundles.coding, evidence), PASS); + const stale = structuredClone(evidence); + stale.reply.text += ` ${marker("coding", number - 1)}`; + assert.match( + reasonsOf("coding", stale), + new RegExp(`stale revision revision_${number - 1}`), + ); +}); + +test("exact tokens are case sensitive and folded tokens are not", () => { + const evidence = evidenceFor("coding", CP_STATE); evidence.reply.text = evidence.reply.text.replace( - marker("coding", 5), - marker("coding", 5).toUpperCase(), + marker("coding", 7), + marker("coding", 7).toUpperCase(), ); assert.match( judgeCheckpoint(bundles.coding, evidence).reasons.join("\n"), - /reply lacks revision_05/, + /reply lacks revision_07/, ); const folded = evidenceFor("coding", CP_STATE); folded.reply.text = folded.reply.text.replace( From c101b087995faba71c850d7681c063f79d8078d8 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 6 Nov 2026 02:27:24 +0800 Subject: [PATCH 5/7] Move web launch and serving into the library The server's launch configuration and its serving tail lived in the binary crate, interleaved with the socket bind, so no test fixture could build and serve a server the way codetrial web does; one that copied them would only resemble the product. Move the flag, environment and config-file handling, the provider pool and server config constructors, the startup refusals, the startup phases and the serving function into src/launch.rs, and call them from run_web in the original order. The product passes a shutdown that never resolves and no drain bound, and behaves as before. The moved code is unchanged apart from paths and visibility. Change-Id: Ie1dddf276870f929708d1b0e2233e77fd30c3611 --- .claude/skills/codetrial-review/rules/paths | 1 + docs/desktop.md | 9 +- src/launch.rs | 995 ++++++++++++++++++++ src/{main_peer.rs => launch/peer.rs} | 6 +- src/lib.rs | 1 + src/main.rs | 950 +------------------ tests/cli.rs | 28 +- tests/unit/bin.rs | 114 ++- 8 files changed, 1166 insertions(+), 938 deletions(-) create mode 100644 src/launch.rs rename src/{main_peer.rs => launch/peer.rs} (87%) diff --git a/.claude/skills/codetrial-review/rules/paths b/.claude/skills/codetrial-review/rules/paths index 631a1b400..f12dedb70 100644 --- a/.claude/skills/codetrial-review/rules/paths +++ b/.claude/skills/codetrial-review/rules/paths @@ -10,6 +10,7 @@ src/gemini* room.md src/gemini* agent.md src/dispatch.rs room.md src/main.rs room.md +src/launch.rs room.md src/config* room.md src/web/setup.rs room.md src/web/token.rs room.md diff --git a/docs/desktop.md b/docs/desktop.md index 09c066d29..0c6b01b5b 100644 --- a/docs/desktop.md +++ b/docs/desktop.md @@ -17,10 +17,11 @@ server in: it runs the `codetrial` executable that ships beside it as a child process, and its window shows the page that child serves on `http://127.0.0.1:3101`. -A child process rather than a library call, because the server's startup lives -in `src/main.rs` (config search, the Setup handover, every refusal it makes -before serving) and moving that into the library would have been a rewrite of -the file the CLI's behavior is pinned to. As a child it is the binary the CLI +A child process rather than a library call, because the server's startup is +the binary's: `src/main.rs` runs the Setup handover and binds the socket, and +builds the server from `src/launch.rs` (config search, every refusal it makes +before serving). Calling that from the app would have meant a second launcher +beside the one the CLI's behavior is pinned to. As a child it is the binary the CLI tests already cover, started with five things a launcher needs that the CLI did not have. Each is decided in the server, so the app keeps no copy of the server's rules: diff --git a/src/launch.rs b/src/launch.rs new file mode 100644 index 000000000..17475bb3d --- /dev/null +++ b/src/launch.rs @@ -0,0 +1,995 @@ +//! How a launch turns its flags, environment and config file into a server. +//! +//! Lives in the library, not the binary, so a test fixture builds the server +//! from the same constructors `codetrial web` does instead of a copy that only +//! resembles them. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +mod peer; + +pub use peer::{PeerServices, peer_services}; + +use crate::config::{DEFAULT_DURATION_MIN, DEFAULT_ROOM_PREFIX, DEFAULT_WEB_DIR}; +use crate::web::WebServerConfig; + +/// One installation keeps its config file and its account database together in +/// `config/`, under whichever root it was installed as: a checkout, or the +/// folder a released binary was unpacked into. Beside the executable is not +/// that root for a checkout -- the executable is in `target/`, which +/// `make clean` deletes -- so the two are named separately and joined per case. +const CONFIG_DIR: &str = "config"; + +const CONFIG_FILE_NAME: &str = "codetrial.env.local"; + +/// Tracked, so it exists in a checkout and in no release archive. That makes it +/// the one thing that tells a `config/` belonging to this project apart from a +/// directory of the same name that happens to sit where the binary was run. +const CONFIG_EXAMPLE_NAME: &str = "codetrial.env.example"; + +const DEFAULT_ACCOUNT_DB_PATH: &str = "codetrial.db"; + +pub const DEFAULT_SESSION_SECRET: &str = "codetrial-local-session"; + +/// Where a `--data-dir` install keeps the key its session cookies are signed +/// with, when its config file does not set `SESSION_SECRET`. +pub const SESSION_SECRET_FILE: &str = "session-secret"; + +#[derive(Debug, Default, Clone)] +pub struct CliOptions { + pub config_path: Option, + pub data_dir: Option, + pub import_config: Option, + pub web_addr: Option, + pub web_dir: Option, + pub allowed_hosts: Option, + pub room_prefix: Option, + pub duration_min: Option, + pub task_mode: bool, + pub no_update_check: bool, +} + +/// Validates operator settings before a listener or background task starts. +pub fn web_server_config( + values: &std::collections::BTreeMap, + pool: crate::config::ProviderPool, + production: bool, + options: &CliOptions, +) -> Result { + // After the pool is complete, so the "is this project in the pool" check + // sees the pool the server will actually serve, and before the listener + // does any work: a half-configured recording block is a startup failure, + // never a surprise at the moment a candidate's interview ends. + let recording = + crate::config::load_recording(values, &pool, interview_duration_min(values), production) + .map_err(|error| error.to_string())?; + recording_needs_verified_identity( + recording.is_some(), + nonempty(values, "GITHUB_CLIENT_ID").is_some() + && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), + )?; + recording_can_deliver(recording.as_ref())?; + let (peer, peer_observer) = peer::peer_services( + values, + nonempty(values, "GITHUB_CLIENT_ID").is_some() + && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), + )? + .unzip(); + + let config = WebServerConfig { + tasks: None, + task_mode_requested: options.task_mode, + web_dir: match nonempty(values, "CODETRIAL_WEB_DIR") { + Some(dir) => PathBuf::from(dir), + + // Under the data directory rather than the working one, which a + // launcher does not choose: a `web/` that happens to sit where the + // app was started would otherwise replace the embedded pages. + None => match &options.data_dir { + Some(dir) => dir.join(DEFAULT_WEB_DIR), + None => PathBuf::from(DEFAULT_WEB_DIR), + }, + }, + github_client_id: nonempty(values, "GITHUB_CLIENT_ID"), + github_client_secret: nonempty(values, "GITHUB_CLIENT_SECRET"), + session_secret: Some(value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET)), + db_path: Some(account_db_path(values, options)), + github_oauth_base_url: None, + github_api_base_url: None, + room_prefix: value_or(values, "CODETRIAL_ROOM_PREFIX", DEFAULT_ROOM_PREFIX), + fixed_room_name: nonempty(values, "INTERVIEW_ROOM_NAME"), + production, + trusted_proxy_hops: trusted_proxy_hops(values), + allowed_hosts: allowed_hosts(values)?, + compiler_explorer_enabled: compiler_explorer_enabled(values), + guest_mode_enabled: guest_mode_enabled(values), + recording, + peer_observer, + peer, + pool, + probe_provider_quota: true, + + // Not a function of its own: off macOS this is always `None`, so no + // test on the Linux CI could tell a replaced body from this one. + update_check_commit: crate::config::update_check_commit( + values.get("CODETRIAL_UPDATE_CHECK").map(String::as_str), + env!("CODETRIAL_RELEASE_COMMIT"), + cfg!(target_os = "macos"), + ), + provision_rooms: true, + }; + if let Some(observer) = &config.peer_observer { + observer.validate_audio_storage(&config.web_dir)?; + } + Ok(config) +} + +/// The pool this deployment will serve. +/// +/// Built straight from the values rather than through `load_from_pairs`: the +/// web mode serves HTTP and mints LiveKit tokens, and has no use for the Gemini +/// key that a full agent config insists on. Demanding it here turned a web-only +/// deployment into an empty pool, silently. +fn web_provider_pool( + values: &BTreeMap, + options: &CliOptions, + production: bool, +) -> Result { + let mut pool = crate::config::ProviderPool::default(); + let (url, api_key, api_secret) = match ( + nonempty(values, "LIVEKIT_URL"), + nonempty(values, "LIVEKIT_API_KEY"), + nonempty(values, "LIVEKIT_API_SECRET"), + ) { + (Some(url), Some(api_key), Some(api_secret)) => (url, api_key, api_secret), + + // Names the file that was actually read. A fixed `config/` in this + // message sends anyone running with `--config` or a bare + // `./codetrial.env.local` to edit a file the server never opened. + _ => { + return Err(format!( + "missing required LiveKit credentials: set LIVEKIT_URL, LIVEKIT_API_KEY and LIVEKIT_API_SECRET in {}", + primary_config_path(options)?.display() + )); + } + }; + crate::config::validate_livekit_url(&url, production)?; + pool.providers.push(crate::config::Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url, + api_key, + api_secret, + google_api_keys: crate::config::google_api_keys(values)?, + }); + extend_pool( + &mut pool, + production, + &config_dir(options), + &value_or(values, crate::config::PROVIDER_ORDER_KEY, ""), + ); + Ok(pool) +} + +/// Why a production launch may not sign session cookies, or `None` where it +/// may. The built-in default is a published string, so in production it is not +/// a weak signing key, it is a known one. +/// +/// A function rather than an inline check because `serve_setup` asks it too. +/// The verdict needs nothing from the submission, and a check that could have +/// been made before the page was served but is made after it has taken +/// credentials, written them and answered 200 is a launch that exits with the +/// config already on disk -- which is what stops the next launch being a cold +/// start, so the page never comes back to say why. +pub fn production_secret_refusal(values: &BTreeMap) -> Option { + if crate::config::is_production(values) + && value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) == DEFAULT_SESSION_SECRET + { + return Some( + "SESSION_SECRET must be set when NODE_ENV=production; the built-in \ + default is a published value that would let anyone forge a session cookie" + .to_string(), + ); + } + None +} + +/// The environment and the flags, which in a cold start is everything +/// `load_values` has: the file it reads between them is the file that does not +/// exist yet. +/// +/// Its own function so `run_web` does not name the process environment. Reading +/// a deployment key straight from it there is what +/// `binary_web_reads_deployment_keys_from_the_config_file` refuses, because a +/// key set in the config file was once silently a no-op. +pub fn environment_and_flags(options: &CliOptions) -> BTreeMap { + let mut values = environment(options); + apply_options(&mut values, options); + values +} + +/// A first `--data-dir` launch started where the credentials Setup asks for +/// are already exported writes them into the folder's config, as Setup would +/// have, instead of asking for them again; every later launch then finds the +/// file, including one started from a desktop that exports nothing. +/// +/// The one exception to `environment` ignoring the process environment, and a +/// narrow one: only Setup's own keys, only while the folder has no config, and +/// written down rather than read live, so the file stays the whole of the +/// install's configuration. +pub fn data_dir_config_from_environment( + options: &CliOptions, + production: bool, +) -> Result<(), String> { + let Some(dir) = &options.data_dir else { + return Ok(()); + }; + let path = dir.join(CONFIG_FILE_NAME); + if path.exists() { + return Ok(()); + } + let exported = std::env::vars().collect(); + let Some(contents) = + crate::web::config_from_values(&exported, production).map_err(|error| { + format!("the exported Setup keys cannot be written to a config: {error}") + })? + else { + return Ok(()); + }; + std::fs::create_dir_all(dir) + .and_then(|()| crate::create_owner_only(&path, contents.as_bytes())) + .map_err(|error| format!("cannot write {}: {error}", path.display()))?; + println!( + "codetrial: wrote {} from the exported LiveKit and Google keys", + path.display() + ); + Ok(()) +} + +/// The process environment, or nothing for a `--data-dir` launch. +/// +/// A launcher that names the folder is saying the install's configuration is +/// in it, and an environment it was started from is not part of that: a shell +/// that exported `CODETRIAL_DB_PATH` or `NODE_ENV` for a CLI deployment would +/// otherwise move this install's reports, or change its rules, depending on +/// how it was started. Decided here, once, rather than by every launcher +/// keeping its own list of the keys this program reads. +pub fn environment(options: &CliOptions) -> BTreeMap { + if options.data_dir.is_some() { + return BTreeMap::new(); + } + std::env::vars().collect() +} + +pub fn load_values(options: &CliOptions) -> Result, String> { + let mut values = environment(options); + let path = primary_config_path(options)?; + crate::config::apply_config_file(&mut values, crate::config::read_config_file(&path)?); + apply_options(&mut values, options); + if let Some(dir) = &options.data_dir + && nonempty(&values, "SESSION_SECRET").is_none() + { + let secret = data_dir_session_secret(&dir.join(SESSION_SECRET_FILE))?; + values.insert("SESSION_SECRET".to_string(), secret); + } + Ok(values) +} + +/// The key a `--data-dir` install signs session cookies with, made on its first +/// launch and reused, so a relaunch keeps everyone signed in. +/// +/// Not the built-in default: that one is published, and any program on the +/// machine can reach a loopback port to present a cookie signed with it. +/// Owner-only on Unix for the same reason. A file that is not a key this wrote +/// is refused rather than replaced, because replacing it signs everyone out +/// and hides whatever damaged it. +pub fn data_dir_session_secret(path: &std::path::Path) -> Result { + match std::fs::read_to_string(path) { + Ok(text) => { + let secret = text.trim(); + if secret.len() >= 32 + && secret + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') + { + Ok(secret.to_string()) + } else { + Err(format!( + "{} is not a session key this program wrote. Move it aside to make a new \ + one; everyone will be signed out.", + path.display() + )) + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let secret = crate::accounts::random_token(32) + .map_err(|error| format!("cannot generate a session key: {error}"))?; + crate::create_owner_only(path, secret.as_bytes()) + .map_err(|error| format!("cannot write {}: {error}", path.display()))?; + Ok(secret) + } + Err(error) => Err(format!("cannot read {}: {error}", path.display())), + } +} + +/// The flags, over whatever the environment and the config file said. +/// +/// Split out because a cold start has to build the same map without the file +/// it does not have yet: `serve_setup` decides where to bind and what a +/// submission must survive, and both answers have to be the ones `run_web` +/// will reach a moment later. Spelled twice, this precedence is free to drift. +pub fn apply_options(values: &mut BTreeMap, options: &CliOptions) { + if let Some(value) = &options.web_addr { + values.insert("CODETRIAL_WEB_ADDR".to_string(), value.clone()); + } + if let Some(value) = &options.web_dir { + values.insert("CODETRIAL_WEB_DIR".to_string(), value.clone()); + } + if let Some(value) = &options.allowed_hosts { + values.insert("CODETRIAL_ALLOWED_HOSTS".to_string(), value.clone()); + } + if let Some(value) = &options.room_prefix { + values.insert("CODETRIAL_ROOM_PREFIX".to_string(), value.clone()); + } + if let Some(value) = options.duration_min { + values.insert("CODETRIAL_DURATION_MIN".to_string(), value.to_string()); + } + + // Over the config file as well: the desktop app is published as its own + // release, so the one the check reads never names its build. + if options.no_update_check { + values.insert("CODETRIAL_UPDATE_CHECK".to_string(), "false".to_string()); + } +} + +pub fn primary_config_path(options: &CliOptions) -> Result { + // The one place searched, and not one place among several: the working + // directory and the executable's folder are whatever the launcher left them + // as, so a config found there would belong to some other install. + if let Some(dir) = &options.data_dir { + let path = dir.join(CONFIG_FILE_NAME); + if path.is_file() { + return Ok(path); + } + return Err(format!( + "required configuration file is missing: {}; write one with LIVEKIT_URL, \ + LIVEKIT_API_KEY and LIVEKIT_API_SECRET", + path.display() + )); + } + if let Some(path) = &options.config_path { + let path = PathBuf::from(path); + if !path.is_file() { + return Err(format!( + "required configuration file is missing: {}", + path.display() + )); + } + return Ok(path); + } + + // The working directory first, which is a checkout's `config/` and an + // operator's deployment directory, then the folder the executable sits in. + // A release binary is unpacked into a folder of its own and its config is + // written there, so it has to be findable from a shortcut or a terminal + // opened somewhere else, not only from a double-click. + // + // Within each of those two roots, `config/` before the bare filename. The + // bare ones are where a release used to keep its config and where an + // operator may still keep one; they are searched, not written. + for path in [ + PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME), + PathBuf::from(CONFIG_FILE_NAME), + crate::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME), + crate::exe_dir().join(CONFIG_FILE_NAME), + ] { + if path.is_file() { + return Ok(path); + } + } + + // The keys, not a file to copy. `config/codetrial.env.example` exists in a + // checkout and nowhere else: the release archives hold the executable and + // nothing beside it, so telling someone who just unpacked one to copy it + // sends them looking for a file they were never given. Naming what the file + // must contain is an instruction both audiences can act on. + Err(format!( + "required configuration file is missing: ./{CONFIG_DIR}/{CONFIG_FILE_NAME}, \ + ./{CONFIG_FILE_NAME}, or {CONFIG_DIR}/{CONFIG_FILE_NAME} beside the \ + executable; write one with LIVEKIT_URL, LIVEKIT_API_KEY and \ + LIVEKIT_API_SECRET ({CONFIG_DIR}/{CONFIG_EXAMPLE_NAME} lists the \ + optional keys in a checkout)" + )) +} + +/// The directory of the config file this launch actually read, which is where +/// everything else belonging to the installation lives: the provider files it +/// discovers, and the account database it writes. +/// +/// So `--config /etc/codetrial/prod.env` discovers +/// `/etc/codetrial/codetrial.env.` rather than whatever `config/` happens +/// to sit in the current directory. Both halves of a deployment are launched +/// with the same `--config`, and that is what makes them agree on the pool. +/// +/// The database is written here rather than read, which is the one asymmetry: +/// a `--config` pointing somewhere a process may not write is a startup +/// failure that names the directory, and `CODETRIAL_DB_PATH` is the way out. +pub fn config_dir(options: &CliOptions) -> PathBuf { + if let Some(dir) = &options.data_dir { + return dir.clone(); + } + + // The search order comes from primary_config_path rather than being written + // out again here. Spelled twice, it was free to drift, and the drift is + // silent: the server keeps reading its own config while the pool quietly + // collapses to the primary project. + // + // A named file still decides, even when it is missing. The operator said + // where the config lives, and load_values is what refuses a path that is + // not there; answering with a different directory would be this function + // second-guessing that. + let path = options + .config_path + .as_deref() + .map(PathBuf::from) + .or_else(|| primary_config_path(options).ok()) + .unwrap_or_else(|| PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME)); + match path.parent() { + // A bare filename has an empty parent, and its siblings are in the + // working directory, not in `config/`. + Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(), + _ => PathBuf::from("."), + } +} + +/// The pool this process routes with. Discovery happens here, once, and never +/// inside `load_from_pairs`: a config loader that reads the current directory +/// gives two processes started from two directories two different pools, and a +/// positional index into those pools then names two different LiveKit projects. +pub fn extend_pool( + pool: &mut crate::config::ProviderPool, + production: bool, + config_dir: &std::path::Path, + order: &str, +) { + let (providers, warnings) = crate::config::discover_providers(config_dir, production); + for warning in warnings { + eprintln!("{warning}"); + } + pool.providers.extend(providers); + + // Outside the `discover` guard: an order naming the environment provider is + // still an order, and a deployment with discovery off should not silently + // ignore the one it was given. + for warning in crate::config::order_providers(&mut pool.providers, order) { + eprintln!("{warning}"); + } + + // Said once, at startup, in rotation order. Every provider question asked + // of a running server so far has been answerable only by reading the config + // directory and the argv back to itself and guessing which won. + if !pool.providers.is_empty() { + let (summary, warnings) = crate::config::pool_summary(&pool.providers); + eprintln!("{summary}"); + for warning in warnings { + eprintln!("WARNING: {warning}"); + } + } +} + +/// Refuses to start on a database that will not migrate, before anything is +/// served. The listener is already bound by this point, so the socket exists +/// and its backlog fills, but nothing accepts from it and the process exits +/// non-zero instead. The router runs the same migration again on the connection +/// it keeps, where the only thing it can do about a failure is answer 503. +pub fn initialize_accounts(config: &WebServerConfig) -> Result<(), String> { + let Some(login) = crate::web::login_config(config) else { + return Ok(()); + }; + + // The router opens the long-lived connection and performs the startup sweep + // on it; this first pass makes migration failure fatal before the server + // starts accepting. + crate::web::initialize_account_database(&login.db_path) + .map_err(|error| format!("account database {}: {error}", login.db_path.display()))?; + Ok(()) +} + +/// The dispatcher cap, with anything unusable about it said out loud. +/// +/// One helper for both modes: a warning printed by web and swallowed by serve +/// would be the same misconfiguration reported twice differently. +fn read_max_concurrent(values: &BTreeMap) -> usize { + let mut warnings = Vec::new(); + let max_concurrent = crate::config::max_concurrent_interviews(values, &mut warnings); + for warning in &warnings { + eprintln!("{warning}"); + } + max_concurrent +} + +/// Zero unless the operator states how many proxies front this server, so the +/// default never trusts a forwarded client address. +/// Recording without an OAuth app is a server that refuses every interview. +/// +/// The delivery target is the primary verified address GitHub returns, and +/// without the OAuth flow every account is self-declared, so `/api/token` +/// answers 403 to all of them. Said at startup, where an operator can act on +/// it, rather than one candidate at a time. +/// +/// `oauth` is both credentials, not just the id. `login_config` enables OAuth +/// only when it has the pair, so an id with no secret is exactly as unable to +/// verify anyone as no id at all, and would have started a server that refused +/// every interview while looking configured. +/// A deployment that records has to be able to hand the file over. +/// +/// The key is parsed here, before the listener does any work, because the +/// alternative is a server that records interviews and discovers at the first +/// delivery that it can never sign for one. Configuration already checked the +/// shape; this is the credential itself. +pub fn recording_can_deliver( + recording: Option<&crate::config::RecordingConfig>, +) -> Result<(), String> { + let Some(recording) = recording else { + return Ok(()); + }; + crate::delivery::GoogleDelivery::new( + &recording.service_account_json, + &recording.gcs_bucket, + &recording.drive_id, + std::sync::Arc::new(|| 0), + ) + .map(|_| ()) + .map_err(|error| format!("CODETRIAL_RECORDING_SERVICE_ACCOUNT_JSON: {error}")) +} + +pub fn recording_needs_verified_identity(recording: bool, oauth: bool) -> Result<(), String> { + if recording && !oauth { + return Err( + "CODETRIAL_RECORDING_ENABLED=true needs GITHUB_CLIENT_ID and \ + GITHUB_CLIENT_SECRET: a recording is delivered to the verified address \ + GitHub returns, and without the OAuth app every account is self-declared \ + and every interview would be refused" + .to_string(), + ); + } + Ok(()) +} + +/// The configured interview length, which is what a recording's maximum +/// duration has to clear. Read here rather than through `load_from_pairs` +/// because `codetrial web` deliberately runs without a full agent config. +fn interview_duration_min(values: &BTreeMap) -> u32 { + nonempty(values, "CODETRIAL_DURATION_MIN") + .and_then(|value| value.parse().ok()) + .unwrap_or(DEFAULT_DURATION_MIN) +} + +/// `CODETRIAL_ALLOWED_HOSTS`, a comma-separated list of `Host` values. +/// +/// Unset or empty means no check. A value that holds only separators is +/// refused instead: it was meant to name a host, and reading it as no list +/// would answer every `Host` the person believes is refused. +pub fn allowed_hosts(values: &BTreeMap) -> Result, String> { + let hosts: Vec = value_or(values, "CODETRIAL_ALLOWED_HOSTS", "") + .split(',') + .map(str::trim) + .filter(|host| !host.is_empty()) + .map(str::to_string) + .collect(); + if hosts.is_empty() && nonempty(values, "CODETRIAL_ALLOWED_HOSTS").is_some() { + return Err("CODETRIAL_ALLOWED_HOSTS names no Host".to_string()); + } + Ok(hosts) +} + +fn trusted_proxy_hops(values: &BTreeMap) -> u32 { + nonempty(values, "CODETRIAL_TRUSTED_PROXY_HOPS") + .and_then(|value| value.parse().ok()) + .unwrap_or(0) +} + +/// What a `codetrial web` start without `NODE_ENV=production` is allowing, in +/// the operator's words rather than the code's. +/// +/// Every guard this reports on is keyed on `NODE_ENV`, so the one mistake none +/// of them can catch is the deployment that sets every credential correctly and +/// never sets `NODE_ENV` at all. The refusal in `run_web` fires only once the +/// operator has already said this is production; a server that was meant to be +/// production and does not say so gets the local defaults and no complaint. +/// This is the only signal it ever gets. +/// +/// `config::is_production` matches the exact string, so a `NODE_ENV=Production` +/// typo lands here too, which is the other way a deployment ends up local +/// without meaning to. +pub fn relaxed_for_local_use(values: &BTreeMap) -> Vec { + if crate::config::is_production(values) { + return Vec::new(); + } + + let mut warnings = vec![ + "NODE_ENV is not production: the page's CSP allows loopback origins and a \ + plaintext ws:// LiveKit URL is accepted. Set NODE_ENV=production to deploy." + .to_string(), + ]; + if nonempty(values, "SESSION_SECRET").is_none() { + warnings.push(format!( + "SESSION_SECRET is unset: signing session cookies with the built-in \ + {DEFAULT_SESSION_SECRET:?}, which is published in this repository and lets \ + anyone forge a session." + )); + } + warnings +} + +/// Why the built-in `SESSION_SECRET` may not sign cookies on `bound`, or `None` +/// where it may. +/// +/// Split out for the reason `relaxed_for_local_use` is: the caller can only +/// return the string, and a refusal reachable solely by starting a process that +/// otherwise never exits is a branch no test can assert on without hanging. +/// +/// `NODE_ENV` is a claim; the address this process bound is a fact. A server +/// other machines can reach is not the local run the published default exists +/// for, whether or not anyone remembered to say so, which is the mistake the +/// warning in `relaxed_for_local_use` can name but not prevent. +/// +/// Unspecified addresses (`0.0.0.0`, `::`) are not loopback and must not be +/// treated as such: binding every interface is how a container serves the +/// world. +/// +/// `to_canonical` before the test, because `Ipv6Addr::is_loopback` is true of +/// `::1` and of nothing else: a dual-stack socket that landed on +/// `::ffff:127.0.0.1`, which is a name for the same interface, would otherwise +/// be refused an address only this machine can reach. +/// How a listener on `bound` can be reached from somewhere other than this +/// machine, or `None` where it cannot. +/// +/// Shared, because two guards ask it and a third answer added to one of them +/// would drift silently -- in the direction that leaves the page taking +/// credentials open. What each guard says about it is not shared: the remedies +/// differ, an address is passed on the command line and a declared proxy is a +/// variable in the environment. +#[derive(Clone, Copy)] +pub enum Reachable { + /// The bind address is not loopback. + Address, + /// `CODETRIAL_TRUSTED_PROXY_HOPS` is the operator saying requests arrive + /// through something in front, which is the same admission: a server on + /// loopback behind nginx or a tunnel is as public as one on `0.0.0.0`. + DeclaredProxy, +} + +pub fn reachable_from_elsewhere( + values: &BTreeMap, + bound: std::net::SocketAddr, +) -> Option { + if !bound.ip().to_canonical().is_loopback() { + Some(Reachable::Address) + } else if trusted_proxy_hops(values) > 0 { + Some(Reachable::DeclaredProxy) + } else { + None + } +} + +pub fn task_mode_allowed(reachable: Option, fixed_room: bool, recording: bool) -> bool { + task_mode_refusal(reachable, fixed_room, recording).is_none() +} + +/// Why task mode may not run on this server, in the words of what to change, or +/// `None` where it may. +pub fn task_mode_refusal( + reachable: Option, + fixed_room: bool, + recording: bool, +) -> Option<&'static str> { + match (reachable, fixed_room, recording) { + (Some(Reachable::Address), _, _) => Some( + "it is bound to an address other than loopback; start it with --web-addr 127.0.0.1:3000", + ), + (Some(Reachable::DeclaredProxy), _, _) => { + Some("CODETRIAL_TRUSTED_PROXY_HOPS says requests arrive through a proxy; unset it") + } + (None, true, _) => Some("INTERVIEW_ROOM_NAME shares one fixed room; unset it"), + (None, false, true) => { + Some("recording is on; task mode does not run on a recording server") + } + (None, false, false) => None, + } +} + +/// Task mode is on when `--task-mode` asked for it and the server is one it may +/// run on; either alone is not enough. +pub fn task_mode_enabled( + requested: bool, + reachable: Option, + fixed_room: bool, + recording: bool, +) -> bool { + requested && task_mode_allowed(reachable, fixed_room, recording) +} + +pub fn published_secret_refusal( + values: &BTreeMap, + bound: std::net::SocketAddr, +) -> Option { + // What the cookie signer will actually use, not whether the key was + // mentioned. `value_or` trims, so `SESSION_SECRET=" codetrial-local-session + // "` is the published key spelled with whitespace rather than a secret of + // the operator's own, and a guard that only asked whether the variable was + // set would wave it through. + if value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) != DEFAULT_SESSION_SECRET { + return None; + } + + let reason = match reachable_from_elsewhere(values, bound)? { + Reachable::Address => format!("bind {bound}"), + Reachable::DeclaredProxy => "serve from behind a declared proxy".to_string(), + }; + + Some(format!( + "SESSION_SECRET must be set to {reason}: {DEFAULT_SESSION_SECRET:?} is published in \ + this repository, so every session cookie this server signs would be forgeable by \ + anyone who can reach it" + )) +} + +fn compiler_explorer_enabled(values: &BTreeMap) -> bool { + crate::config::compiler_explorer_enabled( + values + .get("CODETRIAL_COMPILER_EXPLORER_ENABLED") + .map(String::as_str), + ) +} + +pub fn guest_mode_recording_conflict(guest_mode_enabled: bool, recording_enabled: bool) -> bool { + guest_mode_enabled && recording_enabled +} + +pub fn guest_mode_enabled(values: &BTreeMap) -> bool { + crate::config::guest_mode_enabled( + values + .get("CODETRIAL_GUEST_MODE_ENABLED") + .map(String::as_str), + ) +} + +pub fn nonempty(values: &BTreeMap, key: &str) -> Option { + values + .get(key) + .map(|value| value.trim()) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +pub fn value_or(values: &BTreeMap, key: &str, default: &str) -> String { + nonempty(values, key).unwrap_or_else(|| default.to_string()) +} + +/// One place decides where accounts live, because `web` and `serve` both open +/// the same database and disagreeing about it would split a person's reports. +/// +/// Beside the config file rather than in the working directory: the database +/// has to be the same file on the next launch, and the directory a process was +/// started from is not. It used to be a bare filename, which meant a launch +/// from a shortcut or from another terminal found the config it wrote and then +/// opened an empty database next to wherever it had been started. +pub fn account_db_path(values: &BTreeMap, options: &CliOptions) -> PathBuf { + match nonempty(values, "CODETRIAL_DB_PATH") { + Some(path) => PathBuf::from(path), + None => config_dir(options).join(DEFAULT_ACCOUNT_DB_PATH), + } +} + +/// Where a cold start writes the config it just took, which is the same +/// `config/` the search above looks in first for the root this is installed as. +/// +/// A checkout is told apart by the template it ships, not by `config/` merely +/// existing: a released binary run from a directory that happens to hold one +/// would otherwise write the credentials there and lose them the moment it was +/// next started from somewhere else. +pub fn cold_start_config_path(options: &CliOptions) -> PathBuf { + if let Some(dir) = &options.data_dir { + return dir.join(CONFIG_FILE_NAME); + } + let in_checkout = PathBuf::from(CONFIG_DIR); + if in_checkout.join(CONFIG_EXAMPLE_NAME).is_file() { + return in_checkout.join(CONFIG_FILE_NAME); + } + crate::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME) +} + +/// What a launch settles before it binds a socket: the refusals and warnings +/// that depend only on the values, the provider pool, the server config and +/// the account database. +pub struct Prepared { + pub config: WebServerConfig, + pub max_concurrent: usize, +} + +pub fn prepare( + values: &BTreeMap, + options: &CliOptions, +) -> Result { + if let Some(refusal) = production_secret_refusal(values) { + return Err(refusal); + } + for warning in relaxed_for_local_use(values) { + eprintln!("{warning}"); + } + + // Beside the other startup warnings and not next to the dispatcher that + // consumes it. A cap the operator wrote and this cannot honor is worth + // saying before the first thing that can fail: read late, a bad bind or a + // half-configured pool ends the process first and the operator never learns + // the number in their config file was ignored. + let max_concurrent = read_max_concurrent(values); + + let production = crate::config::is_production(values); + let pool = web_provider_pool(values, options, production)?; + + let config = web_server_config(values, pool, production, options)?; + + if guest_mode_recording_conflict(config.guest_mode_enabled, config.recording.is_some()) { + eprintln!( + "WARNING: CODETRIAL_GUEST_MODE_ENABLED=true is ignored while recording is enabled; \\ + guest mode is unavailable" + ); + } + + initialize_accounts(&config)?; + Ok(Prepared { + config, + max_concurrent, + }) +} + +/// Who started this server. A launcher that owns it (the desktop app, a test +/// fixture) offers no Setup page; an operator who started it is offered one. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LaunchedBy { + Parent, + Operator, +} + +/// What a launch settles once its socket is bound and its address is known: +/// the published-secret refusal, the Setup page and task mode, both of which +/// depend on where the server is reachable. +pub fn finish( + mut config: WebServerConfig, + values: &BTreeMap, + options: &CliOptions, + bound: std::net::SocketAddr, + launched_by: LaunchedBy, +) -> Result<(WebServerConfig, Option), String> { + if let Some(refusal) = published_secret_refusal(values, bound) { + return Err(refusal); + } + let setup = crate::web::SetupContext::new( + primary_config_path(options)?, + values.clone(), + bound, + launched_by == LaunchedBy::Operator, + config.trusted_proxy_hops, + )?; + // Same reason `serve_setup` prints this: say where to go. + println!("codetrial: open http://{bound} in your browser"); + if let Some(setup) = &setup { + println!( + "codetrial: to change keys at http://{bound}/setup, enter {}", + setup.code() + ); + } + + // Task mode is off unless `--task-mode` asks for it. It serves the learner + // at this machine and nobody else: it holds decrypted reference solutions + // in memory and trusts its page to report the test rules. Reachable from + // anywhere else, or set up to record or to share one fixed room, its routes + // say why they are off. A declared proxy makes a loopback bind as public as + // any other. + let reachable = reachable_from_elsewhere(values, bound); + let (fixed_room, recording) = (config.fixed_room_name.is_some(), config.recording.is_some()); + config.tasks = task_mode_enabled(config.task_mode_requested, reachable, fixed_room, recording) + .then(crate::tasks::access::TaskService::new); + + // Said at startup, because the address printed above is the interview lobby + // on any other server, and a learner who asked for task mode would not + // otherwise know what to open or why nothing changed. + if config.tasks.is_some() { + println!( + "codetrial: task mode is on; open http://{bound} for the default task, or the assignment link your instructor sent" + ); + } else if config.task_mode_requested + && let Some(reason) = task_mode_refusal(reachable, fixed_room, recording) + { + eprintln!("codetrial: --task-mode is off: {reason}"); + } + + Ok((config, setup)) +} + +pub fn local_agent_config( + values: BTreeMap, +) -> Result, String> { + // Whether this process also hosts interviewers. A full agent config, which + // is a Gemini key on top of what the web side needs, means yes; without it + // this is the web half of a split deployment and agents arrive from + // elsewhere. Deciding it by what is configured, rather than by a flag, + // keeps a single-host production deployment working out of the box while + // leaving the split one exactly as it was. No `extend_pool` here on + // purpose. The dispatcher never looks a provider up: it is handed the one + // the token was minted from, so a second scan could only introduce a pool + // that disagrees with the web side's. + // + // Only keys that are absent make it the web half. An entry that is present + // but invalid is a mistake in this host's config, and serving on without + // the local interviewer would leave every interview waiting forever. + let agent_config = match crate::config::load_from_pairs(values) { + Ok(config) => Some(config), + Err(error) if error.invalid_entries.is_empty() => None, + Err(error) => return Err(error.to_string()), + }; + if agent_config.is_none() { + eprintln!( + "no GOOGLE_API_KEY or GOOGLE_API_KEYS: serving the web side only. Interviews will wait forever unless \ + an agent joins each room from elsewhere." + ); + } + Ok(agent_config) +} + +/// Serves the web service, and the local interviewer when there is an agent +/// config, until `shutdown` resolves and the connections in flight have +/// finished. `codetrial web` passes a future that never resolves; a fixture +/// passes its stop signal. `drain` bounds how long open connections may hold +/// the shutdown up once it has begun (`None` waits for them): a client holding +/// a half-sent request would otherwise keep a fixture from ever reporting. The +/// listener must already be non-blocking. The runtime is dropped before this +/// returns, which ends any room task still running. +pub fn serve( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + agent_config: Option, + max_concurrent: usize, + shutdown: impl std::future::Future + Send + 'static, + drain: Option, +) -> Result<(), String> { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + runtime + .block_on(async { + let listener = tokio::net::TcpListener::from_std(listener)?; + let dispatcher = agent_config.map(|config| { + std::sync::Arc::new(crate::dispatch::LocalDispatcher { + config, + runtime: tokio::runtime::Handle::current(), + state: std::sync::Arc::default(), + max_concurrent, + }) as std::sync::Arc + }); + let (begun, shutting_down) = tokio::sync::oneshot::channel::<()>(); + let served = axum::serve( + listener, + crate::web::web_service_with_setup(config, dispatcher, setup), + ) + .with_graceful_shutdown(async move { + shutdown.await; + let _ = begun.send(()); + }); + let overdue = async { + match (shutting_down.await, drain) { + (Ok(()), Some(bound)) => tokio::time::sleep(bound).await, + _ => std::future::pending().await, + } + }; + tokio::select! { + result = served => result, + () = overdue => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "connections were still open when the shutdown bound passed", + )), + } + }) + .map_err(|error| format!("web server failed: {error}")) +} diff --git a/src/main_peer.rs b/src/launch/peer.rs similarity index 87% rename from src/main_peer.rs rename to src/launch/peer.rs index 349ac2aa4..66736c666 100644 --- a/src/main_peer.rs +++ b/src/launch/peer.rs @@ -1,14 +1,14 @@ //! Startup rules for reciprocal practice, offered once its prerequisites exist. -use codetrial::config::{ +use crate::config::{ MAX_CAPTURE_SECONDS, PeerConfig, PeerObserverConfig, load_peer, load_peer_observer, }; use std::collections::BTreeMap; -pub(super) type PeerServices = Option<(PeerConfig, PeerObserverConfig)>; +pub type PeerServices = Option<(PeerConfig, PeerObserverConfig)>; /// Reciprocal practice needs GitHub sign-in for member identity and the Google /// keys for the silent observer. Without either it is simply not offered. -pub(super) fn peer_services( +pub fn peer_services( values: &BTreeMap, oauth: bool, ) -> Result { diff --git a/src/lib.rs b/src/lib.rs index 4bae31362..b3d49d751 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,6 +4,7 @@ pub mod config; pub mod delivery; pub mod dispatch; pub mod gemini; +pub mod launch; pub mod livekit; mod peer_audio; pub mod recording; diff --git a/src/main.rs b/src/main.rs index dabb37b14..4a24e87b9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,44 +1,22 @@ -mod main_peer; - use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::Arc; -use codetrial::config::{ - AgentConfig, DEFAULT_DURATION_MIN, DEFAULT_ROOM_PREFIX, DEFAULT_WEB_ADDR, DEFAULT_WEB_DIR, +use codetrial::config::{AgentConfig, DEFAULT_WEB_ADDR, is_production}; +use codetrial::launch::{ + CliOptions, LaunchedBy, Reachable, cold_start_config_path, config_dir, + data_dir_config_from_environment, environment_and_flags, extend_pool, finish, load_values, + local_agent_config, nonempty, prepare, primary_config_path, production_secret_refusal, + reachable_from_elsewhere, serve, value_or, +}; +#[cfg(test)] +use codetrial::launch::{ + DEFAULT_SESSION_SECRET, SESSION_SECRET_FILE, account_db_path, allowed_hosts, apply_options, + data_dir_session_secret, environment, guest_mode_enabled, guest_mode_recording_conflict, + published_secret_refusal, recording_can_deliver, recording_needs_verified_identity, + relaxed_for_local_use, task_mode_allowed, task_mode_enabled, task_mode_refusal, + web_server_config, }; -use codetrial::web::{RoomDispatcher, WebServerConfig}; - -/// One installation keeps its config file and its account database together in -/// `config/`, under whichever root it was installed as: a checkout, or the -/// folder a released binary was unpacked into. Beside the executable is not -/// that root for a checkout -- the executable is in `target/`, which -/// `make clean` deletes -- so the two are named separately and joined per case. -const CONFIG_DIR: &str = "config"; -const CONFIG_FILE_NAME: &str = "codetrial.env.local"; -/// Tracked, so it exists in a checkout and in no release archive. That makes it -/// the one thing that tells a `config/` belonging to this project apart from a -/// directory of the same name that happens to sit where the binary was run. -const CONFIG_EXAMPLE_NAME: &str = "codetrial.env.example"; -const DEFAULT_ACCOUNT_DB_PATH: &str = "codetrial.db"; -const DEFAULT_SESSION_SECRET: &str = "codetrial-local-session"; -/// Where a `--data-dir` install keeps the key its session cookies are signed -/// with, when its config file does not set `SESSION_SECRET`. -const SESSION_SECRET_FILE: &str = "session-secret"; - -#[derive(Debug, Default, Clone)] -struct CliOptions { - config_path: Option, - data_dir: Option, - import_config: Option, - web_addr: Option, - web_dir: Option, - allowed_hosts: Option, - room_prefix: Option, - duration_min: Option, - task_mode: bool, - no_update_check: bool, -} /// Every mode this binary has: its name, how many positionals it takes counting /// the mode word, how it is called, and what runs it. One table and not one @@ -300,92 +278,17 @@ fn bind_web_listener(addr: &str) -> Result { Ok(listener) } -/// Validates operator settings before a listener or background task starts. -fn web_server_config( - values: &std::collections::BTreeMap, - pool: codetrial::config::ProviderPool, - production: bool, - options: &CliOptions, -) -> Result { - // After the pool is complete, so the "is this project in the pool" check - // sees the pool the server will actually serve, and before the listener - // does any work: a half-configured recording block is a startup failure, - // never a surprise at the moment a candidate's interview ends. - let recording = codetrial::config::load_recording( - values, - &pool, - interview_duration_min(values), - production, - ) - .map_err(|error| error.to_string())?; - recording_needs_verified_identity( - recording.is_some(), - nonempty(values, "GITHUB_CLIENT_ID").is_some() - && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), - )?; - recording_can_deliver(recording.as_ref())?; - let (peer, peer_observer) = main_peer::peer_services( - values, - nonempty(values, "GITHUB_CLIENT_ID").is_some() - && nonempty(values, "GITHUB_CLIENT_SECRET").is_some(), - )? - .unzip(); - - let config = WebServerConfig { - tasks: None, - task_mode_requested: options.task_mode, - web_dir: match nonempty(values, "CODETRIAL_WEB_DIR") { - Some(dir) => PathBuf::from(dir), - - // Under the data directory rather than the working one, which a - // launcher does not choose: a `web/` that happens to sit where the - // app was started would otherwise replace the embedded pages. - None => match &options.data_dir { - Some(dir) => dir.join(DEFAULT_WEB_DIR), - None => PathBuf::from(DEFAULT_WEB_DIR), - }, - }, - github_client_id: nonempty(values, "GITHUB_CLIENT_ID"), - github_client_secret: nonempty(values, "GITHUB_CLIENT_SECRET"), - session_secret: Some(value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET)), - db_path: Some(account_db_path(values, options)), - github_oauth_base_url: None, - github_api_base_url: None, - room_prefix: value_or(values, "CODETRIAL_ROOM_PREFIX", DEFAULT_ROOM_PREFIX), - fixed_room_name: nonempty(values, "INTERVIEW_ROOM_NAME"), - production, - trusted_proxy_hops: trusted_proxy_hops(values), - allowed_hosts: allowed_hosts(values)?, - compiler_explorer_enabled: compiler_explorer_enabled(values), - guest_mode_enabled: guest_mode_enabled(values), - recording, - peer_observer, - peer, - pool, - probe_provider_quota: true, - - // Not a function of its own: off macOS this is always `None`, so no - // test on the Linux CI could tell a replaced body from this one. - update_check_commit: codetrial::config::update_check_commit( - values.get("CODETRIAL_UPDATE_CHECK").map(String::as_str), - env!("CODETRIAL_RELEASE_COMMIT"), - cfg!(target_os = "macos"), - ), - provision_rooms: true, - }; - if let Some(observer) = &config.peer_observer { - observer.validate_audio_storage(&config.web_dir)?; - } - Ok(config) -} - fn run_web(options: CliOptions) -> Result<(), String> { // Falls through: once `serve_setup` returns, the config exists and the rest // of this function is an ordinary launch, on the socket Setup served on. // Why that socket is carried here rather than rebound: `serve_setup`. Read // once, so the question and the page it opens answer from the same map. let before_config = environment_and_flags(&options); - let launched_as_child = codetrial::exit_when_stdin_closes(); + let launched_by = if codetrial::exit_when_stdin_closes() { + LaunchedBy::Parent + } else { + LaunchedBy::Operator + }; data_dir_config_from_environment(&options, is_production(&before_config))?; let handed_over = if is_cold_start(&options, &before_config) { Some(serve_setup(&before_config, &options)?) @@ -394,33 +297,9 @@ fn run_web(options: CliOptions) -> Result<(), String> { }; let values = load_values(&options)?; - if let Some(refusal) = production_secret_refusal(&values) { - return Err(refusal); - } - for warning in relaxed_for_local_use(&values) { - eprintln!("{warning}"); - } - - // Beside the other startup warnings and not next to the dispatcher that - // consumes it. A cap the operator wrote and this cannot honor is worth - // saying before the first thing that can fail: read late, a bad bind or a - // half-configured pool ends the process first and the operator never learns - // the number in their config file was ignored. - let max_concurrent = read_max_concurrent(&values); - - let production = is_production(&values); - let pool = web_provider_pool(&values, &options, production)?; - - let mut config = web_server_config(&values, pool, production, &options)?; - - if guest_mode_recording_conflict(config.guest_mode_enabled, config.recording.is_some()) { - eprintln!( - "WARNING: CODETRIAL_GUEST_MODE_ENABLED=true is ignored while recording is enabled; \\ - guest mode is unavailable" - ); - } - - initialize_accounts(&config)?; + let prepared = prepare(&values, &options)?; + let max_concurrent = prepared.max_concurrent; + let config = prepared.config; // The address is not re-read for a cold start: `serve_setup` read it from // the same environment and flags, and the config file written since holds @@ -430,15 +309,16 @@ fn run_web(options: CliOptions) -> Result<(), String> { None => bind_web_listener(&value_or(&values, "CODETRIAL_WEB_ADDR", DEFAULT_WEB_ADDR))?, }; - // The other half of the `SESSION_SECRET` guard above, and the half that - // does not depend on the operator having said anything. + // The other half of the `SESSION_SECRET` guard in `launch::prepare`, and + // the half that does not depend on the operator having said anything. // // After the bind rather than before it, because the address may be a // hostname: `CODETRIAL_WEB_ADDR=interviews.example:3000` is resolved by - // `ToSocketAddrs` and only the socket knows what it landed on. That puts it - // beside `initialize_accounts`, which refuses after binding for the same - // reason and with the same consequence: the socket exists and its backlog - // fills, nothing accepts from it, and the process exits non-zero. + // `ToSocketAddrs` and only the socket knows what it landed on. A refusal + // here therefore comes after the bind, with the consequence that follows: + // the socket exists and its backlog fills, nothing accepts from it, and the + // process exits non-zero. (`launch::prepare`, with `initialize_accounts`, + // runs before the bind and has no such cost.) // // Fails closed when the socket cannot name itself, which a just-bound // listener does not do. Skipping the check on an unreadable address would @@ -447,48 +327,7 @@ fn run_web(options: CliOptions) -> Result<(), String> { let bound = listener .local_addr() .map_err(|error| format!("bound listener has no address to check: {error}"))?; - if let Some(refusal) = published_secret_refusal(&values, bound) { - return Err(refusal); - } - let setup = codetrial::web::SetupContext::new( - primary_config_path(&options)?, - values.clone(), - bound, - !launched_as_child, - config.trusted_proxy_hops, - )?; - // Same reason `serve_setup` prints this: say where to go. - println!("codetrial: open http://{bound} in your browser"); - if let Some(setup) = &setup { - println!( - "codetrial: to change keys at http://{bound}/setup, enter {}", - setup.code() - ); - } - - // Task mode is off unless `--task-mode` asks for it. It serves the learner - // at this machine and nobody else: it holds decrypted reference solutions - // in memory and trusts its page to report the test rules. Reachable from - // anywhere else, or set up to record or to share one fixed room, its routes - // say why they are off. A declared proxy makes a loopback bind as public as - // any other. - let reachable = reachable_from_elsewhere(&values, bound); - let (fixed_room, recording) = (config.fixed_room_name.is_some(), config.recording.is_some()); - config.tasks = task_mode_enabled(config.task_mode_requested, reachable, fixed_room, recording) - .then(codetrial::tasks::access::TaskService::new); - - // Said at startup, because the address printed above is the interview lobby - // on any other server, and a learner who asked for task mode would not - // otherwise know what to open or why nothing changed. - if config.tasks.is_some() { - println!( - "codetrial: task mode is on; open http://{bound} for the default task, or the assignment link your instructor sent" - ); - } else if config.task_mode_requested - && let Some(reason) = task_mode_refusal(reachable, fixed_room, recording) - { - eprintln!("codetrial: --task-mode is off: {reason}"); - } + let (config, setup) = finish(config, &values, &options, bound, launched_by)?; // Past every startup check, so whatever a previous run left in the log is // no longer true. Written on failure and removed on none, the file outlives @@ -498,96 +337,17 @@ fn run_web(options: CliOptions) -> Result<(), String> { // at nothing else. clear_web_error_log(&options); - // Whether this process also hosts interviewers. A full agent config, which - // is a Gemini key on top of what the web side needs, means yes; without it - // this is the web half of a split deployment and agents arrive from - // elsewhere. Deciding it by what is configured, rather than by a flag, - // keeps a single-host production deployment working out of the box while - // leaving the split one exactly as it was. No `extend_pool` here on - // purpose. The dispatcher never looks a provider up: it is handed the one - // the token was minted from, so a second scan could only introduce a pool - // that disagrees with the web side's. - // - // Only keys that are absent make it the web half. An entry that is present - // but invalid is a mistake in this host's config, and serving on without - // the local interviewer would leave every interview waiting forever. - let agent_config = match codetrial::config::load_from_pairs(values) { - Ok(config) => Some(config), - Err(error) if error.invalid_entries.is_empty() => None, - Err(error) => return Err(error.to_string()), - }; - if agent_config.is_none() { - eprintln!( - "no GOOGLE_API_KEY or GOOGLE_API_KEYS: serving the web side only. Interviews will wait forever unless \ - an agent joins each room from elsewhere." - ); - } - - let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); - runtime - .block_on(async { - let listener = tokio::net::TcpListener::from_std(listener)?; - let dispatcher = agent_config.map(|config| { - Arc::new(codetrial::dispatch::LocalDispatcher { - config, - runtime: tokio::runtime::Handle::current(), - state: Arc::default(), - max_concurrent, - }) as Arc - }); - axum::serve( - listener, - codetrial::web::web_service_with_setup(config, dispatcher, setup), - ) - .await - }) - .map_err(|error| format!("web server failed: {error}")) -} + let agent_config = local_agent_config(values)?; -/// The pool this deployment will serve. -/// -/// Built straight from the values rather than through `load_from_pairs`: the -/// web mode serves HTTP and mints LiveKit tokens, and has no use for the Gemini -/// key that a full agent config insists on. Demanding it here turned a web-only -/// deployment into an empty pool, silently. -fn web_provider_pool( - values: &BTreeMap, - options: &CliOptions, - production: bool, -) -> Result { - let mut pool = codetrial::config::ProviderPool::default(); - let (url, api_key, api_secret) = match ( - nonempty(values, "LIVEKIT_URL"), - nonempty(values, "LIVEKIT_API_KEY"), - nonempty(values, "LIVEKIT_API_SECRET"), - ) { - (Some(url), Some(api_key), Some(api_secret)) => (url, api_key, api_secret), - - // Names the file that was actually read. A fixed `config/` in this - // message sends anyone running with `--config` or a bare - // `./codetrial.env.local` to edit a file the server never opened. - _ => { - return Err(format!( - "missing required LiveKit credentials: set LIVEKIT_URL, LIVEKIT_API_KEY and LIVEKIT_API_SECRET in {}", - primary_config_path(options)?.display() - )); - } - }; - codetrial::config::validate_livekit_url(&url, production)?; - pool.providers.push(codetrial::config::Provider { - id: codetrial::config::PRIMARY_PROVIDER_ID.to_string(), - url, - api_key, - api_secret, - google_api_keys: codetrial::config::google_api_keys(values)?, - }); - extend_pool( - &mut pool, - production, - &config_dir(options), - &value_or(values, codetrial::config::PROVIDER_ORDER_KEY, ""), - ); - Ok(pool) + serve( + listener, + config, + setup, + agent_config, + max_concurrent, + std::future::pending(), + None, + ) } /// The solo self-serve cold start: no `--config` given, none of the searched @@ -695,29 +455,6 @@ fn serve_setup( Ok(retained) } -/// Why a production launch may not sign session cookies, or `None` where it -/// may. The built-in default is a published string, so in production it is not -/// a weak signing key, it is a known one. -/// -/// A function rather than an inline check because `serve_setup` asks it too. -/// The verdict needs nothing from the submission, and a check that could have -/// been made before the page was served but is made after it has taken -/// credentials, written them and answered 200 is a launch that exits with the -/// config already on disk -- which is what stops the next launch being a cold -/// start, so the page never comes back to say why. -fn production_secret_refusal(values: &BTreeMap) -> Option { - if is_production(values) - && value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) == DEFAULT_SESSION_SECRET - { - return Some( - "SESSION_SECRET must be set when NODE_ENV=production; the built-in \ - default is a published value that would let anyone forge a session cookie" - .to_string(), - ); - } - None -} - /// Why Setup may not serve on `bound`, or `None` where it may. /// /// Setup writes LiveKit credentials to disk for whoever posts them and has @@ -878,613 +615,6 @@ fn load_agent_config(options: &CliOptions) -> Result { Ok(config) } -/// The environment and the flags, which in a cold start is everything -/// `load_values` has: the file it reads between them is the file that does not -/// exist yet. -/// -/// Its own function so `run_web` does not name the process environment. Reading -/// a deployment key straight from it there is what -/// `binary_web_reads_deployment_keys_from_the_config_file` refuses, because a -/// key set in the config file was once silently a no-op. -fn environment_and_flags(options: &CliOptions) -> BTreeMap { - let mut values = environment(options); - apply_options(&mut values, options); - values -} - -/// A first `--data-dir` launch started where the credentials Setup asks for -/// are already exported writes them into the folder's config, as Setup would -/// have, instead of asking for them again; every later launch then finds the -/// file, including one started from a desktop that exports nothing. -/// -/// The one exception to `environment` ignoring the process environment, and a -/// narrow one: only Setup's own keys, only while the folder has no config, and -/// written down rather than read live, so the file stays the whole of the -/// install's configuration. -fn data_dir_config_from_environment(options: &CliOptions, production: bool) -> Result<(), String> { - let Some(dir) = &options.data_dir else { - return Ok(()); - }; - let path = dir.join(CONFIG_FILE_NAME); - if path.exists() { - return Ok(()); - } - let exported = std::env::vars().collect(); - let Some(contents) = - codetrial::web::config_from_values(&exported, production).map_err(|error| { - format!("the exported Setup keys cannot be written to a config: {error}") - })? - else { - return Ok(()); - }; - std::fs::create_dir_all(dir) - .and_then(|()| codetrial::create_owner_only(&path, contents.as_bytes())) - .map_err(|error| format!("cannot write {}: {error}", path.display()))?; - println!( - "codetrial: wrote {} from the exported LiveKit and Google keys", - path.display() - ); - Ok(()) -} - -/// The process environment, or nothing for a `--data-dir` launch. -/// -/// A launcher that names the folder is saying the install's configuration is -/// in it, and an environment it was started from is not part of that: a shell -/// that exported `CODETRIAL_DB_PATH` or `NODE_ENV` for a CLI deployment would -/// otherwise move this install's reports, or change its rules, depending on -/// how it was started. Decided here, once, rather than by every launcher -/// keeping its own list of the keys this program reads. -fn environment(options: &CliOptions) -> BTreeMap { - if options.data_dir.is_some() { - return BTreeMap::new(); - } - std::env::vars().collect() -} - -fn load_values(options: &CliOptions) -> Result, String> { - let mut values = environment(options); - let path = primary_config_path(options)?; - codetrial::config::apply_config_file(&mut values, codetrial::config::read_config_file(&path)?); - apply_options(&mut values, options); - if let Some(dir) = &options.data_dir - && nonempty(&values, "SESSION_SECRET").is_none() - { - let secret = data_dir_session_secret(&dir.join(SESSION_SECRET_FILE))?; - values.insert("SESSION_SECRET".to_string(), secret); - } - Ok(values) -} - -/// The key a `--data-dir` install signs session cookies with, made on its first -/// launch and reused, so a relaunch keeps everyone signed in. -/// -/// Not the built-in default: that one is published, and any program on the -/// machine can reach a loopback port to present a cookie signed with it. -/// Owner-only on Unix for the same reason. A file that is not a key this wrote -/// is refused rather than replaced, because replacing it signs everyone out -/// and hides whatever damaged it. -fn data_dir_session_secret(path: &std::path::Path) -> Result { - match std::fs::read_to_string(path) { - Ok(text) => { - let secret = text.trim(); - if secret.len() >= 32 - && secret - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') - { - Ok(secret.to_string()) - } else { - Err(format!( - "{} is not a session key this program wrote. Move it aside to make a new \ - one; everyone will be signed out.", - path.display() - )) - } - } - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - let secret = codetrial::accounts::random_token(32) - .map_err(|error| format!("cannot generate a session key: {error}"))?; - codetrial::create_owner_only(path, secret.as_bytes()) - .map_err(|error| format!("cannot write {}: {error}", path.display()))?; - Ok(secret) - } - Err(error) => Err(format!("cannot read {}: {error}", path.display())), - } -} - -/// The flags, over whatever the environment and the config file said. -/// -/// Split out because a cold start has to build the same map without the file -/// it does not have yet: `serve_setup` decides where to bind and what a -/// submission must survive, and both answers have to be the ones `run_web` -/// will reach a moment later. Spelled twice, this precedence is free to drift. -fn apply_options(values: &mut BTreeMap, options: &CliOptions) { - if let Some(value) = &options.web_addr { - values.insert("CODETRIAL_WEB_ADDR".to_string(), value.clone()); - } - if let Some(value) = &options.web_dir { - values.insert("CODETRIAL_WEB_DIR".to_string(), value.clone()); - } - if let Some(value) = &options.allowed_hosts { - values.insert("CODETRIAL_ALLOWED_HOSTS".to_string(), value.clone()); - } - if let Some(value) = &options.room_prefix { - values.insert("CODETRIAL_ROOM_PREFIX".to_string(), value.clone()); - } - if let Some(value) = options.duration_min { - values.insert("CODETRIAL_DURATION_MIN".to_string(), value.to_string()); - } - - // Over the config file as well: the desktop app is published as its own - // release, so the one the check reads never names its build. - if options.no_update_check { - values.insert("CODETRIAL_UPDATE_CHECK".to_string(), "false".to_string()); - } -} - -fn primary_config_path(options: &CliOptions) -> Result { - // The one place searched, and not one place among several: the working - // directory and the executable's folder are whatever the launcher left them - // as, so a config found there would belong to some other install. - if let Some(dir) = &options.data_dir { - let path = dir.join(CONFIG_FILE_NAME); - if path.is_file() { - return Ok(path); - } - return Err(format!( - "required configuration file is missing: {}; write one with LIVEKIT_URL, \ - LIVEKIT_API_KEY and LIVEKIT_API_SECRET", - path.display() - )); - } - if let Some(path) = &options.config_path { - let path = PathBuf::from(path); - if !path.is_file() { - return Err(format!( - "required configuration file is missing: {}", - path.display() - )); - } - return Ok(path); - } - - // The working directory first, which is a checkout's `config/` and an - // operator's deployment directory, then the folder the executable sits in. - // A release binary is unpacked into a folder of its own and its config is - // written there, so it has to be findable from a shortcut or a terminal - // opened somewhere else, not only from a double-click. - // - // Within each of those two roots, `config/` before the bare filename. The - // bare ones are where a release used to keep its config and where an - // operator may still keep one; they are searched, not written. - for path in [ - PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME), - PathBuf::from(CONFIG_FILE_NAME), - codetrial::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME), - codetrial::exe_dir().join(CONFIG_FILE_NAME), - ] { - if path.is_file() { - return Ok(path); - } - } - - // The keys, not a file to copy. `config/codetrial.env.example` exists in a - // checkout and nowhere else: the release archives hold the executable and - // nothing beside it, so telling someone who just unpacked one to copy it - // sends them looking for a file they were never given. Naming what the file - // must contain is an instruction both audiences can act on. - Err(format!( - "required configuration file is missing: ./{CONFIG_DIR}/{CONFIG_FILE_NAME}, \ - ./{CONFIG_FILE_NAME}, or {CONFIG_DIR}/{CONFIG_FILE_NAME} beside the \ - executable; write one with LIVEKIT_URL, LIVEKIT_API_KEY and \ - LIVEKIT_API_SECRET ({CONFIG_DIR}/{CONFIG_EXAMPLE_NAME} lists the \ - optional keys in a checkout)" - )) -} - -/// The directory of the config file this launch actually read, which is where -/// everything else belonging to the installation lives: the provider files it -/// discovers, and the account database it writes. -/// -/// So `--config /etc/codetrial/prod.env` discovers -/// `/etc/codetrial/codetrial.env.` rather than whatever `config/` happens -/// to sit in the current directory. Both halves of a deployment are launched -/// with the same `--config`, and that is what makes them agree on the pool. -/// -/// The database is written here rather than read, which is the one asymmetry: -/// a `--config` pointing somewhere a process may not write is a startup -/// failure that names the directory, and `CODETRIAL_DB_PATH` is the way out. -fn config_dir(options: &CliOptions) -> PathBuf { - if let Some(dir) = &options.data_dir { - return dir.clone(); - } - - // The search order comes from primary_config_path rather than being written - // out again here. Spelled twice, it was free to drift, and the drift is - // silent: the server keeps reading its own config while the pool quietly - // collapses to the primary project. - // - // A named file still decides, even when it is missing. The operator said - // where the config lives, and load_values is what refuses a path that is - // not there; answering with a different directory would be this function - // second-guessing that. - let path = options - .config_path - .as_deref() - .map(PathBuf::from) - .or_else(|| primary_config_path(options).ok()) - .unwrap_or_else(|| PathBuf::from(CONFIG_DIR).join(CONFIG_FILE_NAME)); - match path.parent() { - // A bare filename has an empty parent, and its siblings are in the - // working directory, not in `config/`. - Some(parent) if !parent.as_os_str().is_empty() => parent.to_path_buf(), - _ => PathBuf::from("."), - } -} - -/// The pool this process routes with. Discovery happens here, once, and never -/// inside `load_from_pairs`: a config loader that reads the current directory -/// gives two processes started from two directories two different pools, and a -/// positional index into those pools then names two different LiveKit projects. -fn extend_pool( - pool: &mut codetrial::config::ProviderPool, - production: bool, - config_dir: &std::path::Path, - order: &str, -) { - let (providers, warnings) = codetrial::config::discover_providers(config_dir, production); - for warning in warnings { - eprintln!("{warning}"); - } - pool.providers.extend(providers); - - // Outside the `discover` guard: an order naming the environment provider is - // still an order, and a deployment with discovery off should not silently - // ignore the one it was given. - for warning in codetrial::config::order_providers(&mut pool.providers, order) { - eprintln!("{warning}"); - } - - // Said once, at startup, in rotation order. Every provider question asked - // of a running server so far has been answerable only by reading the config - // directory and the argv back to itself and guessing which won. - if !pool.providers.is_empty() { - let (summary, warnings) = codetrial::config::pool_summary(&pool.providers); - eprintln!("{summary}"); - for warning in warnings { - eprintln!("WARNING: {warning}"); - } - } -} - -/// Refuses to start on a database that will not migrate, before anything is -/// served. The listener is already bound by this point, so the socket exists -/// and its backlog fills, but nothing accepts from it and the process exits -/// non-zero instead. The router runs the same migration again on the connection -/// it keeps, where the only thing it can do about a failure is answer 503. -fn initialize_accounts(config: &WebServerConfig) -> Result<(), String> { - let Some(login) = codetrial::web::login_config(config) else { - return Ok(()); - }; - - // The router opens the long-lived connection and performs the startup sweep - // on it; this first pass makes migration failure fatal before the server - // starts accepting. - codetrial::web::initialize_account_database(&login.db_path) - .map_err(|error| format!("account database {}: {error}", login.db_path.display()))?; - Ok(()) -} - -/// The dispatcher cap, with anything unusable about it said out loud. -/// -/// One helper for both modes: a warning printed by web and swallowed by serve -/// would be the same misconfiguration reported twice differently. -fn read_max_concurrent(values: &BTreeMap) -> usize { - let mut warnings = Vec::new(); - let max_concurrent = codetrial::config::max_concurrent_interviews(values, &mut warnings); - for warning in &warnings { - eprintln!("{warning}"); - } - max_concurrent -} - -fn is_production(values: &BTreeMap) -> bool { - codetrial::config::is_production(values) -} - -/// Zero unless the operator states how many proxies front this server, so the -/// default never trusts a forwarded client address. -/// Recording without an OAuth app is a server that refuses every interview. -/// -/// The delivery target is the primary verified address GitHub returns, and -/// without the OAuth flow every account is self-declared, so `/api/token` -/// answers 403 to all of them. Said at startup, where an operator can act on -/// it, rather than one candidate at a time. -/// -/// `oauth` is both credentials, not just the id. `login_config` enables OAuth -/// only when it has the pair, so an id with no secret is exactly as unable to -/// verify anyone as no id at all, and would have started a server that refused -/// every interview while looking configured. -/// A deployment that records has to be able to hand the file over. -/// -/// The key is parsed here, before the listener does any work, because the -/// alternative is a server that records interviews and discovers at the first -/// delivery that it can never sign for one. Configuration already checked the -/// shape; this is the credential itself. -fn recording_can_deliver( - recording: Option<&codetrial::config::RecordingConfig>, -) -> Result<(), String> { - let Some(recording) = recording else { - return Ok(()); - }; - codetrial::delivery::GoogleDelivery::new( - &recording.service_account_json, - &recording.gcs_bucket, - &recording.drive_id, - std::sync::Arc::new(|| 0), - ) - .map(|_| ()) - .map_err(|error| format!("CODETRIAL_RECORDING_SERVICE_ACCOUNT_JSON: {error}")) -} - -fn recording_needs_verified_identity(recording: bool, oauth: bool) -> Result<(), String> { - if recording && !oauth { - return Err( - "CODETRIAL_RECORDING_ENABLED=true needs GITHUB_CLIENT_ID and \ - GITHUB_CLIENT_SECRET: a recording is delivered to the verified address \ - GitHub returns, and without the OAuth app every account is self-declared \ - and every interview would be refused" - .to_string(), - ); - } - Ok(()) -} - -/// The configured interview length, which is what a recording's maximum -/// duration has to clear. Read here rather than through `load_from_pairs` -/// because `codetrial web` deliberately runs without a full agent config. -fn interview_duration_min(values: &BTreeMap) -> u32 { - nonempty(values, "CODETRIAL_DURATION_MIN") - .and_then(|value| value.parse().ok()) - .unwrap_or(DEFAULT_DURATION_MIN) -} - -/// `CODETRIAL_ALLOWED_HOSTS`, a comma-separated list of `Host` values. -/// -/// Unset or empty means no check. A value that holds only separators is -/// refused instead: it was meant to name a host, and reading it as no list -/// would answer every `Host` the person believes is refused. -fn allowed_hosts(values: &BTreeMap) -> Result, String> { - let hosts: Vec = value_or(values, "CODETRIAL_ALLOWED_HOSTS", "") - .split(',') - .map(str::trim) - .filter(|host| !host.is_empty()) - .map(str::to_string) - .collect(); - if hosts.is_empty() && nonempty(values, "CODETRIAL_ALLOWED_HOSTS").is_some() { - return Err("CODETRIAL_ALLOWED_HOSTS names no Host".to_string()); - } - Ok(hosts) -} - -fn trusted_proxy_hops(values: &BTreeMap) -> u32 { - nonempty(values, "CODETRIAL_TRUSTED_PROXY_HOPS") - .and_then(|value| value.parse().ok()) - .unwrap_or(0) -} - -/// What a `codetrial web` start without `NODE_ENV=production` is allowing, in -/// the operator's words rather than the code's. -/// -/// Every guard this reports on is keyed on `NODE_ENV`, so the one mistake none -/// of them can catch is the deployment that sets every credential correctly and -/// never sets `NODE_ENV` at all. The refusal in `run_web` fires only once the -/// operator has already said this is production; a server that was meant to be -/// production and does not say so gets the local defaults and no complaint. -/// This is the only signal it ever gets. -/// -/// `config::is_production` matches the exact string, so a `NODE_ENV=Production` -/// typo lands here too, which is the other way a deployment ends up local -/// without meaning to. -fn relaxed_for_local_use(values: &BTreeMap) -> Vec { - if is_production(values) { - return Vec::new(); - } - - let mut warnings = vec![ - "NODE_ENV is not production: the page's CSP allows loopback origins and a \ - plaintext ws:// LiveKit URL is accepted. Set NODE_ENV=production to deploy." - .to_string(), - ]; - if nonempty(values, "SESSION_SECRET").is_none() { - warnings.push(format!( - "SESSION_SECRET is unset: signing session cookies with the built-in \ - {DEFAULT_SESSION_SECRET:?}, which is published in this repository and lets \ - anyone forge a session." - )); - } - warnings -} - -/// Why the built-in `SESSION_SECRET` may not sign cookies on `bound`, or `None` -/// where it may. -/// -/// Split out for the reason `relaxed_for_local_use` is: the caller can only -/// return the string, and a refusal reachable solely by starting a process that -/// otherwise never exits is a branch no test can assert on without hanging. -/// -/// `NODE_ENV` is a claim; the address this process bound is a fact. A server -/// other machines can reach is not the local run the published default exists -/// for, whether or not anyone remembered to say so, which is the mistake the -/// warning in `relaxed_for_local_use` can name but not prevent. -/// -/// Unspecified addresses (`0.0.0.0`, `::`) are not loopback and must not be -/// treated as such: binding every interface is how a container serves the -/// world. -/// -/// `to_canonical` before the test, because `Ipv6Addr::is_loopback` is true of -/// `::1` and of nothing else: a dual-stack socket that landed on -/// `::ffff:127.0.0.1`, which is a name for the same interface, would otherwise -/// be refused an address only this machine can reach. -/// How a listener on `bound` can be reached from somewhere other than this -/// machine, or `None` where it cannot. -/// -/// Shared, because two guards ask it and a third answer added to one of them -/// would drift silently -- in the direction that leaves the page taking -/// credentials open. What each guard says about it is not shared: the remedies -/// differ, an address is passed on the command line and a declared proxy is a -/// variable in the environment. -#[derive(Clone, Copy)] -enum Reachable { - /// The bind address is not loopback. - Address, - /// `CODETRIAL_TRUSTED_PROXY_HOPS` is the operator saying requests arrive - /// through something in front, which is the same admission: a server on - /// loopback behind nginx or a tunnel is as public as one on `0.0.0.0`. - DeclaredProxy, -} - -fn reachable_from_elsewhere( - values: &BTreeMap, - bound: std::net::SocketAddr, -) -> Option { - if !bound.ip().to_canonical().is_loopback() { - Some(Reachable::Address) - } else if trusted_proxy_hops(values) > 0 { - Some(Reachable::DeclaredProxy) - } else { - None - } -} - -fn task_mode_allowed(reachable: Option, fixed_room: bool, recording: bool) -> bool { - task_mode_refusal(reachable, fixed_room, recording).is_none() -} - -/// Why task mode may not run on this server, in the words of what to change, or -/// `None` where it may. -fn task_mode_refusal( - reachable: Option, - fixed_room: bool, - recording: bool, -) -> Option<&'static str> { - match (reachable, fixed_room, recording) { - (Some(Reachable::Address), _, _) => Some( - "it is bound to an address other than loopback; start it with --web-addr 127.0.0.1:3000", - ), - (Some(Reachable::DeclaredProxy), _, _) => { - Some("CODETRIAL_TRUSTED_PROXY_HOPS says requests arrive through a proxy; unset it") - } - (None, true, _) => Some("INTERVIEW_ROOM_NAME shares one fixed room; unset it"), - (None, false, true) => { - Some("recording is on; task mode does not run on a recording server") - } - (None, false, false) => None, - } -} - -/// Task mode is on when `--task-mode` asked for it and the server is one it may -/// run on; either alone is not enough. -fn task_mode_enabled( - requested: bool, - reachable: Option, - fixed_room: bool, - recording: bool, -) -> bool { - requested && task_mode_allowed(reachable, fixed_room, recording) -} - -fn published_secret_refusal( - values: &BTreeMap, - bound: std::net::SocketAddr, -) -> Option { - // What the cookie signer will actually use, not whether the key was - // mentioned. `value_or` trims, so `SESSION_SECRET=" codetrial-local-session - // "` is the published key spelled with whitespace rather than a secret of - // the operator's own, and a guard that only asked whether the variable was - // set would wave it through. - if value_or(values, "SESSION_SECRET", DEFAULT_SESSION_SECRET) != DEFAULT_SESSION_SECRET { - return None; - } - - let reason = match reachable_from_elsewhere(values, bound)? { - Reachable::Address => format!("bind {bound}"), - Reachable::DeclaredProxy => "serve from behind a declared proxy".to_string(), - }; - - Some(format!( - "SESSION_SECRET must be set to {reason}: {DEFAULT_SESSION_SECRET:?} is published in \ - this repository, so every session cookie this server signs would be forgeable by \ - anyone who can reach it" - )) -} - -fn compiler_explorer_enabled(values: &BTreeMap) -> bool { - codetrial::config::compiler_explorer_enabled( - values - .get("CODETRIAL_COMPILER_EXPLORER_ENABLED") - .map(String::as_str), - ) -} - -fn guest_mode_recording_conflict(guest_mode_enabled: bool, recording_enabled: bool) -> bool { - guest_mode_enabled && recording_enabled -} - -fn guest_mode_enabled(values: &BTreeMap) -> bool { - codetrial::config::guest_mode_enabled( - values - .get("CODETRIAL_GUEST_MODE_ENABLED") - .map(String::as_str), - ) -} - -fn nonempty(values: &BTreeMap, key: &str) -> Option { - values - .get(key) - .map(|value| value.trim()) - .filter(|value| !value.is_empty()) - .map(str::to_string) -} - -fn value_or(values: &BTreeMap, key: &str, default: &str) -> String { - nonempty(values, key).unwrap_or_else(|| default.to_string()) -} - -/// One place decides where accounts live, because `web` and `serve` both open -/// the same database and disagreeing about it would split a person's reports. -/// -/// Beside the config file rather than in the working directory: the database -/// has to be the same file on the next launch, and the directory a process was -/// started from is not. It used to be a bare filename, which meant a launch -/// from a shortcut or from another terminal found the config it wrote and then -/// opened an empty database next to wherever it had been started. -fn account_db_path(values: &BTreeMap, options: &CliOptions) -> PathBuf { - match nonempty(values, "CODETRIAL_DB_PATH") { - Some(path) => PathBuf::from(path), - None => config_dir(options).join(DEFAULT_ACCOUNT_DB_PATH), - } -} - -/// Where a cold start writes the config it just took, which is the same -/// `config/` the search above looks in first for the root this is installed as. -/// -/// A checkout is told apart by the template it ships, not by `config/` merely -/// existing: a released binary run from a directory that happens to hold one -/// would otherwise write the credentials there and lose them the moment it was -/// next started from somewhere else. -fn cold_start_config_path(options: &CliOptions) -> PathBuf { - if let Some(dir) = &options.data_dir { - return dir.join(CONFIG_FILE_NAME); - } - let in_checkout = PathBuf::from(CONFIG_DIR); - if in_checkout.join(CONFIG_EXAMPLE_NAME).is_file() { - return in_checkout.join(CONFIG_FILE_NAME); - } - codetrial::exe_dir().join(CONFIG_DIR).join(CONFIG_FILE_NAME) -} - #[cfg(test)] // `bin.rs`, not `main.rs`: cargo reads `tests//main.rs` as an // integration-test target of its own, which would compile this file a second diff --git a/tests/cli.rs b/tests/cli.rs index cb7ef30f2..71ae58915 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2175,8 +2175,9 @@ fn binary_web_serves_in_production_with_a_session_secret() { /// startup refusal read from the same values, for real. #[test] fn binary_web_reads_deployment_keys_from_the_config_file() { - let source = std::fs::read_to_string("src/main.rs") - .unwrap() + let source = ["src/main.rs", "src/launch.rs"] + .map(|path| std::fs::read_to_string(path).unwrap()) + .concat() .replace("\r\n", "\n"); // Ends at the closing brace in column zero rather than at whatever item @@ -2193,21 +2194,14 @@ fn binary_web_reads_deployment_keys_from_the_config_file() { .to_string() }; - // Both halves of the startup path. The configuration is assembled in - // `web_server_config` and the socket is bound in `run_web`, and the - // property is about the pair: what this deployment runs on comes out of the - // config file the operator wrote, never out of the ambient environment. - // Splitting the function moved the first assertion's subject and would have - // left this reading an empty haystack had it kept naming only one of them. - let serve = body("run_web"); - let configure = body("web_server_config"); - - assert!( - configure.contains("trusted_proxy_hops(values)"), - "{configure}" - ); - assert!(!serve.contains("std::env::var"), "{serve}"); - assert!(!configure.contains("std::env::var"), "{configure}"); + // Every phase of the startup path: assembled in `web_server_config`, + // `prepare` and `finish` (src/launch.rs), bound in `run_web` (src/main.rs) + // and served by `serve`, all from the config file, never the environment. + assert!(body("web_server_config").contains("trusted_proxy_hops(values)")); + let phases = "run_web prepare finish local_agent_config web_server_config serve"; + for name in phases.split(' ') { + assert!(!body(name).contains("std::env::var"), "{name}"); + } } /// The log is the one place the error text sends a reader with no console, so diff --git a/tests/unit/bin.rs b/tests/unit/bin.rs index ea9a41808..59e440b0d 100644 --- a/tests/unit/bin.rs +++ b/tests/unit/bin.rs @@ -847,12 +847,12 @@ fn task_mode_runs_only_on_a_private_server_that_neither_records_nor_shares_a_roo #[test] fn peer_is_offered_once_oauth_and_observer_keys_exist() { let oauth = |settings: &[(&str, &str)]| { - super::main_peer::peer_services(&values(settings), true).unwrap() + codetrial::launch::peer_services(&values(settings), true).unwrap() }; assert!(oauth(&[]).is_none()); assert!(oauth(&[("GOOGLE_API_KEY", "test-key")]).is_some()); assert!( - super::main_peer::peer_services(&values(&[("GOOGLE_API_KEY", "test-key")]), false) + codetrial::launch::peer_services(&values(&[("GOOGLE_API_KEY", "test-key")]), false) .unwrap() .is_none() ); @@ -899,7 +899,7 @@ fn peer_observer_rejects_sessions_longer_than_its_two_hour_capture() { ("CODETRIAL_PEER_HARD_MAX_MINUTES", "150"), ]); assert!( - super::main_peer::peer_services(&long, true) + codetrial::launch::peer_services(&long, true) .unwrap_err() .contains("at most 120") ); @@ -907,5 +907,111 @@ fn peer_observer_rejects_sessions_longer_than_its_two_hour_capture() { ("GOOGLE_API_KEY", "test-key"), ("CODETRIAL_PEER_HARD_MAX_MINUTES", "120"), ]); - assert!(super::main_peer::peer_services(&limit, true).is_ok()); + assert!(codetrial::launch::peer_services(&limit, true).is_ok()); +} + +/// A folder with an empty config file in it and values pointing the account +/// database into it, so `prepare` touches nothing outside the test. +fn launch_fixture(label: &str) -> (std::path::PathBuf, super::CliOptions) { + let dir = std::env::temp_dir().join(format!( + "codetrial-bin-launch-{label}-{}-{}", + std::process::id(), + codetrial::current_epoch_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + let config = dir.join("codetrial.env.local"); + std::fs::write(&config, "").unwrap(); + let options = super::CliOptions { + config_path: Some(config.display().to_string()), + ..Default::default() + }; + (dir, options) +} + +fn livekit_values(dir: &std::path::Path) -> super::BTreeMap { + let db = dir.join("accounts.db").display().to_string(); + values(&[ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("CODETRIAL_DB_PATH", &db), + ]) +} + +#[test] +fn prepare_builds_the_server_config_and_opens_the_account_database() { + let (dir, options) = launch_fixture("prepare"); + let prepared = super::prepare(&livekit_values(&dir), &options).unwrap(); + assert_eq!(prepared.config.pool.providers.len(), 1); + assert!(prepared.max_concurrent > 0); + assert!(dir.join("accounts.db").is_file(), "the database is opened"); + assert!( + prepared.config.tasks.is_none(), + "task mode waits for the bind" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn prepare_refuses_a_launch_without_livekit_credentials() { + let (dir, options) = launch_fixture("prepare-missing"); + let error = super::prepare(&values(&[]), &options).err().unwrap(); + assert!( + error.contains("missing required LiveKit credentials"), + "{error}" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn finish_refuses_the_published_secret_on_a_public_address() { + let (dir, options) = launch_fixture("finish-public"); + let values = livekit_values(&dir); + let prepared = super::prepare(&values, &options).unwrap(); + let refusal = super::finish( + prepared.config, + &values, + &options, + addr("203.0.113.9:3000"), + super::LaunchedBy::Operator, + ) + .err() + .unwrap(); + assert!(!refusal.is_empty()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn finish_offers_a_launchers_child_no_setup_page_and_no_task_mode() { + let (dir, options) = launch_fixture("finish-child"); + let mut values = livekit_values(&dir); + values.insert( + "SESSION_SECRET".into(), + "a-real-secret-for-this-test-only-0123".into(), + ); + let prepared = super::prepare(&values, &options).unwrap(); + let (config, setup) = super::finish( + prepared.config, + &values, + &options, + addr("127.0.0.1:3000"), + super::LaunchedBy::Parent, + ) + .unwrap(); + assert!(setup.is_none(), "a child has no terminal anyone reads"); + assert!(config.tasks.is_none(), "task mode was not asked for"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_local_interviewer_needs_a_gemini_key_and_a_valid_config() { + let (dir, _) = launch_fixture("agent"); + let mut values = livekit_values(&dir); + assert!(super::local_agent_config(values.clone()).unwrap().is_none()); + values.insert("GOOGLE_API_KEY".into(), "google-key".into()); + assert!(super::local_agent_config(values.clone()).unwrap().is_some()); + values.insert("GEMINI_CONTEXT_TRIGGER_TOKENS".into(), "1000".into()); + let error = super::local_agent_config(values).err().unwrap(); + assert!(error.contains("GEMINI_CONTEXT"), "{error}"); + let _ = std::fs::remove_dir_all(&dir); } From d03a5b09193acceef20579b994189af466049f4c Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Wed, 4 Nov 2026 21:52:02 +0800 Subject: [PATCH 6/7] Host the interview in a fixture that records it A paid long-interview run could only be judged from what the page saw. Add a test-only acquisition capture that records the tool answers each batch executed and their send outcomes before any asynchronous write, and an in-process serving fixture that hosts the real web service and interviewer through the startup and serving functions codetrial web uses. A test-only arming observer takes the room from the dispatcher's accepted assignment and the session from the first bind, and fails for good on anything else. A stopping server now closes admission and may wait, within a bound, for admitted rooms and reports to finish, so the fixture's report is not cut off; a start refused then is told the server is stopping. codetrial web passes no bound and behaves as before. The fixture binds loopback only, clears an earlier run's files first and writes its report even when the server ends badly. Change-Id: If54cd807ffef988484c6376d47139bc8c49778c6 --- docs/provider-cost-and-degradation.md | 6 +- src/dispatch.rs | 70 +++ src/launch.rs | 191 ++++-- src/livekit/latency.rs | 13 + src/livekit/probe.rs | 5 + src/livekit/probe/acquisition.rs | 574 +++++++++++++++++ src/livekit/probe/acquisition/owner.rs | 129 ++++ src/livekit/session.rs | 151 +---- src/livekit/session/observations.rs | 15 +- src/livekit/session/tool_calls.rs | 177 ++++++ src/main.rs | 6 +- src/web/mod.rs | 12 +- src/web/token.rs | 3 + tests/cli.rs | 8 +- tests/unit/bin.rs | 136 ++++ tests/unit/dispatch.rs | 80 ++- tests/unit/launch.rs | 98 +++ tests/unit/livekit/probe/acquisition.rs | 594 ++++++++++++++++++ tests/unit/livekit/probe/serving.rs | 583 +++++++++++++++++ tests/unit/livekit/recovery.rs | 2 + .../recovery_parts/acquisition_tool_sends.rs | 422 +++++++++++++ tests/unit/livekit/session.rs | 58 +- tests/unit/livekit/session/observations.rs | 116 ++++ .../livekit/session_parts/tool_execution.rs | 61 ++ 24 files changed, 3238 insertions(+), 272 deletions(-) create mode 100644 src/livekit/probe/acquisition.rs create mode 100644 src/livekit/probe/acquisition/owner.rs create mode 100644 src/livekit/session/tool_calls.rs create mode 100644 tests/unit/launch.rs create mode 100644 tests/unit/livekit/probe/acquisition.rs create mode 100644 tests/unit/livekit/probe/serving.rs create mode 100644 tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs create mode 100644 tests/unit/livekit/session_parts/tool_execution.rs diff --git a/docs/provider-cost-and-degradation.md b/docs/provider-cost-and-degradation.md index a1652b4db..b79433927 100644 --- a/docs/provider-cost-and-degradation.md +++ b/docs/provider-cost-and-degradation.md @@ -512,7 +512,11 @@ completed interviews awaiting recovery; admission still refuses excess starts rather than exceeding this bound. A fixed local room refuses a new start while its existing agent is finalizing or awaiting report recovery, as `dispatch_refused ... reason=finalizing` rather than `at_capacity`; it cannot -reuse that agent as an interviewer for another candidate. Provider projects are +reuse that agent as an interviewer for another candidate. A server that has +begun to stop refuses every start that would admit a new room as +`reason=closed`, and the candidate is told it is shutting down; admitted rooms +keep running, and a start for a room still being assessed joins it; a live +room otherwise gets the refusal it had before the stop. Provider projects are rotated and their LiveKit connection-minute quota is refreshed in the background, and known-exhausted projects are skipped. diff --git a/src/dispatch.rs b/src/dispatch.rs index 1bcbed7fc..9428bdc4e 100644 --- a/src/dispatch.rs +++ b/src/dispatch.rs @@ -21,6 +21,9 @@ pub struct DispatchState { /// Whose each reserved room is and whether they have arrived, so a page /// that gives up its start can end the wait it began and nobody else's. starts: HashMap, + /// Set once by `close_admission`: no new room is admitted after it, under + /// the same lock that decides every admission and `is_idle`. + closed: bool, } struct Start { @@ -65,6 +68,55 @@ pub struct LocalDispatcher { pub max_concurrent: usize, } +/// Test-only observer of the assignment boundary: told each room the dispatcher +/// admits as new, after admission and before its task is spawned, so a fixture +/// can name the room it will see without choosing the name. It changes no +/// dispatch behavior. +#[cfg(test)] +pub(crate) type AssignmentHook = Arc; +#[cfg(test)] +static ASSIGNED: Mutex> = Mutex::new(None); + +#[cfg(test)] +pub(crate) fn set_assignment_hook(hook: Option) { + *ASSIGNED.lock().unwrap_or_else(|error| error.into_inner()) = hook; +} + +/// Whether `hook` is the one installed. Compares the allocations, not the +/// vtables. +#[cfg(test)] +pub(crate) fn assignment_hook_is(hook: &AssignmentHook) -> bool { + ASSIGNED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .as_ref() + .is_some_and(|now| std::ptr::addr_eq(Arc::as_ptr(now), Arc::as_ptr(hook))) +} + +/// Removes `hook` only if it is still the installed one, so a teardown never +/// clears a successor's. +#[cfg(test)] +pub(crate) fn clear_assignment_hook(hook: &AssignmentHook) { + let mut slot = ASSIGNED.lock().unwrap_or_else(|error| error.into_inner()); + if slot + .as_ref() + .is_some_and(|now| std::ptr::addr_eq(Arc::as_ptr(now), Arc::as_ptr(hook))) + { + *slot = None; + } +} + +#[cfg(test)] +pub(crate) fn notify_assigned(room: &str) { + let hook = ASSIGNED + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if let Some(hook) = hook { + hook(room); + } +} + impl RoomDispatcher for LocalDispatcher { /// A room stays in `live` from admission until its task ends, so one whose /// report is still being written counts too. @@ -77,6 +129,13 @@ impl RoomDispatcher for LocalDispatcher { true } + fn close_admission(&self) { + self.state + .lock() + .unwrap_or_else(|error| error.into_inner()) + .closed = true; + } + fn candidate_video(&self) -> Option { Some(self.config.gemini_candidate_video_enabled) } @@ -162,9 +221,15 @@ impl LocalDispatcher { eprintln!("codetrial dispatch_refused room={room_name} reason=finalizing"); return Err(DispatchRefusal::Finalizing); } + Reservation::Closed => { + eprintln!("codetrial dispatch_refused room={room_name} reason=closed"); + return Err(DispatchRefusal::Stopping); + } Reservation::New(slot) => slot, }; let config = agent_config_for(&self.config, provider); + #[cfg(test)] + notify_assigned(&slot.room_name); // Spawned, not awaited: this runs on the request path, and the task // outlives the response by the length of the interview or attempt. @@ -262,6 +327,7 @@ enum Reservation { Full { limit: usize, guest_cap: bool }, GuestRoomInUse, Finalizing, + Closed, } impl LocalDispatcher { @@ -300,6 +366,10 @@ impl LocalDispatcher { }; } + if state.closed { + return Reservation::Closed; + } + if state.live.len() >= self.max_concurrent { return Reservation::Full { limit: self.max_concurrent, diff --git a/src/launch.rs b/src/launch.rs index 17475bb3d..f9f646e56 100644 --- a/src/launch.rs +++ b/src/launch.rs @@ -465,16 +465,27 @@ pub fn extend_pool( eprintln!("{warning}"); } - // Said once, at startup, in rotation order. Every provider question asked - // of a running server so far has been answerable only by reading the config - // directory and the argv back to itself and guessing which won. - if !pool.providers.is_empty() { - let (summary, warnings) = crate::config::pool_summary(&pool.providers); - eprintln!("{summary}"); - for warning in warnings { - eprintln!("WARNING: {warning}"); - } + for line in pool_report(&pool.providers) { + eprintln!("{line}"); + } +} + +/// What startup says about the pool, in rotation order, and nothing for an +/// empty one. Every provider question asked of a running server so far has been +/// answerable only by reading the config directory and the argv back to itself +/// and guessing which won. +fn pool_report(providers: &[crate::config::Provider]) -> Vec { + if providers.is_empty() { + return Vec::new(); } + let (summary, warnings) = crate::config::pool_summary(providers); + std::iter::once(summary) + .chain( + warnings + .into_iter() + .map(|warning| format!("WARNING: {warning}")), + ) + .collect() } /// Refuses to start on a database that will not migrate, before anything is @@ -939,14 +950,25 @@ pub fn local_agent_config( Ok(agent_config) } +/// How long a stopping server may wait on what it still holds. +#[derive(Clone, Copy, Debug, Default)] +pub struct Bounds { + /// Open connections holding the shutdown up once it has begun (`None` + /// waits for them): a client holding a half-sent request would otherwise + /// keep a fixture from ever reporting. + pub drain: Option, + /// Admitted rooms and the reports they are still writing, after the last + /// connection has gone and nothing new can be admitted (`None` does not + /// wait, and ends them when the runtime drops). + pub settle: Option, +} + /// Serves the web service, and the local interviewer when there is an agent /// config, until `shutdown` resolves and the connections in flight have -/// finished. `codetrial web` passes a future that never resolves; a fixture -/// passes its stop signal. `drain` bounds how long open connections may hold -/// the shutdown up once it has begun (`None` waits for them): a client holding -/// a half-sent request would otherwise keep a fixture from ever reporting. The -/// listener must already be non-blocking. The runtime is dropped before this -/// returns, which ends any room task still running. +/// finished. `codetrial web` passes a future that never resolves and no +/// bounds; a fixture passes its stop signal and bounds. The listener must +/// already be non-blocking. The runtime is dropped before this returns, which +/// ends any room task still running. pub fn serve( listener: std::net::TcpListener, config: WebServerConfig, @@ -954,42 +976,109 @@ pub fn serve( agent_config: Option, max_concurrent: usize, shutdown: impl std::future::Future + Send + 'static, - drain: Option, + bounds: Bounds, ) -> Result<(), String> { let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); - runtime - .block_on(async { - let listener = tokio::net::TcpListener::from_std(listener)?; - let dispatcher = agent_config.map(|config| { - std::sync::Arc::new(crate::dispatch::LocalDispatcher { - config, - runtime: tokio::runtime::Handle::current(), - state: std::sync::Arc::default(), - max_concurrent, - }) as std::sync::Arc - }); - let (begun, shutting_down) = tokio::sync::oneshot::channel::<()>(); - let served = axum::serve( - listener, - crate::web::web_service_with_setup(config, dispatcher, setup), - ) - .with_graceful_shutdown(async move { - shutdown.await; - let _ = begun.send(()); - }); - let overdue = async { - match (shutting_down.await, drain) { - (Ok(()), Some(bound)) => tokio::time::sleep(bound).await, - _ => std::future::pending().await, - } - }; - tokio::select! { - result = served => result, - () = overdue => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "connections were still open when the shutdown bound passed", - )), - } - }) - .map_err(|error| format!("web server failed: {error}")) + let dispatcher = agent_config.map(|config| { + std::sync::Arc::new(crate::dispatch::LocalDispatcher { + config, + runtime: runtime.handle().clone(), + state: std::sync::Arc::default(), + max_concurrent, + }) as std::sync::Arc + }); + runtime.block_on(run(listener, config, setup, dispatcher, shutdown, bounds)) } + +/// `serve` with the dispatcher supplied, for a caller that has its own. +pub fn serve_with( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + dispatcher: Option>, + shutdown: impl std::future::Future + Send + 'static, + bounds: Bounds, +) -> Result<(), String> { + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + runtime.block_on(run(listener, config, setup, dispatcher, shutdown, bounds)) +} + +async fn run( + listener: std::net::TcpListener, + config: WebServerConfig, + setup: Option, + dispatcher: Option>, + shutdown: impl std::future::Future + Send + 'static, + bounds: Bounds, +) -> Result<(), String> { + let listener = tokio::net::TcpListener::from_std(listener) + .map_err(|error| format!("web server failed: {error}"))?; + let watched = dispatcher.clone(); + let closing = dispatcher.clone(); + let (begun, shutting_down) = tokio::sync::oneshot::channel::<()>(); + let served = axum::serve( + listener, + crate::web::web_service_with_setup(config, dispatcher, setup), + ) + .with_graceful_shutdown(async move { + shutdown.await; + + // Closing the listener does not end requests already in flight, and one + // of them could still admit a room while the settle wait below is + // looking at an idle dispatcher. + if let Some(dispatcher) = closing { + dispatcher.close_admission(); + } + let _ = begun.send(()); + }); + let overdue = async { + match (shutting_down.await, bounds.drain) { + (Ok(()), Some(bound)) => tokio::time::sleep(bound).await, + _ => std::future::pending().await, + } + }; + let served = tokio::select! { + result = served => result, + () = overdue => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "connections were still open when the shutdown bound passed", + )), + } + .map_err(|error| format!("web server failed: {error}")); + + // Nothing new is admitted now. Whatever was is allowed to finish, report + // included, before the runtime drops and ends it. + match (bounds.settle, watched) { + (Some(limit), Some(dispatcher)) if !settled(&*dispatcher, limit).await => { + Err(unfinished(served)) + } + _ => served, + } +} + +/// Whether the dispatcher went idle within `limit`. +async fn settled(dispatcher: &dyn crate::web::RoomDispatcher, limit: std::time::Duration) -> bool { + let began = tokio::time::Instant::now(); + while !dispatcher.is_idle() { + if began.elapsed() >= limit { + return false; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + true +} + +/// The error of a stop that found rooms or reports still running at the settle +/// bound: that, and the shutdown's own error as well when it had one, since a +/// drain timeout must not hide that evidence was cut off. +fn unfinished(served: Result<(), String>) -> String { + const UNFINISHED: &str = "rooms or reports were still running when the settle bound passed"; + match served { + Ok(()) => UNFINISHED.into(), + Err(error) => format!("{error}; {UNFINISHED}"), + } +} + +#[cfg(test)] +#[path = "../tests/unit/launch.rs"] +mod tests; diff --git a/src/livekit/latency.rs b/src/livekit/latency.rs index 3178181be..23082b64f 100644 --- a/src/livekit/latency.rs +++ b/src/livekit/latency.rs @@ -348,6 +348,8 @@ impl ReplyLatency { self.session = session; self.socket = socket; self.silence_window_ms = silence_window_ms; + #[cfg(test)] + super::probe::acquisition::bound(room, session); } /// A microphone frame was dequeued at `now`. @@ -387,6 +389,17 @@ impl ReplyLatency { self.anchor.map(|anchor| offset_ms(now, anchor)) } + #[cfg(test)] + pub(super) fn acquisition_identity(&self) -> super::probe::acquisition::IdentityView<'_> { + super::probe::acquisition::IdentityView { + room: &self.room, + session: self.session, + socket: self.socket, + trace_turn: self.trace.as_ref().map(|trace| trace.turn), + clock: self.observation_clock(), + } + } + /// A fresh wall/monotonic pair for joining local harness observations. /// Sampling delay and whole-millisecond rounding are uncertainty, not /// drift. diff --git a/src/livekit/probe.rs b/src/livekit/probe.rs index 2efb7018d..5bcb15568 100644 --- a/src/livekit/probe.rs +++ b/src/livekit/probe.rs @@ -20,6 +20,11 @@ //! frame that arrives next, so the hold starts within about one frame (10 ms) //! of being asked for. +pub(super) mod acquisition; + +#[path = "../../tests/unit/livekit/probe/serving.rs"] +mod serving; + use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::Instant; diff --git a/src/livekit/probe/acquisition.rs b/src/livekit/probe/acquisition.rs new file mode 100644 index 000000000..206289ba4 --- /dev/null +++ b/src/livekit/probe/acquisition.rs @@ -0,0 +1,574 @@ +//! Bounded private execution evidence for authored acquisition fixtures. + +pub(in crate::livekit) mod owner; + +use std::io::{self, Write}; +use std::sync::{Arc, Mutex, PoisonError}; + +use serde::ser::{Serialize, SerializeSeq, Serializer}; +use serde_json::Value; + +use crate::gemini::GeminiFunctionCall; +use crate::livekit::latency::ReplyLatency; + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Identity { + pub(in crate::livekit) room: String, + pub(in crate::livekit) session: u64, + pub(in crate::livekit) socket: u64, + pub(in crate::livekit) trace_turn: Option, + pub(in crate::livekit) clock: Value, +} + +#[derive(serde::Serialize)] +pub(in crate::livekit) struct IdentityView<'a> { + pub(in crate::livekit) room: &'a str, + pub(in crate::livekit) session: u64, + pub(in crate::livekit) socket: u64, + pub(in crate::livekit) trace_turn: Option, + pub(in crate::livekit) clock: Value, +} + +#[derive(Clone, Copy)] +pub(in crate::livekit) struct Limits { + pub(in crate::livekit) batches: usize, + pub(in crate::livekit) calls: usize, + /// Serialized JSON bytes of the retained batch array, including + /// punctuation. + pub(in crate::livekit) bytes: usize, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Call { + pub(in crate::livekit) id: String, + pub(in crate::livekit) name: String, + pub(in crate::livekit) result: Value, +} + +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Batch { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) sequence: usize, + pub(in crate::livekit) calls: Vec, + /// None means no local send completion was observed, not a failed delivery. + pub(in crate::livekit) local_send_completed: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(in crate::livekit) enum Failure { + // Room/session admission or an opaque ticket's batch association differs. + Scope, + Limit, + // Missing or reused provider IDs cannot be correlated safely. + AmbiguousId, +} + +#[derive(Clone, Debug, PartialEq)] +pub(in crate::livekit) struct Snapshot { + pub(in crate::livekit) batches: Vec, + pub(in crate::livekit) failure: Option, + pub(in crate::livekit) retained_calls: usize, + pub(in crate::livekit) retained_bytes: usize, +} + +pub(in crate::livekit) struct Capture { + room: String, + session: u64, + limits: Limits, + inner: Mutex, +} + +impl Capture { + pub(in crate::livekit) fn new( + room: &str, + session: u64, + limits: Limits, + ) -> Result, &'static str> { + if room.is_empty() || session == 0 { + return Err("acquisition requires an assigned room and nonzero session"); + } + if limits.batches == 0 || limits.calls == 0 || limits.bytes < 2 { + return Err("acquisition requires positive retention limits"); + } + + // Bound the admitted scope before retaining its room string. Full batch + // accounting still happens at record time, including its actual clock. + let scope = IdentityView { + room, + session, + socket: 0, + trace_turn: None, + clock: Value::Null, + }; + let mut counter = BoundedBytes { + remaining: limits.bytes - 2, + written: 0, + }; + if serde_json::to_writer(&mut counter, &scope).is_err() { + return Err("acquisition scope metadata exceeds byte limit"); + } + Ok(Arc::new(Self { + room: room.into(), + session, + limits, + inner: Mutex::new(Snapshot { + batches: Vec::new(), + failure: None, + retained_calls: 0, + retained_bytes: 2, + }), + })) + } + + pub(in crate::livekit) fn snapshot(&self) -> Snapshot { + self.inner + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn record( + &self, + identity: IdentityView<'_>, + answers: &[(GeminiFunctionCall, Value)], + ) -> Option { + if answers.is_empty() { + return None; + } + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return None; + } + let failure = if identity.room != self.room || identity.session != self.session { + Some(Failure::Scope) + } else if snapshot.batches.len() >= self.limits.batches + || answers.len() > self.limits.calls.saturating_sub(snapshot.retained_calls) + { + Some(Failure::Limit) + } else if answers.iter().enumerate().any(|(index, (call, _))| { + call.id.is_empty() + || answers[..index] + .iter() + .any(|(earlier, _)| earlier.id == call.id) + || snapshot + .batches + .iter() + .any(|batch| batch.calls.iter().any(|old| old.id == call.id)) + }) { + Some(Failure::AmbiguousId) + } else { + None + }; + if let Some(failure) = failure { + snapshot.failure = Some(failure); + return None; + } + let sequence = snapshot.batches.len(); + let comma = usize::from(sequence != 0); + let remaining = self.limits.bytes.saturating_sub(snapshot.retained_bytes); + let mut counter = BoundedBytes { + remaining: remaining.saturating_sub(comma), + written: 0, + }; + let view = BatchView { + identity: &identity, + sequence, + calls: CallsView(answers), + local_send_completed: None, + }; + if remaining < comma || serde_json::to_writer(&mut counter, &view).is_err() { + snapshot.failure = Some(Failure::Limit); + return None; + } + let batch = Batch { + identity: Identity { + room: identity.room.into(), + session: identity.session, + socket: identity.socket, + trace_turn: identity.trace_turn, + clock: identity.clock, + }, + sequence, + calls: answers + .iter() + .map(|(call, result)| Call { + id: call.id.clone(), + name: call.name.clone(), + result: result.clone(), + }) + .collect(), + local_send_completed: None, + }; + snapshot.retained_bytes += counter.written + comma; + snapshot.retained_calls += answers.len(); + snapshot.batches.push(batch); + Some(sequence) + } +} + +/// Admission fixes the batch's identity permanently: batches only append. +/// An original capture and its sequence associate completion without copying +/// another clock object or looking up whichever capture is installed later. +pub(in crate::livekit) struct Ticket { + capture: Arc, + sequence: usize, +} + +impl Ticket { + pub(in crate::livekit) fn complete(self, sent: bool) { + let mut snapshot = self + .capture + .inner + .lock() + .unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if snapshot + .batches + .get(self.sequence) + .is_none_or(|batch| batch.local_send_completed.is_some()) + { + snapshot.failure = Some(Failure::Scope); + return; + } + let extra = usize::from(!sent); + if extra + > self + .capture + .limits + .bytes + .saturating_sub(snapshot.retained_bytes) + { + snapshot.failure = Some(Failure::Limit); + return; + } + // null and true have equal JSON width; false needs one more byte. + snapshot.batches[self.sequence].local_send_completed = Some(sent); + snapshot.retained_bytes += extra; + } +} + +#[derive(serde::Serialize)] +struct BatchView<'a> { + identity: &'a IdentityView<'a>, + sequence: usize, + calls: CallsView<'a>, + local_send_completed: Option, +} + +#[derive(serde::Serialize)] +struct CallView<'a> { + id: &'a str, + name: &'a str, + result: &'a Value, +} + +struct CallsView<'a>(&'a [(GeminiFunctionCall, Value)]); +impl Serialize for CallsView<'_> { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(self.0.len()))?; + for (call, result) in self.0 { + sequence.serialize_element(&CallView { + id: &call.id, + name: &call.name, + result, + })?; + } + sequence.end() + } +} + +struct BoundedBytes { + remaining: usize, + written: usize, +} +impl Write for BoundedBytes { + fn write(&mut self, bytes: &[u8]) -> io::Result { + if bytes.len() > self.remaining { + return Err(io::Error::other("acquisition byte limit")); + } + self.remaining -= bytes.len(); + self.written += bytes.len(); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// Why an arming observer stopped trusting its scope. Latched: a fixture that +/// saw any of these has no evidence to report. +#[derive(Clone, Copy, Debug, PartialEq)] +pub(in crate::livekit) enum ArmFailure { + /// A bind arrived before any assignment named a room. + BindBeforeAssignment, + /// A second, different room was assigned or bound. + UnrelatedRoom, + /// A bind carried session zero, or a different session than the first. + Session, + /// The bound scope could not be admitted by `Capture::new`. + Scope, + /// Another capture already held the single installation slot. + Install, +} + +/// Dispatch-to-bind handoff for a fixture that cannot know the room name or +/// session id in advance. It starts unarmed and admits nothing: +/// +/// - the dispatcher's accepted assignment names the one expected room; +/// - the first bind for that room latches its nonzero session, builds the +/// `Capture` and installs it, so a tool batch can never arrive at an unbound +/// scope; +/// - a bind before any assignment, a second room, session zero, a different +/// later session or an occupied installation slot fails the observer for +/// good, which also uninstalls the capture: evidence from a scope it doubted +/// would mislead. +/// +/// The observer owns the install guard, so removing the observer (`retire`) +/// or dropping it takes the capture out with it, and a callback that was +/// already in flight finds a retired observer and does nothing. +pub(in crate::livekit) struct Arming { + limits: Limits, + /// Whether a bound scope is installed for `observe` to find. Off only for + /// the detached form, which exists so the state machine can be tested + /// without touching the process-wide slot. + publish: bool, + state: Mutex, +} + +enum ArmState { + Unarmed, + Expecting(String), + Bound { + capture: Arc, + _installed: Option, + }, + Failed(ArmFailure), + Retired, +} + +impl ArmState { + fn room(&self) -> Option<&str> { + match self { + Self::Expecting(room) => Some(room), + Self::Bound { capture, .. } => Some(&capture.room), + _ => None, + } + } + + fn is_terminal(&self) -> bool { + matches!(self, Self::Failed(_) | Self::Retired) + } +} + +impl Arming { + pub(in crate::livekit) fn new(limits: Limits) -> Arc { + Self::build(limits, true) + } + + /// The same rules without installing anything. + pub(in crate::livekit) fn detached(limits: Limits) -> Arc { + Self::build(limits, false) + } + + fn build(limits: Limits, publish: bool) -> Arc { + Arc::new(Self { + limits, + publish, + state: Mutex::new(ArmState::Unarmed), + }) + } + + fn state(&self) -> std::sync::MutexGuard<'_, ArmState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub(in crate::livekit) fn on_assignment(&self, room: &str) { + let mut state = self.state(); + if state.is_terminal() { + return; + } + match state.room() { + None => *state = ArmState::Expecting(room.into()), + Some(expected) if expected == room => {} + Some(_) => *state = ArmState::Failed(ArmFailure::UnrelatedRoom), + } + } + + pub(in crate::livekit) fn on_bind(&self, room: &str, session: u64) { + let mut state = self.state(); + if state.is_terminal() { + return; + } + if let Some(next) = self.after_bind(&state, room, session) { + *state = next; + } + } + + /// The state a bind moves to, or `None` when it changes nothing. + fn after_bind(&self, state: &ArmState, room: &str, session: u64) -> Option { + let fail = |failure| Some(ArmState::Failed(failure)); + match state { + ArmState::Unarmed => fail(ArmFailure::BindBeforeAssignment), + _ if state.room() != Some(room) => fail(ArmFailure::UnrelatedRoom), + ArmState::Bound { capture, .. } if capture.session != session => { + fail(ArmFailure::Session) + } + ArmState::Expecting(_) if session == 0 => fail(ArmFailure::Session), + ArmState::Expecting(_) => { + let Ok(capture) = Capture::new(room, session, self.limits) else { + return fail(ArmFailure::Scope); + }; + let installed = if self.publish { + match install(&capture) { + Ok(installed) => Some(installed), + Err(_) => return fail(ArmFailure::Install), + } + } else { + None + }; + Some(ArmState::Bound { + capture, + _installed: installed, + }) + } + _ => None, + } + } + + /// Ends the observer: whatever it installed is removed and later events + /// are ignored. + fn retire(&self) { + *self.state() = ArmState::Retired; + } + + /// The admitted scope, once bound and for as long as nothing has failed. + pub(in crate::livekit) fn capture(&self) -> Option> { + match &*self.state() { + ArmState::Bound { capture, .. } => Some(Arc::clone(capture)), + _ => None, + } + } + + /// What this observer saw, for a fixture to leave behind: the failure that + /// ended its trust, if any, and the admitted scope's retained evidence. + pub(in crate::livekit) fn report(&self) -> serde_json::Value { + let capture = self.capture().map(|capture| { + let snapshot = capture.snapshot(); + serde_json::json!({ + "batches": snapshot.batches, + "failure": snapshot.failure.map(|failure| format!("{failure:?}")), + "retained_calls": snapshot.retained_calls, + "retained_bytes": snapshot.retained_bytes, + }) + }); + serde_json::json!({ + "arming_failure": self.failure().map(|failure| format!("{failure:?}")), + "capture": capture, + }) + } + + pub(in crate::livekit) fn failure(&self) -> Option { + match &*self.state() { + ArmState::Failed(failure) => Some(*failure), + _ => None, + } + } +} + +static ARMING: Mutex>> = Mutex::new(None); +static ACTIVE: Mutex>> = Mutex::new(None); + +/// Uninstall stops new lookups, but admitted tickets can still complete. +/// Fixtures must join admitted writers before taking a final snapshot. +pub(in crate::livekit) struct Installed(Arc); +impl Drop for Installed { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &self.0)) { + *active = None; + } + } +} + +pub(in crate::livekit) fn install(capture: &Arc) -> Result { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.is_some() { + return Err("an acquisition capture is already installed"); + } + *active = Some(Arc::clone(capture)); + Ok(Installed(Arc::clone(capture))) +} + +/// Removes the arming observer and anything it installed when dropped. +pub(in crate::livekit) struct ArmingInstalled { + arming: Arc, + hook: crate::dispatch::AssignmentHook, +} +impl Drop for ArmingInstalled { + fn drop(&mut self) { + // One critical section, the same lock `install_arming` takes: a + // replacement observer can only install once this one is entirely gone, + // so this teardown never clears the replacement's hook or meets this + // observer's capture still installed. + let mut arming = ARMING.lock().unwrap_or_else(PoisonError::into_inner); + if arming + .as_ref() + .is_some_and(|now| Arc::ptr_eq(now, &self.arming)) + { + *arming = None; + } + crate::dispatch::clear_assignment_hook(&self.hook); + self.arming.retire(); + } +} + +/// Starts watching the dispatcher's assignments and the latency binds for the +/// one room the fixture will be given. Nothing is captured until both agree. +pub(in crate::livekit) fn install_arming( + arming: &Arc, +) -> Result { + let mut slot = ARMING.lock().unwrap_or_else(PoisonError::into_inner); + if slot.is_some() { + return Err("an acquisition arming observer is already installed"); + } + *slot = Some(Arc::clone(arming)); + let watching = Arc::clone(arming); + let hook: crate::dispatch::AssignmentHook = Arc::new(move |room: &str| { + watching.on_assignment(room); + }); + crate::dispatch::set_assignment_hook(Some(Arc::clone(&hook))); + Ok(ArmingInstalled { + arming: Arc::clone(arming), + hook, + }) +} + +/// The latency bind boundary: called with every room and session the loop +/// binds. +pub(in crate::livekit) fn bound(room: &str, session: u64) { + let arming = ARMING + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + if let Some(arming) = arming { + arming.on_bind(room, session); + } +} + +pub(in crate::livekit) fn observe( + latency: &ReplyLatency, + answers: &[(GeminiFunctionCall, Value)], +) -> Option { + let capture = ACTIVE + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(Arc::clone)?; + let sequence = capture.record(latency.acquisition_identity(), answers)?; + Some(Ticket { capture, sequence }) +} + +#[path = "../../../tests/unit/livekit/probe/acquisition.rs"] +mod tests; diff --git a/src/livekit/probe/acquisition/owner.rs b/src/livekit/probe/acquisition/owner.rs new file mode 100644 index 000000000..7bafbcd3d --- /dev/null +++ b/src/livekit/probe/acquisition/owner.rs @@ -0,0 +1,129 @@ +//! Controlled owner-await gates, distinct from physical transport pressure. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use tokio::sync::Notify; + +use super::Capture; +use crate::agent::ThinkingHold; +use crate::livekit::session::GeminiEventContext; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(in crate::livekit) enum Stage { + Acknowledgement, + BoardSettlement, + BoardResend, + ToolSend, +} + +#[derive(Clone, Copy)] +pub(in crate::livekit) enum Transition { + Hold(bool), + Discard(bool), +} + +pub(in crate::livekit) struct Gate { + scope: Arc, + socket: u64, + stage: Stage, + transition: Option, + entered: Notify, + release: Notify, +} + +impl Gate { + pub(in crate::livekit) fn new( + scope: &Arc, + socket: u64, + stage: Stage, + transition: Option, + ) -> Result, &'static str> { + if socket == 0 { + return Err("owner gate requires an assigned socket"); + } + if transition.is_some() && stage != Stage::ToolSend { + return Err("owner transition requires the scoped tool-send boundary"); + } + Ok(Arc::new(Self { + scope: Arc::clone(scope), + socket, + stage, + transition, + entered: Notify::new(), + release: Notify::new(), + })) + } + + pub(in crate::livekit) async fn entered(&self) -> Result<(), tokio::time::error::Elapsed> { + tokio::time::timeout(Duration::from_secs(2), self.entered.notified()).await + } + + pub(in crate::livekit) fn release(&self) { + self.release.notify_one(); + } +} + +static ACTIVE: Mutex>> = Mutex::new(None); + +pub(in crate::livekit) struct Installed(Arc); +impl Drop for Installed { + fn drop(&mut self) { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &self.0)) { + *active = None; + } + } +} + +pub(in crate::livekit) fn install(gate: &Arc) -> Result { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if active.is_some() { + return Err("an owner gate is already installed"); + } + *active = Some(Arc::clone(gate)); + Ok(Installed(Arc::clone(gate))) +} + +pub(in crate::livekit) async fn before(stage: Stage, context: &mut GeminiEventContext<'_>) { + let gate = ACTIVE + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(Arc::clone); + let Some(gate) = gate else { return }; + let matches = { + let identity = context.activity.latency.acquisition_identity(); + gate.stage == stage + && gate.scope.room == identity.room + && gate.scope.session == identity.session + && gate.socket == identity.socket + }; + if !matches { + return; + } + { + let mut active = ACTIVE.lock().unwrap_or_else(PoisonError::into_inner); + if !active.as_ref().is_some_and(|now| Arc::ptr_eq(now, &gate)) { + return; + } + *active = None; + } + gate.entered.notify_one(); + gate.release.notified().await; + match gate.transition { + Some(Transition::Hold(held)) => { + context.state.thinking_hold = if held { + ThinkingHold::Held { + since: Instant::now(), + } + } else { + ThinkingHold::Off + }; + } + Some(Transition::Discard(discarded)) => { + context.activity.socket.discarding_output = discarded; + } + None => {} + } +} diff --git a/src/livekit/session.rs b/src/livekit/session.rs index b7e4b3019..f7ec851f1 100644 --- a/src/livekit/session.rs +++ b/src/livekit/session.rs @@ -16,6 +16,7 @@ mod floor; mod observations; mod recovery; +mod tool_calls; use floor::write_floor; pub(super) use floor::{observe_floor, reconcile_floor}; @@ -26,6 +27,7 @@ pub(super) use recovery::{ apply_recovery_transcript, drain_retired_usage, prepare_input, prepare_listening_handoff, prepare_plain_context, }; +use tool_calls::on_tool_calls; use std::collections::HashMap; use std::time::{Duration, Instant}; @@ -1013,155 +1015,6 @@ pub(super) fn settle_hold_at_turn_end(state: &mut RuntimeState, activity: &mut R } } -/// Answers every call in the batch in one message, and republishes the -/// checklist once when any of them moved it. -async fn on_tool_calls( - room: &RoomLink, - context: &mut GeminiEventContext<'_>, - calls: Vec, -) -> Result<(), Box> { - if calls.is_empty() { - return Ok(()); - } - // A filler that fails to queue is not worth the tool round it precedes. - if let Err(error) = play_tool_ack(context, &calls).await { - eprintln!("tool acknowledgement not played: {error}"); - } - if context.state.example_drawing_enabled - && !context.state.interview_mode.is_whiteboard() - && calls.iter().any(|call| call.name == TOOL_READ_BOARD) - { - context.board.settle_for_read(context.state).await; - } - let timed = !context.state.interviewer_silenced() && !context.activity.socket.discarding_output; - let reviewed_revision = context.activity.substantive_revision_at_last_review; - let tool_started = Instant::now(); - let hints_before = HintCounters::capture(context.state); - let answers = execute_timed_tool_batch( - context.state, - context.activity, - calls, - reviewed_revision, - timed, - ); - let tool_execute_ms = crate::agent::saturating_ms(tool_started.elapsed()); - let editor_review_revision = answers - .iter() - .any(|(_, answer)| answer.get("editorReview").is_some()) - .then_some(context.state.evidence_ledger.code.substantive_revision); - - // What `read_board` could not put in its own response. Once per batch, so a - // batch that asked twice puts the board up once, and before the responses - // rather than after: Gemini starts its reply as soon as it has them, so a - // board sent after would arrive under an answer already being spoken about - // the board before it, while the tool has said this one is in front of it. - // - // Not `?`: an image the socket would not take is worth a line and a turn - // that answers from the board it already had, not an interview ended on the - // write. - let board_started = Instant::now(); - if std::mem::take(&mut context.state.board_resend_requested) - && let Err(error) = board::resend(context.board, context.gemini).await - { - eprintln!("board resend failed ({error}); waiting for the close to be reported"); - } - let board_resend_ms = crate::agent::saturating_ms(board_started.elapsed()); - - // Not `?`. A response that did not go out is owed nothing back, so the flag - // stays down, and the socket it failed on is replaced when its close is - // reported; the checklist below still reflects the calls. - let send_started = Instant::now(); - let sent = context.gemini.send_tool_responses(&answers).await; - let tool_send_ms = crate::agent::saturating_ms(send_started.elapsed()); - if timed { - context - .activity - .turn_timing - .note_tool_response(send_started, Instant::now(), sent.is_ok()); - } - - // Gemini resumes the moment the responses land, so serializing them again - // for the lines below waits until after the send. - if !context.activity.socket.discarding_output && !context.state.interviewer_silenced() { - let timing = observations::tool_timing( - context.activity, - context.state, - &answers, - hints_before, - ToolTiming { - batch_ms: crate::agent::saturating_ms(tool_started.elapsed()), - execute_ms: tool_execute_ms, - board_ms: board_resend_ms, - send_ms: tool_send_ms, - sent: sent.is_ok(), - }, - ); - latency::log(&timing); - } - for (call, response) in &answers { - let name = known_tool_name(&call.name); - eprintln!( - "codetrial live_tool_response room={} tool={name} bytes={} accepted={}", - room.name(), - response.to_string().len(), - response.get("error").is_none(), - ); - - // The phase as the checklist spells it, and only when it is one: the - // argument is model text, and the rest of the call stays out of the log - // for the reason the checklist carries no summary. - if call.name == TOOL_RECORD_FRAMEWORK_EVIDENCE { - let phase = call - .args - .get("phase") - .and_then(serde_json::Value::as_str) - .filter(|phase| { - crate::agent::REACTO_PHASE_IDS.contains(phase) - || crate::agent::STAR_PHASE_IDS.contains(phase) - }) - .unwrap_or("?"); - eprintln!( - "evidence: at={} phase={phase} accepted={} room={}", - log_clock(context.state), - response.get("error").is_none(), - room.name() - ); - } - } - - if let Some(revision) = editor_review_revision { - eprintln!( - "codetrial live_editor_review room={} revision={revision} sent={}", - room.name(), - sent.is_ok(), - ); - } - context.activity.settle_tool_editor_review( - editor_review_revision, - sent.is_ok(), - Instant::now(), - ); - - // A reminder of open earlier steps in a response that never went out has - // not been read, so it may be named again. - context.state.settle_earlier_steps(sent.is_ok()); - match sent { - // Gemini now owes a generation for this, and will deliver it on this - // socket or not at all. - Ok(()) => { - context.gemini.input_cause = Some(TurnCause::Tool.label()); - context.activity.note_tool_response(Instant::now()); - } - Err(error) => { - eprintln!( - "Gemini tool response failed ({error}); waiting for the close to be reported" - ); - } - } - flush_framework_progress(room, context.state).await; - Ok(()) -} - /// A tool round holds back the reply a candidate is waiting for by a model /// round trip; when enabled, a short acknowledgement in Jim's voice fills it. /// Queued, not counted as the reply: the watchdog and the latency trace still diff --git a/src/livekit/session/observations.rs b/src/livekit/session/observations.rs index 6dbffcd0f..aecb0d0c9 100644 --- a/src/livekit/session/observations.rs +++ b/src/livekit/session/observations.rs @@ -1,4 +1,7 @@ -//! Content-free tool context for the observational replay harness. +//! Production tool telemetry stays content-free; cfg(test) acquisition +//! separately +//! captures private authored-fixture answers before subsequent asynchronous +//! work. use std::time::Instant; use serde_json::Value; @@ -32,8 +35,9 @@ pub(super) fn execute_timed_tool_batch( calls: Vec, reviewed_revision: u64, timed: bool, + #[cfg(test)] ticket: &mut Option, ) -> Vec<(GeminiFunctionCall, Value)> { - super::execute_tool_batch_observed( + let answers = super::execute_tool_batch_observed( state, calls, reviewed_revision, @@ -46,7 +50,12 @@ pub(super) fn execute_timed_tool_batch( activity.latency.closing_requested(started, finished); } }, - ) + ); + #[cfg(test)] + { + *ticket = super::super::probe::acquisition::observe(&activity.latency, &answers); + } + answers } pub(super) struct HintCounters { diff --git a/src/livekit/session/tool_calls.rs b/src/livekit/session/tool_calls.rs new file mode 100644 index 000000000..88280469b --- /dev/null +++ b/src/livekit/session/tool_calls.rs @@ -0,0 +1,177 @@ +//! Executes and sends one interview tool batch. + +use super::{ + GeminiEventContext, GeminiFunctionCall, HintCounters, Instant, RoomLink, TOOL_READ_BOARD, + TOOL_RECORD_FRAMEWORK_EVIDENCE, ToolTiming, TurnCause, board, execute_timed_tool_batch, + flush_framework_progress, known_tool_name, latency, log_clock, observations, play_tool_ack, +}; + +#[cfg(test)] +use crate::livekit::probe::acquisition::owner::{self, Stage}; + +/// Answers every call in the batch in one message, and republishes the +/// checklist once when any of them moved it. +pub(super) async fn on_tool_calls( + room: &RoomLink, + context: &mut GeminiEventContext<'_>, + calls: Vec, +) -> Result<(), Box> { + if calls.is_empty() { + return Ok(()); + } + // A filler that fails to queue is not worth the tool round it precedes. + #[cfg(test)] + owner::before(Stage::Acknowledgement, context).await; + if let Err(error) = play_tool_ack(context, &calls).await { + eprintln!("tool acknowledgement not played: {error}"); + } + if context.state.example_drawing_enabled + && !context.state.interview_mode.is_whiteboard() + && calls.iter().any(|call| call.name == TOOL_READ_BOARD) + { + #[cfg(test)] + owner::before(Stage::BoardSettlement, context).await; + context.board.settle_for_read(context.state).await; + } + let timed = !context.state.interviewer_silenced() && !context.activity.socket.discarding_output; + let reviewed_revision = context.activity.substantive_revision_at_last_review; + let tool_started = Instant::now(); + let hints_before = HintCounters::capture(context.state); + #[cfg(test)] + let mut ticket = None; + let answers = execute_timed_tool_batch( + context.state, + context.activity, + calls, + reviewed_revision, + timed, + #[cfg(test)] + &mut ticket, + ); + let tool_execute_ms = crate::agent::saturating_ms(tool_started.elapsed()); + let editor_review_revision = answers + .iter() + .any(|(_, answer)| answer.get("editorReview").is_some()) + .then_some(context.state.evidence_ledger.code.substantive_revision); + + // What `read_board` could not put in its own response. Once per batch, so a + // batch that asked twice puts the board up once, and before the responses + // rather than after: Gemini starts its reply as soon as it has them, so a + // board sent after would arrive under an answer already being spoken about + // the board before it, while the tool has said this one is in front of it. + // + // Not `?`: an image the socket would not take is worth a line and a turn + // that answers from the board it already had, not an interview ended on the + // write. + let board_started = Instant::now(); + if std::mem::take(&mut context.state.board_resend_requested) + && let Err(error) = { + #[cfg(test)] + owner::before(Stage::BoardResend, context).await; + board::resend(context.board, context.gemini).await + } + { + eprintln!("board resend failed ({error}); waiting for the close to be reported"); + } + let board_resend_ms = crate::agent::saturating_ms(board_started.elapsed()); + + // Not `?`. A response that did not go out is owed nothing back, so the flag + // stays down, and the socket it failed on is replaced when its close is + // reported; the checklist below still reflects the calls. + #[cfg(test)] + owner::before(Stage::ToolSend, context).await; + let send_started = Instant::now(); + let sent = context.gemini.send_tool_responses(&answers).await; + let tool_send_ms = crate::agent::saturating_ms(send_started.elapsed()); + #[cfg(test)] + if let Some(ticket) = ticket { + ticket.complete(sent.is_ok()); + } + if timed { + context + .activity + .turn_timing + .note_tool_response(send_started, Instant::now(), sent.is_ok()); + } + + // Gemini resumes the moment the responses land, so serializing them again + // for the lines below waits until after the send. + if !context.activity.socket.discarding_output && !context.state.interviewer_silenced() { + let timing = observations::tool_timing( + context.activity, + context.state, + &answers, + hints_before, + ToolTiming { + batch_ms: crate::agent::saturating_ms(tool_started.elapsed()), + execute_ms: tool_execute_ms, + board_ms: board_resend_ms, + send_ms: tool_send_ms, + sent: sent.is_ok(), + }, + ); + latency::log(&timing); + } + for (call, response) in &answers { + let name = known_tool_name(&call.name); + eprintln!( + "codetrial live_tool_response room={} tool={name} bytes={} accepted={}", + room.name(), + response.to_string().len(), + response.get("error").is_none(), + ); + + // The phase as the checklist spells it, and only when it is one: the + // argument is model text, and the rest of the call stays out of the log + // for the reason the checklist carries no summary. + if call.name == TOOL_RECORD_FRAMEWORK_EVIDENCE { + let phase = call + .args + .get("phase") + .and_then(serde_json::Value::as_str) + .filter(|phase| { + crate::agent::REACTO_PHASE_IDS.contains(phase) + || crate::agent::STAR_PHASE_IDS.contains(phase) + }) + .unwrap_or("?"); + eprintln!( + "evidence: at={} phase={phase} accepted={} room={}", + log_clock(context.state), + response.get("error").is_none(), + room.name() + ); + } + } + + if let Some(revision) = editor_review_revision { + eprintln!( + "codetrial live_editor_review room={} revision={revision} sent={}", + room.name(), + sent.is_ok(), + ); + } + context.activity.settle_tool_editor_review( + editor_review_revision, + sent.is_ok(), + Instant::now(), + ); + + // A reminder of open earlier steps in a response that never went out has + // not been read, so it may be named again. + context.state.settle_earlier_steps(sent.is_ok()); + match sent { + // Gemini now owes a generation for this, and will deliver it on this + // socket or not at all. + Ok(()) => { + context.gemini.input_cause = Some(TurnCause::Tool.label()); + context.activity.note_tool_response(Instant::now()); + } + Err(error) => { + eprintln!( + "Gemini tool response failed ({error}); waiting for the close to be reported" + ); + } + } + flush_framework_progress(room, context.state).await; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 4a24e87b9..426b4bbe4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use codetrial::config::{AgentConfig, DEFAULT_WEB_ADDR, is_production}; use codetrial::launch::{ - CliOptions, LaunchedBy, Reachable, cold_start_config_path, config_dir, + Bounds, CliOptions, LaunchedBy, Reachable, cold_start_config_path, config_dir, data_dir_config_from_environment, environment_and_flags, extend_pool, finish, load_values, local_agent_config, nonempty, prepare, primary_config_path, production_secret_refusal, reachable_from_elsewhere, serve, value_or, @@ -14,7 +14,7 @@ use codetrial::launch::{ DEFAULT_SESSION_SECRET, SESSION_SECRET_FILE, account_db_path, allowed_hosts, apply_options, data_dir_session_secret, environment, guest_mode_enabled, guest_mode_recording_conflict, published_secret_refusal, recording_can_deliver, recording_needs_verified_identity, - relaxed_for_local_use, task_mode_allowed, task_mode_enabled, task_mode_refusal, + relaxed_for_local_use, serve_with, task_mode_allowed, task_mode_enabled, task_mode_refusal, web_server_config, }; @@ -346,7 +346,7 @@ fn run_web(options: CliOptions) -> Result<(), String> { agent_config, max_concurrent, std::future::pending(), - None, + Bounds::default(), ) } diff --git a/src/web/mod.rs b/src/web/mod.rs index c9ee8a9f7..f0aff159c 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -833,6 +833,13 @@ pub trait RoomDispatcher: Send + Sync + 'static { false } + /// Stops admitting new rooms; those already admitted keep running. A + /// stopping server calls it as its shutdown begins, so "nothing live" can + /// mean nothing will be. The default does nothing, for a dispatcher that + /// cannot refuse; with `is_idle` also left at its default, a server that + /// bounds its stop waits out the whole bound and reports rooms running. + fn close_admission(&self) {} + fn ensure_guest_agent( &self, _room_name: &str, @@ -856,13 +863,14 @@ pub enum AgentJob { }, } -/// Why no interviewer will come. The two answer the candidate differently: a +/// Why no interviewer will come. Each answers the candidate differently: a /// full server frees up as any interview ends, a finalizing room only when its -/// own report is out. +/// own report is out, and a stopping server not until it is started again. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DispatchRefusal { AtCapacity, Finalizing, + Stopping, } pub fn web_service(config: WebServerConfig) -> IntoMakeServiceWithConnectInfo { diff --git a/src/web/token.rs b/src/web/token.rs index 4d5991729..080836e27 100644 --- a/src/web/token.rs +++ b/src/web/token.rs @@ -632,6 +632,9 @@ async fn dispatch_or_release_consent( super::DispatchRefusal::Finalizing => { "This room is still finishing the previous interview's report. Try again in a few minutes." } + super::DispatchRefusal::Stopping => { + "The server is shutting down and is not starting new interviews. Try again later." + } }; Some(json_response( diff --git a/tests/cli.rs b/tests/cli.rs index 71ae58915..0e3417219 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -2194,11 +2194,11 @@ fn binary_web_reads_deployment_keys_from_the_config_file() { .to_string() }; - // Every phase of the startup path: assembled in `web_server_config`, - // `prepare` and `finish` (src/launch.rs), bound in `run_web` (src/main.rs) - // and served by `serve`, all from the config file, never the environment. + // Every phase of the startup path, in src/launch.rs and src/main.rs, takes + // its deployment from the operator's config file, never the environment. assert!(body("web_server_config").contains("trusted_proxy_hops(values)")); - let phases = "run_web prepare finish local_agent_config web_server_config serve"; + let phases = "run_web prepare finish local_agent_config web_server_config \ + serve serve_with run"; for name in phases.split(' ') { assert!(!body(name).contains("std::env::var"), "{name}"); } diff --git a/tests/unit/bin.rs b/tests/unit/bin.rs index 59e440b0d..095508d14 100644 --- a/tests/unit/bin.rs +++ b/tests/unit/bin.rs @@ -1015,3 +1015,139 @@ fn the_local_interviewer_needs_a_gemini_key_and_a_valid_config() { assert!(error.contains("GEMINI_CONTEXT"), "{error}"); let _ = std::fs::remove_dir_all(&dir); } + +/// A dispatcher that says whether an interview is still running or writing its +/// report, notes that admission was closed, and starts nothing. +struct Busy { + busy: std::sync::atomic::AtomicBool, + closed: std::sync::atomic::AtomicBool, +} + +impl codetrial::web::RoomDispatcher for Busy { + fn ensure_agent( + &self, + _: &str, + _: &codetrial::config::Provider, + _: codetrial::web::AgentJob, + ) -> Result<(), codetrial::web::DispatchRefusal> { + Ok(()) + } + + fn is_idle(&self) -> bool { + !self.busy.load(std::sync::atomic::Ordering::SeqCst) + } + + fn close_admission(&self) { + self.closed.store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +/// Serves with a stop that has already happened and a dispatcher that is busy +/// for `busy_for` (`None`: forever), and returns how the run ended, how long it +/// took and whether admission had been closed. +fn run_stopped( + label: &str, + busy_for: Option, + settle: Option, +) -> (Result<(), String>, std::time::Duration, bool) { + let (dir, options) = launch_fixture(label); + let values = livekit_values(&dir); + let prepared = super::prepare(&values, &options).unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let bound = listener.local_addr().unwrap(); + let (config, setup) = super::finish( + prepared.config, + &values, + &options, + bound, + super::LaunchedBy::Parent, + ) + .unwrap(); + let dispatcher = std::sync::Arc::new(Busy { + busy: std::sync::atomic::AtomicBool::new(true), + closed: std::sync::atomic::AtomicBool::new(false), + }); + if let Some(after) = busy_for { + let dispatcher = std::sync::Arc::clone(&dispatcher); + std::thread::spawn(move || { + std::thread::sleep(after); + dispatcher + .busy + .store(false, std::sync::atomic::Ordering::SeqCst); + }); + } + let serving = std::sync::Arc::clone(&dispatcher); + let began = std::time::Instant::now(); + + // On its own thread with a deadline, so a bound that stops working fails + // this test instead of hanging the suite. + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(super::serve_with( + listener, + config, + setup, + Some(serving), + async {}, + super::Bounds { + drain: None, + settle, + }, + )); + }); + let result = finished + .recv_timeout(std::time::Duration::from_secs(30)) + .expect("serve_with returned within its bounds"); + let elapsed = began.elapsed(); + let _ = std::fs::remove_dir_all(&dir); + let closed = dispatcher.closed.load(std::sync::atomic::Ordering::SeqCst); + (result, elapsed, closed) +} + +#[test] +fn a_stopping_server_waits_for_its_rooms_and_reports_to_finish() { + let (result, elapsed, closed) = run_stopped( + "settle-waits", + Some(std::time::Duration::from_millis(600)), + Some(std::time::Duration::from_secs(20)), + ); + result.unwrap(); + assert!( + closed, + "nothing new is admitted once the shutdown has begun" + ); + assert!( + elapsed >= std::time::Duration::from_millis(500), + "{elapsed:?}" + ); +} + +#[test] +fn rooms_still_running_at_the_settle_bound_are_reported_not_hidden() { + let (result, elapsed, closed) = run_stopped( + "settle-bound", + None, + Some(std::time::Duration::from_millis(300)), + ); + assert!(closed); + let error = result.unwrap_err(); + assert!(error.contains("settle bound"), "{error}"); + // It waited out the bound, and no longer. + assert!( + elapsed >= std::time::Duration::from_millis(250), + "{elapsed:?}" + ); + assert!(elapsed < std::time::Duration::from_secs(10), "{elapsed:?}"); +} + +#[test] +fn without_a_settle_bound_the_product_stops_without_waiting() { + let (result, elapsed, closed) = run_stopped("settle-none", None, None); + assert!( + closed, + "the product closes admission as its shutdown begins too" + ); + result.unwrap(); + assert!(elapsed < std::time::Duration::from_secs(5), "{elapsed:?}"); +} diff --git a/tests/unit/dispatch.rs b/tests/unit/dispatch.rs index 22c9ce3a0..e76c84bac 100644 --- a/tests/unit/dispatch.rs +++ b/tests/unit/dispatch.rs @@ -363,8 +363,8 @@ async fn a_concurrent_burst_never_overbooks_and_released_capacity_returns() { Reservation::GuestRoomInUse => { panic!("signed-in reservations must not hit the guest room limit") } - Reservation::Existing | Reservation::Finalizing => { - panic!("burst ids are unique") + Reservation::Existing | Reservation::Finalizing | Reservation::Closed => { + panic!("burst ids are unique and the dispatcher is open") } } })); @@ -943,3 +943,79 @@ async fn a_released_start_gives_its_consent_and_slot_back() { assert!(dispatcher.state.lock().unwrap().starts.is_empty()); drop(accounts); } + +/// Closing is part of the one admission decision, so after it no way of +/// starting a room gets in, while a room already admitted is still served and +/// still counts as live until it ends. +#[tokio::test] +async fn a_closed_dispatcher_admits_no_new_room_but_keeps_the_ones_it_has() { + let dispatcher = release_dispatcher(8); + let provider = Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let admitted = started(&dispatcher, "interview-before-close", false, "candidate"); + assert!(!dispatcher.is_idle()); + + dispatcher.close_admission(); + + assert!(matches!( + dispatcher.reserve_with_kind("interview-after-close", false), + Reservation::Closed + )); + assert!(matches!( + dispatcher.reserve_with_kind("guest-after-close", true), + Reservation::Closed + )); + assert_eq!( + dispatcher.ensure_agent("interview-via-ensure", &provider, AgentJob::Interview), + Err(DispatchRefusal::Stopping) + ); + assert_eq!( + dispatcher.ensure_guest_agent("guest-via-ensure", &provider, "candidate"), + Err(DispatchRefusal::Stopping) + ); + assert_eq!( + dispatcher.ensure_agent_with_no_show_release("no-show", &provider, "candidate", None), + Err(DispatchRefusal::Stopping) + ); + // The room that was admitted is still the dispatcher's, and still live. + assert!(matches!( + dispatcher.reserve_with_kind("interview-before-close", false), + Reservation::Existing + )); + assert!(!dispatcher.is_idle()); + drop(admitted); + assert!(dispatcher.is_idle()); + assert_eq!( + dispatcher.state.lock().unwrap().live.len(), + 0, + "refusals left no room behind" + ); +} + +#[test] +fn a_dispatcher_with_nothing_to_close_ignores_the_request() { + struct Plain; + impl RoomDispatcher for Plain { + fn ensure_agent(&self, _: &str, _: &Provider, _: AgentJob) -> Result<(), DispatchRefusal> { + Ok(()) + } + } + let provider = Provider { + id: crate::config::PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let plain = Plain; + plain.close_admission(); + assert_eq!( + plain.ensure_agent("room", &provider, AgentJob::Interview), + Ok(()) + ); +} diff --git a/tests/unit/launch.rs b/tests/unit/launch.rs new file mode 100644 index 000000000..280091233 --- /dev/null +++ b/tests/unit/launch.rs @@ -0,0 +1,98 @@ +use super::*; + +#[test] +fn unfinished_rooms_are_reported_together_with_the_shutdowns_own_error() { + let alone = unfinished(Ok(())); + assert!( + alone.starts_with("rooms or reports were still running"), + "{alone}" + ); + assert_eq!( + unfinished(Err("web server failed: slow".into())), + "web server failed: slow; rooms or reports were still running when the settle bound passed" + ); +} + +/// A config directory holding one provider file per id, removed on drop. +struct Providers(std::path::PathBuf); + +impl Providers { + fn new(label: &str, ids: &[&str]) -> Self { + let dir = + std::env::temp_dir().join(format!("codetrial-launch-{label}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + for id in ids { + std::fs::write( + dir.join(format!("codetrial.env.{id}")), + format!( + "LIVEKIT_URL=wss://{id}.example\nLIVEKIT_API_KEY=k-{id}\nLIVEKIT_API_SECRET=s-{id}\n" + ), + ) + .unwrap(); + } + Self(dir) + } +} + +impl Drop for Providers { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn the_pool_gains_the_discovered_providers_in_the_order_asked_for() { + let dir = Providers::new("pool", &["eu", "tokyo"]); + let mut pool = crate::config::ProviderPool::default(); + extend_pool(&mut pool, false, &dir.0, "tokyo"); + let ids: Vec<_> = pool.providers.iter().map(|p| p.id.as_str()).collect(); + assert_eq!(ids, ["tokyo", "eu"]); + let report = pool_report(&pool.providers); + assert_eq!(report.len(), 1, "{report:?}"); + assert!( + report[0].starts_with("provider pooling: 2 provider(s) over 2 project(s): tokyo="), + "{report:?}" + ); + assert!(pool_report(&[]).is_empty()); +} + +#[test] +fn a_project_listed_twice_is_reported_as_a_warning() { + let dir = Providers::new("twice", &["eu"]); + let mut pool = crate::config::ProviderPool::default(); + extend_pool(&mut pool, false, &dir.0, ""); + let twin = pool.providers[0].clone(); + pool.providers.push(crate::config::Provider { + id: "copy".into(), + ..twin + }); + let report = pool_report(&pool.providers); + assert_eq!(report.len(), 2, "{report:?}"); + assert!(report[1].starts_with("WARNING: eu and copy are the same LiveKit project")); +} + +#[test] +fn the_launch_reads_its_cap_length_and_compiler_switch_from_the_values() { + let values = |pairs: &[(&str, &str)]| -> BTreeMap { + pairs + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + assert_eq!( + read_max_concurrent(&values(&[("CODETRIAL_MAX_CONCURRENT_INTERVIEWS", "5")])), + 5 + ); + assert_eq!( + interview_duration_min(&values(&[("CODETRIAL_DURATION_MIN", "45")])), + 45 + ); + assert_eq!(interview_duration_min(&values(&[])), DEFAULT_DURATION_MIN); + for (value, enabled) in [("true", true), ("false", false)] { + assert_eq!( + compiler_explorer_enabled(&values(&[("CODETRIAL_COMPILER_EXPLORER_ENABLED", value)])), + enabled + ); + } +} diff --git a/tests/unit/livekit/probe/acquisition.rs b/tests/unit/livekit/probe/acquisition.rs new file mode 100644 index 000000000..00d37a881 --- /dev/null +++ b/tests/unit/livekit/probe/acquisition.rs @@ -0,0 +1,594 @@ +use super::*; +use crate::livekit::latency::ReplyLatency; +use serde_json::json; +use std::time::Instant; + +fn limits() -> Limits { + Limits { + batches: 3, + calls: 4, + bytes: 8192, + } +} +fn identity() -> IdentityView<'static> { + IdentityView { + room: "fixture", + session: 7, + socket: 1, + trace_turn: None, + clock: json!({"clock_uncertainty_ms":null}), + } +} +fn answers(id: &str, result: Value) -> Vec<(GeminiFunctionCall, Value)> { + vec![( + GeminiFunctionCall { + id: id.into(), + name: "read_editor".into(), + args: json!({"private":"not retained"}), + }, + result, + )] +} + +#[test] +fn assigned_scope_and_positive_limits_are_required() { + assert!(Capture::new("", 7, limits()).is_err()); + assert!(Capture::new("fixture", 0, limits()).is_err()); + for bad in [ + Limits { + batches: 0, + ..limits() + }, + Limits { + calls: 0, + ..limits() + }, + Limits { + bytes: 1, + ..limits() + }, + ] { + assert!(Capture::new("fixture", 7, bad).is_err()); + } +} + +#[test] +fn room_and_session_admit_but_socket_changes_are_retained() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &answers("a", json!("authored"))); + let mut next = identity(); + next.socket = 2; + let _ = capture.record(next, &answers("b", json!("second"))); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches[1].identity.socket, 2); + assert_eq!(snapshot.batches[1].sequence, 1); + assert_eq!(snapshot.retained_calls, 2); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); + assert!( + !serde_json::to_string(&snapshot.batches) + .unwrap() + .contains("not retained") + ); + for (room, session) in [("", 0), ("foreign", 7), ("fixture", 8)] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let mut view = identity(); + view.room = room; + view.session = session; + let _ = capture.record(view, &answers("a", json!(null))); + let _ = capture.record(identity(), &answers("b", json!(null))); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + assert!(capture.snapshot().batches.is_empty()); + } +} + +#[test] +fn byte_budget_counts_utf8_escaping_and_array_punctuation_before_cloning() { + let executed = answers("a", json!({"value":"é\n\\\""})); + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &executed); + let exact = capture.snapshot().retained_bytes; + for (bytes, accepted) in [(exact, true), (exact - 1, false)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + let _ = capture.record(identity(), &executed); + assert_eq!(capture.snapshot().batches.len(), usize::from(accepted)); + assert_eq!( + capture.snapshot().failure, + (!accepted).then_some(Failure::Limit) + ); + } + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: exact, + ..limits() + }, + ) + .unwrap(); + let _ = capture.record(identity(), &executed); + let _ = capture.record(identity(), &answers("b", json!("second"))); + assert_eq!(capture.snapshot().failure, Some(Failure::Limit)); + assert_eq!(capture.snapshot().retained_bytes, exact); +} + +#[test] +fn count_limits_and_ambiguous_ids_latch_failure() { + for caps in [ + Limits { + batches: 1, + ..limits() + }, + Limits { + calls: 1, + ..limits() + }, + ] { + let capture = Capture::new("fixture", 7, caps).unwrap(); + let _ = capture.record(identity(), &answers("a", json!(null))); + let _ = capture.record(identity(), &answers("b", json!(null))); + let _ = capture.record(identity(), &[]); + assert_eq!(capture.snapshot().failure, Some(Failure::Limit)); + assert_eq!(capture.snapshot().retained_calls, 1); + } + for ids in [vec![""], vec!["same", "same"]] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let batch = ids + .into_iter() + .flat_map(|id| answers(id, json!(null))) + .collect::>(); + let _ = capture.record(identity(), &batch); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + assert!(capture.snapshot().batches.is_empty()); + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _ = capture.record(identity(), &answers("same", json!(null))); + let _ = capture.record(identity(), &answers("same", json!(null))); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + assert_eq!(capture.snapshot().batches.len(), 1); +} + +#[test] +fn identity_view_preserves_trace_and_existing_paired_clock_policy() { + let now = Instant::now(); + let mut latency = ReplyLatency::default(); + assert_eq!(latency.acquisition_identity().session, 0); + latency.bind("fixture", 7, 1, 1500); + let before = latency.acquisition_identity(); + assert_eq!(before.trace_turn, None); + assert!(before.clock["emitted_ms"].is_null()); + assert!(before.clock["start_unix_ms"].is_null()); + assert!(before.clock["emitted_unix_ms"].as_u64().is_some()); + assert_eq!(before.clock["clock_uncertainty_ms"], Value::Null); + latency.anchor(now); + let after = latency.acquisition_identity(); + assert_eq!(after.trace_turn, None); + assert!(after.clock["start_unix_ms"].as_u64().is_some()); + assert!(after.clock["emitted_unix_ms"].as_u64().is_some()); + assert!(after.clock["emitted_ms"].as_i64().is_some()); + assert!(after.clock["clock_uncertainty_ms"].as_u64().is_some()); + latency.prompt(now); + let turn = latency.acquisition_identity().trace_turn; + assert!(turn.is_some()); + assert_eq!(latency.acquisition_identity().trace_turn, turn); +} + +#[test] +fn constructor_rejects_scope_metadata_before_cloning_it() { + assert!( + Capture::new( + "fixture", + 7, + Limits { + bytes: 2, + ..limits() + } + ) + .is_err() + ); + let escaped_room = "\"".repeat(1024); + assert!( + Capture::new( + &escaped_room, + 7, + Limits { + bytes: 2048, + ..limits() + } + ) + .is_err() + ); + let multibyte_room = "é".repeat(1024); + assert!( + Capture::new( + &multibyte_room, + 7, + Limits { + bytes: 1536, + ..limits() + } + ) + .is_err() + ); + let large_room = "x".repeat(8192); + assert!( + Capture::new( + &large_room, + 7, + Limits { + bytes: 256, + ..limits() + } + ) + .is_err() + ); +} + +fn admitted_ticket(capture: &Arc, id: &str) -> Ticket { + let sequence = capture + .record(identity(), &answers(id, json!("executed"))) + .unwrap(); + Ticket { + capture: Arc::clone(capture), + sequence, + } +} + +#[test] +fn completion_preserves_exact_array_bytes_and_original_object_clock() { + let prototype = Capture::new("fixture", 7, limits()).unwrap(); + let _ = admitted_ticket(&prototype, "a"); + let null_bytes = prototype.snapshot().retained_bytes; + for (sent, allowance, accepted) in [(true, 0, true), (false, 0, false), (false, 1, true)] { + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: null_bytes + allowance, + ..limits() + }, + ) + .unwrap(); + let ticket = admitted_ticket(&capture, "a"); + let before = capture.snapshot(); + assert!(before.batches[0].identity.clock.is_object()); + ticket.complete(sent); + let after = capture.snapshot(); + assert_eq!(after.batches[0].identity, before.batches[0].identity); + assert_eq!(after.batches[0].sequence, 0); + assert_eq!(after.batches[0].calls, before.batches[0].calls); + assert_eq!(after.retained_calls, 1); + assert_eq!( + after.batches[0].local_send_completed, + accepted.then_some(sent) + ); + assert_eq!(after.failure, (!accepted).then_some(Failure::Limit)); + assert_eq!( + after.retained_bytes, + before.retained_bytes + usize::from(accepted && !sent) + ); + assert_eq!( + after.retained_bytes, + serde_json::to_vec(&after.batches).unwrap().len() + ); + } +} + +#[test] +fn rejected_or_latched_admission_cannot_update_an_earlier_batch() { + for rejected in [Failure::Scope, Failure::Limit, Failure::AmbiguousId] { + let caps = Limits { + batches: if rejected == Failure::Limit { 1 } else { 3 }, + ..limits() + }; + let capture = Capture::new("fixture", 7, caps).unwrap(); + assert!(capture.record(identity(), &[]).is_none()); + assert_eq!(capture.snapshot().failure, None); + let ticket = admitted_ticket(&capture, "first"); + let mut view = identity(); + if rejected == Failure::Scope { + view.room = "foreign"; + } + let id = if rejected == Failure::AmbiguousId { + "first" + } else { + "second" + }; + assert!( + capture + .record(view, &answers(id, json!("rejected"))) + .is_none() + ); + assert!( + capture + .record(identity(), &answers("later", json!("latched"))) + .is_none() + ); + let before = capture.snapshot(); + assert_eq!(before.failure, Some(rejected)); + ticket.complete(false); + let after = capture.snapshot(); + assert_eq!( + after, before, + "a latched capture refuses completion of an admitted ticket too" + ); + } +} + +#[test] +fn sequence_mismatch_and_duplicate_completion_fail_closed() { + for sequence in [0, 1] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let ticket = admitted_ticket(&capture, "first"); + if sequence == 0 { + ticket.complete(true); + } else { + drop(ticket); + } + let before = capture.snapshot(); + Ticket { + capture: Arc::clone(&capture), + sequence, + } + .complete(false); + let after = capture.snapshot(); + assert_eq!(after.failure, Some(Failure::Scope)); + assert_eq!(after.batches, before.batches); + assert_eq!(after.retained_bytes, before.retained_bytes); + } +} + +#[test] +fn dropping_a_ticket_leaves_completion_unknown() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let ticket = admitted_ticket(&capture, "first"); + let before = capture.snapshot(); + drop(ticket); + assert_eq!(capture.snapshot(), before); + assert_eq!(capture.snapshot().batches[0].local_send_completed, None); +} + +#[test] +fn owner_gate_changes_are_scoped_to_the_post_sample_tool_send_boundary() { + use super::owner::{Gate, Stage, Transition}; + let capture = Capture::new("fixture", 7, limits()).unwrap(); + assert!(Gate::new(&capture, 0, Stage::ToolSend, None).is_err()); + for stage in [ + Stage::Acknowledgement, + Stage::BoardSettlement, + Stage::BoardResend, + ] { + assert!(Gate::new(&capture, 1, stage, Some(Transition::Hold(true))).is_err()); + } + assert!(Gate::new(&capture, 1, Stage::ToolSend, Some(Transition::Hold(true))).is_ok()); +} + +#[test] +fn completion_can_finish_admitted_sequences_out_of_order() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let first = admitted_ticket(&capture, "first"); + let second = admitted_ticket(&capture, "second"); + second.complete(false); + first.complete(true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches[0].calls[0].id, "first"); + assert_eq!(snapshot.batches[0].local_send_completed, Some(true)); + assert_eq!(snapshot.batches[1].calls[0].id, "second"); + assert_eq!(snapshot.batches[1].local_send_completed, Some(false)); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); +} + +/// Runs the calling test again in a child process so the module's global +/// installation slots cannot meet another test's. +fn isolated(test: &str, env: &str) -> bool { + let name = format!("{}::{test}", module_path!().split_once("::").unwrap().1); + crate::test_isolation::isolated_checked(&name, env, false, |_| {}) +} + +fn bound_scope_records(capture: &Arc, room: &str, session: u64) -> bool { + let mut id = identity(); + id.room = room; + id.session = session; + capture + .record(id, &answers("a", json!("authored"))) + .is_some() +} + +#[test] +fn an_unarmed_observer_never_arms_itself_from_a_bind() { + let arming = Arming::detached(limits()); + arming.on_bind("fixture", 7); + assert!(arming.capture().is_none()); + assert_eq!(arming.failure(), Some(ArmFailure::BindBeforeAssignment)); + arming.on_assignment("fixture"); + assert!(arming.capture().is_none(), "a failed observer stays failed"); +} + +#[test] +fn an_assignment_alone_admits_nothing_until_its_session_binds() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + assert!(arming.capture().is_none()); + assert_eq!(arming.failure(), None); + arming.on_bind("fixture", 7); + let capture = arming + .capture() + .expect("the expected room's bind admits it"); + assert!(bound_scope_records(&capture, "fixture", 7)); + assert!(!bound_scope_records(&capture, "fixture", 8)); + assert!(!bound_scope_records(&capture, "other", 7)); +} + +#[test] +fn a_second_room_or_a_foreign_bind_is_a_fixture_failure() { + for foreign in ["assign", "bind"] { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + match foreign { + "assign" => arming.on_assignment("other"), + _ => arming.on_bind("other", 7), + } + assert_eq!(arming.failure(), Some(ArmFailure::UnrelatedRoom)); + arming.on_bind("fixture", 7); + assert!(arming.capture().is_none(), "{foreign}"); + } +} + +#[test] +fn the_first_bound_session_is_latched_and_zero_is_refused() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 0); + assert_eq!(arming.failure(), Some(ArmFailure::Session)); + + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 7); + arming.on_bind("fixture", 7); + assert!( + arming.capture().is_some(), + "repeats of the same scope are fine" + ); + assert_eq!(arming.failure(), None); + arming.on_bind("fixture", 8); + assert_eq!(arming.failure(), Some(ArmFailure::Session)); + assert!(arming.capture().is_none()); +} + +#[test] +fn the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope() { + use crate::config::{PRIMARY_PROVIDER_ID, Provider}; + use crate::dispatch::LocalDispatcher; + use crate::web::{AgentJob, RoomDispatcher}; + + if isolated( + "the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope", + "CODETRIAL_ACQUISITION_ARMING_TEST", + ) { + return; + } + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + assert!(install_arming(&arming).is_err()); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + let dispatcher = LocalDispatcher { + config: crate::config::load_from_pairs([ + ("LIVEKIT_URL", "wss://primary.example"), + ("LIVEKIT_API_KEY", "primary-key"), + ("LIVEKIT_API_SECRET", "primary-secret"), + ("GOOGLE_API_KEY", "primary-google"), + ]) + .unwrap(), + runtime: runtime.handle().clone(), + state: Arc::default(), + max_concurrent: 2, + }; + let provider = Provider { + id: PRIMARY_PROVIDER_ID.to_string(), + url: "wss://primary.example".to_string(), + api_key: "primary-key".to_string(), + api_secret: "primary-secret".to_string(), + google_api_keys: Vec::new(), + }; + let mut latency = ReplyLatency::default(); + + // Nothing is observed before the assignment and the bind. + assert!(observe(&latency, &answers("early", json!("x"))).is_none()); + + dispatcher + .ensure_agent("fixture-room", &provider, AgentJob::Interview) + .unwrap(); + assert_eq!( + arming.failure(), + None, + "the expected room's own bind comes first" + ); + latency.bind("fixture-room", 7, 1, 1500); + assert!(observe(&latency, &answers("a", json!("x"))).is_some()); + + // A second room reaching dispatch fails the fixture and stops capture. + dispatcher + .ensure_agent("other-room", &provider, AgentJob::Interview) + .unwrap(); + assert_eq!(arming.failure(), Some(ArmFailure::UnrelatedRoom)); + assert!(observe(&latency, &answers("b", json!("x"))).is_none()); + + drop(installed); + drop(runtime); +} + +#[test] +fn a_blocked_or_removed_observer_leaves_no_capture_installed() { + if isolated( + "a_blocked_or_removed_observer_leaves_no_capture_installed", + "CODETRIAL_ACQUISITION_ARMING_BLOCKED_TEST", + ) { + return; + } + + // Another capture holds the one slot: the fixture must see a failure rather + // than believe its scope went live. + let foreign = Capture::new("foreign", 1, limits()).unwrap(); + let held = install(&foreign).unwrap(); + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + arming.on_assignment("fixture"); + bound("fixture", 7); + assert_eq!(arming.failure(), Some(ArmFailure::Install)); + drop(installed); + drop(held); + + // Removing the observer takes its capture out, and an event that was + // already in flight afterwards installs nothing, because that capture would + // have no owner left to remove it. + let arming = Arming::new(limits()); + let installed = install_arming(&arming).unwrap(); + arming.on_assignment("fixture"); + bound("fixture", 7); + assert!(arming.capture().is_some()); + drop(installed); + arming.on_bind("fixture", 7); + arming.on_assignment("fixture"); + let probe = Capture::new("probe", 1, limits()).unwrap(); + assert!(install(&probe).is_ok(), "a removed observer left a capture"); +} + +#[test] +fn a_teardown_clears_only_its_own_dispatch_hook() { + if isolated( + "a_teardown_clears_only_its_own_dispatch_hook", + "CODETRIAL_ACQUISITION_ARMING_HOOK_TEST", + ) { + return; + } + let first = Arming::new(limits()); + let installed = install_arming(&first).unwrap(); + // A successor that slipped in while this observer was being removed. + let successor: crate::dispatch::AssignmentHook = Arc::new(|_| {}); + crate::dispatch::set_assignment_hook(Some(Arc::clone(&successor))); + drop(installed); + assert!(crate::dispatch::assignment_hook_is(&successor)); + crate::dispatch::clear_assignment_hook(&successor); + assert!(!crate::dispatch::assignment_hook_is(&successor)); + + // And a plain reinstall after a removal is armed from the first event. + let second = Arming::new(limits()); + let guard = install_arming(&second).unwrap(); + crate::dispatch::notify_assigned("room"); + bound("room", 7); + assert_eq!(second.failure(), None); + assert!(second.capture().is_some()); + drop(guard); +} diff --git a/tests/unit/livekit/probe/serving.rs b/tests/unit/livekit/probe/serving.rs new file mode 100644 index 000000000..bd0b4ce37 --- /dev/null +++ b/tests/unit/livekit/probe/serving.rs @@ -0,0 +1,583 @@ +//! Serves the real web service and the local interviewer in this process, built +//! by the same startup phases and the same `launch::serve` that `codetrial web` +//! runs, with the arming observer installed before the first request, so a +//! browser run can leave behind the tool-call evidence of the interview it +//! drove. +//! +//! `serve_web_with_arming` is the ignored entry a launcher runs; the ordinary +//! tests beside it drive `serve` with a throwaway config so the fixture itself +//! is checked without credentials or a browser. +//! +//! Where it differs from `codetrial web`, on purpose: it is always a launcher's +//! child (no Setup page, as with the desktop app's server), it does not clear +//! the operator's web error log, task mode is off, and it reads the process +//! environment exactly as the product does, so an ambient `NODE_ENV` or +//! `CODETRIAL_*` setting reaches it too. It waits up to its settle bound for +//! admitted rooms and reports to finish before the snapshot, and a room still +//! running at the bound fails the run. + +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde_json::json; + +use super::acquisition::{Arming, Limits, install_arming}; +use crate::launch::{ + Bounds, CliOptions, LaunchedBy, finish, load_values, local_agent_config, prepare, + serve as serve_web, +}; +use crate::livekit::latency::ReplyLatency; + +/// A launcher gives a config file and a directory the fixture owns; the +/// account database, the ready file, the stop file and the report all live in +/// that directory under fixed names. The account database therefore has no +/// default to fall back on: the default is the operator's own `codetrial.db`. +struct Settings { + config: PathBuf, + dir: PathBuf, + addr: String, + deadline: Duration, + /// How long open connections may hold the shutdown up before the run + /// reports without them. + drain: Duration, + /// How long admitted rooms and their reports may take to finish after the + /// last connection has gone, before the run reports them as unfinished. + settle: Duration, + /// Skips the startup probe of the provider's quota. Only the + /// credential-free tests set it; the entry a launcher runs keeps the + /// product's behavior. + offline: bool, +} + +impl Settings { + fn from_lookup(get: &dyn Fn(&str) -> Option) -> Result { + let required = |name: &str| { + get(name) + .filter(|value| !value.trim().is_empty()) + .map(PathBuf::from) + .ok_or_else(|| format!("{name} must be set")) + }; + let seconds = |name: &str, default: u64| { + get(name) + .map(|text| text.parse().map_err(|_| format!("{name} is not seconds"))) + .transpose() + .map(|value| Duration::from_secs(value.unwrap_or(default))) + }; + Ok(Self { + config: required("CODETRIAL_FIXTURE_CONFIG")?, + dir: required("CODETRIAL_FIXTURE_DIR")?, + addr: get("CODETRIAL_FIXTURE_ADDR").unwrap_or_else(|| "127.0.0.1:0".into()), + deadline: seconds("CODETRIAL_FIXTURE_DEADLINE_S", 7200)?, + + // From the launcher, which sizes its stop wait from this and the + // settle; the default is what it passes. + drain: seconds("CODETRIAL_FIXTURE_DRAIN_S", 10)?, + settle: seconds("CODETRIAL_FIXTURE_SETTLE_S", 120)?, + offline: false, + }) + } + + fn db(&self) -> PathBuf { + self.dir.join("accounts.db") + } + + fn ready(&self) -> PathBuf { + self.dir.join("ready") + } + + fn stop(&self) -> PathBuf { + self.dir.join("stop") + } + + fn out(&self) -> PathBuf { + self.dir.join("out.json") + } +} + +/// Serves until `stop` resolves, then writes what the observer saw and how the +/// server ended to `out.json`. The ready file holds the bound address; it is +/// written once the listener is bound, so a request that arrives before the +/// accept loop has run waits in the socket's backlog rather than being refused. +fn serve( + settings: &Settings, + stop: impl Future + Send + 'static, +) -> Result<(), String> { + clear_previous(settings)?; + + // Loopback only: it runs with a throwaway session secret and an account + // database nobody else should reach. + let addr: std::net::SocketAddr = settings + .addr + .parse() + .map_err(|error| format!("{} is not an address: {error}", settings.addr))?; + + // Canonical first, as `launch::reachable_from_elsewhere` does: a dual-stack + // socket names 127.0.0.1 as ::ffff:127.0.0.1. + if !addr.ip().to_canonical().is_loopback() { + return Err(format!("{addr} is not a loopback address")); + } + let options = CliOptions { + config_path: Some(settings.config.display().to_string()), + ..Default::default() + }; + let mut values = load_values(&options)?; + values.insert( + "CODETRIAL_DB_PATH".into(), + settings.db().display().to_string(), + ); + + let prepared = prepare(&values, &options)?; + let listener = std::net::TcpListener::bind(addr) + .map_err(|error| format!("cannot bind {addr}: {error}"))?; + listener + .set_nonblocking(true) + .map_err(|error| error.to_string())?; + let bound = listener.local_addr().map_err(|error| error.to_string())?; + let (mut config, setup) = finish( + prepared.config, + &values, + &options, + bound, + LaunchedBy::Parent, + )?; + config.probe_provider_quota &= !settings.offline; + let agent = local_agent_config(values)? + .ok_or("the fixture hosts the interviewer and needs a Gemini key")?; + + let arming = Arming::new(Limits { + batches: 512, + calls: 2048, + bytes: 4 << 20, + }); + let installed = install_arming(&arming)?; + publish(&settings.ready(), &bound.to_string())?; + let outcome = serve_web( + listener, + config, + setup, + Some(agent), + prepared.max_concurrent, + stop, + Bounds { + drain: Some(settings.drain), + settle: Some(settings.settle), + }, + ); + + let mut report = arming.report(); + report["bound"] = json!(bound.to_string()); + report["outcome"] = json!(outcome.clone().err().unwrap_or_else(|| "ok".into())); + drop(installed); + ended(outcome, publish(&settings.out(), &format!("{report:#}\n"))) +} + +/// How a run ended, once its report was or was not written: when both failed, +/// both are said, the serving failure first. +fn ended(outcome: Result<(), String>, written: Result<(), String>) -> Result<(), String> { + match (outcome, written) { + (Err(served), Err(unwritten)) => Err(format!("{served}; and {unwritten}")), + (outcome, written) => outcome.and(written), + } +} + +/// Written beside and renamed into place, so a reader never sees half a file. +fn publish(path: &Path, text: &str) -> Result<(), String> { + let partial = path.with_extension("partial"); + std::fs::write(&partial, text) + .and_then(|()| std::fs::rename(&partial, path)) + .map_err(|error| format!("cannot write {}: {error}", path.display())) +} + +/// Resolves once the stop file exists or the deadline passes. +async fn stop_file(path: PathBuf, deadline: Duration) { + let began = Instant::now(); + while !path.exists() && began.elapsed() < deadline { + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +/// Before anything can fail: a run that never reports must not leave an +/// earlier run's report to be read as its own. +fn clear_previous(settings: &Settings) -> Result<(), String> { + for stale in [settings.ready(), settings.out()] { + match std::fs::remove_file(&stale) { + Err(error) if error.kind() != std::io::ErrorKind::NotFound => { + return Err(format!("cannot clear {}: {error}", stale.display())); + } + _ => {} + } + } + Ok(()) +} + +fn serve_until_stop_file(settings: &Settings) -> Result<(), String> { + clear_previous(settings)?; + if settings.stop().exists() { + return Err(format!( + "{} already exists and would end the run before it began", + settings.stop().display() + )); + } + serve(settings, stop_file(settings.stop(), settings.deadline)) +} + +#[test] +#[ignore = "serves until stopped; an interview run through it spends provider quota"] +fn serve_web_with_arming() { + // The observer's hooks are process-wide, so another test's room or bind in + // this process would fail it for good. The launcher runs this one alone. + assert!( + std::env::args().any(|arg| arg == "--exact"), + "run this with --exact so no other test shares the process" + ); + let settings = Settings::from_lookup(&|name| std::env::var(name).ok()).unwrap(); + serve_until_stop_file(&settings).unwrap(); +} + +fn scratch(label: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "codetrial-serving-{label}-{}-{}", + std::process::id(), + crate::current_epoch_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// Settings for a throwaway directory: a fake config, loopback port zero, and +/// no quota probe. +fn throwaway(dir: &Path) -> Settings { + let config = dir.join("codetrial.env.local"); + std::fs::write( + &config, + "LIVEKIT_URL=wss://primary.example\nLIVEKIT_API_KEY=primary-key\n\ + LIVEKIT_API_SECRET=primary-secret\nGOOGLE_API_KEY=google-key\n", + ) + .unwrap(); + Settings { + config, + dir: dir.to_path_buf(), + addr: "127.0.0.1:0".into(), + deadline: Duration::from_secs(60), + drain: Duration::from_secs(10), + settle: Duration::from_secs(10), + offline: true, + } +} + +fn get(addr: &str, path: &str) -> String { + use std::io::{Read, Write}; + let mut stream = std::net::TcpStream::connect(addr).unwrap(); + write!( + stream, + "GET {path} HTTP/1.1\r\nHost: {addr}\r\nConnection: close\r\n\r\n" + ) + .unwrap(); + let mut response = String::new(); + stream.read_to_string(&mut response).unwrap(); + response +} + +fn wait_for(path: &Path) -> String { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(30) { + if let Ok(text) = std::fs::read_to_string(path) + && !text.is_empty() + { + return text; + } + std::thread::sleep(Duration::from_millis(50)); + } + panic!("{} never appeared", path.display()); +} + +fn lookup( + config: Option<&str>, + dir: Option<&str>, + deadline: Option<&str>, +) -> impl Fn(&str) -> Option { + let (config, dir, deadline) = ( + config.map(String::from), + dir.map(String::from), + deadline.map(String::from), + ); + move |name| match name { + "CODETRIAL_FIXTURE_CONFIG" => config.clone(), + "CODETRIAL_FIXTURE_DIR" => dir.clone(), + "CODETRIAL_FIXTURE_DEADLINE_S" => deadline.clone(), + _ => None, + } +} + +#[test] +fn the_fixture_owns_one_directory_and_never_defaults_the_database() { + let settings = Settings::from_lookup(&lookup(Some("/c"), Some("/d"), None)).unwrap(); + assert_eq!(settings.db(), Path::new("/d/accounts.db")); + for (name, path) in [ + ("ready", settings.ready()), + ("stop", settings.stop()), + ("out.json", settings.out()), + ] { + assert_eq!(path, Path::new("/d").join(name)); + } + + // A blank directory counts as missing: a blank database path is read as + // absent by the launch code and would open the operator's own database. + for blank in [None, Some(""), Some(" "), Some("\t")] { + for (config, dir, missing) in [ + (blank, Some("/d"), "CODETRIAL_FIXTURE_CONFIG"), + (Some("/c"), blank, "CODETRIAL_FIXTURE_DIR"), + ] { + assert_eq!( + Settings::from_lookup(&lookup(config, dir, None)) + .err() + .unwrap(), + format!("{missing} must be set") + ); + } + } + assert!(Settings::from_lookup(&lookup(Some("/c"), Some("/d"), Some("soon"))).is_err()); + assert_eq!(settings.settle, Duration::from_secs(120)); + assert_eq!(settings.drain, Duration::from_secs(10)); + for (key, value, read) in [ + ("CODETRIAL_FIXTURE_SETTLE_S", "later", None), + ("CODETRIAL_FIXTURE_DRAIN_S", "soon", None), + ("CODETRIAL_FIXTURE_DRAIN_S", "3", Some(3)), + ] { + let lookup = |name: &str| match name { + "CODETRIAL_FIXTURE_CONFIG" => Some("/c".to_string()), + "CODETRIAL_FIXTURE_DIR" => Some("/d".to_string()), + name if name == key => Some(value.to_string()), + _ => None, + }; + let result = Settings::from_lookup(&lookup); + match read { + None => assert_eq!(result.err().unwrap(), format!("{key} is not seconds")), + Some(seconds) => assert_eq!(result.unwrap().drain, Duration::from_secs(seconds)), + } + } +} + +#[test] +fn a_stop_file_left_from_an_earlier_run_refuses_the_start() { + let dir = scratch("stale-stop"); + let settings = throwaway(&dir); + std::fs::write(settings.stop(), "").unwrap(); + // An earlier run's report goes too, so it cannot be kept as this one's. + std::fs::write(settings.out(), "{\"outcome\":\"ok\"}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + let error = serve_until_stop_file(&settings).unwrap_err(); + assert!( + error.contains("would end the run before it began"), + "{error}" + ); + assert!(!settings.db().exists() && !settings.out().exists()); + assert!(!settings.ready().exists()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_run_that_fails_and_cannot_report_says_both() { + let fail = |why: &str| Err::<(), String>(why.into()); + assert_eq!(ended(Ok(()), Ok(())), Ok(())); + assert_eq!(ended(fail("served"), Ok(())), fail("served")); + assert_eq!(ended(Ok(()), fail("unwritten")), fail("unwritten")); + assert_eq!( + ended(fail("served"), fail("unwritten")), + fail("served; and unwritten") + ); +} + +#[test] +fn an_address_off_loopback_refuses_the_start() { + let dir = scratch("public-addr"); + for (addr, refusal) in [ + ("0.0.0.0:0", "not a loopback address"), + ("192.0.2.1:0", "not a loopback address"), + ("localhost:0", "not an address"), + ] { + let mut settings = throwaway(&dir); + settings.addr = addr.into(); + let error = serve_until_stop_file(&settings).unwrap_err(); + assert!(error.contains(refusal), "{addr}: {error}"); + assert!(!settings.db().exists() && !settings.out().exists()); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_fixture_serves_the_real_service_and_reports_what_the_observer_saw() { + if crate::test_isolation::isolated_checked( + &format!( + "{}::the_fixture_serves_the_real_service_and_reports_what_the_observer_saw", + module_path!().split_once("::").unwrap().1 + ), + "CODETRIAL_SERVING_FIXTURE_TEST", + false, + |_| {}, + ) { + return; + } + let dir = scratch("serve"); + let settings = throwaway(&dir); + let (ready, out, db) = (settings.ready(), settings.out(), settings.db()); + let stop = std::sync::Arc::new(tokio::sync::Notify::new()); + let stopping = std::sync::Arc::clone(&stop); + let server = + std::thread::spawn(move || serve(&settings, async move { stopping.notified().await })); + + let addr = wait_for(&ready); + assert!(get(&addr, "/healthz").starts_with("HTTP/1.1 200"), "{addr}"); + assert!( + get(&addr, "/setup").starts_with("HTTP/1.1 404"), + "a child has no Setup" + ); + assert!(db.is_file(), "the named database is the one opened"); + + // The product's own boundaries: the dispatcher's assignment, the latency + // bind, the executed tool answers and the local send outcome. + crate::dispatch::notify_assigned("fixture-room"); + let mut latency = ReplyLatency::default(); + latency.bind("fixture-room", 7, 1, 1500); + super::acquisition::observe(&latency, &fixture_answers("call-1")) + .expect("the armed scope records the batch") + .complete(true); + stop.notify_one(); + server.join().unwrap().unwrap(); + + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert_eq!(report["outcome"], "ok", "{report}"); + assert_eq!(report["bound"], addr, "{report}"); + // Present and null, not merely absent: a missing key must not pass. + assert_eq!(report.get("arming_failure"), Some(&serde_json::Value::Null)); + let capture = &report["capture"]; + assert_eq!( + capture.get("failure"), + Some(&serde_json::Value::Null), + "{report}" + ); + assert!(capture["retained_bytes"].as_u64().unwrap() > 0, "{report}"); + assert!( + !report.to_string().contains("not retained"), + "arguments are not kept" + ); + assert_eq!(capture["retained_calls"], 1, "{report}"); + assert_eq!(capture["batches"].as_array().unwrap().len(), 1, "{report}"); + let batch = &capture["batches"][0]; + assert_eq!(batch["identity"]["room"], "fixture-room", "{report}"); + assert_eq!(batch["identity"]["session"], 7, "{report}"); + assert_eq!(batch["calls"][0]["id"], "call-1", "{report}"); + assert_eq!(batch["calls"][0]["name"], "read_editor", "{report}"); + assert_eq!(batch["calls"][0]["result"], "authored result", "{report}"); + assert_eq!(batch["local_send_completed"], true, "{report}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting() { + if crate::test_isolation::isolated_checked( + &format!( + "{}::a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting", + module_path!().split_once("::").unwrap().1 + ), + "CODETRIAL_SERVING_FIXTURE_DRAIN_TEST", + false, + |_| {}, + ) { + return; + } + let dir = scratch("drain"); + let mut settings = throwaway(&dir); + settings.drain = Duration::from_secs(1); + let (ready, out) = (settings.ready(), settings.out()); + let stop = std::sync::Arc::new(tokio::sync::Notify::new()); + let stopping = std::sync::Arc::clone(&stop); + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let result = serve(&settings, async move { stopping.notified().await }); + let _ = done.send(result); + }); + + let addr = wait_for(&ready); + // A login whose body never finishes arriving, held open past the stop. + use std::io::Write; + let mut held = std::net::TcpStream::connect(&addr).unwrap(); + write!( + held, + "POST /api/login HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/json\r\n\ + Content-Length: 1000\r\n\r\n{{\"partial\":" + ) + .unwrap(); + std::thread::sleep(Duration::from_millis(300)); + stop.notify_one(); + let result = finished + .recv_timeout(Duration::from_secs(20)) + .expect("the run reported within its drain bound"); + let error = result.unwrap_err(); + assert!(error.contains("shutdown bound"), "{error}"); + + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert!( + report["outcome"] + .as_str() + .unwrap() + .contains("shutdown bound"), + "{report}" + ); + drop(held); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_run_replaces_the_artifacts_an_earlier_run_left() { + if crate::test_isolation::isolated_checked( + &format!( + "{}::a_run_replaces_the_artifacts_an_earlier_run_left", + module_path!().split_once("::").unwrap().1 + ), + "CODETRIAL_SERVING_FIXTURE_STALE_TEST", + false, + |_| {}, + ) { + return; + } + let dir = scratch("stale-artifacts"); + let settings = throwaway(&dir); + std::fs::write(settings.out(), "{\"outcome\":\"ok\",\"stale\":true}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + // A run that fails to start leaves neither file behind. + let mut broken = throwaway(&dir); + broken.config = dir.join("missing.env.local"); + assert!(serve(&broken, async {}).is_err()); + assert!(!settings.out().exists() && !settings.ready().exists()); + std::fs::write(settings.out(), "{\"outcome\":\"ok\",\"stale\":true}").unwrap(); + std::fs::write(settings.ready(), "127.0.0.1:1").unwrap(); + + // A stop that has already happened: the run starts, serves nothing, + // reports. + serve(&settings, async {}).unwrap(); + let report: serde_json::Value = + serde_json::from_str(&std::fs::read_to_string(settings.out()).unwrap()).unwrap(); + assert!(report.get("stale").is_none(), "{report}"); + assert_eq!(report["outcome"], "ok", "{report}"); + let ready = std::fs::read_to_string(settings.ready()).unwrap(); + assert_ne!(ready, "127.0.0.1:1"); + assert_eq!(report["bound"], ready.as_str(), "{report}"); + assert!( + !settings.dir.join("out.partial").exists(), + "published by rename" + ); + let _ = std::fs::remove_dir_all(&dir); +} + +fn fixture_answers(id: &str) -> Vec<(crate::gemini::GeminiFunctionCall, serde_json::Value)> { + vec![( + crate::gemini::GeminiFunctionCall { + id: id.into(), + name: "read_editor".into(), + args: json!({"private": "not retained"}), + }, + json!("authored result"), + )] +} diff --git a/tests/unit/livekit/recovery.rs b/tests/unit/livekit/recovery.rs index 06ac8afc9..7ec841030 100644 --- a/tests/unit/livekit/recovery.rs +++ b/tests/unit/livekit/recovery.rs @@ -487,3 +487,5 @@ fn a_stream_end_or_a_new_stream_breaks_the_gap_and_replayed_frames_are_not_deque } include!("recovery_parts/report_generation_cardinality.rs"); + +include!("recovery_parts/acquisition_tool_sends.rs"); diff --git a/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs new file mode 100644 index 000000000..0a80fbc77 --- /dev/null +++ b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs @@ -0,0 +1,422 @@ +async fn acquisition_receive_tool(rig: &mut Rig, id: &str, tool: &str) { + let mut context = rig.turn.context( + &mut rig.output, + &mut rig.gemini, + &mut rig.board, + &mut rig.media, + ); + session::handle_gemini_event( + &rig.room, + &mut context, + crate::gemini::GeminiEvent::ToolCall(vec![crate::gemini::GeminiFunctionCall { + id: id.into(), + name: tool.into(), + args: serde_json::json!({"fromLine":1}), + }]), + Interruptible::Yes, + ) + .await + .unwrap(); +} + +async fn acquisition_receive_editor(rig: &mut Rig, id: &str) { + acquisition_receive_tool(rig, id, crate::runtime::TOOL_READ_EDITOR).await; +} + +async fn acquisition_actual_tool_send(failed: bool) { + use crate::livekit::probe::acquisition::{Capture, Limits, install}; + for (held, discarded) in [(false, false), (true, false), (false, true)] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + rig.turn.state.code = "def solve(): return 'send-result-sentinel'".into(); + if held { + rig.turn.state.thinking_hold = crate::agent::ThinkingHold::Held { + since: Instant::now(), + }; + } + rig.turn.activity.socket.discarding_output = discarded; + let now = Instant::now(); + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + rig.turn.activity.latency.anchor(now); + rig.turn.activity.latency.prompt(now); + let capture = Capture::new( + "recovery-test", + 7, + Limits { + batches: 1, + calls: 1, + bytes: 65536, + }, + ) + .unwrap(); + let installed = install(&capture).unwrap(); + if failed { + rig.gemini.shutdown().await.unwrap(); + } + acquisition_receive_editor(&mut rig, "actual-tool-send").await; + // Join the real peer before any assertion can end this fixture. + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 1); + let batch = &snapshot.batches[0]; + assert_eq!(batch.calls.len(), 1); + assert_eq!(batch.calls[0].id, "actual-tool-send"); + assert_eq!(batch.calls[0].name, crate::runtime::TOOL_READ_EDITOR); + assert!( + batch.calls[0].result["result"] + .as_str() + .unwrap() + .contains("send-result-sentinel") + ); + assert_eq!( + batch.local_send_completed, + Some(!failed), + "actual owner await result, independent of held/discarded timing gates" + ); + assert_eq!( + snapshot.retained_bytes, + serde_json::to_vec(&snapshot.batches).unwrap().len() + ); + let responses = frames + .iter() + .filter_map(|frame| frame.pointer("/toolResponse/functionResponses")) + .filter_map(serde_json::Value::as_array) + .flatten() + .collect::>(); + assert_eq!(responses.len(), usize::from(!failed)); + if let Some(response) = responses.first() { + assert_eq!(response["id"], "actual-tool-send"); + assert!( + response["response"] == batch.calls[0].result, + "actual wire result matches captured execution" + ); + } + } +} + +#[tokio::test] +async fn acquisition_tool_send_ok_records_real_completion() { + if isolated("acquisition_tool_send_ok_records_real_completion") { + return; + } + acquisition_actual_tool_send(false).await; +} + +#[tokio::test] +async fn acquisition_tool_send_error_records_returned_failure() { + if isolated("acquisition_tool_send_error_records_returned_failure") { + return; + } + acquisition_actual_tool_send(true).await; +} + +fn acquisition_gate_capture( + rig: &mut Rig, +) -> std::sync::Arc { + use crate::livekit::probe::acquisition::{Capture, Limits}; + let now = Instant::now(); + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + rig.turn.activity.latency.anchor(now); + rig.turn.activity.latency.prompt(now); + Capture::new( + "recovery-test", + 7, + Limits { + batches: 2, + calls: 2, + bytes: 65536, + }, + ) + .unwrap() +} + +#[tokio::test] +async fn acquisition_controlled_owner_await_cancellation_preserves_execution_boundary() { + if isolated("acquisition_controlled_owner_await_cancellation_preserves_execution_boundary") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + for stage in [ + Stage::Acknowledgement, + Stage::BoardSettlement, + Stage::BoardResend, + Stage::ToolSend, + ] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let tool = match stage { + Stage::BoardSettlement => { + rig.turn.state.example_drawing_enabled = true; + crate::runtime::TOOL_READ_BOARD + } + Stage::BoardResend => { + rig.turn.state.interview_mode = crate::agent::InterviewMode::Whiteboard; + rig.board.seed_recovery_image(vec![0x5a; 64]); + crate::runtime::TOOL_READ_BOARD + } + _ => crate::runtime::TOOL_READ_EDITOR, + }; + let capture = acquisition_gate_capture(&mut rig); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, stage, None).unwrap(); + let held = owner::install(&gate).unwrap(); + let mut call = Box::pin(acquisition_receive_tool(&mut rig, "cancelled-owner", tool)); + tokio::select! { + entered = gate.entered() => { entered.unwrap(); }, + () = &mut call => panic!("owner must reach its selected controlled await"), + () = tokio::time::sleep(Duration::from_secs(2)) => panic!("controlled owner gate did not enter"), + } + drop(call); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + let executed = matches!(stage, Stage::BoardResend | Stage::ToolSend); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), usize::from(executed), "{stage:?}"); + if let Some(batch) = snapshot.batches.first() { + assert_eq!(batch.calls[0].id, "cancelled-owner"); + assert_eq!( + batch.local_send_completed, None, + "controlled cancellation is unknown, not Err" + ); + } + assert!( + frames.is_empty(), + "controlled gate is before underlying I/O polling" + ); + } +} + +#[tokio::test] +async fn acquisition_admitted_ticket_completes_original_capture_after_reinstallation() { + if isolated("acquisition_admitted_ticket_completes_original_capture_after_reinstallation") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let original = acquisition_gate_capture(&mut rig); + let replacement = acquisition_gate_capture(&mut rig); + let installed = install(&original).unwrap(); + let gate = Gate::new(&original, 1, Stage::ToolSend, None).unwrap(); + let held = owner::install(&gate).unwrap(); + let actor = async { + gate.entered().await.unwrap(); + assert_eq!(original.snapshot().batches.len(), 1); + drop(installed); + let replacement_installed = install(&replacement).unwrap(); + gate.release(); + replacement_installed + }; + let ((), replacement_installed) = tokio::join!( + acquisition_receive_editor(&mut rig, "original-ticket"), + actor + ); + let frames = rig.old_frames().await; + assert_eq!(tool_response_ids(&frames), ["original-ticket"]); + let first = original.snapshot(); + let second = replacement.snapshot(); + drop(replacement_installed); + drop(held); + assert_eq!(first.failure, None); + assert_eq!(first.batches[0].local_send_completed, Some(true)); + assert_eq!(first.batches[0].calls[0].id, "original-ticket"); + assert_eq!(second.failure, None); + assert!( + second.batches.is_empty(), + "completion must never resolve the current ACTIVE capture" + ); +} + +async fn acquisition_timing_gate_divergence( + initial: bool, + transition: crate::livekit::probe::acquisition::owner::Transition, +) { + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage, Transition}, + }; + for failed in [false, true] { + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + match transition { + Transition::Hold(_) => { + if !initial { + rig.turn.state.thinking_hold = crate::agent::ThinkingHold::Held { + since: Instant::now(), + }; + } + } + Transition::Discard(_) => rig.turn.activity.socket.discarding_output = !initial, + } + let capture = acquisition_gate_capture(&mut rig); + rig.turn + .activity + .turn_timing + .note_transcript(Instant::now()); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, Stage::ToolSend, Some(transition)).unwrap(); + let held = owner::install(&gate).unwrap(); + if failed { + rig.gemini.shutdown().await.unwrap(); + } + let actor = async { + gate.entered().await.unwrap(); + gate.release(); + }; + let ((), ()) = tokio::join!( + acquisition_receive_editor(&mut rig, "divergent-gates"), + actor + ); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + let later = + !rig.turn.state.interviewer_silenced() && !rig.turn.activity.socket.discarding_output; + assert_eq!( + later, !initial, + "the synthetic transition must actually diverge the two gates" + ); + let timing = rig + .turn + .activity + .turn_timing + .finish( + Instant::now(), + crate::livekit::timing::TimingOutcome::NoAudio, + ) + .unwrap(); + assert_eq!( + timing.tool_calls, + u64::from(initial), + "initial timed sample independently controls execution accounting" + ); + assert_eq!(timing.tool_response_errors, u64::from(initial && failed)); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 1); + assert_eq!(snapshot.batches[0].local_send_completed, Some(!failed)); + assert_eq!(tool_response_ids(&frames).len(), usize::from(!failed)); + } +} + +fn acquisition_check_later_log(output: &std::process::Output, later: bool) { + let stderr = String::from_utf8_lossy(&output.stderr); + let logged = stderr + .lines() + .filter_map(|line| line.strip_prefix("codetrial latency ")) + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|record| record["stage"] == "tool_response") + .count(); + assert_eq!( + logged, + if later { 2 } else { 0 }, + "later log gate observed separately from the initial timed sample" + ); +} + +#[tokio::test] +async fn acquisition_tool_send_hold_turns_timing_off_after_initial_sample() { + if isolated_checked( + "acquisition_tool_send_hold_turns_timing_off_after_initial_sample", + |out| acquisition_check_later_log(out, false), + ) { + return; + } + acquisition_timing_gate_divergence( + true, + crate::livekit::probe::acquisition::owner::Transition::Hold(true), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_release_turns_logging_on_after_untimed_execution() { + if isolated_checked( + "acquisition_tool_send_release_turns_logging_on_after_untimed_execution", + |out| acquisition_check_later_log(out, true), + ) { + return; + } + acquisition_timing_gate_divergence( + false, + crate::livekit::probe::acquisition::owner::Transition::Hold(false), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_discard_turns_logging_off_after_timed_execution() { + if isolated_checked( + "acquisition_tool_send_discard_turns_logging_off_after_timed_execution", + |out| acquisition_check_later_log(out, false), + ) { + return; + } + acquisition_timing_gate_divergence( + true, + crate::livekit::probe::acquisition::owner::Transition::Discard(true), + ) + .await; +} + +#[tokio::test] +async fn acquisition_tool_send_discard_end_turns_logging_on_after_untimed_execution() { + if isolated_checked( + "acquisition_tool_send_discard_end_turns_logging_on_after_untimed_execution", + |out| acquisition_check_later_log(out, true), + ) { + return; + } + acquisition_timing_gate_divergence( + false, + crate::livekit::probe::acquisition::owner::Transition::Discard(false), + ) + .await; +} + +#[tokio::test] +async fn acquisition_consumed_gate_does_not_hold_a_later_tool_batch() { + if isolated("acquisition_consumed_gate_does_not_hold_a_later_tool_batch") { + return; + } + use crate::livekit::probe::acquisition::{ + install, + owner::{self, Gate, Stage}, + }; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + let capture = acquisition_gate_capture(&mut rig); + let installed = install(&capture).unwrap(); + let gate = Gate::new(&capture, 1, Stage::ToolSend, None).unwrap(); + let held = owner::install(&gate).unwrap(); + assert!(owner::install(&gate).is_err()); + let actor = async { + gate.entered().await.unwrap(); + gate.release(); + }; + let ((), ()) = tokio::join!(acquisition_receive_editor(&mut rig, "first-gated"), actor); + tokio::time::timeout( + Duration::from_secs(2), + acquisition_receive_editor(&mut rig, "later-ungated"), + ) + .await + .unwrap(); + let frames = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(held); + drop(installed); + assert_eq!(tool_response_ids(&frames), ["first-gated", "later-ungated"]); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 2); + assert!( + snapshot + .batches + .iter() + .all(|batch| batch.local_send_completed == Some(true)) + ); +} diff --git a/tests/unit/livekit/session.rs b/tests/unit/livekit/session.rs index 435e168b4..af24e53d5 100644 --- a/tests/unit/livekit/session.rs +++ b/tests/unit/livekit/session.rs @@ -362,63 +362,7 @@ fn continuity_reminders_leave_opening_default_and_inactive_inputs_alone() { } } -#[test] -fn real_tool_dispatch_records_only_tools_before_reply_audio() { - let start = Instant::now(); - let mut activity = RuntimeActivity::new(start); - let mut state = RuntimeState { - code: "return 1;".into(), - ..RuntimeState::default() - }; - let call = GeminiFunctionCall { - id: "read-1".into(), - name: TOOL_READ_EDITOR.into(), - args: serde_json::json!({}), - }; - activity.note_candidate_finished(start, false); - let response = execute_timed_tool_batch( - &mut state, - &mut activity, - vec![call.clone(), call.clone()], - 0, - true, - ); - assert!( - response[0].1["result"] - .as_str() - .is_some_and(|code| code.contains("return 1;")) - ); - let first = activity - .turn_timing - .note_audio(start + Duration::from_secs(1), true) - .unwrap(); - assert_eq!(first.tool_calls, 2); - assert_eq!(first.tool_wait, Some(first.tool_exec)); - assert!(first.pre_tool.is_some()); - execute_timed_tool_batch( - &mut state, - &mut activity, - vec![call.clone(), call.clone()], - 0, - true, - ); - assert!( - activity - .turn_timing - .finish(start + Duration::from_secs(2), TimingOutcome::NoAudio) - .is_none() - ); - activity.note_candidate_finished(start + Duration::from_secs(3), false); - execute_timed_tool_batch(&mut state, &mut activity, vec![call], 0, false); - let next = activity - .turn_timing - .note_audio(start + Duration::from_secs(4), true) - .unwrap(); - assert_eq!( - next.tool_calls, 0, - "discarded/held tool responses must not be carried forward" - ); -} +include!("session_parts/tool_execution.rs"); #[test] fn timing_transcripts_follow_the_existing_late_fragment_guard() { diff --git a/tests/unit/livekit/session/observations.rs b/tests/unit/livekit/session/observations.rs index cc3b80d46..7b640bd58 100644 --- a/tests/unit/livekit/session/observations.rs +++ b/tests/unit/livekit/session/observations.rs @@ -1,3 +1,5 @@ +use crate::test_isolation as isolation; + use super::*; use crate::livekit::session::{execute_tool_batch_observed, execute_tool_call}; use serde_json::json; @@ -147,6 +149,7 @@ fn accepted_close_request_is_observed_before_tool_batch_response_timing() { ], 0, true, + &mut None, ); assert_eq!(answers.len(), 2); assert!( @@ -216,3 +219,116 @@ fn accepted_close_request_is_observed_before_tool_batch_response_timing() { ); } } + +#[test] +fn executed_tool_capture_precedes_send_and_telemetry_gates() { + let name = format!( + "{}::executed_tool_capture_precedes_send_and_telemetry_gates", + module_path!().split_once("::").unwrap().1 + ); + if isolation::isolated_checked(&name, "CODETRIAL_ACQUISITION_CAPTURE_TEST", false, |_| {}) { + return; + } + use crate::livekit::probe::acquisition::{Capture, Failure, Limits, install}; + let capture = Capture::new( + "acquisition-fixture", + 7, + Limits { + batches: 3, + calls: 4, + bytes: 16384, + }, + ) + .unwrap(); + let installed = install(&capture).unwrap(); + assert!(install(&capture).is_err()); + let mut state = RuntimeState { + code: "def solve(items):\n return 'authored-result-sentinel'\n".into(), + ..RuntimeState::default() + }; + state.evidence_ledger.code.parser_observation = Some(crate::agent::CodeObservation::Parsed); + state.evidence_ledger.code.substantive_revision = 1; + let now = Instant::now(); + let mut activity = RuntimeActivity::for_interview(now, 0, 7); + activity.latency.bind("acquisition-fixture", 7, 1, 1500); + activity.latency.anchor(now); + activity.latency.prompt(now); + let call = |id: &str| GeminiFunctionCall { + id: id.into(), + name: TOOL_READ_EDITOR.into(), + args: json!({"fromLine":1}), + }; + let answers = execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("actual-received-call-id"), call("second-id")], + 0, + false, + &mut None, + ); + let snapshot = capture.snapshot(); + assert_eq!( + snapshot.batches.len(), + 1, + "executed answers must survive without any later send/timing observation" + ); + let batch = &snapshot.batches[0]; + for (captured, (executed, result)) in batch.calls.iter().zip(&answers) { + assert_eq!(captured.id, executed.id); + assert_eq!(captured.name, executed.name); + assert_eq!(&captured.result, result); + } + assert_eq!(batch.calls.len(), 2); + assert!( + batch.calls[0].result["result"] + .as_str() + .unwrap() + .contains("authored-result-sentinel") + ); + assert!(batch.calls[0].result.get("editorReview").is_none()); + assert!(batch.calls[1].result["editorReview"].as_str().is_some()); + assert_eq!(batch.local_send_completed, None); + assert_eq!(batch.identity.socket, 1); + assert!(batch.identity.trace_turn.is_some()); + let record = tool_timing( + &mut activity, + &state, + &answers, + HintCounters::capture(&state), + timing(false), + ); + assert!(!record.to_string().contains("actual-received-call-id")); + assert!(!record.to_string().contains("authored-result-sentinel")); + activity.latency.set_socket(2); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("replacement-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().batches[1].identity.socket, 2); + activity.latency.bind("foreign-room", 8, 3, 1500); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("foreign-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + assert_eq!(capture.snapshot().batches.len(), 2); + drop(installed); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call("uninstalled-id")], + 1, + false, + &mut None, + ); + assert_eq!(capture.snapshot().batches.len(), 2); + let _reinstalled = install(&capture).unwrap(); +} diff --git a/tests/unit/livekit/session_parts/tool_execution.rs b/tests/unit/livekit/session_parts/tool_execution.rs new file mode 100644 index 000000000..68bd41005 --- /dev/null +++ b/tests/unit/livekit/session_parts/tool_execution.rs @@ -0,0 +1,61 @@ +// Included in the session test module so existing test paths stay unchanged. + +#[test] +fn real_tool_dispatch_records_only_tools_before_reply_audio() { + let start = Instant::now(); + let mut activity = RuntimeActivity::new(start); + let mut state = RuntimeState { + code: "return 1;".into(), + ..RuntimeState::default() + }; + let call = GeminiFunctionCall { + id: "read-1".into(), + name: TOOL_READ_EDITOR.into(), + args: serde_json::json!({}), + }; + activity.note_candidate_finished(start, false); + let response = execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call.clone(), call.clone()], + 0, + true, + &mut None, + ); + assert!( + response[0].1["result"] + .as_str() + .is_some_and(|code| code.contains("return 1;")) + ); + let first = activity + .turn_timing + .note_audio(start + Duration::from_secs(1), true) + .unwrap(); + assert_eq!(first.tool_calls, 2); + assert_eq!(first.tool_wait, Some(first.tool_exec)); + assert!(first.pre_tool.is_some()); + execute_timed_tool_batch( + &mut state, + &mut activity, + vec![call.clone(), call.clone()], + 0, + true, + &mut None, + ); + assert!( + activity + .turn_timing + .finish(start + Duration::from_secs(2), TimingOutcome::NoAudio) + .is_none() + ); + activity.note_candidate_finished(start + Duration::from_secs(3), false); + execute_timed_tool_batch(&mut state, &mut activity, vec![call], 0, false, &mut None); + let next = activity + .turn_timing + .note_audio(start + Duration::from_secs(4), true) + .unwrap(); + assert_eq!( + next.tool_calls, 0, + "discarded/held tool responses must not be carried forward" + ); +} From 95bf2656f14c5ddc8a6c0d96b39d3523c3d20934 Mon Sep 17 00:00:00 2001 From: Jim Huang Date: Fri, 6 Nov 2026 10:51:33 +0800 Subject: [PATCH 7/7] Judge long interviews from the fixture's evidence Run the serving fixture as the browser check's server from the pinned test binary, and capture what it needs to judge reply content: each received tool call, each interviewer segment and each turn ending with its socket and what preceded it. After the fixture stops, keep its report and log owner only, and judge the capture's reply content without changing the run's exit status or qualification. A checkpoint is credited only when the facts leave one reading; otherwise it is not_assessed with the reason. The fixture takes a TERM as a stop request and its stop is bounded, and stand-ins run the whole path from the manifest to the exit handler without a browser. Tests that rerun in a child process now take their own name from the test. Change-Id: Ibfe233474f69a3f316eb1d0991619a8d2fd99e4e --- Cargo.toml | 5 +- docs/development.md | 22 +- docs/latency-observability.md | 13 +- scripts/browser-check.cjs | 1 + scripts/browser-check.sh | 175 ++++- scripts/fixture-server.sh | 188 +++++ scripts/interview-run-budget.py | 10 +- scripts/live-sdk-fixture.sh | 6 +- scripts/long-interview-driver.cjs | 6 + scripts/long-interview-evidence.cjs | 600 +++++++++++++++ src/livekit/probe.rs | 1 + src/livekit/probe/acquisition.rs | 398 +++++++++- src/livekit/session.rs | 21 +- src/livekit/session/observations.rs | 37 +- src/livekit/session/tool_calls.rs | 7 + tests/browser/long-interview-driver.test.js | 6 + tests/browser/long-interview-evidence.test.js | 723 ++++++++++++++++++ tests/fixtures/README.md | 1 + tests/fixtures/acquisition-report.json | 208 +++++ .../stubs/interview-run-budget/fake_worker.py | 1 + .../interview-run-budget/stand-in-fixture.py | 73 ++ .../interview-run-budget/stand-in-node.sh | 10 + tests/test_interview_run_budget.py | 525 +++++++++++++ tests/unit/gemini/socket_loopback.rs | 6 +- tests/unit/isolation.rs | 19 + tests/unit/livekit/handoff.rs | 11 +- tests/unit/livekit/probe/acquisition.rs | 517 ++++++++++++- tests/unit/livekit/probe/serving.rs | 87 ++- .../recovery_parts/acquisition_tool_sends.rs | 254 ++++++ tests/unit/livekit/session/observations.rs | 6 +- 30 files changed, 3806 insertions(+), 131 deletions(-) create mode 100755 scripts/fixture-server.sh create mode 100644 scripts/long-interview-evidence.cjs create mode 100644 tests/browser/long-interview-evidence.test.js create mode 100644 tests/fixtures/acquisition-report.json create mode 100644 tests/stubs/interview-run-budget/stand-in-fixture.py create mode 100644 tests/stubs/interview-run-budget/stand-in-node.sh diff --git a/Cargo.toml b/Cargo.toml index f4866e30d..c99687383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,8 +134,9 @@ livekit-protocol = "0.8.2" # LiveKit's transport, reached directly to check the vendored proxy path. livekit-net = "0.1.4" # The paused clock, so the Live socket's keepalive and idle limits are tested -# at their real values instead of waiting them out. -tokio = { version = "1.54.1", features = ["test-util"] } +# at their real values instead of waiting them out; signals, so the serving +# fixture can treat TERM as a stop request. +tokio = { version = "1.54.1", features = ["signal", "test-util"] } # The continuity fixture installs a logger to read the SDK's queue-overflow # line; the SDK already depends on this crate. log = "0.4.34" diff --git a/docs/development.md b/docs/development.md index 9aaa5c760..d9b9063cb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1069,8 +1069,10 @@ python3 scripts/interview-run-budget.py run \ `validate` is free and should always precede `run`. `run` passes the worker `BROWSER_CHECK_RUN_MANIFEST`, `BROWSER_CHECK_RUN_ID`, `BROWSER_CHECK_RUN_MANIFEST_SHA256` (the hash of the manifest bytes), -`BROWSER_CHECK_BUDGET_SOCKET`, `BROWSER_CHECK_BUDGET_TELEMETRY` and -`BROWSER_CHECK_BUDGET_STOP`. `scripts/browser-check.sh` refuses a partial set, a +`BROWSER_CHECK_BUDGET_SOCKET`, `BROWSER_CHECK_BUDGET_TELEMETRY`, +`BROWSER_CHECK_BUDGET_STOP` and `BROWSER_CHECK_BUDGET_REAP_AT` (when its reap +begins, in Unix seconds, which the serving fixture's own deadline is set +before). `scripts/browser-check.sh` refuses a partial set, a run ID or hash that disagrees with the manifest, a binary whose sha256 changed, and `CODETRIAL_WEB_URL`, since an external server is shared and was not built from the pinned executable. With `BROWSER_CHECK_VALIDATE_ENV_ONLY=1` it stops @@ -1132,6 +1134,22 @@ reserve is revalidated before launch against the resolved handover grace at or after the boundary is `late_idle` or `late_terminal_preempted` and never counts toward the two completed rotations a run needs. +`scripts/long-interview-evidence.cjs` reads the acquisition report the test +fixture leaves (receipts, executed batches, interviewer segments and turn +endings, each with its socket and how many of the others preceded it) and gives +`judgeCheckpoint` its evidence. It claims an association only when the facts +leave one reading: a reply cites the calls received on its own socket after the +previous ending, executed before it began, and not received while output was +being discarded. A swallowed or missing ending, a segment sealed by a cut, a +reply that spans a tool call, a socket replacement, a rewritten segment, two +replies in one window, a window edge inside the clock uncertainty, a reply +that began after the question but before its estimated end (a recording can +finish speaking sooner than its word count says) or a latched capture failure +makes the checkpoint `not_assessed` with the reason, never a pass. The +interview is `verified` only when every checkpoint passed. +`tests/fixtures/acquisition-report.json` is real producer output, regenerated +with `CODETRIAL_UPDATE_FIXTURES=1` on the test that writes it. + What a reply is judged on is what its question asks and no more (`scripts/long-interview-schedule.cjs`). An inspect question asks about the duplicate case, so the revision is judged in what the tool read (the answer diff --git a/docs/latency-observability.md b/docs/latency-observability.md index 61bc3e51c..da8cb9272 100644 --- a/docs/latency-observability.md +++ b/docs/latency-observability.md @@ -220,15 +220,18 @@ Run the replay and the long-interview report checks without credentials: ```sh python3 scripts/run-python-tests.py tests/test_analyze_latency.py -node --test tests/browser/soak-judge.test.js tests/browser/long-interview-driver.test.js +node --test tests/browser/soak-judge.test.js tests/browser/long-interview-driver.test.js \ + tests/browser/long-interview-evidence.test.js ``` A long-interview report's `assessment_scope` is `structural`, even when `run_kind` declares a qualification workload. Its `qualification` remains -`not_evaluated`: checkpoint `reply_content` is `not_assessed` by this driver, -and post-swap `decoded_signal` (`verified`, `not_required`, `failed` or -`not_assessed`) is reported beside it and changes neither the qualification -nor the exit code. Separate evidence is required before qualifying a build. `exposure_s` ends at the harness's observation boundary; +`not_evaluated`. Checkpoint `reply_content` is `not_assessed` by this driver; +in serving-fixture mode `scripts/browser-check.sh` judges it from the +fixture's report once the fixture has stopped and writes the verdict into the +capture. Post-swap `decoded_signal` (`verified`, `not_required`, `failed` or +`not_assessed`) is reported beside it. Neither changes the qualification or the +exit code. Separate evidence is required before qualifying a build. `exposure_s` ends at the harness's observation boundary; the analyzer's exposure can extend through the terminal socket's end and must not be substituted for it. `end_permit_granted` records permission, while `end_clicked` records the page action; the countdown can win after the permit diff --git a/scripts/browser-check.cjs b/scripts/browser-check.cjs index a66411666..c8b7c6541 100644 --- a/scripts/browser-check.cjs +++ b/scripts/browser-check.cjs @@ -937,6 +937,7 @@ async function longInterview( log.endsWith("\n"); const finished = finalizeLongRun({ plan, + mode: prepared.mode, schedule: prepared.schedule, playback: prepared.playback, clipSource: prepared.clipSource, diff --git a/scripts/browser-check.sh b/scripts/browser-check.sh index 64a64862c..7936d1e79 100755 --- a/scripts/browser-check.sh +++ b/scripts/browser-check.sh @@ -137,11 +137,56 @@ LOGIN_DB_PATH="$TMP/accounts.db" LOGIN_SESSION_SECRET=browser-check-session CONFIG_PATH="$TMP/codetrial.env.local" +# The in-process serving fixture (scripts/fixture-server.sh): a pinned test +# executable run as the server in place of `codetrial web`, so the interview +# leaves behind what the arming observer saw. Off unless a test name is given. +FIXTURE_TEST=${BROWSER_CHECK_FIXTURE_TEST:-} +FIXTURE_DIR="$TMP/fixture" +FIXTURE_STOP_WAIT_S=${BROWSER_CHECK_FIXTURE_STOP_WAIT_S:-150} + +# The fixture's connection drain, passed to it, and the slack left after its +# settle: the settle is whatever the stop wait has left after both. +FIXTURE_DRAIN_S=10 +FIXTURE_SLACK_S=20 + +# Asked to stop and waited for, not killed: the serving fixture has connections +# to drain, rooms and reports to finish and a report of its own to write, and +# that report is kept before the scratch directory goes. +cleanup_fixture() +{ + fixture_status=0 + "$ROOT/scripts/fixture-server.sh" stop "$FIXTURE_DIR" "$SERVER_PID" "$FIXTURE_STOP_WAIT_S" \ + || fixture_status=1 + wait "$SERVER_PID" 2> /dev/null || { + echo "The serving fixture exited with a failure; its report may say why." >&2 + fixture_status=1 + } + + "$ROOT/scripts/fixture-server.sh" keep "$FIXTURE_DIR" "$BROWSER_CHECK_FIXTURE_OUT" "$SERVER_LOG" \ + || fixture_status=1 + return "$fixture_status" +} + +# The reply content is judged once the serving fixture has stopped and its +# report is kept: the worker finished before that and could not read it. The +# verdict is written into the capture file and never into this script's own +# status, so a run that could not be judged is still the run it was. +judge_reply_content() +{ + [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -f "$BROWSER_CHECK_CAPTURE" ] || return 0 + node "$ROOT/scripts/long-interview-evidence.cjs" \ + --capture "$BROWSER_CHECK_CAPTURE" --fixture "$BROWSER_CHECK_FIXTURE_OUT" \ + || echo "Reply content was not judged; the run's own result stands." >&2 +} + cleanup() { cleanup_status=$? trap - EXIT INT TERM - if [ "${SERVER_PID:-}" ]; then + if [ "${SERVER_PID:-}" ] && [ -n "$FIXTURE_TEST" ]; then + cleanup_fixture || cleanup_status=1 + judge_reply_content + elif [ "${SERVER_PID:-}" ]; then kill "$SERVER_PID" 2> /dev/null || true wait "$SERVER_PID" 2> /dev/null || true fi @@ -278,6 +323,82 @@ if [ -n "$BUDGET_VARS" ]; then --manifest "$BROWSER_CHECK_RUN_MANIFEST" candidate) || exit 2 fi +if [ -n "$FIXTURE_TEST" ]; then + if [ -z "$PREBUILT_BIN" ] || [ "$BROWSER_CHECK_AGENT" != dispatch ]; then + echo "BROWSER_CHECK_FIXTURE_TEST needs a budgeted run (its pinned executable is the" >&2 + echo "fixture) and BROWSER_CHECK_AGENT=dispatch." >&2 + exit 2 + fi + if [ -z "${BROWSER_CHECK_FIXTURE_OUT:-}" ] || [ -e "$BROWSER_CHECK_FIXTURE_OUT" ] \ + || [ -e "$BROWSER_CHECK_FIXTURE_OUT.server.log" ] \ + || [ -L "$BROWSER_CHECK_FIXTURE_OUT.server.log" ] \ + || [ ! -d "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" ] \ + || [ ! -w "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" ]; then + echo "BROWSER_CHECK_FIXTURE_OUT must name a report file that does not exist yet," >&2 + echo "in a directory that does and can be written." >&2 + exit 2 + fi + + # The verdict is written into the capture after the fixture stops, so a + # capture left by an earlier run would be judged against this run's report. + if [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -e "$BROWSER_CHECK_CAPTURE" ]; then + echo "BROWSER_CHECK_CAPTURE must not exist yet when the serving fixture runs." >&2 + exit 2 + fi + + # Neither exists yet, so their directories are compared: the report kept at + # cleanup would otherwise replace the capture the run just wrote. + if [ -n "${BROWSER_CHECK_CAPTURE:-}" ] && [ -d "$(dirname -- "$BROWSER_CHECK_CAPTURE")" ]; then + capture_at="$(cd -P -- "$(dirname -- "$BROWSER_CHECK_CAPTURE")" && pwd)/$(basename -- "$BROWSER_CHECK_CAPTURE")" + out_at="$(cd -P -- "$(dirname -- "$BROWSER_CHECK_FIXTURE_OUT")" && pwd)/$(basename -- "$BROWSER_CHECK_FIXTURE_OUT")" + + # Each also stages files beside it (the log beside the report, the + # judged capture at .partial), so neither may be a prefix of the other. + case $out_at in "$capture_at"*) collides=1 ;; *) collides= ;; esac + case $capture_at in "$out_at"*) collides=1 ;; esac + if [ -n "$collides" ]; then + echo "BROWSER_CHECK_CAPTURE and BROWSER_CHECK_FIXTURE_OUT must name different files." >&2 + exit 2 + fi + fi + + # A leading zero too: shell arithmetic reads it as octal while the tests + # below read it as decimal, so the reserve check would ask for too little. + case $FIXTURE_STOP_WAIT_S in + '' | *[!0-9]* | 0[0-9]*) + echo "BROWSER_CHECK_FIXTURE_STOP_WAIT_S must be a number of seconds." >&2 + exit 2 + ;; + esac + + # One number drives both sides: the fixture settles for this long less its + # connection drain and the slack, so a longer settle cannot make the wait + # below give up on a fixture that is still working. At least 10 s of it. + fixture_min_wait_s=$((FIXTURE_DRAIN_S + FIXTURE_SLACK_S + 10)) + if [ "$FIXTURE_STOP_WAIT_S" -lt "$fixture_min_wait_s" ]; then + echo "BROWSER_CHECK_FIXTURE_STOP_WAIT_S must be at least $fixture_min_wait_s seconds." >&2 + exit 2 + fi + + # The stop waits for a drain and a settle, so the cleanup window has to + # outlast it with the launcher's own cleanup still to come. + reserve=$("$BUDGET_PYTHON" "$BUDGET_TOOL" validate \ + --manifest "$BROWSER_CHECK_RUN_MANIFEST" --print cleanup_reserve_s) || exit 2 + if [ "$reserve" -lt $((FIXTURE_STOP_WAIT_S + 60)) ]; then + echo "cleanup_reserve_s is $reserve; the serving fixture's stop wait of" >&2 + echo "${FIXTURE_STOP_WAIT_S}s needs at least $((FIXTURE_STOP_WAIT_S + 60))s of it." >&2 + exit 2 + fi + + # When the budget runner starts its reap, for the fixture's own deadline. + case ${BROWSER_CHECK_BUDGET_REAP_AT:-} in + '' | *[!0-9]*) + echo "BROWSER_CHECK_FIXTURE_TEST needs BROWSER_CHECK_BUDGET_REAP_AT from the budget runner." >&2 + exit 2 + ;; + esac +fi + # The duration, run kind, end mode and observation target of a long interview # are judged before anything is built or started. The same module answers when # browser-check.cjs is run directly. @@ -292,6 +413,11 @@ if [ "${BROWSER_CHECK_VALIDATE_ENV_ONLY:-}" ]; then exit 0 fi +# Runs the pin, so it waits until validation-only runs are out of the way. +if [ -n "$FIXTURE_TEST" ]; then + "$ROOT/scripts/fixture-server.sh" check "$PREBUILT_BIN" "$FIXTURE_TEST" || exit 2 +fi + # Anything that serves `web/` needs the fetched vendor binaries, and this script # launches its own server rather than going through the Makefile. Without this a # fresh clone serves a MediaPipe loader whose .wasm and model 404, and face @@ -400,10 +526,49 @@ else # mode that a missing dispatch breaks, and nothing else covers it. unset INTERVIEW_ROOM_NAME DISPATCH_MODE=1 - SESSION_SECRET="$LOGIN_SESSION_SECRET" \ - CODETRIAL_DB_PATH="$LOGIN_DB_PATH" \ - "$@" web --config "$CONFIG_PATH" --web-addr "127.0.0.1:$PORT" --web-dir "$ROOT/web" \ - > "$SERVER_LOG" 2>&1 & + if [ -n "$FIXTURE_TEST" ]; then + + # The pinned test executable, running only the fixture. Its + # account database is inside its own directory by construction. + mkdir "$FIXTURE_DIR" + + # In a session of its own where it can be, so a signal sent to + # this script's process group reaches this script, whose trap + # asks the fixture to stop and waits, and not the fixture first: + # an INT would end it without a report, and a TERM in its first + # moments, before it can take one as a stop request, likewise. + # This script has no job control, so setsid runs the fixture in + # place and $! is the fixture; were it to fork, wait_for_server + # would see the launcher exit and fail the run at once. + set -- "$PREBUILT_BIN" --ignored --exact "$FIXTURE_TEST" --nocapture --test-threads=1 + if command -v setsid > /dev/null 2>&1; then + set -- setsid "$@" + fi + + # A backstop only: cleanup asks it to stop first. As late as it + # can be and still leave the stop wait and some margin before + # the runner's reap, counted from now, since the fixture counts + # its deadline from its own start. + FIXTURE_DEADLINE_S=$((BROWSER_CHECK_BUDGET_REAP_AT - $(date +%s) - \ + FIXTURE_STOP_WAIT_S - FIXTURE_SLACK_S)) + if [ "$FIXTURE_DEADLINE_S" -le 0 ]; then + echo "No wall time is left for the serving fixture to run and stop." >&2 + exit 2 + fi + SESSION_SECRET="$LOGIN_SESSION_SECRET" \ + CODETRIAL_FIXTURE_CONFIG="$CONFIG_PATH" \ + CODETRIAL_FIXTURE_DIR="$FIXTURE_DIR" \ + CODETRIAL_FIXTURE_ADDR="127.0.0.1:$PORT" \ + CODETRIAL_FIXTURE_DEADLINE_S="$FIXTURE_DEADLINE_S" \ + CODETRIAL_FIXTURE_DRAIN_S="$FIXTURE_DRAIN_S" \ + CODETRIAL_FIXTURE_SETTLE_S="$((FIXTURE_STOP_WAIT_S - FIXTURE_DRAIN_S - FIXTURE_SLACK_S))" \ + "$@" > "$SERVER_LOG" 2>&1 & + else + SESSION_SECRET="$LOGIN_SESSION_SECRET" \ + CODETRIAL_DB_PATH="$LOGIN_DB_PATH" \ + "$@" web --config "$CONFIG_PATH" --web-addr "127.0.0.1:$PORT" --web-dir "$ROOT/web" \ + > "$SERVER_LOG" 2>&1 & + fi ;; rust) INTERVIEW_ROOM_NAME=${INTERVIEW_ROOM_NAME:-interview-browser-$(date +%s)-$$} diff --git a/scripts/fixture-server.sh b/scripts/fixture-server.sh new file mode 100755 index 000000000..ef2b0363a --- /dev/null +++ b/scripts/fixture-server.sh @@ -0,0 +1,188 @@ +#!/bin/sh + +# The two steps scripts/browser-check.sh needs around the in-process serving +# fixture (tests/unit/livekit/probe/serving.rs), kept here so they can be run +# against a stand-in executable without a browser. +# +# fixture-server.sh check PIN TEST +# PIN is the pinned test executable. Fails unless it lists TEST as +# exactly one ignored test, so the run cannot start some other test, or +# none, under a name that only looks right. It runs the pin with --list, +# which builds nothing. +# +# fixture-server.sh stop DIR PID WAIT_SECONDS +# Asks the fixture running as PID to stop by creating DIR/stop, then waits +# up to WAIT_SECONDS for it to finish its drain and settle and write its +# report. A fixture that has not exited by then is terminated (killed after +# FIXTURE_SERVER_TERM_GRACE_S, default 5), and that is a failure: its +# report cannot be trusted to be complete. The fixture also treats a TERM +# that reaches it as the same request, but only until it has begun to stop: +# after the stop file it ignores TERM, so the escalation is the kill. +# When DIR/stop cannot be written, the request is a TERM with the same +# wait, and that too is a failure. +# +# fixture-server.sh keep DIR OUT SERVER_LOG +# Keeps what the fixture left before the scratch directory goes: the +# server log beside the report as OUT.server.log (owner only), and DIR/out.json +# as OUT. When the fixture wrote no report, OUT holds one that says so +# and the log tail is printed. Fails when anything could not be kept, and +# when the kept report lacks the shape the judge reads or does not say +# the fixture ended well. + +set -u + +here=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +usage() +{ + echo "usage: ${0##*/} check PIN TEST | stop DIR PID WAIT_SECONDS | keep DIR OUT SERVER_LOG" >&2 + exit 2 +} + +# A child that has exited but not been waited for still answers kill -0, and the +# caller is the one that waits for it, so once the cheap probe says yes its +# state is asked for. +alive() +{ + kill -0 "$1" 2> /dev/null || return 1 + state=$(ps -o stat= -p "$1" 2> /dev/null) || return 1 + + # Some systems pad the column. + state=${state#"${state%%[! ]*}"} + case $state in + '' | [Zz]*) return 1 ;; + *) return 0 ;; + esac +} + +# Whether $1 exited within $2 seconds. +exited_within() +{ + waited=0 + while alive "$1" && [ "$waited" -lt "$2" ]; do + sleep 1 + waited=$((waited + 1)) + done + ! alive "$1" +} + +check() +{ + [ $# -eq 2 ] || usage + pin=$1 + test_name=$2 + case $test_name in + '' | *[[:space:]]*) + echo "fixture-server: the test name must be one exact name" >&2 + return 2 + ;; + esac + if [ ! -x "$pin" ]; then + echo "fixture-server: $pin is not an executable" >&2 + return 2 + fi + listed=$("$pin" --list --ignored 2> /dev/null) || { + echo "fixture-server: $pin would not list its ignored tests" >&2 + return 2 + } + count=$(printf '%s\n' "$listed" | grep -cxF "$test_name: test") + if [ "$count" != 1 ]; then + echo "fixture-server: $pin lists $count ignored tests named $test_name, not 1" >&2 + return 2 + fi +} + +stop() +{ + [ $# -eq 3 ] || usage + dir=$1 + pid=$2 + wait_seconds=$3 + + # Not 0 or a leading zero, since `kill 0` signals this script's whole + # process group rather than the fixture, and not 1, a container's init. + case $pid in + '' | *[!0-9]* | 0* | 1) usage ;; + esac + case $wait_seconds in + '' | *[!0-9]*) usage ;; + esac + grace=${FIXTURE_SERVER_TERM_GRACE_S:-5} + + # Not `:`, whose failed redirection would end this script, a special + # built-in's, and leave the caller waiting on a fixture nothing asked to + # stop. + if ! true > "$dir/stop"; then + echo "fixture-server: could not write the stop file; terminating the fixture" >&2 + + # While it serves, the fixture takes a TERM as the same request as the + # stop file, so the TERM gets the whole wait before the kill. + grace=$wait_seconds + elif exited_within "$pid" "$wait_seconds"; then + return 0 + else + echo "fixture-server: the fixture had not stopped after ${wait_seconds}s; terminating it" >&2 + fi + kill -TERM "$pid" 2> /dev/null || true + exited_within "$pid" "$grace" || kill -KILL "$pid" 2> /dev/null || true + return 1 +} + +keep() +{ + [ $# -eq 3 ] || usage + dir=$1 + out=$2 + server_log=$3 + kept_status=0 + + # Beside the report, because the scratch directory goes next and the + # server's own log is the only account of why a fixture misbehaved. Every + # write here refuses a file that is already there (set -C), so nothing kept + # can replace another run's file or the capture this one just wrote. + if ! (umask 077 && set -C && cat "$server_log" > "$out.server.log") 2> /dev/null; then + echo "fixture-server: could not keep the server log beside the report" >&2 + kept_status=1 + fi + if [ ! -f "$dir/out.json" ]; then + echo "fixture-server: the serving fixture left no report; the end of its log:" >&2 + tail -n 40 "$server_log" >&2 || true + + # A report that says so, so a reader of the report path never finds + # nothing where a verdict was owed. + (umask 077 && set -C && printf '%s\n' '{"outcome":"the serving fixture left no report","arming_failure":null,"capture":null}' \ + > "$out") 2> /dev/null \ + || echo "fixture-server: could not write that at $out; a file already there is not this run's report" >&2 + return 1 + fi + + # Owner only, like the log: it holds the interviewer's replies and the tool + # results. + if ! (umask 077 && set -C && cat "$dir/out.json" > "$out"); then + + # The scratch directory goes next, and this is the only copy. + echo "fixture-server: could not keep the serving fixture's report; it said:" >&2 + cat "$dir/out.json" >&2 + return 1 + fi + + # Accepted only if it has the shape the judge reads and says it ended well. + node "$here/long-interview-evidence.cjs" --check-fixture "$out" || kept_status=1 + return "$kept_status" +} + +case ${1:-} in + check) + shift + check "$@" + ;; + stop) + shift + stop "$@" + ;; + keep) + shift + keep "$@" + ;; + *) usage ;; +esac diff --git a/scripts/interview-run-budget.py b/scripts/interview-run-budget.py index e7936f811..2773883f0 100644 --- a/scripts/interview-run-budget.py +++ b/scripts/interview-run-budget.py @@ -1385,6 +1385,9 @@ def launch(args, command): os.chmod(socket_dir, 0o700) socket_path = os.path.join(socket_dir, "permit.sock") t0 = time.monotonic() + # When the reap begins, on the wall clock, for a worker that must be done + # before it (the serving fixture's own deadline). + reap_at = int(time.time() + reap_start) faults = [] service = PermitService( manifest, @@ -1405,6 +1408,7 @@ def launch(args, command): BROWSER_CHECK_BUDGET_SOCKET=socket_path, BROWSER_CHECK_BUDGET_TELEMETRY=telemetry_path, BROWSER_CHECK_BUDGET_STOP=stop_file, + BROWSER_CHECK_BUDGET_REAP_AT=str(reap_at), ) # A launcher that is told to stop must stop the worker with it: the worker # sits in its own session and would otherwise keep spending. @@ -1698,6 +1702,8 @@ def command_validate(args): print(digest) elif args.print_field == "run_id": print(manifest["run_id"]) + elif args.print_field == "cleanup_reserve_s": + print(manifest["cleanup_reserve_s"]) else: print(f"ok {manifest['run_id']} {digest}") return 0 @@ -1725,7 +1731,9 @@ def main(argv=None): validate = commands.add_parser("validate", help="check a manifest") validate.add_argument("--manifest", required=True) validate.add_argument("--verify-binaries", action="store_true") - validate.add_argument("--print", dest="print_field", choices=("sha", "run_id")) + validate.add_argument( + "--print", dest="print_field", choices=("sha", "run_id", "cleanup_reserve_s") + ) validate.add_argument( "--min-cleanup-reserve-s", type=int, default=MIN_CLEANUP_RESERVE_S ) diff --git a/scripts/live-sdk-fixture.sh b/scripts/live-sdk-fixture.sh index 7ee1da6a3..1182f66ac 100755 --- a/scripts/live-sdk-fixture.sh +++ b/scripts/live-sdk-fixture.sh @@ -126,9 +126,9 @@ launch() || die "the pinned binary changed since build" artifacts=$dir/artifacts-$letter [ ! -e "$artifacts" ] || die "$artifacts exists; a scenario runs once per directory" - name=$("$bin" --list --ignored 2> /dev/null \ - | sed -n "s/: test\$//p" | grep "^${live_test}\$") \ - || die "the pinned binary lists no live fixture" + name=$live_test + sh "$ROOT/scripts/fixture-server.sh" check "$bin" "$name" \ + || die "the pinned binary does not list $name exactly once" writer=$("$bin" --list 2> /dev/null \ | sed -n "s/: test\$//p" | grep "^${TEST_FILTER}live_write_manifest\$") \ || die "the pinned binary lists no manifest writer" diff --git a/scripts/long-interview-driver.cjs b/scripts/long-interview-driver.cjs index c1579b062..5458afc51 100644 --- a/scripts/long-interview-driver.cjs +++ b/scripts/long-interview-driver.cjs @@ -595,6 +595,12 @@ function finalizeLongRun(o) { cleanup_assessed: cleanupAssessed, end_permit_granted: run.endPermitGranted, end_clicked: run.endClicked, + // Where the schedule's clock began, so a reader after teardown can put + // the interview clock's readings on it. + schedule_origin_unix_ms: run.joinedAtMs, + // The surface the schedule was generated for, which the plan alone does + // not name and a reader rebuilding the schedule needs. + schedule_mode: o.mode, schedule_start_delay_s: (run.joinedAtMs - run.interviewStartMs) / 1000, exposure_s: stamps.exposureS, timestamps: stamps.timestamps, diff --git a/scripts/long-interview-evidence.cjs b/scripts/long-interview-evidence.cjs new file mode 100644 index 000000000..94f9b0e06 --- /dev/null +++ b/scripts/long-interview-evidence.cjs @@ -0,0 +1,600 @@ +"use strict"; + +// Turns the fixture's acquisition report into the evidence `judgeCheckpoint` +// reads, and says so plainly when it cannot. The report holds ordered facts +// only (receipts, batches, reply segments and turn endings, each with the +// socket it came from and how many of the others preceded it); this module +// is where they become a claim, and a claim is made only when the facts leave +// one reading. Anything else is `not_assessed` with the reason, never a pass +// and never a guess at a failure. + +const fs = require("node:fs"); + +const sched = require("./long-interview-schedule.cjs"); + +const { generateSchedule, judgeCheckpoint } = sched; + +// A reading whose sampling uncertainty exceeds this cannot place an event +// against a window edge. +const MAX_CLOCK_UNCERTAINTY_MS = 1000; + +// The lists a capture holds, which the judge and the acceptance check both read. +const CAPTURE_LISTS = ["receipts", "batches", "replies", "boundaries"]; + +const isCount = (value) => Number.isSafeInteger(value) && value >= 0; + +/// Why the report cannot be read at all, or an empty list. +function reportProblems(report) { + const problems = []; + if (!report || typeof report !== "object") return ["no report"]; + if (report.arming_failure != null) { + problems.push(`the observer failed: ${report.arming_failure}`); + } + const capture = report.capture; + if (!capture || typeof capture !== "object") { + problems.push("no capture was admitted"); + return problems; + } + if (capture.failure != null) { + problems.push(`the capture latched ${capture.failure}`); + } + for (const name of CAPTURE_LISTS) { + if (!Array.isArray(capture[name])) problems.push(`${name} is missing`); + } + if (problems.length) return problems; + // A row the judge walks is readable when it is there, carries its + // identity and passes its own check; one that is not makes the capture one + // it cannot judge, never an exception halfway through the merge. + const readable = (rows, ok) => + rows.every((row) => row && row.identity && ok(row)); + const counted = (rows, fields) => + readable(rows, (row) => fields.every((field) => isCount(row[field]))); + if ( + !counted(capture.replies, [ + "receipts_before", + "batches_before", + "receipts_last", + "batches_last", + ]) + ) { + problems.push("a reply has no positions"); + } + if ( + !counted(capture.boundaries, [ + "receipts_before", + "batches_before", + "replies_before", + ]) + ) { + problems.push("a boundary has no positions"); + } + const hasId = (row) => row && typeof row.id === "string"; + if ( + !readable( + capture.batches, + (batch) => Array.isArray(batch.calls) && batch.calls.every(hasId), + ) + ) { + problems.push("a batch has no calls"); + } + if (!readable(capture.receipts, hasId)) { + problems.push("a receipt has no id"); + } + return problems; +} + +/// A reading of the interview clock as seconds since the schedule began with +/// its own sampling uncertainty, or null when it cannot place an event. +function readClock(identity, scheduleStartUnixMs, maxUncertaintyMs) { + const clock = identity?.clock; + if (!clock || !Number.isFinite(clock.emitted_unix_ms)) return null; + const uncertainty = clock.clock_uncertainty_ms; + if (!Number.isFinite(uncertainty) || uncertainty > maxUncertaintyMs) { + return null; + } + return { + s: (clock.emitted_unix_ms - scheduleStartUnixMs) / 1000, + u: uncertainty / 1000, + }; +} + +/// Where a reading sits against a threshold: -1 before it, 1 after it, 0 when +/// its uncertainty reaches across, which is no answer. +function sideOf(reading, threshold) { + if (reading.s + reading.u < threshold) return -1; + return reading.s - reading.u > threshold ? 1 : 0; +} + +/// Every string a tool result carries, one per line. The fact matchers look +/// for tokens in text, and JSON escaping would hide a token that spans a +/// newline or holds a quote. +function resultText(value) { + if (typeof value === "string") return value; + if (Array.isArray(value)) return value.map(resultText).join("\n"); + if (value && typeof value === "object") { + return Object.values(value).map(resultText).join("\n"); + } + return ""; +} + +/// Where the turn reply `index` belongs to began: the last ending that the +/// loop took on its socket and that arrived before the reply did. An ending of +/// another generation (swallowed, or from another socket) that falls inside the +/// turn leaves ownership of everything in it unknown. +function turnStart(capture, index) { + const socket = capture.replies[index].identity.socket; + let receipts = 0; + let replies = 0; + const strays = []; + for (const boundary of capture.boundaries) { + if (boundary.replies_before > index) continue; + if (boundary.accepted && boundary.identity.socket === socket) { + receipts = Math.max(receipts, boundary.receipts_before); + replies = Math.max(replies, boundary.replies_before); + } else { + strays.push(boundary); + } + } + const stray = strays.some((boundary) => boundary.receipts_before > receipts); + return { receipts, replies, stray }; +} + +/// The segments of `socket` with index in [from, to). +const segmentsOf = (capture, socket, from, to) => + capture.replies + .slice(from, to) + .filter((reply) => reply.identity.socket === socket); + +/// How reply `index` ended: "complete" or "interrupted" when the first +/// provider ending after it, on its socket and in its generation, was taken +/// and nothing else sealed it first, otherwise "unknown" with the reason. +function endingOf(capture, index) { + const reply = capture.replies[index]; + const socket = reply.identity.socket; + for (const boundary of capture.boundaries) { + if (boundary.identity.socket !== socket) continue; + if (boundary.replies_before <= index) continue; + if ( + segmentsOf(capture, socket, index + 1, boundary.replies_before).length + ) { + return { + kind: "unknown", + why: "a cut or rollover sealed the reply before any provider ending", + }; + } + if (!boundary.accepted) { + return { + kind: "unknown", + why: "its ending was swallowed by the interruption guard", + }; + } + if ( + reply.identity.trace_turn == null || + boundary.identity.trace_turn !== reply.identity.trace_turn + ) { + return { + kind: "unknown", + why: "the ending belongs to another generation than the reply", + }; + } + return { + kind: boundary.ending === "Interrupted" ? "interrupted" : "complete", + }; + } + return { kind: "unknown", why: "no provider ending followed the reply" }; +} + +/// The tool calls reply `index` may cite, or the reason the facts do not say. +/// Calls count only when they were received on the reply's own socket and +/// generation after the previous ending, executed before the reply began, and +/// not received while output was being discarded. +function citedCalls(capture, index) { + const reply = capture.replies[index]; + const socket = reply.identity.socket; + if ( + reply.receipts_before !== reply.receipts_last || + reply.batches_before !== reply.batches_last + ) { + return { why: "the reply continued across a tool call" }; + } + const turn = turnStart(capture, index); + if (turn.stray) { + return { why: "an ending of another generation lies inside the turn" }; + } + if (segmentsOf(capture, socket, turn.replies, index).length) { + return { why: "an earlier segment of the same turn may own the calls" }; + } + const window = capture.receipts.slice(turn.receipts, reply.receipts_before); + if (window.some((receipt) => receipt.identity.socket !== socket)) { + return { + why: "a socket replacement lies between the question and the reply", + }; + } + const calls = []; + for (const receipt of window) { + if (receipt.discarded) continue; + if (receipt.batch == null || receipt.batch >= reply.batches_before) { + continue; + } + if ( + reply.identity.trace_turn == null || + receipt.identity.trace_turn !== reply.identity.trace_turn + ) { + return { why: "a call belongs to another generation than the reply" }; + } + calls.push(receipt); + } + return { calls }; +} + +/// The evidence for one cited call, or the reason it cannot be read. +function toolCall(capture, receipt, cp, key, options) { + const batch = capture.batches.find((b) => b.sequence === receipt.batch); + const call = batch?.calls.find((candidate) => candidate.id === receipt.id); + if (!call) return { why: `call ${receipt.id} has no executed batch` }; + if (batch.local_send_completed == null) { + return { why: `the send of call ${receipt.id} was not observed` }; + } + const reading = readClock( + batch.identity, + options.scheduleStartUnixMs, + options.maxClockUncertaintyMs, + ); + if (!reading) { + return { why: `call ${receipt.id} has no usable clock reading` }; + } + const earliest = Math.max( + cp.at_s, + ...(key.tools ?? [key.tool]) + .filter((need) => need.name === call.name) + .map((need) => need.not_before_s), + ); + const latest = cp.stimulus_end_s + cp.response_window_s; + if (sideOf(reading, earliest) === 0 || sideOf(reading, latest) === 0) { + return { + why: `call ${receipt.id} lies within the clock uncertainty of a limit`, + }; + } + const failed = + call.result && typeof call.result === "object" && "error" in call.result; + return { + call: { + call_id: call.id, + name: call.name, + at_s: reading.s, + ok: !failed && batch.local_send_completed === true, + result_text: resultText(call.result), + }, + }; +} + +const notAssessed = (checkpointId, reasons) => ({ + checkpoint_id: checkpointId, + verdict: "not_assessed", + reasons, +}); + +const judge = (bundle, evidence) => ({ + checkpoint_id: evidence.checkpoint_id, + ...judgeCheckpoint(bundle, evidence), +}); + +/// The one reply that started inside the checkpoint's window as +/// {index, at}, null when none did, or a reason when the clock cannot say. +function replyInWindow(capture, from, to, settings) { + const inside = []; + for (const [index, reply] of capture.replies.entries()) { + const reading = readClock( + reply.identity, + settings.scheduleStartUnixMs, + settings.maxClockUncertaintyMs, + ); + if (!reading) return { why: "a reply has no usable clock reading" }; + const [start, end] = [sideOf(reading, from), sideOf(reading, to)]; + if (start === 0 || end === 0) { + return { + why: "a reply lies within the clock uncertainty of the window edge", + }; + } + if (start === 1 && end === -1) inside.push({ index, at: reading.s }); + } + if (inside.length > 1) { + return { why: `${inside.length} replies started in the window` }; + } + return { reply: inside[0] ?? null }; +} + +/// One checkpoint's verdict: "pass", "fail" or "not_assessed", with reasons. +function assessCheckpoint(bundle, report, checkpointId, options) { + const settings = { + maxClockUncertaintyMs: MAX_CLOCK_UNCERTAINTY_MS, + ...options, + }; + const problems = reportProblems(report); + if (problems.length) return notAssessed(checkpointId, problems); + if (!Number.isFinite(settings.scheduleStartUnixMs)) { + return notAssessed(checkpointId, ["the schedule origin is unknown"]); + } + const cp = bundle.schedule.checkpoints.find( + (c) => c.checkpoint_id === checkpointId, + ); + const key = bundle.answer_key.checkpoints.find( + (c) => c.checkpoint_id === checkpointId, + ); + if (!cp || !key) return notAssessed(checkpointId, ["unknown checkpoint"]); + const capture = report.capture; + + // The window opens at the question's estimated end, from its word count; a + // recording that finished speaking sooner can be answered before that. Such + // a reply cannot be told from one that talked over the question, so neither + // is held against the reply as a missing answer. + const early = replyInWindow(capture, cp.at_s, cp.stimulus_end_s, settings); + if (early.why) return notAssessed(checkpointId, [early.why]); + if (early.reply) { + return notAssessed(checkpointId, [ + "a reply began before the question's estimated end", + ]); + } + + const found = replyInWindow( + capture, + cp.stimulus_end_s, + cp.stimulus_end_s + cp.response_window_s, + settings, + ); + if (found.why) return notAssessed(checkpointId, [found.why]); + if (!found.reply) return judge(bundle, { checkpoint_id: checkpointId }); + + const { index, at } = found.reply; + const reply = capture.replies[index]; + if (reply.revised) { + return notAssessed(checkpointId, [ + "the reply's text was rewritten while it was shown", + ]); + } + if (!reply.final_segment) { + return notAssessed(checkpointId, ["the reply was never sealed"]); + } + const ending = endingOf(capture, index); + if (ending.kind === "unknown") return notAssessed(checkpointId, [ending.why]); + const cited = citedCalls(capture, index); + if (!cited.calls) return notAssessed(checkpointId, [cited.why]); + const calls = []; + for (const receipt of cited.calls) { + const read = toolCall(capture, receipt, cp, key, settings); + if (!read.call) return notAssessed(checkpointId, [read.why]); + calls.push(read.call); + } + return judge(bundle, { + checkpoint_id: checkpointId, + reply: { + text: reply.text, + started_at_s: at, + completed: ending.kind === "complete", + interrupted: ending.kind === "interrupted", + tool_call_ids: calls.map((call) => call.call_id), + }, + tool_calls: calls, + }); +} + +/// "verified" only when every checkpoint was assessed and passed; "failed" when +/// any assessed one failed; otherwise "not_assessed". +function summarize(checkpoints) { + const failed = checkpoints.some((c) => c.verdict === "fail"); + const verified = + checkpoints.length > 0 && checkpoints.every((c) => c.verdict === "pass"); + return { + status: failed ? "failed" : verified ? "verified" : "not_assessed", + checkpoints, + uncovered: checkpoints + .filter((c) => c.verdict === "not_assessed") + .map((c) => c.checkpoint_id), + }; +} + +function judgeInterview(bundle, report, options) { + return summarize( + bundle.schedule.checkpoints.map((cp) => + assessCheckpoint(bundle, report, cp.checkpoint_id, options), + ), + ); +} + +const MAX_REASONS = 20; + +/// Why a fixture report that is there cannot vouch for the run, read the same +/// by the keep check and by the judge: unreadable, or ended other than well. +function fixtureRefusal(report) { + if (report?.unreadable) { + return `the serving fixture's report could not be read: ${report.unreadable}`; + } + if (typeof report?.outcome === "string" && report.outcome !== "ok") { + return `the serving fixture ended with ${report.outcome}`; + } + return null; +} + +/// Why the serving fixture's report is not one to accept, from its shape and +/// the outcome it names alone. A failure the observer or the capture reports +/// about itself is not a reason here: the judge says not_assessed for it, and +/// the run keeps its own result. A report that holds neither a capture nor an +/// observer failure is malformed, and refuses the run. +function fixtureProblems(report) { + const refusal = fixtureRefusal(report); + if (refusal) return [refusal]; + if (!report || typeof report !== "object" || Array.isArray(report)) { + return ["the serving fixture's report is not an object"]; + } + if (typeof report.outcome !== "string") { + return ["the serving fixture's report names no outcome"]; + } + const problems = []; + for (const key of ["arming_failure", "bound"]) { + if (!(key in report)) problems.push(`the report has no ${key}`); + } + const capture = report.capture; + // An observer that failed admits no capture and says so itself; that is the + // judge's to name. A report with neither is not one an interview left. + if (capture == null && report.arming_failure != null) return problems; + if (!capture || typeof capture !== "object") { + return [...problems, "the report holds no capture"]; + } + if (!("failure" in capture)) + problems.push("the capture has no failure field"); + for (const name of CAPTURE_LISTS) { + if (!Array.isArray(capture[name])) + problems.push(`the capture has no ${name} list`); + } + return problems; +} + +/// Judges a capture against its fixture report, or says in every +/// checkpoint's reason why it could not. +function judgeCapture(capture, fixture) { + const { report } = capture; + let bundle; + try { + bundle = generateSchedule(report.schedule_mode, { + durationMin: capture.plan.durationMin, + observationTargetS: capture.plan.observationTargetS, + }); + } catch (error) { + return summarize([ + notAssessed("interview", [ + `the schedule cannot be rebuilt: ${error.message}`, + ]), + ]); + } + const refuse = (reason) => + summarize( + bundle.schedule.checkpoints.map((cp) => + notAssessed(cp.checkpoint_id, [reason]), + ), + ); + if (report.exit_code !== 0 || report.status !== "complete") { + return refuse("the run did not complete cleanly"); + } + // A tone fixture asked the model nothing, so there is no content to judge. + if (report.validation_fixture !== false) { + return refuse("the run played tone fixtures, not spoken questions"); + } + if (report.anchor?.source !== "server_clock") { + return refuse("the interview clock was not anchored to the server"); + } + if (!fixture) return refuse("the serving fixture left no report"); + const refusal = + fixtureRefusal(fixture) ?? + (fixture.outcome === "ok" + ? null + : "the serving fixture's report names no outcome"); + if (refusal) return refuse(refusal); + if (bundle.schedule.schedule_sha256 !== report.schedule_sha256) { + return refuse("the schedule that ran is not the one rebuilt from the plan"); + } + return judgeInterview(bundle, fixture, { + scheduleStartUnixMs: report.schedule_origin_unix_ms, + }); +} + +/// The three report fields a verdict sets. The reasons are always present, +/// empty for a verified interview, so that rewriting a report with a better +/// verdict does not keep an earlier one's. +function verdictFields(verdict) { + const counts = { pass: 0, fail: 0, not_assessed: 0 }; + for (const checkpoint of verdict.checkpoints) counts[checkpoint.verdict] += 1; + const reasons = verdict.checkpoints + .filter((checkpoint) => checkpoint.verdict !== "pass") + .map( + (checkpoint) => `${checkpoint.checkpoint_id}: ${checkpoint.reasons[0]}`, + ); + return { + reply_content: verdict.status, + reply_content_checkpoints: counts, + reply_content_reasons: reasons.slice(0, MAX_REASONS), + }; +} + +/// The capture file with its long-run report's `reply_content` filled in from +/// the serving fixture's report. Nothing else in the report moves: the exit +/// code and `qualification` were settled before this ran, and a verdict here +/// does not reopen them. +function mergeVerdict(capture, fixture) { + const report = capture?.report; + if (capture?.flow !== "long-interview" || !report) return null; + return { + ...capture, + report: { ...report, ...verdictFields(judgeCapture(capture, fixture)) }, + }; +} + +/// A JSON file, null when it is not there, or why it cannot be read: a report +/// that is there but unreadable is a different failure from one the fixture +/// never wrote, and the verdict says which. +const readJson = (file) => { + try { + return JSON.parse(fs.readFileSync(file, "utf8")); + } catch (error) { + // The kind of failure and no more: a parse error's message quotes the + // text around it, which can be the interviewer's reply. + if (error.code === "ENOENT") return null; + return { unreadable: error.code ?? error.name }; + } +}; + +/// `--capture FILE --fixture FILE`: rewrites the capture with the verdict. +/// Exits 0 when it merged or had nothing to judge, 2 when it could not read +/// the capture or write it back; the caller treats either as no change to the +/// run's own result. +function main(argv, { out = console.log, err = console.error } = {}) { + if (argv[0] === "--check-fixture") { + const problems = fixtureProblems(readJson(argv[1] ?? "")); + problems.forEach((problem) => err(`fixture report: ${problem}`)); + return problems.length ? 1 : 0; + } + const flag = (name) => { + const at = argv.indexOf(name); + return at < 0 ? undefined : argv[at + 1]; + }; + const [captureFile, fixtureFile] = [flag("--capture"), flag("--fixture")]; + if (!captureFile || !fixtureFile) { + err("usage: long-interview-evidence.cjs --capture FILE --fixture FILE"); + return 2; + } + const capture = readJson(captureFile); + if (!capture || capture.unreadable) { + const why = capture?.unreadable ? `: ${capture.unreadable}` : ""; + err(`cannot read the capture ${captureFile}${why}`); + return 2; + } + const merged = mergeVerdict(capture, readJson(fixtureFile)); + if (!merged) return 0; + const partial = `${captureFile}.partial`; + try { + fs.writeFileSync(partial, `${JSON.stringify(merged, null, 2)}\n`); + fs.renameSync(partial, captureFile); + } catch (error) { + err(`cannot write ${captureFile}: ${error.message}`); + return 2; + } + const { reply_content: status, reply_content_checkpoints: counts } = + merged.report; + out( + `long-interview reply_content=${status} pass=${counts.pass} fail=${counts.fail} not_assessed=${counts.not_assessed}`, + ); + return 0; +} + +if (require.main === module) process.exit(main(process.argv.slice(2))); + +module.exports = { + MAX_CLOCK_UNCERTAINTY_MS, + assessCheckpoint, + fixtureProblems, + judgeInterview, + main, + mergeVerdict, + reportProblems, + summarize, + verdictFields, +}; diff --git a/src/livekit/probe.rs b/src/livekit/probe.rs index 5bcb15568..9de7280ea 100644 --- a/src/livekit/probe.rs +++ b/src/livekit/probe.rs @@ -22,6 +22,7 @@ pub(super) mod acquisition; +#[cfg(unix)] #[path = "../../tests/unit/livekit/probe/serving.rs"] mod serving; diff --git a/src/livekit/probe/acquisition.rs b/src/livekit/probe/acquisition.rs index 206289ba4..6ad7f73f3 100644 --- a/src/livekit/probe/acquisition.rs +++ b/src/livekit/probe/acquisition.rs @@ -20,6 +20,18 @@ pub(in crate::livekit) struct Identity { pub(in crate::livekit) clock: Value, } +impl Identity { + fn from_view(view: &IdentityView<'_>) -> Self { + Self { + room: view.room.into(), + session: view.session, + socket: view.socket, + trace_turn: view.trace_turn, + clock: view.clock.clone(), + } + } +} + #[derive(serde::Serialize)] pub(in crate::livekit) struct IdentityView<'a> { pub(in crate::livekit) room: &'a str, @@ -33,8 +45,10 @@ pub(in crate::livekit) struct IdentityView<'a> { pub(in crate::livekit) struct Limits { pub(in crate::livekit) batches: usize, pub(in crate::livekit) calls: usize, - /// Serialized JSON bytes of the retained batch array, including - /// punctuation. + /// Serialized JSON bytes, punctuation included, that each retained array + /// (batches, receipts, replies and turn endings) is held to on its own, so + /// a report can hold up to four times this. `calls` likewise bounds the + /// receipts, the replies and the turn endings each. pub(in crate::livekit) bytes: usize, } @@ -54,6 +68,76 @@ pub(in crate::livekit) struct Batch { pub(in crate::livekit) local_send_completed: Option, } +/// A tool call as the handler received it, before the acknowledgement and +/// board awaits that can still cancel the batch. Received, executed and +/// locally sent are three separate facts, and none of them is the provider's +/// receipt. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Receipt { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) id: String, + pub(in crate::livekit) name: String, + /// Delivered after a hold or discard had cut its generation off. Such a + /// call is still answered, so it is kept, but it is never to be tied to a + /// later reply. + pub(in crate::livekit) discarded: bool, + /// Sequence of the executed batch that answered it; `None` for a call that + /// was received and never executed. Stands in for a stage: received, + /// executed and locally sent are one lifecycle that the reply capture will + /// fold into a single per-call record. + pub(in crate::livekit) batch: Option, +} + +/// The newest view of one interviewer transcript segment, as queued for the +/// page (the writer may coalesce waiting views, so `views` can exceed what the +/// page received). Only ordered facts are kept: how many receipts and batches +/// existed +/// when the segment first appeared says what could have preceded it, and +/// nothing here says what it was about. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Reply { + /// Where the segment first appeared; later views keep it. + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) id: String, + pub(in crate::livekit) text: String, + pub(in crate::livekit) final_segment: bool, + /// A view was not an extension of the one before it, or a final segment + /// was published as open again: what the page showed was rewritten. + pub(in crate::livekit) revised: bool, + pub(in crate::livekit) views: usize, + pub(in crate::livekit) receipts_before: usize, + pub(in crate::livekit) batches_before: usize, + /// The same counts at the newest view. A turn spans tool calls, so a + /// segment whose two counts differ straddles them and says nothing about + /// which call its later words follow. + pub(in crate::livekit) receipts_last: usize, + pub(in crate::livekit) batches_last: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) enum Ending { + Complete, + Interrupted, +} + +/// A turn ending the provider sent, and whether the room loop took it. A +/// swallowed ending belongs to a generation already cut off, so it closes +/// nothing the page or the model saw; it is kept because the boundary a reply +/// would otherwise be bounded by is then missing. Local cuts and socket +/// replacements are not endings: they show as a sealed segment or a new socket +/// in the identity. An ending the old socket sent while a speech handoff +/// collected its events is consumed there and never seen here, so a socket +/// change between two replies is an unknown boundary, never an absent one. +#[derive(Clone, Debug, PartialEq, serde::Serialize)] +pub(in crate::livekit) struct Boundary { + pub(in crate::livekit) identity: Identity, + pub(in crate::livekit) ending: Ending, + pub(in crate::livekit) accepted: bool, + pub(in crate::livekit) receipts_before: usize, + pub(in crate::livekit) batches_before: usize, + pub(in crate::livekit) replies_before: usize, +} + #[derive(Clone, Copy, Debug, PartialEq)] pub(in crate::livekit) enum Failure { // Room/session admission or an opaque ticket's batch association differs. @@ -69,6 +153,32 @@ pub(in crate::livekit) struct Snapshot { pub(in crate::livekit) failure: Option, pub(in crate::livekit) retained_calls: usize, pub(in crate::livekit) retained_bytes: usize, + pub(in crate::livekit) receipts: Vec, + /// Serialized length of the receipt array, brackets and commas included, + /// held to the same byte limit as the batches but counted apart from them. + pub(in crate::livekit) retained_receipt_bytes: usize, + pub(in crate::livekit) replies: Vec, + pub(in crate::livekit) boundaries: Vec, + /// Serialized length of each retained reply and of the two arrays, + /// brackets and commas included, so a newer view of a reply, which + /// arrives for every transcript chunk, is charged for its own change + /// rather than by serializing every reply again under the lock. + reply_bytes: Vec, + retained_reply_bytes: usize, + retained_boundary_bytes: usize, +} + +impl Snapshot { + /// The receipt a call executed under `identity` answers: same session and + /// same socket, so a call received on one socket and executed on another + /// is left unlinked as evidence of a replacement. + fn receipt_mut(&mut self, identity: &IdentityView<'_>, id: &str) -> Option<&mut Receipt> { + self.receipts.iter_mut().find(|receipt| { + receipt.id == id + && receipt.identity.session == identity.session + && receipt.identity.socket == identity.socket + }) + } } pub(in crate::livekit) struct Capture { @@ -116,10 +226,35 @@ impl Capture { failure: None, retained_calls: 0, retained_bytes: 2, + receipts: Vec::new(), + // The empty array, so the count is the serialized length. + retained_receipt_bytes: 2, + replies: Vec::new(), + boundaries: Vec::new(), + reply_bytes: Vec::new(), + retained_reply_bytes: 2, + retained_boundary_bytes: 2, }), })) } + /// Whether the identity is the room and session this capture was admitted + /// for; the socket may change under it. + fn admits(&self, identity: &IdentityView<'_>) -> bool { + identity.room == self.room && identity.session == self.session + } + + /// Whether an array that serialized to `retained` bytes still fits once + /// `value` is added to it, and its length if so. + fn charged(&self, retained: usize, added: usize, value: &impl Serialize) -> Option { + let mut counter = BoundedBytes { + remaining: self.limits.bytes.checked_sub(retained + added)?, + written: 0, + }; + serde_json::to_writer(&mut counter, value).ok()?; + Some(counter.written) + } + pub(in crate::livekit) fn snapshot(&self) -> Snapshot { self.inner .lock() @@ -127,6 +262,177 @@ impl Capture { .clone() } + /// Records the calls of one batch at the handler's entry. Scoped like a + /// batch, bounded by the call and byte limits, and all or nothing: a batch + /// that does not fit latches the failure and keeps none of its receipts. + pub(in crate::livekit) fn receive( + &self, + identity: IdentityView<'_>, + calls: &[GeminiFunctionCall], + discarded: bool, + ) { + if calls.is_empty() { + return; + } + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + let failure = if !self.admits(&identity) { + Some(Failure::Scope) + } else if calls.len() > self.limits.calls.saturating_sub(snapshot.receipts.len()) { + Some(Failure::Limit) + } else if calls.iter().enumerate().any(|(index, call)| { + call.id.is_empty() + || calls[..index].iter().any(|earlier| earlier.id == call.id) + || snapshot.receipts.iter().any(|old| old.id == call.id) + }) { + Some(Failure::AmbiguousId) + } else { + None + }; + if let Some(failure) = failure { + snapshot.failure = Some(failure); + return; + } + let mut retained = snapshot.retained_receipt_bytes; + let mut fresh = Vec::with_capacity(calls.len()); + for (index, call) in calls.iter().enumerate() { + // A comma before every receipt but the very first. + let comma = usize::from(snapshot.receipts.len() + index != 0); + let receipt = Receipt { + identity: Identity::from_view(&identity), + id: call.id.clone(), + name: call.name.clone(), + discarded, + batch: None, + }; + let Some(bytes) = self.charged(retained, comma, &receipt) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + retained += comma + bytes; + fresh.push(receipt); + } + snapshot.retained_receipt_bytes = retained; + snapshot.receipts.extend(fresh); + } + + /// Notes a published interviewer segment. A segment is identified by its + /// session, socket and id, and only its newest text is kept, so the byte + /// limit bounds the serialized array of newest views. A view that does not + /// fit latches the failure and leaves the earlier ones as they were. + pub(in crate::livekit) fn reply( + &self, + identity: IdentityView<'_>, + id: &str, + text: &str, + final_segment: bool, + ) { + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if !self.admits(&identity) { + snapshot.failure = Some(Failure::Scope); + return; + } + let at = snapshot.replies.iter().position(|reply| { + reply.id == id + && reply.identity.session == identity.session + && reply.identity.socket == identity.socket + }); + let (reply, old) = if let Some(at) = at { + // Built anew rather than cloned: the old text, the largest field, + // is about to be replaced. + let was = &snapshot.replies[at]; + let reply = Reply { + identity: was.identity.clone(), + id: was.id.clone(), + text: text.into(), + final_segment, + revised: was.revised + || !text.starts_with(&was.text) + || (was.final_segment && !final_segment), + views: was.views + 1, + receipts_before: was.receipts_before, + batches_before: was.batches_before, + receipts_last: snapshot.receipts.len(), + batches_last: snapshot.batches.len(), + }; + (reply, snapshot.reply_bytes[at]) + } else if snapshot.replies.len() >= self.limits.calls { + snapshot.failure = Some(Failure::Limit); + return; + } else { + let reply = Reply { + identity: Identity::from_view(&identity), + id: id.into(), + text: text.into(), + final_segment, + revised: false, + views: 1, + receipts_before: snapshot.receipts.len(), + batches_before: snapshot.batches.len(), + receipts_last: snapshot.receipts.len(), + batches_last: snapshot.batches.len(), + }; + (reply, 0) + }; + // A comma before every reply but the first. + let comma = usize::from(at.is_none() && !snapshot.replies.is_empty()); + let rest = snapshot.retained_reply_bytes - old; + let Some(bytes) = self.charged(rest, comma, &reply) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + snapshot.retained_reply_bytes = rest + comma + bytes; + if let Some(at) = at { + snapshot.replies[at] = reply; + snapshot.reply_bytes[at] = bytes; + } else { + snapshot.replies.push(reply); + snapshot.reply_bytes.push(bytes); + } + } + + /// Notes a turn ending in arrival order. Bounded like the replies: by the + /// call limit in count and the byte limit in serialized size. + pub(in crate::livekit) fn boundary( + &self, + identity: IdentityView<'_>, + ending: Ending, + accepted: bool, + ) { + let mut snapshot = self.inner.lock().unwrap_or_else(PoisonError::into_inner); + if snapshot.failure.is_some() { + return; + } + if !self.admits(&identity) { + snapshot.failure = Some(Failure::Scope); + return; + } + let boundary = Boundary { + identity: Identity::from_view(&identity), + ending, + accepted, + receipts_before: snapshot.receipts.len(), + batches_before: snapshot.batches.len(), + replies_before: snapshot.replies.len(), + }; + if snapshot.boundaries.len() >= self.limits.calls { + snapshot.failure = Some(Failure::Limit); + return; + } + let comma = usize::from(!snapshot.boundaries.is_empty()); + let Some(bytes) = self.charged(snapshot.retained_boundary_bytes, comma, &boundary) else { + snapshot.failure = Some(Failure::Limit); + return; + }; + snapshot.retained_boundary_bytes += comma + bytes; + snapshot.boundaries.push(boundary); + } + fn record( &self, identity: IdentityView<'_>, @@ -139,7 +445,7 @@ impl Capture { if snapshot.failure.is_some() { return None; } - let failure = if identity.room != self.room || identity.session != self.session { + let failure = if !self.admits(&identity) { Some(Failure::Scope) } else if snapshot.batches.len() >= self.limits.batches || answers.len() > self.limits.calls.saturating_sub(snapshot.retained_calls) @@ -165,29 +471,18 @@ impl Capture { } let sequence = snapshot.batches.len(); let comma = usize::from(sequence != 0); - let remaining = self.limits.bytes.saturating_sub(snapshot.retained_bytes); - let mut counter = BoundedBytes { - remaining: remaining.saturating_sub(comma), - written: 0, - }; let view = BatchView { identity: &identity, sequence, calls: CallsView(answers), local_send_completed: None, }; - if remaining < comma || serde_json::to_writer(&mut counter, &view).is_err() { + let Some(bytes) = self.charged(snapshot.retained_bytes, comma, &view) else { snapshot.failure = Some(Failure::Limit); return None; - } + }; let batch = Batch { - identity: Identity { - room: identity.room.into(), - session: identity.session, - socket: identity.socket, - trace_turn: identity.trace_turn, - clock: identity.clock, - }, + identity: Identity::from_view(&identity), sequence, calls: answers .iter() @@ -199,9 +494,23 @@ impl Capture { .collect(), local_send_completed: None, }; - snapshot.retained_bytes += counter.written + comma; + snapshot.retained_bytes += comma + bytes; snapshot.retained_calls += answers.len(); snapshot.batches.push(batch); + + // Linked only to a receipt from this same socket: a call received on + // one socket and executed on another is evidence of a replacement, not + // of an answer. + let digits = sequence.to_string().len(); + let mut receipt_bytes = snapshot.retained_receipt_bytes; + for (call, _) in answers { + if let Some(receipt) = snapshot.receipt_mut(&identity, &call.id) { + receipt.batch = Some(sequence); + // `null` (four bytes) becomes the sequence number. + receipt_bytes = (receipt_bytes + digits).saturating_sub(4); + } + } + snapshot.retained_receipt_bytes = receipt_bytes; Some(sequence) } } @@ -461,6 +770,10 @@ impl Arming { "failure": snapshot.failure.map(|failure| format!("{failure:?}")), "retained_calls": snapshot.retained_calls, "retained_bytes": snapshot.retained_bytes, + "receipts": snapshot.receipts, + "retained_receipt_bytes": snapshot.retained_receipt_bytes, + "replies": snapshot.replies, + "boundaries": snapshot.boundaries, }) }); serde_json::json!({ @@ -557,18 +870,57 @@ pub(in crate::livekit) fn bound(room: &str, session: u64) { } } +fn active() -> Option> { + ACTIVE + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .map(Arc::clone) +} + pub(in crate::livekit) fn observe( latency: &ReplyLatency, answers: &[(GeminiFunctionCall, Value)], ) -> Option { - let capture = ACTIVE - .lock() - .unwrap_or_else(PoisonError::into_inner) - .as_ref() - .map(Arc::clone)?; + let capture = active()?; let sequence = capture.record(latency.acquisition_identity(), answers)?; Some(Ticket { capture, sequence }) } +/// The handler's entry, before anything that can cancel the batch. +pub(in crate::livekit) fn receive( + latency: &ReplyLatency, + calls: &[GeminiFunctionCall], + discarded: bool, +) { + if let Some(capture) = active() { + capture.receive(latency.acquisition_identity(), calls, discarded); + } +} + +/// A turn ending as the room loop decided it; other events are not endings. +pub(in crate::livekit) fn boundary( + latency: &ReplyLatency, + event: &crate::gemini::GeminiEvent, + accepted: bool, +) { + use crate::gemini::GeminiEvent; + let ending = match event { + GeminiEvent::TurnComplete => Ending::Complete, + GeminiEvent::Interrupted => Ending::Interrupted, + _ => return, + }; + if let Some(capture) = active() { + capture.boundary(latency.acquisition_identity(), ending, accepted); + } +} + +/// An interviewer segment as the room loop publishes it. +pub(in crate::livekit) fn reply(latency: &ReplyLatency, id: &str, text: &str, final_segment: bool) { + if let Some(capture) = active() { + capture.reply(latency.acquisition_identity(), id, text, final_segment); + } +} + #[path = "../../../tests/unit/livekit/probe/acquisition.rs"] mod tests; diff --git a/src/livekit/session.rs b/src/livekit/session.rs index f7ec851f1..8a3954ff9 100644 --- a/src/livekit/session.rs +++ b/src/livekit/session.rs @@ -20,7 +20,9 @@ mod tool_calls; use floor::write_floor; pub(super) use floor::{observe_floor, reconcile_floor}; -use observations::{HintCounters, ToolTiming, execute_timed_tool_batch, known_tool_name}; +use observations::{ + HintCounters, ToolTiming, execute_timed_tool_batch, known_tool_name, silent_completion_kind, +}; use recovery::log_socket_pressure; pub(super) use recovery::{ @@ -724,15 +726,6 @@ fn observe_latency_event( None } -/// What a completion with nothing in it answered, if this event is one. -fn silent_completion_kind(event: &GeminiEvent, activity: &RuntimeActivity) -> Option<&'static str> { - if matches!(event, GeminiEvent::TurnComplete) && !activity.generating { - activity.debt_silence_answers() - } else { - None - } -} - /// `accept_event_with_reply_debt`, plus the silence the event answered. Both /// reads come before the accept, because the accept clears the waits they /// read. A silent completion that settles the reply still closes the turn's @@ -830,6 +823,8 @@ pub(super) async fn handle_gemini_event( let discarding = context.activity.socket.discarding_output; let (accepted, interruption_debt, silent_owed) = accept_event_noting_silence(&event, context.activity, held, Instant::now()); + #[cfg(test)] + super::probe::acquisition::boundary(&context.activity.latency, &event, accepted); let handled = if accepted { if let Some(silent_owed) = silent_owed { eprintln!( @@ -1086,7 +1081,7 @@ fn on_output_transcript(room: &RoomLink, context: &mut GeminiEventContext<'_>, t final_segment: false, identity: None, }; - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } @@ -2677,7 +2672,7 @@ pub(super) fn close_turns(room: &RoomLink, context: &mut GeminiEventContext<'_>) eprintln!("{line}"); } for segment in closing { - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } @@ -2691,7 +2686,7 @@ fn close_interviewer_turn(room: &RoomLink, context: &mut GeminiEventContext<'_>) crate::current_epoch_millis(), None, ) { - context.transcripts.publish(room, segment); + observations::publish_segment(room, context, segment); } } diff --git a/src/livekit/session/observations.rs b/src/livekit/session/observations.rs index aecb0d0c9..19fbb53d4 100644 --- a/src/livekit/session/observations.rs +++ b/src/livekit/session/observations.rs @@ -7,13 +7,15 @@ use std::time::Instant; use serde_json::Value; use crate::agent::RuntimeState; -use crate::gemini::GeminiFunctionCall; +use crate::gemini::{GeminiEvent, GeminiFunctionCall}; use crate::livekit::turn::RuntimeActivity; use crate::runtime::{ TOOL_END_INTERVIEW, TOOL_LOG_HINT, TOOL_READ_BOARD, TOOL_READ_CONVERSATION, TOOL_READ_EDITOR, TOOL_RECORD_FRAMEWORK_EVIDENCE, }; +use super::{GeminiEventContext, RoomLink, Segment}; + /// A declared tool's name, never the model's text for an undeclared one. pub(super) fn known_tool_name(name: &str) -> &'static str { [ @@ -58,6 +60,39 @@ pub(super) fn execute_timed_tool_batch( answers } +/// Publishes a transcript segment, noting the interviewer's for the +/// acquisition capture on the way. +pub(super) fn publish_segment( + room: &RoomLink, + context: &mut GeminiEventContext<'_>, + segment: Segment, +) { + #[cfg(test)] + // The id, not the identity: a candidate's final is published as the agent + // when the candidate has already left. + if segment.identity.is_none() && segment.id.starts_with("interviewer-") { + super::super::probe::acquisition::reply( + &context.activity.latency, + &segment.id, + &segment.text, + segment.final_segment, + ); + } + context.transcripts.publish(room, segment); +} + +/// What a completion with nothing in it answered, if this event is one. +pub(super) fn silent_completion_kind( + event: &GeminiEvent, + activity: &RuntimeActivity, +) -> Option<&'static str> { + if matches!(event, GeminiEvent::TurnComplete) && !activity.generating { + activity.debt_silence_answers() + } else { + None + } +} + pub(super) struct HintCounters { rungs: usize, volunteered: u32, diff --git a/src/livekit/session/tool_calls.rs b/src/livekit/session/tool_calls.rs index 88280469b..4bbb6319c 100644 --- a/src/livekit/session/tool_calls.rs +++ b/src/livekit/session/tool_calls.rs @@ -19,6 +19,13 @@ pub(super) async fn on_tool_calls( if calls.is_empty() { return Ok(()); } + // Before every await below, any of which can cancel the batch. + #[cfg(test)] + crate::livekit::probe::acquisition::receive( + &context.activity.latency, + &calls, + context.activity.socket.discarding_output || context.state.interviewer_silenced(), + ); // A filler that fails to queue is not worth the tool round it precedes. #[cfg(test)] owner::before(Stage::Acknowledgement, context).await; diff --git a/tests/browser/long-interview-driver.test.js b/tests/browser/long-interview-driver.test.js index 16a6d8810..d9d42a29a 100644 --- a/tests/browser/long-interview-driver.test.js +++ b/tests/browser/long-interview-driver.test.js @@ -539,6 +539,12 @@ test("the duration is claimed only from the server clock", async () => { assert.equal(viaToken.report.full_interview, false); assert.equal(finalize(played).report.anchor.source, "server_clock"); assert.equal(finalize(played).report.schedule_start_delay_s, 5); + assert.equal(finalize(played).report.schedule_mode, played.prepared.mode); + // The origin a later reader needs to read the interview clock against. + assert.equal( + finalize(played).report.schedule_origin_unix_ms, + played.joinedAtMs, + ); }); test("a stimulus whose page action throws still spends its timestamp", async () => { diff --git a/tests/browser/long-interview-evidence.test.js b/tests/browser/long-interview-evidence.test.js new file mode 100644 index 000000000..5e9b0987f --- /dev/null +++ b/tests/browser/long-interview-evidence.test.js @@ -0,0 +1,723 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import { fileURLToPath } from "node:url"; + +import evidence_ from "../../scripts/long-interview-evidence.cjs"; +import sched from "../../scripts/long-interview-schedule.cjs"; + +const { + assessCheckpoint, + fixtureProblems, + judgeInterview, + main, + mergeVerdict, + reportProblems, + summarize, + verdictFields, +} = evidence_; +const { answerFor, generateSchedule } = sched; + +const ROOT = fileURLToPath(new URL("../..", import.meta.url)); +const BUNDLE = generateSchedule("coding", { + durationMin: 90, + observationTargetS: 5400, +}); +const CP = BUNDLE.schedule.checkpoints[0]; +const ANSWER = answerFor(BUNDLE, CP.checkpoint_id); +const START = 1_700_000_000_000; + +// What the Rust producer wrote: three replies (the first cites an executed +// read_editor call), then a completed, an interrupted one and a swallowed +// ending. Structure is the producer's; only the clocks and the authored +// content are put back, because a fixture can hold neither. +const FIXTURE = JSON.parse( + fs.readFileSync(`${ROOT}/tests/fixtures/acquisition-report.json`, "utf8"), +); + +function at(identity, seconds, uncertainty = 4) { + identity.clock.emitted_unix_ms = START + Math.round(seconds * 1000); + identity.clock.clock_uncertainty_ms = uncertainty; +} + +/// The fixture as a faithful run of checkpoint 1: reply A answers it after +/// reading the editor, replies B and C come long after the window. +function faithful(edit = () => {}) { + const report = structuredClone(FIXTURE); + const { replies, batches, receipts } = report.capture; + replies[0].text = ANSWER.reply.text; + batches[0].calls[0].name = ANSWER.tool_calls[0].name; + batches[0].calls[0].result = ANSWER.tool_calls[0].result_text; + receipts[0].name = ANSWER.tool_calls[0].name; + at(batches[0].identity, ANSWER.tool_calls[0].at_s); + at(replies[0].identity, ANSWER.reply.started_at_s); + at(replies[1].identity, 400); + at(replies[2].identity, 500); + edit(report); + return report; +} + +const OPTIONS = { scheduleStartUnixMs: START }; +const assess = (report, options = OPTIONS) => + assessCheckpoint(BUNDLE, report, CP.checkpoint_id, options); + +/// One test per row: the edit, applied to a faithful run, must give `verdict` +/// with a reason matching the pattern. +function rows(table, verdict, suffix) { + for (const [name, edit, pattern] of table) { + test(`${name} ${suffix}`, () => { + const result = assess(faithful(edit)); + assert.equal(result.verdict, verdict, reasonsOf(result)); + assert.match(reasonsOf(result), pattern); + }); + } +} + +function reasonsOf(result) { + return result.reasons.join(" | "); +} + +test("a faithful reply that cites an executed read passes", () => { + const result = assess(faithful()); + assert.deepEqual(result, { + checkpoint_id: CP.checkpoint_id, + verdict: "pass", + reasons: [], + }); +}); + +test("the raw producer report cannot place anything in time", () => { + const result = assess(structuredClone(FIXTURE)); + assert.equal(result.verdict, "not_assessed"); + assert.match(reasonsOf(result), /clock/); +}); + +const UNREADABLE = [ + ["an observer failure", (r) => (r.arming_failure = "Scope"), /observer/], + ["a latched capture", (r) => (r.capture.failure = "Limit"), /latched Limit/], + ["no capture", (r) => (r.capture = null), /no capture/], + [ + "a reply without positions", + (r) => delete r.capture.replies[0].receipts_last, + /positions/, + ], + ["a batch without calls", (r) => delete r.capture.batches[0].calls, /calls/], + [ + "a batch with an unreadable call", + (r) => (r.capture.batches[0].calls = [null]), + /calls/, + ], + [ + "a receipt without an id", + (r) => delete r.capture.receipts[0].id, + /receipt/, + ], +]; +rows(UNREADABLE, "not_assessed", "makes every checkpoint not_assessed"); + +test("an unknown schedule origin or checkpoint is not assessed", () => { + assert.equal(assess(faithful(), {}).verdict, "not_assessed"); + assert.equal( + assessCheckpoint(BUNDLE, faithful(), "nope", OPTIONS).verdict, + "not_assessed", + ); +}); + +test("a clock reading too uncertain to place an event is unusable", () => { + const result = assess( + faithful((r) => at(r.capture.replies[0].identity, 140, 5000)), + ); + assert.equal(result.verdict, "not_assessed"); +}); + +const UNKNOWN = [ + [ + "a swallowed ending", + (r) => (r.capture.boundaries[0].accepted = false), + /swallowed/, + ], + [ + "a reply sealed by a cut before any ending", + (r) => (r.capture.boundaries[0].replies_before = 2), + /sealed the reply/, + ], + [ + "no ending at all", + (r) => (r.capture.boundaries = []), + /no provider ending/, + ], + [ + "a reply that continued across a tool call", + (r) => (r.capture.replies[0].receipts_last = 2), + /continued across/, + ], + [ + "a call received on another socket", + (r) => (r.capture.receipts[0].identity.socket = 2), + /replacement/, + ], + [ + "a rewritten reply", + (r) => (r.capture.replies[0].revised = true), + /rewritten/, + ], + [ + "a reply never sealed", + (r) => (r.capture.replies[0].final_segment = false), + /never sealed/, + ], + [ + "a second reply inside the window", + (r) => at(r.capture.replies[1].identity, 150), + /2 replies/, + ], + [ + "a reply just before the window, within the uncertainty", + (r) => at(r.capture.replies[1].identity, CP.stimulus_end_s - 0.5, 1000), + /window edge/, + ], + [ + "a reply just after the window, within the uncertainty", + (r) => + at( + r.capture.replies[1].identity, + CP.stimulus_end_s + CP.response_window_s + 0.5, + 1000, + ), + /window edge/, + ], + [ + "a reply within the clock uncertainty of the question's start", + (r) => at(r.capture.replies[2].identity, CP.at_s + 0.2, 1000), + /window edge/, + ], + [ + "the faithful reply, after a recording that ended its speech early", + (r) => at(r.capture.replies[0].identity, CP.stimulus_end_s - 2), + /before the question's estimated end/, + ], + [ + "the faithful reply too close to the start of the window", + (r) => at(r.capture.replies[0].identity, CP.stimulus_end_s + 0.2, 1000), + /window edge/, + ], + [ + "a generation other than the reply's ended it", + (r) => (r.capture.boundaries[0].identity.trace_turn = 9), + /another generation/, + ], + [ + "a call from another generation", + (r) => (r.capture.receipts[0].identity.trace_turn = 9), + /another generation/, + ], + [ + "an ending of another generation inside the turn", + (r) => { + const stray = structuredClone(r.capture.boundaries[0]); + stray.accepted = false; + stray.replies_before = 0; + r.capture.boundaries.unshift(stray); + }, + /another generation lies inside/, + ], + [ + "an ending taken on another socket inside the turn", + (r) => { + const other = structuredClone(r.capture.boundaries[0]); + other.identity.socket = 2; + other.replies_before = 0; + r.capture.boundaries.unshift(other); + }, + /another generation lies inside/, + ], + [ + "a send nobody observed", + (r) => (r.capture.batches[0].local_send_completed = null), + /not observed/, + ], + [ + "a call with no usable clock reading", + (r) => at(r.capture.batches[0].identity, 136, 5000), + /usable clock/, + ], + [ + "a call too close to the limit it must clear", + (r) => at(r.capture.batches[0].identity, 125.2, 1000), + /uncertainty of a limit/, + ], + [ + "a cited call with no executed batch", + (r) => (r.capture.batches[0].calls[0].id = "other"), + /no executed batch/, + ], + [ + "a reply that continued across an executed batch", + (r) => (r.capture.replies[0].batches_last = 2), + /continued across/, + ], + [ + "an earlier segment of the same turn", + (r) => { + // The faithful reply is B, which follows A with no ending between. + const [a, b] = r.capture.replies; + b.text = a.text; + at(b.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + r.capture.boundaries[0].replies_before = 0; + }, + /same turn/, + ], +]; +rows(UNKNOWN, "not_assessed", "is not assessed, never a pass or a fail"); + +const FAILS = [ + [ + "no reply in the window", + (r) => at(r.capture.replies[0].identity, 300), + /no reply was recorded/, + ], + [ + "a reply without the nonce", + (r) => (r.capture.replies[0].text = "something unrelated"), + /nonce/, + ], + [ + "an interrupted reply", + (r) => { + // The faithful reply is the interrupted one (reply C). + const [a, , c] = r.capture.replies; + c.text = a.text; + at(c.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + }, + /interrupted/, + ], + [ + "a call received while output was being discarded", + (r) => (r.capture.receipts[0].discarded = true), + /no read_editor call/, + ], + [ + "a call that was received and never executed", + (r) => (r.capture.receipts[0].batch = null), + /no read_editor call/, + ], + [ + "a call executed after the reply began", + (r) => { + r.capture.replies[0].batches_before = 0; + r.capture.replies[0].batches_last = 0; + }, + /no read_editor call/, + ], + [ + "a response that was not sent", + (r) => (r.capture.batches[0].local_send_completed = false), + /did not succeed/, + ], + [ + "a tool that answered with an error", + (r) => (r.capture.batches[0].calls[0].result = { error: "denied" }), + /did not succeed/, + ], + [ + "a result that lacks the revision", + (r) => (r.capture.batches[0].calls[0].result = "nothing useful"), + /lacks/, + ], +]; +rows(FAILS, "fail", "fails the checkpoint"); + +test("a tool result is matched on its strings, not its JSON escaping", () => { + const report = faithful((r) => { + r.capture.batches[0].calls[0].result = { + frameworkState: { phases: [] }, + result: `line one\n"${ANSWER.tool_calls[0].result_text}"\n`, + }; + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a reply that began before the question was asked does not count against it", () => { + // The previous question's answer, say: before at_s, so outside both the + // early span and the window. + const report = faithful((r) => + at(r.capture.replies[2].identity, CP.at_s - 5), + ); + assert.equal(assess(report).verdict, "pass"); +}); + +test("an ending from another socket does not close the reply", () => { + const report = faithful((r) => { + const other = structuredClone(r.capture.boundaries[0]); + other.identity.socket = 2; + other.ending = "Interrupted"; + r.capture.boundaries.unshift(other); + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a reply on another socket between a reply and its ending is not its sealer", () => { + const report = faithful((r) => { + r.capture.replies[1].identity.socket = 2; + r.capture.boundaries[0].replies_before = 2; + }); + assert.equal(assess(report).verdict, "pass"); +}); + +test("a later reply does not inherit the previous turn's call", () => { + const report = faithful((r) => { + const [a, b] = r.capture.replies; + b.text = a.text; + at(b.identity, ANSWER.reply.started_at_s); + at(a.identity, 400); + }); + const result = assess(report); + assert.equal(result.verdict, "fail", reasonsOf(result)); + assert.match(reasonsOf(result), /no read_editor call/); +}); + +test("a report missing its parts is named", () => { + for (const [edit, pattern] of [ + [(r) => delete r.capture.boundaries, /boundaries is missing/], + [ + (r) => delete r.capture.boundaries[0].replies_before, + /boundary has no positions/, + ], + [(r) => delete r.capture.replies[0].identity, /reply has no positions/], + ]) { + const report = structuredClone(FIXTURE); + edit(report); + assert.match(reportProblems(report).join(" "), pattern); + } +}); + +test("the interview is verified only when every checkpoint passed", () => { + const pass = { checkpoint_id: "a", verdict: "pass", reasons: [] }; + const fail = { checkpoint_id: "b", verdict: "fail", reasons: ["x"] }; + const unknown = { + checkpoint_id: "c", + verdict: "not_assessed", + reasons: ["y"], + }; + assert.equal(summarize([pass, pass]).status, "verified"); + assert.equal(summarize([pass, unknown]).status, "not_assessed"); + assert.deepEqual(summarize([pass, unknown]).uncovered, ["c"]); + assert.equal(summarize([pass, fail, unknown]).status, "failed"); + assert.equal(summarize([]).status, "not_assessed"); +}); + +test("judging the interview covers every scheduled checkpoint", () => { + const verdict = judgeInterview(BUNDLE, faithful(), { + scheduleStartUnixMs: START, + }); + assert.equal(verdict.checkpoints.length, BUNDLE.schedule.checkpoints.length); + assert.equal(verdict.checkpoints[0].verdict, "pass"); + // The run holds one faithful reply, so every other checkpoint found none. + assert.equal(verdict.status, "failed"); + const broken = judgeInterview( + BUNDLE, + faithful((r) => (r.capture.failure = "Limit")), + { + scheduleStartUnixMs: START, + }, + ); + assert.equal(broken.status, "not_assessed"); + assert.equal(broken.uncovered.length, BUNDLE.schedule.checkpoints.length); +}); + +test("an unreadable report is named, not thrown on", () => { + assert.deepEqual(reportProblems(null), ["no report"]); + assert.deepEqual(reportProblems(faithful()), []); +}); + +// The capture file the browser check leaves, with the long-run report fields +// the merge reads. +function capture() { + const plan = { durationMin: 90, observationTargetS: 5400 }; + return { + flow: "long-interview", + plan, + report: { + status: "complete", + exit_code: 0, + anchor: { source: "server_clock" }, + validation_fixture: false, + schedule_mode: "coding", + qualification: "not_evaluated", + reply_content: "not_assessed", + schedule_sha256: BUNDLE.schedule.schedule_sha256, + schedule_origin_unix_ms: START, + }, + }; +} + +/// The serving fixture's report: the acquisition report it left, ended ok. +const servingReport = () => ({ + outcome: "ok", + bound: "127.0.0.1:0", + ...faithful(), +}); + +function merged(edit = () => {}) { + const [file, fixture] = [capture(), servingReport()]; + edit(file, fixture); + return mergeVerdict(file, fixture); +} + +test("the merge fills in reply_content and nothing else", () => { + const result = merged(); + assert.equal(result.report.reply_content, "failed"); + assert.deepEqual(result.report.reply_content_checkpoints, { + pass: 1, + fail: BUNDLE.schedule.checkpoints.length - 1, + not_assessed: 0, + }); + assert.equal(result.report.qualification, "not_evaluated"); + assert.equal(result.report.status, "complete"); + assert.match(result.report.reply_content_reasons[0], /no reply was recorded/); +}); + +const UNJUDGED = [ + [ + "a fixture that did not end ok", + (file, fixture) => (fixture.outcome = "shutdown bound"), + /ended with shutdown bound/, + ], + [ + "a schedule other than the planned one", + (file) => (file.report.schedule_sha256 = "0"), + /not the one rebuilt/, + ], + [ + "an unknown schedule origin", + (file) => delete file.report.schedule_origin_unix_ms, + /origin/, + ], + [ + "a latched capture", + (file, fixture) => (fixture.capture.failure = "Limit"), + /latched/, + ], + [ + "a plan that builds no schedule", + (file) => (file.plan.durationMin = 1), + /cannot be rebuilt/, + ], + [ + "a surface that is not one", + (file) => (file.report.schedule_mode = "dispatch"), + /cannot be rebuilt/, + ], + [ + "a run that failed", + (file) => (file.report.exit_code = 1), + /did not complete cleanly/, + ], + [ + "a run that did not finish", + (file) => (file.report.status = "failed"), + /did not complete cleanly/, + ], + [ + "a clock not anchored to the server", + (file) => (file.report.anchor.source = "first_token_request"), + /not anchored/, + ], + [ + "a run that played tone fixtures", + (file) => (file.report.validation_fixture = true), + /tone fixtures/, + ], + [ + "a capture that does not say what it played", + (file) => delete file.report.validation_fixture, + /tone fixtures/, + ], +]; +for (const [name, edit, pattern] of UNJUDGED) { + test(`${name} leaves reply_content not_assessed with the reason`, () => { + const result = merged(edit); + assert.equal(result.report.reply_content, "not_assessed"); + assert.match(result.report.reply_content_reasons.join(" "), pattern); + }); +} + +test("only the three verdict fields move", () => { + const file = capture(); + const after = mergeVerdict(file, servingReport()); + const VERDICT = [ + "reply_content", + "reply_content_checkpoints", + "reply_content_reasons", + ]; + const without = (report) => + Object.fromEntries( + Object.entries(report).filter(([key]) => !VERDICT.includes(key)), + ); + assert.deepEqual(without(after.report), without(file.report)); + assert.deepEqual( + Object.keys(after.report).filter((key) => key.startsWith("reply_content")), + VERDICT, + ); +}); + +test("a verified interview carries an empty reasons list, not a stale one", () => { + const pass = { checkpoint_id: "a", verdict: "pass", reasons: [] }; + const fields = verdictFields(summarize([pass, pass])); + assert.equal(fields.reply_content, "verified"); + assert.deepEqual(fields.reply_content_reasons, []); + assert.deepEqual(fields.reply_content_checkpoints, { + pass: 2, + fail: 0, + not_assessed: 0, + }); +}); + +test("a missing fixture report is named, and other flows are left alone", () => { + const file = capture(); + const result = mergeVerdict(file, null); + assert.equal(result.report.reply_content, "not_assessed"); + assert.match(result.report.reply_content_reasons[0], /left no report/); + assert.equal(mergeVerdict({ ...file, flow: "soak" }, null), null); +}); + +function withFiles(run) { + const dir = fs.mkdtempSync(`${os.tmpdir()}/evidence-`); + try { + fs.writeFileSync(`${dir}/capture.json`, JSON.stringify(capture())); + fs.writeFileSync(`${dir}/out.json`, JSON.stringify(servingReport())); + run(dir); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +} + +test("the command rewrites the capture in place and reports one line", () => { + withFiles((dir) => { + const lines = []; + const code = main( + ["--capture", `${dir}/capture.json`, "--fixture", `${dir}/out.json`], + { out: (line) => lines.push(line) }, + ); + assert.equal(code, 0); + assert.match(lines[0], /^long-interview reply_content=failed pass=1 /); + const after = JSON.parse(fs.readFileSync(`${dir}/capture.json`, "utf8")); + assert.equal(after.report.reply_content, "failed"); + }); +}); + +test("a fixture report that is there but unreadable is not called missing", () => { + withFiles((dir) => { + const run = (fixture) => { + main(["--capture", `${dir}/capture.json`, "--fixture", fixture], { + out: () => {}, + }); + const after = JSON.parse(fs.readFileSync(`${dir}/capture.json`, "utf8")); + return after.report.reply_content_reasons[0]; + }; + fs.writeFileSync(`${dir}/cut.json`, '{"outcome": "ok", "secret reply te'); + const reason = run(`${dir}/cut.json`); + assert.match(reason, /report could not be read: SyntaxError$/); + assert.doesNotMatch(reason, /secret/); + assert.match(run(`${dir}/never-written.json`), /left no report/); + }); +}); + +test("a capture that is there but unreadable exits 2 and is named so", () => { + withFiles((dir) => { + fs.writeFileSync(`${dir}/capture.json`, "{"); + const errs = []; + const code = main( + ["--capture", `${dir}/capture.json`, "--fixture", `${dir}/out.json`], + { err: (line) => errs.push(line) }, + ); + assert.equal(code, 2); + assert.match(errs.join(" "), /SyntaxError/); + assert.equal(fs.readFileSync(`${dir}/capture.json`, "utf8"), "{"); + }); +}); + +test("the fixture check names a report that is there but unreadable", () => { + withFiles((dir) => { + fs.writeFileSync(`${dir}/cut.json`, '{"outcome"'); + const errs = []; + assert.equal( + main(["--check-fixture", `${dir}/cut.json`], { + err: (line) => errs.push(line), + }), + 1, + ); + assert.match(errs.join(" "), /could not be read: SyntaxError/); + }); +}); + +test("a command line or capture it cannot use exits 2 and writes nothing", () => { + withFiles((dir) => { + const before = fs.readFileSync(`${dir}/capture.json`, "utf8"); + const err = () => {}; + for (const argv of [ + [], + ["--capture", `${dir}/capture.json`], + ["--fixture", `${dir}/out.json`], + ["--capture", `${dir}/nope`, "--fixture", `${dir}/out.json`], + ]) { + assert.equal(main(argv, { err }), 2, argv.join(" ")); + } + assert.equal(fs.readFileSync(`${dir}/capture.json`, "utf8"), before); + }); +}); + +test("a serving fixture report is accepted by its shape and its outcome", () => { + const without = (key) => { + const copy = servingReport(); + delete copy[key]; + return copy; + }; + assert.deepEqual(fixtureProblems(servingReport()), []); + const rows = [ + [null, /not an object/], + [[], /not an object/], + [{}, /no outcome/], + [{ outcome: "shutdown bound" }, /ended with shutdown bound/], + [without("arming_failure"), /no arming_failure/], + [without("bound"), /no bound/], + [{ ...servingReport(), capture: null }, /no capture/], + ]; + for (const [report, pattern] of rows) { + assert.match(fixtureProblems(report).join(" "), pattern); + } + const lacking = servingReport(); + delete lacking.capture.boundaries; + delete lacking.capture.failure; + assert.match(fixtureProblems(lacking).join(" "), /no failure field/); + assert.match(fixtureProblems(lacking).join(" "), /no boundaries list/); + // An observer that failed says so itself, and the judge names it. + const failed = { ...servingReport(), arming_failure: "Scope", capture: null }; + assert.deepEqual(fixtureProblems(failed), []); + // A latched failure inside a well-formed report is the judge's to name. + const latched = servingReport(); + latched.capture.failure = "Limit"; + assert.deepEqual(fixtureProblems(latched), []); +}); + +test("the check command exits 1 on a report it would not accept", () => { + withFiles((dir) => { + const err = []; + const sink = (line) => err.push(line); + assert.equal( + main(["--check-fixture", `${dir}/out.json`], { err: sink }), + 0, + ); + fs.writeFileSync(`${dir}/bad.json`, JSON.stringify({ outcome: "boom" })); + assert.equal( + main(["--check-fixture", `${dir}/bad.json`], { err: sink }), + 1, + ); + assert.match(err.join(" "), /ended with boom/); + assert.equal(main(["--check-fixture", `${dir}/missing`], { err: sink }), 1); + assert.equal(main(["--check-fixture"], { err: sink }), 1); + }); +}); diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 3191b9f89..e7ff0c6dc 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -19,6 +19,7 @@ only thing in the repo that crosses that boundary. | `integrity-chain.json` | `integrityEventPayload` | `apply_integrity` | | `task-action.json`, `task-edit.json`, `task-revision.json`, `task-run.json`, `task-start.json`, `task-end.json`, `task-connected.json`, `task-thinking.json` | `taskActionPayload`, `taskEditPayload`, `taskRevisionPayload`, `taskRunPayload`, `taskStartPayload`, `taskEndPayload`, `taskConnectedPayload`, `taskThinkingPayload` in `web/lib.js` | the typed messages `TaskSession::receive` parses, checked in `tests/unit/tasks/session.rs` | | `task-state.json`, `task-error.json`, `task-review.json` | written by the generator; the Rust side that must agree is `TaskSession::snapshot` (state), `ErrorMessage` (error) and `tasks::report` (review) | `web/task.js`, driven from these files by `tests/browser/task-workspace.test.js` and `task-controller.test.js`; Rust reads them in `tests/unit/tasks/session.rs` and `report.rs` | +| `acquisition-report.json` | `Arming::report` in `src/livekit/probe/acquisition.rs`, written by `acquisition_report_is_the_wire_fixture_the_consumer_reads` (regenerate with `CODETRIAL_UPDATE_FIXTURES=1 cargo test --lib acquisition_report_is_the_wire`) | `scripts/long-interview-evidence.cjs`, read by `tests/browser/long-interview-evidence.test.js` | `tests/fixtures/task-mode/` is not wire data: it is the packaged task set, its encrypted package and the patches both the server and the packaging tool must diff --git a/tests/fixtures/acquisition-report.json b/tests/fixtures/acquisition-report.json new file mode 100644 index 000000000..16a3d9efd --- /dev/null +++ b/tests/fixtures/acquisition-report.json @@ -0,0 +1,208 @@ +{ + "arming_failure": null, + "capture": { + "batches": [ + { + "calls": [ + { + "id": "call-1", + "name": "read_editor", + "result": { + "frameworkState": { + "phases": [] + }, + "result": "BEGIN UNTRUSTED EDITOR (python)\n(the editor is currently empty)\nEND UNTRUSTED EDITOR\nBEGIN UNTRUSTED TEST RUN\nNo test run was recorded; tests may not have been attempted or may not have been available for the selected language/problem yet.\nEND UNTRUSTED TEST RUN\nTIMER: about 45 minutes remain on the candidate's countdown." + } + } + ], + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "local_send_completed": true, + "sequence": 0 + } + ], + "boundaries": [ + { + "accepted": true, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "receipts_before": 1, + "replies_before": 1 + }, + { + "accepted": true, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 2 + }, + "receipts_before": 1, + "replies_before": 2 + }, + { + "accepted": true, + "batches_before": 1, + "ending": "Interrupted", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 3 + }, + "receipts_before": 1, + "replies_before": 3 + }, + { + "accepted": false, + "batches_before": 1, + "ending": "Complete", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": null + }, + "receipts_before": 1, + "replies_before": 3 + } + ], + "failure": null, + "receipts": [ + { + "batch": 0, + "discarded": false, + "id": "call-1", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "name": "read_editor" + } + ], + "replies": [ + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-0", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 1 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-A one. two.", + "views": 3 + }, + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-1", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 2 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-B.", + "views": 2 + }, + { + "batches_before": 1, + "batches_last": 1, + "final_segment": true, + "id": "interviewer-2", + "identity": { + "clock": { + "clock_uncertainty_ms": null, + "emitted_ms": null, + "emitted_unix_ms": 0, + "start_unix_ms": null + }, + "room": "recovery-test", + "session": 7, + "socket": 1, + "trace_turn": 3 + }, + "receipts_before": 1, + "receipts_last": 1, + "revised": false, + "text": "REPLY-C cut", + "views": 2 + } + ], + "retained_bytes": 0, + "retained_calls": 1, + "retained_receipt_bytes": 0 + } +} diff --git a/tests/stubs/interview-run-budget/fake_worker.py b/tests/stubs/interview-run-budget/fake_worker.py index d09b1b2c4..b394b1d1d 100644 --- a/tests/stubs/interview-run-budget/fake_worker.py +++ b/tests/stubs/interview-run-budget/fake_worker.py @@ -78,6 +78,7 @@ def raw_permit_request(guard, action_id, seq): signal.signal(signal.SIGTERM, signal.SIG_IGN) note("pids", str(os.getpid())) +note("reap_at", os.environ["BROWSER_CHECK_BUDGET_REAP_AT"]) telemetry(event="hb") guard = irb.WorkerGuard.from_env() stop = os.environ["BROWSER_CHECK_BUDGET_STOP"] diff --git a/tests/stubs/interview-run-budget/stand-in-fixture.py b/tests/stubs/interview-run-budget/stand-in-fixture.py new file mode 100644 index 000000000..c8a6da887 --- /dev/null +++ b/tests/stubs/interview-run-budget/stand-in-fixture.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import json +import os +import signal +import sys +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +if "--list" in sys.argv: + print("module::serve: test") + sys.exit(0) +directory = os.environ["CODETRIAL_FIXTURE_DIR"] +address = os.environ["CODETRIAL_FIXTURE_ADDR"] +host, port = address.rsplit(":", 1) + + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def do_POST(self): + self.send_response(200) + self.send_header("Set-Cookie", "codetrial_session=stand-in; Path=/") + self.end_headers() + + def log_message(self, *args): + pass + + +server = HTTPServer((host, int(port)), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() +with open(os.path.join(directory, "ready"), "w") as ready: + ready.write(address) +# The two lines the script looks for to know a room got an interviewer. +print("codetrial dispatch room=stand-in", flush=True) +print("starting interview: room=stand-in", flush=True) +terms = [] +signal.signal(signal.SIGTERM, lambda *args: terms.append(1)) +with open(os.environ["STAND_IN_PIDFILE"], "w") as pidfile: + pidfile.write(str(os.getpid())) +# A deadline of its own: a run that never reaches the stop file must not leave +# this behind. +deadline = time.time() + 120 +while ( + not os.path.exists(os.path.join(directory, "stop")) + and not terms + and time.time() < deadline +): + time.sleep(0.05) +print("stand-in fixture stopping", flush=True) +if os.environ.get("STAND_IN_WRITES_REPORT"): + report = { + "outcome": "ok", + "bound": address, + "arming_failure": None, + # What the launcher passed, so the test can hold it to the manifest. + "launched_with": { + name: os.environ.get(f"CODETRIAL_FIXTURE_{name}") + for name in ("DEADLINE_S", "DRAIN_S", "SETTLE_S") + }, + "capture": { + "failure": None, + "receipts": [], + "batches": [], + "replies": [], + "boundaries": [], + }, + } + with open(os.path.join(directory, "out.json"), "w") as out: + json.dump(report, out) diff --git a/tests/stubs/interview-run-budget/stand-in-node.sh b/tests/stubs/interview-run-budget/stand-in-node.sh new file mode 100644 index 000000000..8926817ab --- /dev/null +++ b/tests/stubs/interview-run-budget/stand-in-node.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Stands in for node on browser-check.cjs only: leaves the capture a finished +# long interview would, then exits as told. Everything else runs the real node. +case $1 in + */scripts/browser-check.cjs) + printf '%s' "$STAND_IN_CAPTURE" > "$BROWSER_CHECK_CAPTURE" + exit "${STAND_IN_WORKER_EXIT:-0}" + ;; +esac +exec "$REAL_NODE" "$@" diff --git a/tests/test_interview_run_budget.py b/tests/test_interview_run_budget.py index 7ba83208e..e7f4126f4 100644 --- a/tests/test_interview_run_budget.py +++ b/tests/test_interview_run_budget.py @@ -650,10 +650,15 @@ def test_bounded_cleanup_gives_up_on_cleanup_that_hangs(self): class LauncherRunTests(Scratch): def test_a_clean_run_completes_with_every_action_granted_and_dispatched_once(self): + started = time.time() result, _ = self.launch("normal", env={"FAKE_OBSERVED": "1"}) self.assertEqual(result.returncode, 0, result.stderr) report = self.result() self.assertEqual(report["status"], "complete") + # The worker is told when the reap begins: before the wall cap. + reap_at = int((self.fake / "reap_at").read_text()) + self.assertGreater(reap_at, started - 1) + self.assertLess(reap_at, started + self.manifest()["outer_wall_cap_s"]) self.assertEqual(self.dispatched(), ACTIONS + ["end"]) self.assertEqual(self.grants(), ACTIONS + ["end"]) hard = report["hard"]["scripted_actions"] @@ -1168,6 +1173,526 @@ def test_a_live_run_needs_the_launchers_socket(self): self.assertEqual(result.returncode, 2, result.stderr) self.assertIn("live BROWSER_CHECK_BUDGET_SOCKET", result.stderr) + def dummy_config(self): + """The credentials a dispatch run asks for, from a throwaway config + rather than whatever this machine keeps.""" + config = self.root / "dummy.env.local" + config.write_text( + "LIVEKIT_URL=wss://primary.example\nLIVEKIT_API_KEY=k\n" + "LIVEKIT_API_SECRET=s\nGOOGLE_API_KEY=g\n" + ) + return config + + def test_the_serving_fixture_needs_a_budgeted_dispatch_run_and_a_fresh_report(self): + config = self.dummy_config() + report = self.root / "fixture-report.json" + # A reserve that outlasts the 150 s default stop wait with a minute over. + manifest = self.write_manifest( + outer_wall_cap_s=3600, cleanup_reserve_s=300, selected_duration_s=600 + ) + base = { + "CODETRIAL_CONFIG_ENV": str(config), + "BROWSER_CHECK_RUN_MANIFEST": str(manifest), + "BROWSER_CHECK_RUN_MANIFEST_SHA256": sha256(manifest), + "BROWSER_CHECK_FIXTURE_TEST": "module::serve", + "BROWSER_CHECK_AGENT": "dispatch", + "BROWSER_CHECK_FIXTURE_OUT": str(report), + "BROWSER_CHECK_BUDGET_REAP_AT": str(int(time.time()) + 3600), + } + result = self.check(**base) + self.assertEqual(result.returncode, 0, result.stderr) + # A capture path that is free is fine. + free = self.check(**base, BROWSER_CHECK_CAPTURE=str(self.root / "capture.json")) + self.assertEqual(free.returncode, 0, free.stderr) + no_run = { + "BROWSER_CHECK_RUN_MANIFEST": None, + "BROWSER_CHECK_RUN_ID": None, + "BROWSER_CHECK_RUN_MANIFEST_SHA256": None, + "BROWSER_CHECK_BUDGET_SOCKET": None, + } + existing = self.root / "already-there.json" + existing.write_text("{}") + linked = self.root / "linked" + linked.symlink_to(self.root, target_is_directory=True) + rows = [ + ({**base, "BROWSER_CHECK_AGENT": "home"}, "needs a budgeted run"), + ({**base, **no_run}, "needs a budgeted run"), + ({**base, "BROWSER_CHECK_FIXTURE_OUT": None}, "must name a report file"), + ({**base, "BROWSER_CHECK_BUDGET_REAP_AT": None}, "BUDGET_REAP_AT"), + ({**base, "BROWSER_CHECK_BUDGET_REAP_AT": "soon"}, "BUDGET_REAP_AT"), + ( + {**base, "BROWSER_CHECK_CAPTURE": str(report)}, + "must name different files", + ), + ( + # Through a symlinked directory: the paths are compared once + # resolved, not as they are spelled. + {**base, "BROWSER_CHECK_CAPTURE": str(linked / report.name)}, + "must name different files", + ), + ( + {**base, "BROWSER_CHECK_CAPTURE": f"{report}.server.log"}, + "must name different files", + ), + ( + { + **base, + "BROWSER_CHECK_CAPTURE": str(report.with_suffix("")), + "BROWSER_CHECK_FIXTURE_OUT": f"{report.with_suffix('')}.partial", + }, + "must name different files", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_OUT": str(existing)}, + "must name a report file", + ), + ( + { + **base, + "BROWSER_CHECK_FIXTURE_OUT": str( + self.root / "no-such-dir" / "r.json" + ), + }, + "in a directory that does and can be written", + ), + ( + {**base, "BROWSER_CHECK_CAPTURE": str(existing)}, + "BROWSER_CHECK_CAPTURE must not exist yet", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "soon"}, + "number of seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "0150"}, + "number of seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "39"}, + "at least 40 seconds", + ), + ( + {**base, "BROWSER_CHECK_FIXTURE_STOP_WAIT_S": "241"}, + "needs at least 301s", + ), + ] + for env, expected in rows: + with self.subTest(env=env): + result = self.check(**env) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(expected, result.stderr) + + def test_the_stop_wait_must_fit_inside_the_manifests_cleanup_reserve(self): + config = self.dummy_config() + # The reserve the shared manifest here carries is the accepted minimum, + # far short of a stop wait of 150 s. + result = self.check( + CODETRIAL_CONFIG_ENV=str(config), + BROWSER_CHECK_AGENT="dispatch", + BROWSER_CHECK_FIXTURE_TEST="module::serve", + BROWSER_CHECK_FIXTURE_OUT=str(self.root / "report.json"), + ) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("cleanup_reserve_s is 60", result.stderr) + self.assertIn("needs at least 210s", result.stderr) + + def test_the_fixture_is_started_in_a_session_of_its_own(self): + script = (ROOT / "scripts" / "browser-check.sh").read_text() + # A TERM sent to the script's process group must reach the script, whose + # trap stops the fixture gracefully, and not the fixture first. The + # launch itself needs a browser, so this reads the script. + self.assertRegex(script, r'set -- setsid "\$@"') + self.assertRegex( + script, r'--ignored --exact "\$FIXTURE_TEST" --nocapture --test-threads=1' + ) + + def kill_pidfile(self, path): + if path.exists(): + self.kill(int(path.read_text())) + + GOOD_CAPTURE = { + "flow": "long-interview", + "mode": "dispatch", + "plan": {"durationMin": 90, "observationTargetS": 5400}, + "report": { + "status": "complete", + "exit_code": 0, + "anchor": {"source": "server_clock"}, + "validation_fixture": False, + "schedule_mode": "coding", + "qualification": "not_evaluated", + "reply_content": "not_assessed", + "schedule_origin_unix_ms": 1700000000000, + }, + } + + def good_capture(self): + """The capture a finished long interview leaves, with the schedule hash + the judge will rebuild from the plan.""" + sha = subprocess.run( + [ + shutil.which("node"), + "-e", + "console.log(require('./scripts/long-interview-schedule.cjs')" + ".generateSchedule('coding',{durationMin:90,observationTargetS:5400})" + ".schedule.schedule_sha256)", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + capture = json.loads(json.dumps(self.GOOD_CAPTURE)) + capture["report"]["schedule_sha256"] = sha + return json.dumps(capture) + + def run_fixture_path(self, writes_report=True, worker_exit=0, capture_text=None): + """browser-check.sh from the manifest to the exit trap, with a stand-in + fixture server and a stand-in worker: no browser, no credentials.""" + fixture = self.root / "stand-in-fixture" + support.write_executable( + fixture, support.stub("interview-run-budget/stand-in-fixture.py") + ) + manifest = self.write_manifest( + outer_wall_cap_s=3600, + cleanup_reserve_s=300, + selected_duration_s=600, + binaries={"candidate": {"path": str(fixture), "sha256": sha256(fixture)}}, + ) + shim = self.root / "shim" + shim.mkdir() + # The script keeps its floor and latency logs outside its own scratch + # directory by design; here they go inside this test's. + tmp = self.root / "tmp" + tmp.mkdir() + support.write_executable( + shim / "node", support.stub("interview-run-budget/stand-in-node.sh") + ) + socket_path = self.root / "budget.sock" + # Bound and never listened on: the script only asks that it be a socket. + listener = socket.socket(socket.AF_UNIX) + listener.bind(str(socket_path)) + self.addCleanup(listener.close) + report = self.root / "kept-report.json" + capture = self.root / "capture.json" + pidfile = self.root / "stand-in.pid" + self.addCleanup(self.kill_pidfile, pidfile) + result = self.check( + BROWSER_CHECK_VALIDATE_ENV_ONLY=None, + BROWSER_CHECK_AGENT="dispatch", + BROWSER_CHECK_FIXTURE_TEST="module::serve", + BROWSER_CHECK_FIXTURE_OUT=str(report), + BROWSER_CHECK_FIXTURE_STOP_WAIT_S="40", + BROWSER_CHECK_CAPTURE=str(capture), + BROWSER_CHECK_RUN_MANIFEST=str(manifest), + BROWSER_CHECK_RUN_MANIFEST_SHA256=sha256(manifest), + BROWSER_CHECK_BUDGET_SOCKET=str(socket_path), + BROWSER_CHECK_BUDGET_REAP_AT=str(int(time.time()) + 3600), + CODETRIAL_CONFIG_ENV=str(self.dummy_config()), + PATH=f"{shim}:{os.environ['PATH']}", + TMPDIR=str(tmp), + PORT=None, + INTERVIEW_ROOM_NAME=None, + STAND_IN_PIDFILE=str(pidfile), + REAL_NODE=shutil.which("node"), + STAND_IN_CAPTURE=capture_text or self.good_capture(), + STAND_IN_WORKER_EXIT=str(worker_exit), + STAND_IN_WRITES_REPORT="1" if writes_report else None, + ) + return result, report, capture + + def test_the_fixture_path_runs_from_start_to_a_judged_capture(self): + result, report, capture = self.run_fixture_path() + self.assertEqual(result.returncode, 0, result.stderr) + # The report was kept with a private copy of the server's log. + kept_report = json.loads(report.read_text()) + self.assertEqual(kept_report["outcome"], "ok") + # Its drain and settle add up to the stop wait less the slack, and its + # own deadline leaves the stop wait and the slack before the runner's + # reap (an hour from the launch here), less what the launch used. + launched = kept_report["launched_with"] + self.assertEqual((launched["DRAIN_S"], launched["SETTLE_S"]), ("10", "10")) + ceiling = 3600 - 40 - 20 + self.assertIn(int(launched["DEADLINE_S"]), range(ceiling - 60, ceiling + 1)) + log = Path(str(report) + ".server.log") + self.assertIn("stand-in fixture stopping", log.read_text()) + self.assertEqual(log.stat().st_mode & 0o777, 0o600) + # And judged after it stopped: the verdict is in the capture, the rest + # of the report is as the worker left it. + kept = json.loads(capture.read_text())["report"] + self.assertEqual(kept["reply_content"], "failed", kept) + self.assertEqual(kept["qualification"], "not_evaluated") + self.assertEqual(kept["status"], "complete") + self.assertIn("long-interview reply_content=failed", result.stdout) + + def test_a_judgment_never_turns_a_failed_run_green(self): + failed, _, capture = self.run_fixture_path(worker_exit=1) + self.assertNotEqual(failed.returncode, 0, failed.stderr) + self.assertEqual( + json.loads(capture.read_text())["report"]["reply_content"], "failed" + ) + + def test_a_judgment_that_cannot_run_leaves_a_good_run_good(self): + result, _, capture = self.run_fixture_path(capture_text="not a capture") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("cannot read the capture", result.stderr) + self.assertIn("run's own result stands", result.stderr) + self.assertEqual(capture.read_text(), "not a capture") + + def test_a_fixture_that_wrote_no_report_fails_the_run_and_says_so(self): + result, report, capture = self.run_fixture_path(writes_report=False) + self.assertNotEqual(result.returncode, 0, result.stderr) + self.assertIn("left no report", json.loads(report.read_text())["outcome"]) + kept = json.loads(capture.read_text())["report"] + self.assertEqual(kept["reply_content"], "not_assessed", kept) + + def test_the_sdk_fixture_launcher_pins_its_test_name_through_the_shared_check(self): + script = (ROOT / "scripts" / "live-sdk-fixture.sh").read_text() + self.assertIn('fixture-server.sh" check "$bin" "$name"', script) + + +class FixtureServerScriptTests(Scratch): + """scripts/fixture-server.sh against stand-in executables.""" + + SCRIPT = ROOT / "scripts" / "fixture-server.sh" + + def run_script(self, *args, **env): + environment = dict(os.environ) + environment.update(env) + return subprocess.run( + ["sh", str(self.SCRIPT), *map(str, args)], + env=environment, + capture_output=True, + text=True, + timeout=60, + check=False, + ) + + def pin(self, listing): + pin = self.root / "pin" + support.write_executable( + pin, + '#!/bin/sh\nif [ "$1" = --list ] && [ "$2" = --ignored ]; then\n' + f"cat <<'EOF'\n{listing}EOF\nfi\n", + ) + return pin + + GOOD_REPORT = { + "outcome": "ok", + "bound": "127.0.0.1:1", + "arming_failure": None, + "capture": { + "failure": None, + "receipts": [], + "batches": [], + "replies": [], + "boundaries": [], + }, + } + + def keep(self, report, log="server output\n", before=None): + """Runs `keep` over a stand-in fixture directory; returns the result and + the path the report was kept at.""" + directory = Path(tempfile.mkdtemp(dir=self.root)) + if report is not None: + (directory / "out.json").write_text(json.dumps(report)) + server_log = directory / "web.log" + server_log.write_text(log) + out = directory / "kept.json" + if before: + before(out) + result = self.run_script("keep", directory, out, server_log) + return result, out + + def test_keep_holds_a_good_report_and_a_private_copy_of_the_log(self): + result, out = self.keep(self.GOOD_REPORT) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(out.read_text())["outcome"], "ok") + # Both private: the report holds the interviewer's replies and the + # tool results. + self.assertEqual(out.stat().st_mode & 0o777, 0o600) + kept = Path(str(out) + ".server.log") + self.assertEqual(kept.read_text(), "server output\n") + self.assertEqual(kept.stat().st_mode & 0o777, 0o600) + + def test_keep_never_replaces_a_file_already_at_the_report_path(self): + # A capture written there by this run, or another run's report, is + # left as it was and the keep fails. + result, out = self.keep( + self.GOOD_REPORT, before=lambda out: out.write_text("the capture\n") + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(out.read_text(), "the capture\n") + + def test_keep_replaces_neither_a_log_nor_a_report_already_there(self): + result, out = self.keep( + self.GOOD_REPORT, + before=lambda out: Path(str(out) + ".server.log").write_text("earlier\n"), + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(Path(str(out) + ".server.log").read_text(), "earlier\n") + result, out = self.keep(None, before=lambda out: out.write_text("earlier\n")) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(out.read_text(), "earlier\n") + self.assertIn("not this run's report", result.stderr) + + def test_keep_leaves_an_envelope_when_the_fixture_wrote_nothing(self): + result, out = self.keep(None) + self.assertEqual(result.returncode, 1, result.stderr) + envelope = json.loads(out.read_text()) + self.assertEqual(out.stat().st_mode & 0o777, 0o600) + self.assertIn("left no report", envelope["outcome"]) + self.assertIsNone(envelope["capture"]) + self.assertIn("server output", result.stderr) + + def test_keep_refuses_a_report_the_judge_could_not_read(self): + reports = ( + {**self.GOOD_REPORT, "outcome": "shutdown bound"}, + {k: v for k, v in self.GOOD_REPORT.items() if k != "bound"}, + {**self.GOOD_REPORT, "capture": None}, + ) + for report in reports: + with self.subTest(report=report): + result, out = self.keep(report) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertTrue(out.exists()) + + def test_keep_fails_when_the_log_cannot_be_kept(self): + # A directory sits where the log copy has to go. + result, _ = self.keep( + self.GOOD_REPORT, before=lambda out: Path(str(out) + ".server.log").mkdir() + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("could not keep the server log", result.stderr) + + def test_the_pin_must_list_the_test_exactly_once(self): + pin = self.pin("a::one: test\nmodule::serve: test\n") + self.assertEqual(self.run_script("check", pin, "module::serve").returncode, 0) + rows = [ + ("module::missing", "lists 0"), + ("serve", "lists 0"), + ] + for name, expected in rows: + with self.subTest(name=name): + result = self.run_script("check", pin, name) + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn(expected, result.stderr) + twice = self.pin("module::serve: test\nmodule::serve: test\n") + result = self.run_script("check", twice, "module::serve") + self.assertEqual(result.returncode, 2, result.stderr) + self.assertIn("lists 2", result.stderr) + + def test_a_pin_that_is_not_runnable_or_a_name_that_is_not_one_name_is_refused(self): + plain = self.root / "plain" + plain.write_text("not executable") + failing = self.root / "failing" + support.write_executable(failing, "#!/bin/sh\nexit 1\n") + for args in [ + (plain, "module::serve"), + (failing, "module::serve"), + (self.pin("module::serve: test\n"), ""), + (self.pin("module::serve: test\n"), "two names"), + ]: + with self.subTest(args=args): + self.assertEqual(self.run_script("check", *args).returncode, 2) + + def test_a_fixture_that_stops_on_request_is_waited_for_and_leaves_its_report(self): + fixture = self.root / "fixture" + support.write_executable( + fixture, + '#!/bin/sh\nwhile [ ! -e "$1/stop" ]; do sleep 0.1; done\n' + 'sleep 1\necho \'{"outcome":"ok"}\' > "$1/out.json"\n', + ) + directory = self.root / "fixture-dir" + directory.mkdir() + process = subprocess.Popen([str(fixture), str(directory)]) + self.spawned.append(process.pid) + result = self.run_script("stop", directory, process.pid, 20) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((directory / "stop").exists()) + # It finished its own work first: the stop did not cut it short. The + # process is still unreaped here, which the script must not mistake for + # running. + self.assertEqual( + (directory / "out.json").read_text().strip(), '{"outcome":"ok"}' + ) + self.assertEqual(process.wait(timeout=10), 0) + + def test_a_fixture_that_will_not_stop_is_terminated_and_that_is_a_failure(self): + stubborn = self.root / "stubborn" + support.write_executable( + stubborn, "#!/bin/sh\ntrap '' TERM\nwhile :; do sleep 1; done\n" + ) + directory = self.root / "stubborn-dir" + directory.mkdir() + process = subprocess.Popen([str(stubborn)]) + self.spawned.append(process.pid) + result = self.run_script( + "stop", directory, process.pid, 0, FIXTURE_SERVER_TERM_GRACE_S="1" + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("had not stopped after 0s", result.stderr) + process.wait(timeout=10) + self.assertIsNotNone(process.returncode) + + def test_a_stop_file_that_cannot_be_written_falls_back_to_a_bounded_term(self): + # TERM is a stop request to the fixture: it finishes and exits cleanly + # inside the wait, past the kill grace, so only the wait lets it. + fixture = self.root / "on-term" + support.write_executable( + fixture, + "#!/bin/sh\ntrap 'sleep 2; exit 0' TERM\nwhile :; do sleep 0.1; done\n", + ) + process = subprocess.Popen([str(fixture)]) + self.spawned.append(process.pid) + time.sleep(0.5) + started = time.monotonic() + result = self.run_script( + "stop", + self.root / "no-such-dir", + process.pid, + 20, + FIXTURE_SERVER_TERM_GRACE_S="1", + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("could not write the stop file", result.stderr) + self.assertEqual(process.wait(timeout=10), 0, "it was asked, not killed") + self.assertLess(time.monotonic() - started, 15) + + def test_a_padded_zombie_state_from_ps_still_counts_as_exited(self): + # A system that pads the state column (" Z") must not leave the stop + # waiting out its whole limit for a process that has already exited. + bin_dir = self.root / "bin" + bin_dir.mkdir() + support.write_executable(bin_dir / "ps", "#!/bin/sh\necho ' Z+'\n") + sleeper = subprocess.Popen(["sleep", "300"]) + self.spawned.append(sleeper.pid) + directory = self.root / "padded-dir" + directory.mkdir() + started = time.monotonic() + result = self.run_script( + "stop", + directory, + sleeper.pid, + 30, + PATH=f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertLess(time.monotonic() - started, 10) + + def test_stop_refuses_a_pid_or_wait_that_is_not_a_number(self): + for args in [ + ("stop", self.root, "abc", 1), + ("stop", self.root, 0, 1), + ("stop", self.root, 1, 1), + ("stop", self.root, "007", 1), + ("stop", self.root, 1, "soon"), + ("stop", self.root), + ("bogus",), + ]: + with self.subTest(args=args): + self.assertEqual(self.run_script(*args).returncode, 2) + class WorkerGuardNodeTests(Scratch): """scripts/run-budget-guard.cjs against the real Python permit service.""" diff --git a/tests/unit/gemini/socket_loopback.rs b/tests/unit/gemini/socket_loopback.rs index 2eb98ca55..5914cc7e8 100644 --- a/tests/unit/gemini/socket_loopback.rs +++ b/tests/unit/gemini/socket_loopback.rs @@ -21,11 +21,7 @@ fn loopback_registration_rejects_every_non_fixture_destination() { #[test] fn loopback_registration_is_scoped_one_shot_and_drops_without_a_connection() { - let name = format!( - "{}::loopback_registration_is_scoped_one_shot_and_drops_without_a_connection", - module_path!().split_once("::").unwrap().1, - ); - if isolation::isolated_checked(&name, "CODETRIAL_SOCKET_FIXTURE_TEST", false, |_| {}) { + if isolation::isolated!("CODETRIAL_SOCKET_FIXTURE_TEST") { return; } let endpoint = "ws://127.0.0.1:9/?key=sequence-test-key"; diff --git a/tests/unit/isolation.rs b/tests/unit/isolation.rs index 24f8e78d4..5f27ed02d 100644 --- a/tests/unit/isolation.rs +++ b/tests/unit/isolation.rs @@ -1,5 +1,24 @@ use std::time::{Duration, Instant}; +/// Runs the calling test again in a child process, alone, under `marker`, and +/// is true in the parent once the child has passed: the test returns then. The +/// name is read from the path of an item declared inside the test, so it +/// cannot fall out of step with the test's own; one that selected no test +/// would fail `isolated_checked` rather than pass vacuously. +macro_rules! isolated { + ($marker:expr) => {{ + fn here() {} + let test = std::any::type_name_of_val(&here) + .strip_suffix("::here") + .map(|path| path.trim_end_matches("::{{closure}}")) + .and_then(|path| path.split_once("::")) + .map(|(_, test)| test) + .expect("an item inside a test has the test's path"); + $crate::test_isolation::isolated_checked(test, $marker, false, |_| {}) + }}; +} +pub(crate) use isolated; + pub(crate) fn isolated_checked( name: &str, marker: &str, diff --git a/tests/unit/livekit/handoff.rs b/tests/unit/livekit/handoff.rs index ed492fc8b..1cd2e461b 100644 --- a/tests/unit/livekit/handoff.rs +++ b/tests/unit/livekit/handoff.rs @@ -413,16 +413,7 @@ async fn a_tick_recovers_when_long_quiet_or_a_failed_request_leaves_the_watchdog #[tokio::test] async fn a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tick() { - let name = format!( - "{}::a_failed_listening_note_keeps_the_cold_brief_due_and_never_arms_the_tick", - module_path!().split_once("::").unwrap().1 - ); - if isolation::isolated_checked( - &name, - "CODETRIAL_HANDOFF_LISTENING_NOTE_TEST", - false, - |_| {}, - ) { + if isolation::isolated!("CODETRIAL_HANDOFF_LISTENING_NOTE_TEST") { return; } let mut fixture = TickFixture::new(1500, 10).await; diff --git a/tests/unit/livekit/probe/acquisition.rs b/tests/unit/livekit/probe/acquisition.rs index 00d37a881..6d6bddea8 100644 --- a/tests/unit/livekit/probe/acquisition.rs +++ b/tests/unit/livekit/probe/acquisition.rs @@ -19,15 +19,15 @@ fn identity() -> IdentityView<'static> { clock: json!({"clock_uncertainty_ms":null}), } } +fn call(id: &str) -> GeminiFunctionCall { + GeminiFunctionCall { + id: id.into(), + name: "read_editor".into(), + args: json!({"private":"not retained"}), + } +} fn answers(id: &str, result: Value) -> Vec<(GeminiFunctionCall, Value)> { - vec![( - GeminiFunctionCall { - id: id.into(), - name: "read_editor".into(), - args: json!({"private":"not retained"}), - }, - result, - )] + vec![(call(id), result)] } #[test] @@ -385,13 +385,6 @@ fn completion_can_finish_admitted_sequences_out_of_order() { ); } -/// Runs the calling test again in a child process so the module's global -/// installation slots cannot meet another test's. -fn isolated(test: &str, env: &str) -> bool { - let name = format!("{}::{test}", module_path!().split_once("::").unwrap().1); - crate::test_isolation::isolated_checked(&name, env, false, |_| {}) -} - fn bound_scope_records(capture: &Arc, room: &str, session: u64) -> bool { let mut id = identity(); id.room = room; @@ -469,17 +462,17 @@ fn the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope() { use crate::dispatch::LocalDispatcher; use crate::web::{AgentJob, RoomDispatcher}; - if isolated( - "the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope", - "CODETRIAL_ACQUISITION_ARMING_TEST", - ) { + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_TEST") { return; } let arming = Arming::new(limits()); let installed = install_arming(&arming).unwrap(); assert!(install_arming(&arming).is_err()); - let runtime = tokio::runtime::Builder::new_multi_thread() + // A current-thread runtime that is never driven: the room task the + // dispatcher spawns is queued and dropped with it, so it can never reach + // for the network, however the scheduler happens to fall. + let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build() .unwrap(); @@ -531,10 +524,7 @@ fn the_real_dispatch_and_bind_boundaries_admit_only_the_expected_scope() { #[test] fn a_blocked_or_removed_observer_leaves_no_capture_installed() { - if isolated( - "a_blocked_or_removed_observer_leaves_no_capture_installed", - "CODETRIAL_ACQUISITION_ARMING_BLOCKED_TEST", - ) { + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_BLOCKED_TEST") { return; } @@ -567,10 +557,7 @@ fn a_blocked_or_removed_observer_leaves_no_capture_installed() { #[test] fn a_teardown_clears_only_its_own_dispatch_hook() { - if isolated( - "a_teardown_clears_only_its_own_dispatch_hook", - "CODETRIAL_ACQUISITION_ARMING_HOOK_TEST", - ) { + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_ARMING_HOOK_TEST") { return; } let first = Arming::new(limits()); @@ -592,3 +579,477 @@ fn a_teardown_clears_only_its_own_dispatch_hook() { assert!(second.capture().is_some()); drop(guard); } + +fn calls(ids: &[&str]) -> Vec { + ids.iter().map(|id| call(id)).collect() +} + +#[test] +fn a_received_call_is_recorded_before_it_executes() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a", "b"]), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.batches.len(), 0); + assert_eq!(snapshot.receipts.len(), 2); + let first = &snapshot.receipts[0]; + assert_eq!( + (first.id.as_str(), first.name.as_str()), + ("a", "read_editor") + ); + assert_eq!((first.identity.session, first.identity.socket), (7, 1)); + assert!(!first.discarded); + assert!( + snapshot + .receipts + .iter() + .all(|receipt| receipt.batch.is_none()) + ); + // The arguments are never retained, only what identifies the call. + assert!( + !serde_json::to_string(&snapshot.receipts) + .unwrap() + .contains("not retained") + ); +} + +#[test] +fn executing_a_batch_links_only_the_receipts_it_answers() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a", "b"]), false); + capture + .record(identity(), &answers("a", json!("result"))) + .unwrap(); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.receipts[0].batch, Some(0)); + assert_eq!(snapshot.receipts[1].batch, None, "received, never executed"); +} + +#[test] +fn a_call_received_during_a_discard_is_kept_and_marked() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["late"]), true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert!(snapshot.receipts[0].discarded); +} + +#[test] +fn receipts_fail_closed_on_scope_reuse_and_limits() { + let scoped = Capture::new("fixture", 7, limits()).unwrap(); + let mut elsewhere = identity(); + elsewhere.session = 8; + scoped.receive(elsewhere, &calls(&["a"]), false); + assert_eq!(scoped.snapshot().failure, Some(Failure::Scope)); + assert!(scoped.snapshot().receipts.is_empty()); + + let reused = Capture::new("fixture", 7, limits()).unwrap(); + reused.receive(identity(), &calls(&["a"]), false); + reused.receive(identity(), &calls(&["a"]), false); + assert_eq!(reused.snapshot().failure, Some(Failure::AmbiguousId)); + + let twice = Capture::new("fixture", 7, limits()).unwrap(); + twice.receive(identity(), &calls(&["a", "a"]), false); + assert_eq!(twice.snapshot().failure, Some(Failure::AmbiguousId)); + + let blank = Capture::new("fixture", 7, limits()).unwrap(); + blank.receive(identity(), &calls(&[""]), false); + assert_eq!(blank.snapshot().failure, Some(Failure::AmbiguousId)); + + let many = Capture::new("fixture", 7, limits()).unwrap(); + many.receive(identity(), &calls(&["1", "2", "3", "4", "5"]), false); + assert_eq!(many.snapshot().failure, Some(Failure::Limit)); + assert!( + many.snapshot().receipts.is_empty(), + "nothing partial is kept" + ); + + let tiny = Capture::new( + "fixture", + 7, + Limits { + bytes: 150, + ..limits() + }, + ) + .unwrap(); + tiny.receive( + identity(), + &calls(&["a-long-call-id-that-will-not-fit-in-the-byte-budget"]), + false, + ); + assert_eq!(tiny.snapshot().failure, Some(Failure::Limit)); +} + +#[test] +fn a_latched_failure_stops_receipts_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.receive(identity(), &calls(&["a"]), false); + assert_eq!(capture.snapshot().failure, Some(Failure::AmbiguousId)); + capture.receive(identity(), &calls(&["fresh"]), false); + assert_eq!(capture.snapshot().receipts.len(), 1); +} + +fn receipt_bytes(capture: &Capture) -> (usize, usize) { + let snapshot = capture.snapshot(); + ( + snapshot.retained_receipt_bytes, + serde_json::to_vec(&snapshot.receipts).unwrap().len(), + ) +} + +#[test] +fn receipt_bytes_are_the_serialized_array_before_and_after_linking() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + assert_eq!( + receipt_bytes(&capture), + (2, 2), + "an empty array is two bytes" + ); + capture.receive(identity(), &calls(&["a"]), false); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual); + capture.receive(identity(), &calls(&["b", "c"]), false); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual, "commas between receipts are counted"); + capture + .record(identity(), &answers("b", json!("result"))) + .unwrap(); + let (counted, actual) = receipt_bytes(&capture); + assert_eq!(counted, actual, "null becomes a sequence number"); + assert_eq!(capture.snapshot().failure, None); +} + +#[test] +fn the_receipt_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.receive(identity(), &calls(&["a"]), false); + let one = receipt_bytes(&probe).1; + for (bytes, fits) in [(one, true), (one - 1, false)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + assert_eq!(capture.snapshot().failure.is_none(), fits, "limit {bytes}"); + } + // A second receipt that does not fit leaves the first alone. + let capture = Capture::new( + "fixture", + 7, + Limits { + bytes: one, + ..limits() + }, + ) + .unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + let before = receipt_bytes(&capture); + capture.receive(identity(), &calls(&["b"]), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.receipts.len(), 1); + assert_eq!(receipt_bytes(&capture), before); +} + +#[test] +fn a_batch_answers_only_the_receipt_from_its_own_socket() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + let mut later = identity(); + later.socket = 2; + capture + .record(later, &answers("a", json!("result"))) + .unwrap(); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!( + snapshot.receipts[0].batch, None, + "received on socket 1, executed on 2" + ); + assert_eq!( + snapshot.batches.len(), + 1, + "the execution itself is still recorded" + ); +} + +#[test] +fn a_reply_keeps_its_newest_view_and_where_it_first_appeared() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.reply(identity(), "interviewer-0", "Hel", false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hello", false); + capture.receive(identity(), &calls(&["b"]), false); + let mut later = identity(); + later.trace_turn = Some(9); + later.clock = json!({"clock_uncertainty_ms":5}); + capture.reply(later, "interviewer-0", "Hello there", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + let [reply] = snapshot.replies.as_slice() else { + panic!("one segment, one reply: {:?}", snapshot.replies); + }; + assert_eq!(reply.text, "Hello there"); + assert!(reply.final_segment && !reply.revised); + assert_eq!(reply.views, 3); + // First seen after the receipt and before the batch that answered it. + assert_eq!((reply.receipts_before, reply.batches_before), (1, 0)); + // The turn went on across a second call and an answer. + assert_eq!((reply.receipts_last, reply.batches_last), (2, 1)); + assert_eq!(reply.identity, Identity::from_view(&identity())); +} + +#[test] +fn a_rewritten_view_is_marked_and_a_reopened_final_one_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.reply(identity(), "interviewer-0", "Hello there", false); + capture.reply(identity(), "interviewer-0", "Hello", false); + capture.reply(identity(), "interviewer-1", "Fine", true); + capture.reply(identity(), "interviewer-1", "Fine", false); + capture.reply(identity(), "interviewer-2", "Fine", true); + capture.reply(identity(), "interviewer-2", "Fine and", true); + // Ordinary growth afterwards does not make the page's view trustworthy. + capture.reply(identity(), "interviewer-0", "Hello there, again", true); + capture.reply(identity(), "interviewer-1", "Fine, thanks", true); + let revised: Vec<_> = capture + .snapshot() + .replies + .iter() + .map(|reply| reply.revised) + .collect(); + assert_eq!(revised, [true, true, false]); +} + +#[test] +fn a_segment_id_on_another_socket_is_another_reply() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.reply(identity(), "interviewer-0", "one", true); + let mut next = identity(); + next.socket = 2; + capture.reply(next, "interviewer-0", "two", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!(snapshot.replies.len(), 2); + assert_eq!(snapshot.replies[0].text, "one"); + assert_eq!(snapshot.replies[1].identity.socket, 2); +} + +#[test] +fn replies_fail_closed_on_scope_count_and_bytes() { + for (room, session) in [("foreign", 7), ("fixture", 8)] { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let mut view = identity(); + view.room = room; + view.session = session; + capture.reply(view, "interviewer-0", "x", true); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Scope)); + assert!(snapshot.replies.is_empty()); + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + for index in 0..5 { + capture.reply(identity(), &format!("interviewer-{index}"), "x", true); + } + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.replies.len(), 4); + + // A view that no longer fits is dropped whole, and the earlier one stays. + let small = Limits { + bytes: 400, + ..limits() + }; + let capture = Capture::new("fixture", 7, small).unwrap(); + capture.reply(identity(), "interviewer-0", "short", false); + capture.reply(identity(), "interviewer-0", &"long".repeat(200), false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.replies[0].text, "short"); +} + +#[test] +fn the_reply_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.reply(identity(), "interviewer-0", "exact", true); + let exact = serde_json::to_vec(&probe.snapshot().replies).unwrap().len(); + for (bytes, failure) in [(exact, None), (exact - 1, Some(Failure::Limit))] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.reply(identity(), "interviewer-0", "exact", true); + assert_eq!(capture.snapshot().failure, failure); + } +} + +#[test] +fn the_running_byte_counts_are_the_serialized_lengths() { + // Each view is charged for its own change, so the running count has to stay + // what serializing the whole array would give, through revisions that grow, + // shrink and reopen a reply. + let roomy = Limits { + calls: 8, + ..limits() + }; + let capture = Capture::new("fixture", 7, roomy).unwrap(); + for (id, text, last) in [ + ("interviewer-0", "Hel", false), + ("interviewer-0", "Hello there", true), + ("interviewer-1", "Fine", true), + ("interviewer-0", "Hi", false), + ("interviewer-2", "\"quoted\" \u{3b1}", true), + ] { + capture.reply(identity(), id, text, last); + capture.boundary(identity(), Ending::Complete, last); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + assert_eq!( + snapshot.retained_reply_bytes, + serde_json::to_vec(&snapshot.replies).unwrap().len() + ); + assert_eq!(snapshot.reply_bytes.len(), snapshot.replies.len()); + assert_eq!( + snapshot.retained_boundary_bytes, + serde_json::to_vec(&snapshot.boundaries).unwrap().len() + ); + } +} + +#[test] +fn the_boundary_byte_limit_is_exact() { + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.boundary(identity(), Ending::Complete, true); + probe.boundary(identity(), Ending::Interrupted, false); + let exact = serde_json::to_vec(&probe.snapshot().boundaries) + .unwrap() + .len(); + for (bytes, failure, kept) in [(exact, None, 2), (exact - 1, Some(Failure::Limit), 1)] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + capture.boundary(identity(), Ending::Interrupted, false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, failure); + assert_eq!(snapshot.boundaries.len(), kept); + } +} + +#[test] +fn a_latched_failure_stops_replies_too() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&[""]), false); + capture.reply(identity(), "interviewer-0", "x", true); + assert!(capture.snapshot().replies.is_empty()); +} + +#[test] +fn a_boundary_keeps_arrival_order_and_what_preceded_it() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + capture.receive(identity(), &calls(&["a"]), false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hi", false); + let mut next = identity(); + next.socket = 2; + capture.boundary(next, Ending::Interrupted, false); + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, None); + let seen: Vec<_> = snapshot + .boundaries + .iter() + .map(|boundary| { + ( + boundary.ending, + boundary.accepted, + boundary.identity.socket, + boundary.receipts_before, + boundary.batches_before, + boundary.replies_before, + ) + }) + .collect(); + assert_eq!( + seen, + [ + (Ending::Complete, true, 1, 0, 0, 0), + (Ending::Interrupted, false, 2, 1, 1, 1) + ] + ); +} + +#[test] +fn boundaries_fail_closed_on_scope_count_and_bytes() { + let mut view = identity(); + view.session = 8; + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.boundary(view, Ending::Complete, true); + assert_eq!(capture.snapshot().failure, Some(Failure::Scope)); + + let capture = Capture::new("fixture", 7, limits()).unwrap(); + for _ in 0..5 { + capture.boundary(identity(), Ending::Complete, true); + } + let snapshot = capture.snapshot(); + assert_eq!(snapshot.failure, Some(Failure::Limit)); + assert_eq!(snapshot.boundaries.len(), 4); + + let probe = Capture::new("fixture", 7, limits()).unwrap(); + probe.boundary(identity(), Ending::Complete, true); + let exact = serde_json::to_vec(&probe.snapshot().boundaries) + .unwrap() + .len(); + for (bytes, failure) in [(exact, None), (exact - 1, Some(Failure::Limit))] { + let capture = Capture::new("fixture", 7, Limits { bytes, ..limits() }).unwrap(); + capture.boundary(identity(), Ending::Complete, true); + assert_eq!(capture.snapshot().failure, failure); + } +} + +#[test] +fn only_turn_endings_are_boundaries() { + use crate::gemini::GeminiEvent; + if crate::test_isolation::isolated!("CODETRIAL_ACQUISITION_BOUNDARY_TEST") { + return; + } + let capture = Capture::new("fixture", 7, limits()).unwrap(); + let _installed = install(&capture).unwrap(); + let mut latency = ReplyLatency::default(); + latency.bind("fixture", 7, 1, 1500); + boundary(&latency, &GeminiEvent::OutputTranscript("x".into()), true); + boundary(&latency, &GeminiEvent::TurnComplete, true); + boundary(&latency, &GeminiEvent::Interrupted, false); + let endings: Vec<_> = capture + .snapshot() + .boundaries + .iter() + .map(|boundary| (boundary.ending, boundary.accepted)) + .collect(); + assert_eq!( + endings, + [(Ending::Complete, true), (Ending::Interrupted, false)] + ); +} + +#[test] +fn a_single_view_reply_starts_with_matching_counts() { + let capture = Capture::new("fixture", 7, limits()).unwrap(); + capture.receive(identity(), &calls(&["a"]), false); + capture.record(identity(), &answers("a", json!("ok"))); + capture.reply(identity(), "interviewer-0", "Hi", true); + let reply = &capture.snapshot().replies[0]; + assert_eq!((reply.receipts_before, reply.batches_before), (1, 1)); + assert_eq!((reply.receipts_last, reply.batches_last), (1, 1)); +} + +#[test] +fn the_report_carries_replies_and_boundaries() { + let arming = Arming::detached(limits()); + arming.on_assignment("fixture"); + arming.on_bind("fixture", 7); + let capture = arming.capture().unwrap(); + capture.reply(identity(), "interviewer-0", "Hi", true); + capture.boundary(identity(), Ending::Interrupted, false); + let report = arming.report(); + assert_eq!(report["capture"]["replies"][0]["text"], "Hi"); + let boundary = &report["capture"]["boundaries"][0]; + assert_eq!(boundary["ending"], "Interrupted"); + assert_eq!(boundary["accepted"], false); + assert_eq!(boundary["replies_before"], 1); +} diff --git a/tests/unit/livekit/probe/serving.rs b/tests/unit/livekit/probe/serving.rs index bd0b4ce37..83e2f0c95 100644 --- a/tests/unit/livekit/probe/serving.rs +++ b/tests/unit/livekit/probe/serving.rs @@ -8,6 +8,9 @@ //! tests beside it drive `serve` with a throwaway config so the fixture itself //! is checked without credentials or a browser. //! +//! Unix only (`probe.rs` gates the module): a TERM is a stop request, which +//! needs Unix signals, and the launcher keeps it in a session of its own. +//! //! Where it differs from `codetrial web`, on purpose: it is always a launcher's //! child (no Setup page, as with the desktop app's server), it does not clear //! the operator's web error log, task mode is off, and it reads the process @@ -189,6 +192,9 @@ fn publish(path: &Path, text: &str) -> Result<(), String> { .map_err(|error| format!("cannot write {}: {error}", path.display())) } +/// Set once a TERM can be received, so a test sends one only then. +static TERM_REGISTERED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + /// Resolves once the stop file exists or the deadline passes. async fn stop_file(path: PathBuf, deadline: Duration) { let began = Instant::now(); @@ -197,6 +203,26 @@ async fn stop_file(path: PathBuf, deadline: Duration) { } } +/// The stop file, or a TERM, which a launcher that cannot reach the file sends +/// to ask the same thing: drain, settle and write the report. Registered when +/// first polled, inside the fixture's runtime, which is after the ready file is +/// written: a TERM in that first moment still finds the default action and ends +/// the fixture without a report, so a launcher asks with the stop file and +/// keeps the fixture out of group-wide signals. Fails loudly if it cannot be +/// registered, since the fixture would otherwise carry on without its contract. +async fn stop_requested(path: PathBuf, deadline: Duration) { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("the TERM handler registers"); + TERM_REGISTERED.store(true, std::sync::atomic::Ordering::SeqCst); + + // Once this returns the handler stays installed and a later TERM does + // nothing, so a launcher's escalation after the stop file is the KILL. + tokio::select! { + () = stop_file(path, deadline) => {} + _ = term.recv() => {} + } +} + /// Before anything can fail: a run that never reports must not leave an /// earlier run's report to be read as its own. fn clear_previous(settings: &Settings) -> Result<(), String> { @@ -219,7 +245,7 @@ fn serve_until_stop_file(settings: &Settings) -> Result<(), String> { settings.stop().display() )); } - serve(settings, stop_file(settings.stop(), settings.deadline)) + serve(settings, stop_requested(settings.stop(), settings.deadline)) } #[test] @@ -408,15 +434,7 @@ fn an_address_off_loopback_refuses_the_start() { #[test] fn the_fixture_serves_the_real_service_and_reports_what_the_observer_saw() { - if crate::test_isolation::isolated_checked( - &format!( - "{}::the_fixture_serves_the_real_service_and_reports_what_the_observer_saw", - module_path!().split_once("::").unwrap().1 - ), - "CODETRIAL_SERVING_FIXTURE_TEST", - false, - |_| {}, - ) { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_TEST") { return; } let dir = scratch("serve"); @@ -474,17 +492,38 @@ fn the_fixture_serves_the_real_service_and_reports_what_the_observer_saw() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn a_term_is_a_stop_request_and_the_run_still_reports() { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_TEST") { + return; + } + let dir = scratch("term"); + let settings = throwaway(&dir); + let (ready, out) = (settings.ready(), settings.out()); + let server = std::thread::spawn(move || serve_until_stop_file(&settings)); + wait_for(&ready); + + // The handler is registered when the stop future is first polled, which the + // bound address does not wait for. + let began = Instant::now(); + while !TERM_REGISTERED.load(std::sync::atomic::Ordering::SeqCst) { + assert!(began.elapsed() < Duration::from_secs(30), "no TERM handler"); + std::thread::sleep(Duration::from_millis(20)); + } + let status = std::process::Command::new("kill") + .args(["-TERM", &std::process::id().to_string()]) + .status() + .unwrap(); + assert!(status.success()); + server.join().unwrap().unwrap(); + let report: serde_json::Value = serde_json::from_str(&wait_for(&out)).unwrap(); + assert_eq!(report["outcome"], "ok", "{report}"); + let _ = std::fs::remove_dir_all(&dir); +} + #[test] fn a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting() { - if crate::test_isolation::isolated_checked( - &format!( - "{}::a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting", - module_path!().split_once("::").unwrap().1 - ), - "CODETRIAL_SERVING_FIXTURE_DRAIN_TEST", - false, - |_| {}, - ) { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_DRAIN_TEST") { return; } let dir = scratch("drain"); @@ -531,15 +570,7 @@ fn a_client_holding_a_half_sent_request_cannot_keep_the_run_from_reporting() { #[test] fn a_run_replaces_the_artifacts_an_earlier_run_left() { - if crate::test_isolation::isolated_checked( - &format!( - "{}::a_run_replaces_the_artifacts_an_earlier_run_left", - module_path!().split_once("::").unwrap().1 - ), - "CODETRIAL_SERVING_FIXTURE_STALE_TEST", - false, - |_| {}, - ) { + if crate::test_isolation::isolated!("CODETRIAL_SERVING_FIXTURE_STALE_TEST") { return; } let dir = scratch("stale-artifacts"); diff --git a/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs index 0a80fbc77..db9342a0f 100644 --- a/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs +++ b/tests/unit/livekit/recovery_parts/acquisition_tool_sends.rs @@ -62,6 +62,16 @@ async fn acquisition_actual_tool_send(failed: bool) { let batch = &snapshot.batches[0]; assert_eq!(batch.calls.len(), 1); assert_eq!(batch.calls[0].id, "actual-tool-send"); + + // Every executed call was received first, and a call that arrived under + // a discard says so. + assert_eq!(snapshot.receipts.len(), 1); + assert_eq!(snapshot.receipts[0].id, "actual-tool-send"); + assert_eq!(snapshot.receipts[0].batch, Some(0)); + + // A hold silences the interviewer as a discard does: the call is still + // answered, and neither kind is to be tied to a later reply. + assert_eq!(snapshot.receipts[0].discarded, held || discarded); assert_eq!(batch.calls[0].name, crate::runtime::TOOL_READ_EDITOR); assert!( batch.calls[0].result["result"] @@ -111,6 +121,26 @@ async fn acquisition_tool_send_error_records_returned_failure() { acquisition_actual_tool_send(true).await; } +async fn acquisition_reply_rig() -> ( + Rig, + std::sync::Arc, +) { + use crate::livekit::probe::acquisition::{Capture, Limits}; + let mut rig = Rig::new("ws://127.0.0.1:9/".into()).await; + rig.turn.activity.latency.bind("recovery-test", 7, 1, 1500); + let capture = Capture::new( + "recovery-test", + 7, + Limits { + batches: 2, + calls: 4, + bytes: 65536, + }, + ) + .unwrap(); + (rig, capture) +} + fn acquisition_gate_capture( rig: &mut Rig, ) -> std::sync::Arc { @@ -177,6 +207,17 @@ async fn acquisition_controlled_owner_await_cancellation_preserves_execution_bou let executed = matches!(stage, Stage::BoardResend | Stage::ToolSend); assert_eq!(snapshot.failure, None); assert_eq!(snapshot.batches.len(), usize::from(executed), "{stage:?}"); + + // Received before the acknowledgement gate, so cancellation at any gate + // leaves the call on record, executed only if the batch got that far. + assert_eq!(snapshot.receipts.len(), 1, "{stage:?}"); + assert_eq!(snapshot.receipts[0].id, "cancelled-owner"); + assert_eq!( + snapshot.receipts[0].batch, + executed.then_some(0), + "{stage:?}" + ); + assert!(!snapshot.receipts[0].discarded); if let Some(batch) = snapshot.batches.first() { assert_eq!(batch.calls[0].id, "cancelled-owner"); assert_eq!( @@ -420,3 +461,216 @@ async fn acquisition_consumed_gate_does_not_hold_a_later_tool_batch() { .all(|batch| batch.local_send_completed == Some(true)) ); } + +async fn acquisition_event(rig: &mut Rig, event: crate::gemini::GeminiEvent) { + let mut context = rig.turn.context( + &mut rig.output, + &mut rig.gemini, + &mut rig.board, + &mut rig.media, + ); + session::handle_gemini_event(&rig.room, &mut context, event, Interruptible::Yes) + .await + .unwrap(); +} + +#[tokio::test] +async fn acquisition_keeps_the_interviewer_segments_the_page_was_sent() { + if isolated("acquisition_keeps_the_interviewer_segments_the_page_was_sent") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_receive_editor(&mut rig, "before-reply").await; + acquisition_event( + &mut rig, + GeminiEvent::OutputTranscript("The answer ".into()), + ) + .await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("is 42.".into())).await; + // The candidate's own speech closes with the same turn and is not a reply. + rig.media.identity = Some("candidate".into()); + acquisition_event(&mut rig, GeminiEvent::InputTranscript("my words".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Next.".into())).await; + let _ = rig.old_frames().await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + let replies: Vec<_> = snapshot + .replies + .iter() + .map(|reply| (reply.text.as_str(), reply.final_segment, reply.views)) + .collect(); + + // The first turn's newest view, sealed by the turn's end; the next turn's + // segment is still open and is its own reply. + assert_eq!( + replies, + [("The answer is 42.", true, 3), ("Next.", false, 1)] + ); + assert_ne!(snapshot.replies[0].id, snapshot.replies[1].id); + assert!(snapshot.replies.iter().all(|reply| !reply.revised)); + // The tool call preceded both, and was answered before either. + assert!( + snapshot + .replies + .iter() + .all(|reply| (reply.receipts_before, reply.batches_before) == (1, 1)) + ); +} + +#[tokio::test] +async fn acquisition_seals_the_line_a_cut_marked_before_the_next_words() { + if isolated("acquisition_seals_the_line_a_cut_marked_before_the_next_words") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Cut words.".into())).await; + rig.turn + .state + .interviewer_delivery + .cancel(crate::agent::DeliveryCut::Generation, Instant::now()); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("New words.".into())).await; + let snapshot = capture.snapshot(); + drop(installed); + let replies: Vec<_> = snapshot + .replies + .iter() + .map(|reply| (reply.text.as_str(), reply.final_segment)) + .collect(); + assert_eq!(replies, [("Cut words.", true), ("New words.", false)]); +} + +#[tokio::test] +async fn acquisition_marks_an_ending_the_guard_swallowed() { + if isolated("acquisition_marks_an_ending_the_guard_swallowed") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::{Ending, install}; + let (mut rig, capture) = acquisition_reply_rig().await; + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("Said.".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("More.".into())).await; + acquisition_event(&mut rig, GeminiEvent::Interrupted).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let snapshot = capture.snapshot(); + drop(installed); + let seen: Vec<_> = snapshot + .boundaries + .iter() + .map(|boundary| (boundary.ending, boundary.accepted, boundary.replies_before)) + .collect(); + + // The ending that follows an interruption closes nothing: the provider + // sends it for the generation the interruption already cut. + assert_eq!( + seen, + [ + (Ending::Complete, true, 1), + (Ending::Interrupted, true, 2), + (Ending::Complete, false, 2) + ] + ); +} + +#[tokio::test] +async fn acquisition_does_not_take_a_departed_candidates_words_for_a_reply() { + if isolated("acquisition_does_not_take_a_departed_candidates_words_for_a_reply") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::install; + let (mut rig, capture) = acquisition_reply_rig().await; + rig.media.identity = Some("candidate".into()); + let installed = install(&capture).unwrap(); + acquisition_event(&mut rig, GeminiEvent::InputTranscript("my words".into())).await; + + // The candidate leaves before the turn closes, so the close publishes their + // final segment without an identity, as the agent. + rig.media.identity = None; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let snapshot = capture.snapshot(); + drop(installed); + assert_eq!(snapshot.failure, None); + assert!(snapshot.replies.is_empty(), "{:?}", snapshot.replies); +} + +/// The report the fixture leaves behind, from a script that exercises every +/// shape the JS consumer reads: a reply that cites an executed call, a plain +/// completed reply, and an interrupted one whose late ending is swallowed. +/// Wall-clock readings and the byte counts that include them are replaced with +/// zero, because a fixture cannot hold the time the test ran; the consumer's +/// tests put times back. +#[tokio::test] +async fn acquisition_report_is_the_wire_fixture_the_consumer_reads() { + if isolated("acquisition_report_is_the_wire_fixture_the_consumer_reads") { + return; + } + use crate::gemini::GeminiEvent; + use crate::livekit::probe::acquisition::{Arming, Limits, install}; + let (mut rig, _) = acquisition_reply_rig().await; + let arming = Arming::detached(Limits { + batches: 8, + calls: 8, + bytes: 65536, + }); + arming.on_assignment("recovery-test"); + arming.on_bind("recovery-test", 7); + let capture = arming.capture().unwrap(); + let installed = install(&capture).unwrap(); + acquisition_receive_editor(&mut rig, "call-1").await; + for text in ["REPLY-A one. ", "two."] { + acquisition_event(&mut rig, GeminiEvent::OutputTranscript(text.into())).await; + } + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event(&mut rig, GeminiEvent::OutputTranscript("REPLY-B.".into())).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + acquisition_event( + &mut rig, + GeminiEvent::OutputTranscript("REPLY-C cut".into()), + ) + .await; + acquisition_event(&mut rig, GeminiEvent::Interrupted).await; + acquisition_event(&mut rig, GeminiEvent::TurnComplete).await; + let _ = rig.old_frames().await; + let mut report = arming.report(); + drop(installed); + fn zero_clocks(value: &mut serde_json::Value) { + match value { + serde_json::Value::Object(map) => { + for (key, inner) in map.iter_mut() { + // Byte counts embed the digits of the clock they carry; the + // call count does not, and is kept. + if key == "emitted_unix_ms" + || (key.starts_with("retained_") && key.ends_with("_bytes")) + { + *inner = 0.into(); + } else { + zero_clocks(inner); + } + } + } + serde_json::Value::Array(items) => items.iter_mut().for_each(zero_clocks), + _ => {} + } + } + zero_clocks(&mut report); + let path = "tests/fixtures/acquisition-report.json"; + let rendered = format!("{}\n", serde_json::to_string_pretty(&report).unwrap()); + if std::env::var_os("CODETRIAL_UPDATE_FIXTURES").is_some() { + std::fs::write(path, &rendered).unwrap(); + } + assert_eq!( + std::fs::read_to_string(path).unwrap(), + rendered, + "regenerate with CODETRIAL_UPDATE_FIXTURES=1" + ); +} diff --git a/tests/unit/livekit/session/observations.rs b/tests/unit/livekit/session/observations.rs index 7b640bd58..f38a1d2c7 100644 --- a/tests/unit/livekit/session/observations.rs +++ b/tests/unit/livekit/session/observations.rs @@ -222,11 +222,7 @@ fn accepted_close_request_is_observed_before_tool_batch_response_timing() { #[test] fn executed_tool_capture_precedes_send_and_telemetry_gates() { - let name = format!( - "{}::executed_tool_capture_precedes_send_and_telemetry_gates", - module_path!().split_once("::").unwrap().1 - ); - if isolation::isolated_checked(&name, "CODETRIAL_ACQUISITION_CAPTURE_TEST", false, |_| {}) { + if isolation::isolated!("CODETRIAL_ACQUISITION_CAPTURE_TEST") { return; } use crate::livekit::probe::acquisition::{Capture, Failure, Limits, install};