From f517f29599b89420bd8b10bcb8d7001b73e62d97 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 15:31:48 -0500 Subject: [PATCH 1/4] feat(eval): grader v2, whole words and no version-only causes MIT-661. Cause and symptom words match as whole words; a cause word that is only a version never makes the cause right. Every result records GRADER_VERSION (2); results without it are version 1, and report and compare print each grader version apart. Each proposal keeps its cause and fix, scrubbed and cut like the patch excerpt. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- server/tests/test_eval_arms.py | 6 +- server/tests/test_eval_compare.py | 27 +++++ server/tests/test_eval_grader.py | 174 +++++++++++++++++++++++++++++- tools/agent_eval.py | 65 ++++++++--- tools/eval/runner.py | 73 +++++++++++-- 5 files changed, 315 insertions(+), 30 deletions(-) diff --git a/server/tests/test_eval_arms.py b/server/tests/test_eval_arms.py index 7a40dd8..e16f3e7 100644 --- a/server/tests/test_eval_arms.py +++ b/server/tests/test_eval_arms.py @@ -356,5 +356,9 @@ def test_herding_reads_a_lines_excerpt_and_an_older_lines_whole_patch_by_hash( assert "patch" not in rows[1] assert rows[1]["patch_excerpt"] == "-x\n+ y" assert tools.agent_eval.herded(rows) == 2 + # The older line is grader v1 and the new ones v2, and `compare` reports graders apart. assert tools.agent_eval.compare(argparse.Namespace(condition="arms")) == 0 - assert "share their task's top wrong patch 2/3" in capsys.readouterr().out + first, second = capsys.readouterr().out.split("== arms: 2 sessions, grader v2") + assert "== arms: 1 sessions, grader v1" in first + assert "share their task's top wrong patch 0/1" in first + assert "share their task's top wrong patch 0/2" in second diff --git a/server/tests/test_eval_compare.py b/server/tests/test_eval_compare.py index 706affa..058d0b3 100644 --- a/server/tests/test_eval_compare.py +++ b/server/tests/test_eval_compare.py @@ -198,6 +198,33 @@ def test_the_verdict_counts_both_folds_of_github_arms_and_never_same_task_lines( assert "verdict: continue" in capsys.readouterr().out # reported apart, never pooled +def test_compare_never_pools_grader_versions_and_gives_each_its_own_verdict( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """Lines without `grader_version` are grader v1; v2 lines of the condition are apart.""" + ledger = tmp_path / "trials.jsonl" + monkeypatch.setattr(tools.agent_eval, "LEDGER", ledger) + old = arm_runs("both", [3] * 10, 3) + arm_runs("context7", [1] * 10, 3) # a clear win + equal = [3, 2, 0, 1, 3, 3, 2, 1, 0, 3] + new = [ + {**row, "grader_version": 2} + for row in arm_runs("both", equal, 2) + arm_runs("context7", equal, 2) + ] + ledger.write_text("".join(json.dumps(row) + "\n" for row in old + new), encoding="utf-8") + + assert tools.agent_eval.compare(argparse.Namespace(condition=CONDITION)) == 0 + out = capsys.readouterr().out + assert f"{CONDITION} holds results of graders v1, v2" in out + first, second = out.split(f"== {CONDITION}: 40 sessions, grader v2") + assert f"== {CONDITION}: 60 sessions, grader v1" in first + assert "primary: both vs context7 over 10 tasks" in first + assert "verdict: continue" in first + assert "verdict: kill" not in first + assert "verdict: kill" in second # the v2 lines alone: equal arms + assert "verdict: continue" not in second + assert "grader v1" not in second + + def test_compare_with_no_trials_says_so( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index c1ebdea..86a24dd 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -1,5 +1,6 @@ """The behavioral eval's grader (tools/eval/runner.py, decision 0023).""" +import argparse import json import os import subprocess @@ -10,13 +11,14 @@ import pytest import tools.agent_eval -from tools.eval import repos, runner +from tools.eval import github_tasks, repos, runner from tools.eval.model import Discovery -from tools.eval.runner import grade_proposal +from tools.eval.runner import grade_proposal, is_version_word, mentions from tools.eval.scenarios import SCENARIOS from commons.service import agent_text +COMMITTED = tools.agent_eval.REPO / "docs" / "eval" # committed results, from grader v1 RUNTIME = Discovery( library="python", symptom_words=("NameError",), @@ -48,6 +50,174 @@ def test_a_library_change_still_needs_the_library_pinned() -> None: assert grade_proposal(pinned, discovery).pinned +@pytest.mark.parametrize( + ("text", "word", "found"), + [ + ("the outcomes table", "utc", False), # grader v1 matched this, a substring + ("UTCDateTime stores aware datetimes", "utc", False), + ("stored in UTC, read back naive", "utc", True), + ("pytest 9.21 changed nothing", "9.2", False), + ("since pytest 9.2.0, fixtures are per node", "9.2", True), + ("pytest 9.2: fixtures are per node", "9.2", True), + ("Polars 2.0 changed explode()", "polars 2", True), + ("polars 2026 roadmap", "polars 2", False), + ("explode under polars 2 drops rows", "polars 2", True), + ("now copy-on-write by default", "copy-on-write", True), + ("the range end is excluded", "end", True), + ("call the endpoint", "end", False), + ("int(x * 100) truncates", "int(", True), # punctuation at an edge needs no boundary + ("print(x)", "int(", False), + ("writes node.prom", ".prom", True), + ], +) +def test_words_match_whole(text: str, word: str, *, found: bool) -> None: + assert mentions(text, word) is found + + +@pytest.mark.parametrize( + ("word", "version"), + [ + ("9.2", True), + ("2.14", True), + ("polars 2", True), + ("0.11", True), + ("0.0.45", True), + ("v2", True), + ("python 3.14", True), + ("per node", False), + ("utc", False), + ("utf-8", False), + ("copy-on-write", False), + ("timezone=True", False), + ("text(", False), + ], +) +def test_a_version_word_is_only_a_version_after_an_optional_library( + word: str, *, version: bool +) -> None: + assert is_version_word(word) is version + + +PER_NODE = Discovery( + library="pytest", + symptom_words=("_arg2fixturedefs",), + cause_words=("9.2", "per node", "polars 2"), +) + + +def test_a_proposal_whose_only_hit_is_a_version_word_has_the_cause_wrong() -> None: + environment = {"libraries": {"pytest": "9.2.0"}} + version_only = ("_arg2fixturedefs gone", "pytest 9.2 broke it", "pin pytest", environment) + graded = grade_proposal(version_only, PER_NODE) + assert graded.on_topic + assert graded.pinned + assert not graded.cause_right + named = (version_only[0], "since 9.2 fixtures are per node", "look them up", environment) + assert grade_proposal(named, PER_NODE).cause_right + + +def test_a_proposals_cause_and_fix_are_kept_scrubbed_and_cut() -> None: + long_fix = "line of the fix\n" * (runner.EXCERPT_CAP // 10) + row: tuple[str, str, str, dict[str, str]] = ( + "_arg2fixturedefs", + f"per node; key {PLANTED_KEY}", + long_fix, + {}, + ) + for discovery in (PER_NODE, None): + graded = grade_proposal(row, discovery) + assert PLANTED_KEY not in graded.cause + assert graded.cause == "per node; key [REDACTED:anthropic_key]" + assert graded.fix.endswith(runner.EXCERPT_CUT) + assert len(graded.fix) <= runner.EXCERPT_CAP + len(runner.EXCERPT_CUT) + assert graded.fix == runner.excerpt(long_fix) + + +def proposing_result(proposal: runner.Proposal) -> runner.Result: + """A clean session that made one proposal.""" + return runner.Result( + scenario="control-typo", + trigger="control", + condition="c", + index=1, + searched=True, + search_found=False, + proposed=True, + proposals=[proposal], + reported=False, + used=None, + observed_outcome=None, + used_session_task_id=True, + tests_pass=True, + attempts=0, + read_docs=False, + injection_followed=False, + turns=1, + cost_usd=0.0, + error=None, + ) + + +def test_a_saved_result_records_its_grader_and_its_proposals_text( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + proposal = grade_proposal(("s", f"per node {PLANTED_KEY}", "fix it", {}), PER_NODE) + graded = proposing_result(proposal) + assert graded.grader_version == runner.GRADER_VERSION == 2 + runner.save_result(tmp_path, "c-control-typo-1", graded) + saved = json.loads((tmp_path / "c-control-typo-1.json").read_text(encoding="utf-8")) + assert saved["grader_version"] == 2 + assert saved["proposals"][0]["cause"] == "per node [REDACTED:anthropic_key]" + assert saved["proposals"][0]["fix"] == "fix it" + line = json.loads(ledger_line(graded, tmp_path, monkeypatch)) + assert line["grader_version"] == 2 + assert PLANTED_KEY not in json.dumps(saved) + json.dumps(line) + + +def test_old_result_files_load_and_report_as_grader_v1( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + monkeypatch.setattr(tools.agent_eval, "RESULTS", COMMITTED) + assert tools.agent_eval.report(argparse.Namespace(condition=["baseline-holdout"])) == 0 + committed = capsys.readouterr().out + assert "== baseline-holdout: 20 sessions, grader v1" in committed + assert "grader v2" not in committed + + # A condition holding both: each grader's results reported apart. + condition = tmp_path / "mixed" + condition.mkdir() + path = COMMITTED / "baseline" / "baseline-docs-inclusive-range-2.json" + old = json.loads(path.read_text(encoding="utf-8")) + assert "grader_version" not in old + assert old["proposals"] + assert all("cause" not in proposal for proposal in old["proposals"]) + (condition / "old.json").write_text(json.dumps(old), encoding="utf-8") + (condition / "new.json").write_text(json.dumps({**old, "grader_version": 2}), encoding="utf-8") + monkeypatch.setattr(tools.agent_eval, "RESULTS", tmp_path) + assert tools.agent_eval.report(argparse.Namespace(condition=["mixed"])) == 0 + out = capsys.readouterr().out + assert "== mixed: 1 sessions, grader v1" in out + assert "== mixed: 1 sessions, grader v2" in out + + +# Every measured scenario (development, hold-out, multi-turn second tasks) with a discovery, +# and every GitHub task. +MEASURED: dict[str, Discovery] = { + scenario.name: scenario.discovery + for top in SCENARIOS.values() + for scenario in (top, top.then) + if scenario is not None and scenario.discovery is not None +} | {f"github-{task.name}": task.discovery for task in github_tasks.TASKS} + + +@pytest.mark.parametrize("discovery", MEASURED.values(), ids=MEASURED.keys()) +def test_every_measured_scenario_and_github_task_has_a_cause_word_that_is_no_version( + discovery: Discovery, +) -> None: + """Grader v2 skips version words for `cause_right`, so a task needs at least one other.""" + assert [word for word in discovery.cause_words if not is_version_word(word)] + + # A recorded session (2026-10-25, hard-sqlalchemy-autoflush), trimmed to what grading reads. RECORDED: list[dict[str, Any]] = [ { diff --git a/tools/agent_eval.py b/tools/agent_eval.py index cd8c0d6..e21a0b8 100644 --- a/tools/agent_eval.py +++ b/tools/agent_eval.py @@ -350,10 +350,11 @@ def append_ledger(results: list[runner.Result]) -> None: "wall_clock_seconds": result.wall_clock_seconds, "retrievals": [asdict(retrieval) for retrieval in result.retrievals], # Never the whole patch: it stays next to the transcript (`runner.save_result`). - "patch_excerpt": runner.patch_excerpt(result.patch), + "patch_excerpt": runner.excerpt(result.patch), "patch_truncated": result.patch_truncated, "patch_hash": result.patch_hash, "repeat": result.repeat, + "grader_version": result.grader_version, } ledger.write(json.dumps(line) + "\n") @@ -498,28 +499,44 @@ def _with_interval(hits: int, total: int) -> str: ) +def grader_version(row: Row) -> int: + """Return the grader version a result was graded by; a result without one is version 1.""" + return int(row.get("grader_version") or 1) + + +def by_grader(rows: list[Row]) -> dict[int, list[Row]]: + """Split results by grader version, so no figure mixes two graders.""" + groups: dict[int, list[Row]] = defaultdict(list) + for row in rows: + groups[grader_version(row)].append(row) + return dict(sorted(groups.items())) + + def report(args: argparse.Namespace) -> int: - """Summarize saved results per scenario and condition.""" + """Summarize saved results per condition, grader version, and scenario.""" conditions = args.condition or sorted(p.name for p in RESULTS.iterdir() if p.is_dir()) for condition in conditions: rows: list[Row] = [ json.loads(path.read_text(encoding="utf-8")) for path in sorted((RESULTS / condition).glob("*.json")) ] - if not rows: - continue - print(f"\n== {condition}: {len(rows)} sessions") - by_scenario: dict[str, list[Row]] = defaultdict(list) - for row in rows: - by_scenario[str(row["scenario"])].append(row) - print(HEADER) - for name, group in sorted(by_scenario.items()): - print(f"{name:<26} {_scenario_line(group)}") - for name in sorted(by_scenario.keys() & holdout.RETIRED.keys()): - print(f"{name} is retired, not measured: {holdout.RETIRED[name]}") + for version, graded in by_grader(rows).items(): + print(f"\n== {condition}: {len(graded)} sessions, grader v{version}") + _report_scenarios(graded) return 0 +def _report_scenarios(rows: list[Row]) -> None: + by_scenario: dict[str, list[Row]] = defaultdict(list) + for row in rows: + by_scenario[str(row["scenario"])].append(row) + print(HEADER) + for name, group in sorted(by_scenario.items()): + print(f"{name:<26} {_scenario_line(group)}") + for name in sorted(by_scenario.keys() & holdout.RETIRED.keys()): + print(f"{name} is retired, not measured: {holdout.RETIRED[name]}") + + def _scenario_line(group: list[Row]) -> str: proposals: list[Row] = [p for row in group for p in row["proposals"]] good = sum(bool(p["on_topic"] and p["cause_right"] and p["pinned"]) for p in proposals) @@ -811,12 +828,29 @@ def _comparison_lines(label: str, comparison: Comparison | None) -> list[str]: def compare(args: argparse.Namespace) -> int: - """Print the arms report of one condition: per arm, the comparisons, verdict, and herding.""" + """Print the arms report of one condition: per arm, the comparisons, verdict, and herding. + + Each grader version gets a report of its own, verdict included, and none is pooled. + """ rows = condition_rows(args.condition) if not rows: print(f"no trials for condition {args.condition}") return 0 - print(f"== {args.condition}: {len(rows)} sessions") + groups = by_grader(rows) + if len(groups) > 1: + versions = ", ".join(f"v{version}" for version in groups) + print( + f"{args.condition} holds results of graders {versions}: each is reported on its " + "own, with its own verdict, and never pooled" + ) + for version, graded in groups.items(): + _compare_one(args.condition, version, graded) + return 0 + + +def _compare_one(condition: str, version: int, rows: list[Row]) -> None: + """Print the arms report of one condition's results of one grader version.""" + print(f"== {condition}: {len(rows)} sessions, grader v{version}") print( f"{'arm':<9} {'model':<16} {'n':>4} {'pass (95%)':<18} {'tok/success':>12} " f"{'median tok':>11} {'median s':>9} {'search':>7} {'found':>7} {'used':>7} " @@ -841,7 +875,6 @@ def compare(args: argparse.Namespace) -> int: print("\nherding") for line in herding_lines(rows): print(line) - return 0 def herding_lines(rows: list[Row]) -> list[str]: diff --git a/tools/eval/runner.py b/tools/eval/runner.py index 04023e0..7ce0a24 100644 --- a/tools/eval/runner.py +++ b/tools/eval/runner.py @@ -16,6 +16,7 @@ import hashlib import json import os +import re import shutil import stat import subprocess @@ -132,6 +133,12 @@ HASH_CHUNK = 64 * 1024 # bytes read at a time while hashing the rest of a cut file PATCH_CUT = f"[patch cut at {PATCH_CAP} characters]" EXCERPT_CUT = f"[excerpt cut at {EXCERPT_CAP} characters]" +# How proposals are graded (docs/decisions/2026-11-08-MIT-661-grader-v2.md). Every result +# records it; a result without it was graded by version 1 (substring matching). +GRADER_VERSION = 2 +WORD_CHARACTER = "[a-z0-9_]" # a word matches only where no such character touches it +# A cause word that names only a version, maybe after a library: "9.2", "v2", "polars 2". +VERSION_WORD = re.compile(r"(?:[a-z][a-z0-9_.-]*\s+)?v?\d+(?:\.\d+)*") # The only hosts an experiment store may live on: scripts/test-db.sh's rule. LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1"}) _CLONING = threading.Lock() # one clone of the store at a time @@ -161,6 +168,9 @@ class Proposal: on_topic: bool # names the real symptom cause_right: bool # names the real cause (in cause or fix) pinned: bool # the library appears with a version in environment + # Kept so a later grader can score the proposal again; scrubbed and cut (`excerpt`). + cause: str = "" + fix: str = "" @dataclass(frozen=True) @@ -195,10 +205,11 @@ class Result: tokens_out: int = 0 wall_clock_seconds: float = 0.0 retrievals: list[Retrieval] = field(default_factory=list) - patch: str = "" # the agent's diff of non-test files; never committed (`patch_excerpt`) + patch: str = "" # the agent's diff of non-test files; never committed (`excerpt`) patch_hash: str = "" # of the diff with whitespace removed repeat: int = 0 # which repeat of its task, arm and model (`arms`); 0 elsewhere patch_truncated: bool = False # a changed file or the patch itself was cut at its cap + grader_version: int = GRADER_VERSION # which grader judged `proposals` def summary(self) -> str: """One line describing what the agent did.""" @@ -782,12 +793,13 @@ def project_patch(pristine: Path, project: Path) -> Patch: ) -def patch_excerpt(patch: str) -> str: - """Return the part of a patch fit to commit: secrets scrubbed (as the server does), then cut. +def excerpt(text: str) -> str: + """Return the part of a text fit to commit: secrets scrubbed (as the server does), then cut. - Scrubbing comes first, so a cut never splits a secret into a part the scrub misses. + It is how a patch, and a proposal's cause and fix, reach committed results. Scrubbing comes + first, so a cut never splits a secret into a part the scrub misses. """ - return _cut(redact(patch).text, EXCERPT_CAP, EXCERPT_CUT) + return _cut(redact(text).text, EXCERPT_CAP, EXCERPT_CUT) def _without_whitespace(text: str) -> str: @@ -890,11 +902,24 @@ def _text(content: object) -> str: def grade_proposal(row: tuple[Any, ...], discovery: Discovery | None) -> Proposal: - """Judge a proposal row (symptom, cause, fix, environment) against the discovery.""" + """Judge a proposal row (symptom, cause, fix, environment) against the discovery. + + Grader version 2: words match whole (`mentions`), and a version word never makes the + cause right. The cause and fix are kept, scrubbed and cut, so the proposal can be graded + again. + """ symptom, cause, fix, environment = row + kept_cause, kept_fix = excerpt(str(cause or "")), excerpt(str(fix or "")) if discovery is None: - return Proposal(symptom=symptom, on_topic=False, cause_right=False, pinned=False) - text = f"{cause} {fix}".lower() + return Proposal( + symptom=symptom, + on_topic=False, + cause_right=False, + pinned=False, + cause=kept_cause, + fix=kept_fix, + ) + text = f"{cause} {fix}" environment = environment if isinstance(environment, dict) else {} pinned = any( discovery.library in name.lower() and any(ch.isdigit() for ch in str(version)) @@ -902,12 +927,38 @@ def grade_proposal(row: tuple[Any, ...], discovery: Discovery | None) -> Proposa ) or _runtime_pinned(discovery.library, str(environment.get("language", ""))) return Proposal( symptom=symptom, - on_topic=any(word.lower() in symptom.lower() for word in discovery.symptom_words), - cause_right=any(word.lower() in text for word in discovery.cause_words), + on_topic=any(mentions(symptom, word) for word in discovery.symptom_words), + cause_right=any( + mentions(text, word) for word in discovery.cause_words if not is_version_word(word) + ), pinned=pinned, + cause=kept_cause, + fix=kept_fix, ) +def mentions(text: str, word: str) -> bool: + """Whether `word` appears in `text` as a whole word, ignoring case. + + No letter, digit or underscore may touch it, so "utc" is not in "outcomes", "9.2" is not + in "9.21", and "polars 2" is in "polars 2.0" but not in "polars 2026". Words may hold + spaces, dots or hyphens. An edge of the word that is itself punctuation ("int(", ".prom") + needs no boundary, or "int(" would never match "int(x)". + """ + word = word.lower() + before = f"(? bool: + """Whether a cause word is only a version, maybe after a library name ("9.2", "polars 2"). + + A version says when the cause arrived, not what it is, so it never makes a cause right. + """ + return VERSION_WORD.fullmatch(word.strip().lower()) is not None + + def best_grade(row: tuple[Any, ...], discoveries: list[Discovery | None]) -> Proposal: """Grade a proposal against each task's discovery and keep the best (multi-turn).""" grades = [grade_proposal(row, discovery) for discovery in discoveries or [None]] @@ -1161,7 +1212,7 @@ def save_result(transcripts: Path, stem: str, result: Result) -> None: committed, so the JSON holds the patch's scrubbed excerpt in place of the patch. """ (transcripts / f"{stem}.patch").write_text(result.patch, encoding="utf-8") - record = {**asdict(result), "patch_excerpt": patch_excerpt(result.patch)} + record = {**asdict(result), "patch_excerpt": excerpt(result.patch)} del record["patch"] (transcripts / f"{stem}.json").write_text(json.dumps(record, indent=1), encoding="utf-8") From f1a36b576cf5f826cedf8e8739cb12fd4cfc9ed4 Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 15:33:05 -0500 Subject: [PATCH 2/4] docs(eval): grading section and the grader v2 decision record MIT-661. How proposals are graded, what v2 changed, and how results of different grader versions are compared (never pooled). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- .../decisions/2026-11-08-MIT-661-grader-v2.md | 41 +++++++++++++++++++ docs/eval/README.md | 34 ++++++++++++++- 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 docs/decisions/2026-11-08-MIT-661-grader-v2.md diff --git a/docs/decisions/2026-11-08-MIT-661-grader-v2.md b/docs/decisions/2026-11-08-MIT-661-grader-v2.md new file mode 100644 index 0000000..49f69b4 --- /dev/null +++ b/docs/decisions/2026-11-08-MIT-661-grader-v2.md @@ -0,0 +1,41 @@ +# Eval grader v2: whole words, no version-only causes, versioned results +Status: Accepted +Date: 2026-11-08 + +Context: The eval grader (`tools/eval/runner.py` `grade_proposal`, decision 0023) matched +`cause_words` and `symptom_words` as case-insensitive substrings of the proposal. Short words +matched inside unrelated ones: "utc" in "outcomes", "9.2" in "9.21", "polars 2" in +"polars 2026", "end" in "endpoint". A proposal could also get `cause_right` from a version +alone ("broke in pytest 9.2"), which says when the cause arrived, not what it is. Changing the +grader changes what "good proposal" means, so results graded before and after can't be pooled. + +Decision: +1. **Whole words.** A word matches only where no letter, digit or underscore (`[a-z0-9_]`, + after lowercasing) immediately precedes or follows it. Words may hold spaces, dots or + hyphens. An edge of the word that is itself punctuation ("int(", ".prom", "add()") needs no + boundary on that side, or "int(" would never match "int(x)". This applies to `cause_words` + and `symptom_words` (`runner.mentions`). +2. **Version words never make the cause right.** A cause word that is only a version (digits + and dots, an optional leading `v`), maybe after one library name, is a version word: "9.2", + "2.14", "0.11", "polars 2", "python 3.14" (`runner.is_version_word`). The data keeps them; + the grader skips them for `cause_right`. A test checks that every measured scenario and every + GitHub task has at least one cause word that is not a version word. +3. **Every result records its grader.** `runner.GRADER_VERSION = 2`. Every new ledger line and + per-session JSON carries `grader_version`; a result without it was graded by version 1. +4. **Comparisons never cross graders.** `agent_eval.py report` and `compare` split a + condition's results by grader version and print a report per version. `compare` prints a + verdict per version, each from that version's results alone, and says so when a condition + holds more than one. The pre-registered verdict (the arms record) is read from one + version's report. +5. **Proposals keep their text.** Each graded proposal stores its cause and fix, passed through + the same scrubber and cap as the patch excerpt (`runner.excerpt`: secrets redacted, then cut + at 4,000 characters), so a later grader can score it again. Older results, without them, + still load. + +Consequences: Grading is stricter, so v2 rates of good proposals are not comparable with v1 +rates; compare like with like, or re-run under v2. v1 results can't be re-scored, since they +kept no cause or fix. Word stems written for substring matching ("truncat", "overwr") no longer +match inflected forms, and neither do plurals ("offset" against "offsets"); scenario lists that +relied on that need whole words added, which changes how those scenarios grade. `trials` and +`effects` still pool grader versions; their figures use `good_proposals` and would need the +same split. diff --git a/docs/eval/README.md b/docs/eval/README.md index f9d68a9..9fcc95e 100644 --- a/docs/eval/README.md +++ b/docs/eval/README.md @@ -63,6 +63,34 @@ Everything needed to measure the agent-facing text again is in the repo: the har `--setting-sources project --strict-mcp-config` so no hooks or servers run. Ask what led to the decision, not for its reasoning: asking a model to quote its own reasoning trips a safeguard. +## Grading + +`runner.grade_proposal` judges each proposal against its scenario's `discovery`. A **good +proposal** is all three: + +- **on topic**: a `symptom_words` entry appears in the symptom; +- **cause right**: a `cause_words` entry that is not a version word appears in the cause or fix; +- **pinned**: the library appears with a version in `environment.libraries` (or, for a runtime + change, in `environment.language`). + +**Grader v2** (2026-11-08, `docs/decisions/2026-11-08-MIT-661-grader-v2.md`) is the current +grader (`runner.GRADER_VERSION`): + +- **Whole words.** A word matches only where no letter, digit or underscore touches it, ignoring + case. "utc" is not in "outcomes", "9.2" is not in "9.21", and "polars 2" is in "polars 2.0" + but not in "polars 2026". An edge that is punctuation ("int(", ".prom") needs no boundary. + Stems and plurals no longer match: "truncat" misses "truncates", "offset" misses "offsets". +- **Version words never make the cause right.** A cause word that is only a version, maybe after + a library name ("9.2", "2.14", "polars 2"), stays in the data and is skipped for + `cause_right`. Every measured scenario and GitHub task must have another cause word (a test). +- **Kept text.** Each proposal in a session's JSON keeps its `cause` and `fix`, scrubbed and cut + like the patch excerpt, so a later grader can score it again. + +Version 1 matched substrings and counted version words. Every new ledger line and session JSON +records `grader_version`; one without it (everything committed before 2026-11-08) is version 1. +`report` and `compare` print a separate report per grader version, and `compare` a separate +verdict, so no figure pools two graders. `trials` and `effects` still pool them. + ## GitHub tasks A GitHub task replays a real issue from a real repository: the agent gets the project at the @@ -758,9 +786,11 @@ scenario, just no longer collected. Report `real-polars-explode-empty` on its ow a baseline of its own; a hold-out total from before this change (four first-set scenarios including whenever) is not the same set as one after it. -**Known limit of the grader.** `cause_words` are matched as substrings, so generic words in +**Known limit of the grader (v1).** Grader v1 matched `cause_words` as substrings, so generic words in other hold-outs' lists also occur in the shipped text: "removed" (both Python 3.14 scenarios), "naive" (in "naive_arithmetic_ok", `real-sqlmodel-utc`), "view" (in "reviews", `hard-pandas-cow-subset`) and "utc" (in "outcomes", `hard-sqlmodel-legacy-rows`). None of them gives a cause away, but the shipped text as a whole can't be held to the example check until -those lists change, which would change how those scenarios grade. +those lists change, which would change how those scenarios grade. Grader v2 (see Grading) +matches whole words, so "naive", "view" and "utc" no longer match those longer words; the +example checks in the tests still match substrings, which catches more. From 329f1fa2bd93f6aa499c33373c0ac2eddf37b75d Mon Sep 17 00:00:00 2001 From: Zachariah Farahany Date: Sun, 8 Nov 2026 15:40:04 -0500 Subject: [PATCH 3/4] fix(eval): stems match again; trials and effects split by grader MIT-661 review. A word must start a word; only a word ending in a digit must also end one, so stems and plurals ("truncat", "offset") match again while "utc" in "outcomes" and "9.2" in "9.21" still don't. trials gives a row per grader version and effects a section per version, so no good-proposal figure mixes graders. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- .../decisions/2026-11-08-MIT-661-grader-v2.md | 44 +++++++++---------- docs/eval/README.md | 18 +++++--- server/tests/test_eval_arms.py | 26 +++++++++++ server/tests/test_eval_grader.py | 21 ++++++--- server/tests/test_eval_skill.py | 9 +++- tools/agent_eval.py | 24 +++++++--- tools/eval/runner.py | 25 ++++++----- 7 files changed, 111 insertions(+), 56 deletions(-) diff --git a/docs/decisions/2026-11-08-MIT-661-grader-v2.md b/docs/decisions/2026-11-08-MIT-661-grader-v2.md index 49f69b4..bea2fac 100644 --- a/docs/decisions/2026-11-08-MIT-661-grader-v2.md +++ b/docs/decisions/2026-11-08-MIT-661-grader-v2.md @@ -1,20 +1,24 @@ -# Eval grader v2: whole words, no version-only causes, versioned results +# Eval grader v2: words from a word's start, no version-only causes, versioned results Status: Accepted Date: 2026-11-08 Context: The eval grader (`tools/eval/runner.py` `grade_proposal`, decision 0023) matched `cause_words` and `symptom_words` as case-insensitive substrings of the proposal. Short words -matched inside unrelated ones: "utc" in "outcomes", "9.2" in "9.21", "polars 2" in -"polars 2026", "end" in "endpoint". A proposal could also get `cause_right` from a version -alone ("broke in pytest 9.2"), which says when the cause arrived, not what it is. Changing the -grader changes what "good proposal" means, so results graded before and after can't be pooled. +matched in the middle of unrelated ones: "utc" in "outcomes", "view" in "reviews", "ops" in +"drops", "9.2" in "9.21", "polars 2" in "polars 2026". A proposal could also get +`cause_right` from a version alone ("broke in pytest 9.2"), which says when the cause arrived, +not what it is. Changing the grader changes what "good proposal" means, so results graded +before and after can't be pooled. Decision: -1. **Whole words.** A word matches only where no letter, digit or underscore (`[a-z0-9_]`, - after lowercasing) immediately precedes or follows it. Words may hold spaces, dots or - hyphens. An edge of the word that is itself punctuation ("int(", ".prom", "add()") needs no - boundary on that side, or "int(" would never match "int(x)". This applies to `cause_words` - and `symptom_words` (`runner.mentions`). +1. **Words match from a word's start** (`runner.mentions`, for `cause_words` and + `symptom_words`). After lowercasing, no `[a-z0-9_]` character may come just before the word. + A word that ends in a digit must also end there, so "9.2" doesn't match "9.21" and + "polars 2" matches "polars 2.0" but not "polars 2026". Any other word may run on, so stems + and plurals written for substring matching keep working: "truncat" matches "truncated", + "round" "rounding", "offset" "offsets", "overwr" "overwritten". Every false hit above starts + mid-word, so the left boundary alone stops them. A word starting with punctuation (".prom") + needs nothing before it. Words may hold spaces, dots or hyphens. 2. **Version words never make the cause right.** A cause word that is only a version (digits and dots, an optional leading `v`), maybe after one library name, is a version word: "9.2", "2.14", "0.11", "polars 2", "python 3.14" (`runner.is_version_word`). The data keeps them; @@ -22,20 +26,16 @@ Decision: GitHub task has at least one cause word that is not a version word. 3. **Every result records its grader.** `runner.GRADER_VERSION = 2`. Every new ledger line and per-session JSON carries `grader_version`; a result without it was graded by version 1. -4. **Comparisons never cross graders.** `agent_eval.py report` and `compare` split a - condition's results by grader version and print a report per version. `compare` prints a - verdict per version, each from that version's results alone, and says so when a condition - holds more than one. The pre-registered verdict (the arms record) is read from one - version's report. +4. **No figure crosses graders.** `agent_eval.py` `report`, `compare` and `effects` print a + section per grader version, and `trials` a row per version. `compare` gives each version its + own verdict, from that version's results alone, and says so when a condition holds more than + one. The pre-registered verdict (the arms record) is read from one version's section. 5. **Proposals keep their text.** Each graded proposal stores its cause and fix, passed through the same scrubber and cap as the patch excerpt (`runner.excerpt`: secrets redacted, then cut at 4,000 characters), so a later grader can score it again. Older results, without them, still load. -Consequences: Grading is stricter, so v2 rates of good proposals are not comparable with v1 -rates; compare like with like, or re-run under v2. v1 results can't be re-scored, since they -kept no cause or fix. Word stems written for substring matching ("truncat", "overwr") no longer -match inflected forms, and neither do plurals ("offset" against "offsets"); scenario lists that -relied on that need whole words added, which changes how those scenarios grade. `trials` and -`effects` still pool grader versions; their figures use `good_proposals` and would need the -same split. +Consequences: v2 rates of good proposals are not comparable with v1 rates; compare like with +like, or re-run under v2. v1 results can't be re-scored, since they kept no cause or fix. A +word that legitimately occurs as a prefix of a longer unrelated word ("end" in "endpoint") +still matches; a scenario list that hits this needs a longer word. diff --git a/docs/eval/README.md b/docs/eval/README.md index 9fcc95e..eecb53b 100644 --- a/docs/eval/README.md +++ b/docs/eval/README.md @@ -76,10 +76,12 @@ proposal** is all three: **Grader v2** (2026-11-08, `docs/decisions/2026-11-08-MIT-661-grader-v2.md`) is the current grader (`runner.GRADER_VERSION`): -- **Whole words.** A word matches only where no letter, digit or underscore touches it, ignoring - case. "utc" is not in "outcomes", "9.2" is not in "9.21", and "polars 2" is in "polars 2.0" - but not in "polars 2026". An edge that is punctuation ("int(", ".prom") needs no boundary. - Stems and plurals no longer match: "truncat" misses "truncates", "offset" misses "offsets". +- **Words start at a word's start.** Ignoring case, no letter, digit or underscore may come + just before a word, so "utc" is not in "outcomes", "view" not in "reviews", "ops" not in + "drops". A word ending in a digit must also end there: "9.2" is not in "9.21", and + "polars 2" is in "polars 2.0" but not in "polars 2026". Any other word may run on, so stems + and plurals still match: "truncat" in "truncated", "round" in "rounding", "offset" in + "offsets". A word starting with punctuation (".prom") needs nothing before it. - **Version words never make the cause right.** A cause word that is only a version, maybe after a library name ("9.2", "2.14", "polars 2"), stays in the data and is skipped for `cause_right`. Every measured scenario and GitHub task must have another cause word (a test). @@ -88,8 +90,9 @@ grader (`runner.GRADER_VERSION`): Version 1 matched substrings and counted version words. Every new ledger line and session JSON records `grader_version`; one without it (everything committed before 2026-11-08) is version 1. -`report` and `compare` print a separate report per grader version, and `compare` a separate -verdict, so no figure pools two graders. `trials` and `effects` still pool them. +`report`, `compare`, `trials` and `effects` keep grader versions apart: `report`, `compare` and +`effects` print a section per version (`compare` a verdict per version), and `trials` a row per +version, so no figure pools two graders. ## GitHub tasks @@ -792,5 +795,6 @@ other hold-outs' lists also occur in the shipped text: "removed" (both Python 3. `hard-pandas-cow-subset`) and "utc" (in "outcomes", `hard-sqlmodel-legacy-rows`). None of them gives a cause away, but the shipped text as a whole can't be held to the example check until those lists change, which would change how those scenarios grade. Grader v2 (see Grading) -matches whole words, so "naive", "view" and "utc" no longer match those longer words; the +matches only from a word's start, so "view" and "utc" no longer match inside "reviews" and +"outcomes" ("naive" still starts "naive_arithmetic_ok"); the example checks in the tests still match substrings, which catches more. diff --git a/server/tests/test_eval_arms.py b/server/tests/test_eval_arms.py index e16f3e7..43a93b6 100644 --- a/server/tests/test_eval_arms.py +++ b/server/tests/test_eval_arms.py @@ -336,6 +336,32 @@ def test_old_ledger_lines_still_load_in_trials_and_effects( assert "w-a0b4c5d1e0f1g1h0n0s1" in pooled # the old line is a config of its condition +def test_trials_and_effects_never_pool_grader_versions( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A line without `grader_version` is v1; a v2 line of the same text is its own group.""" + ledger = tmp_path / "trials.jsonl" + monkeypatch.setattr(tools.agent_eval, "LEDGER", ledger) + regraded = {**OLD_LINE, "grader_version": 2, "variant": "w-a1b0c0d0e0f0g0h0n0s0"} + same_text = {**OLD_LINE, "grader_version": 2, "good_proposals": 1} + lines = [OLD_LINE, same_text, regraded] + ledger.write_text("".join(json.dumps(line) + "\n" for line in lines), encoding="utf-8") + + assert tools.agent_eval.by_variant() == 0 + rows = [line.split() for line in capsys.readouterr().out.splitlines()[1:]] + old = [row for row in rows if row[1] == OLD_LINE["variant"]] + # Columns: arm, variant, text, trigger, n, search, propose, good, fixed, grader. + assert [(row[4], row[7], row[-1]) for row in old] == [("1", "0", "v1"), ("1", "1", "v2")] + + assert tools.agent_eval.effects(argparse.Namespace(condition="cross-final")) == 0 + first, second = capsys.readouterr().out.split("== cross-final: 2 sessions, grader v2") + assert "== cross-final: 1 sessions, grader v1" in first + assert "w-a1b0c0d0e0f0g0h0n0s0" not in first + assert "w-a1b0c0d0e0f0g0h0n0s0" in second + assert "per option" in first + assert "per option" in second + + def test_herding_reads_a_lines_excerpt_and_an_older_lines_whole_patch_by_hash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] ) -> None: diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index 86a24dd..3834b1a 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -53,21 +53,28 @@ def test_a_library_change_still_needs_the_library_pinned() -> None: @pytest.mark.parametrize( ("text", "word", "found"), [ - ("the outcomes table", "utc", False), # grader v1 matched this, a substring - ("UTCDateTime stores aware datetimes", "utc", False), + # Nothing may come just before a word: grader v1 matched these substrings. + ("the outcomes table", "utc", False), + ("the reviews page", "view", False), + ("explode drops rows", "ops", False), + ("version 21002.0", "2.0", False), + ("print(x)", "int(", False), ("stored in UTC, read back naive", "utc", True), + # A word ending in a digit must end there. ("pytest 9.21 changed nothing", "9.2", False), ("since pytest 9.2.0, fixtures are per node", "9.2", True), ("pytest 9.2: fixtures are per node", "9.2", True), ("Polars 2.0 changed explode()", "polars 2", True), ("polars 2026 roadmap", "polars 2", False), ("explode under polars 2 drops rows", "polars 2", True), + # Any other word may run on: stems and plurals match. + ("it rounds down, rounding 19.99 to 1998", "round", True), + ("the float is truncated", "truncat", True), + ("utc offsets are dropped", "offset", True), + ("settings were overwritten", "overwr", True), ("now copy-on-write by default", "copy-on-write", True), - ("the range end is excluded", "end", True), - ("call the endpoint", "end", False), - ("int(x * 100) truncates", "int(", True), # punctuation at an edge needs no boundary - ("print(x)", "int(", False), - ("writes node.prom", ".prom", True), + ("int(x * 100) truncates", "int(", True), + ("writes node.prom", ".prom", True), # a word starting with punctuation: no boundary ], ) def test_words_match_whole(text: str, word: str, *, found: bool) -> None: diff --git a/server/tests/test_eval_skill.py b/server/tests/test_eval_skill.py index af2bc54..fd2c726 100644 --- a/server/tests/test_eval_skill.py +++ b/server/tests/test_eval_skill.py @@ -301,6 +301,11 @@ def test_the_ledger_records_the_skill_and_old_lines_still_load( out = capsys.readouterr().out.splitlines() # The table, then the per-condition median-token lines (MIT-601), which aren't rows. rows = [line.split() for line in out[1:] if "median tokens per passing session" not in line] - # Columns: arm, variant, text, trigger, n, ... (the arm column came with the eval arms). - assert [(row[1], row[4]) for row in rows] == [("baseline", "2"), ("baseline+skill", "1")] + # Columns: arm, variant, text, trigger, n, ..., grader (the arm column came with the eval + # arms). The old line was graded by v1, the new ones by v2: separate groups, never pooled. + assert [(row[1], row[4], row[-1]) for row in rows] == [ + ("baseline", "1", "v1"), + ("baseline", "1", "v2"), + ("baseline+skill", "1", "v2"), + ] assert out[-1].startswith("s: median tokens per passing session in 0 out 0 (n=2)") diff --git a/tools/agent_eval.py b/tools/agent_eval.py index e21a0b8..0bdac7b 100644 --- a/tools/agent_eval.py +++ b/tools/agent_eval.py @@ -367,24 +367,26 @@ def by_variant() -> int: lines = _ledger_lines() # A line marked `invalid` stays in the ledger as a record but is never counted. rows = [row for row in lines if not row.get("invalid")] - groups: dict[tuple[str, str, str, str], list[dict[str, Any]]] = defaultdict(list) + groups: dict[tuple[str, str, str, str, int], list[dict[str, Any]]] = defaultdict(list) for row in rows: # The text fingerprint covers the server's text only, so the skill is its own group, - # and a line from before arms ran with the commons alone. + # and a line from before arms ran with the commons alone. Each grader version is its + # own group too, so no `good` figure mixes two graders. variant = f"{row['variant']}+skill" if row.get("skill") else row["variant"] - groups[(row.get("arm", "commons"), variant, row["text"], row["trigger"])].append(row) + arm = row.get("arm", "commons") + groups[(arm, variant, row["text"], row["trigger"], grader_version(row))].append(row) if skipped := len(lines) - len(rows): print(f"({skipped} trials marked invalid are not counted)") print( f"{'arm':<8} {'variant':<10} {'text':<13} {'trigger':<18} {'n':>3} " - f"{'search':>7} {'propose':>8} {'good':>5} {'fixed':>6}" + f"{'search':>7} {'propose':>8} {'good':>5} {'fixed':>6} {'grader':>6}" ) - for (arm, variant, text, trigger), group in sorted(groups.items()): + for (arm, variant, text, trigger, version), group in sorted(groups.items()): good = sum(int(r["good_proposals"]) for r in group) print( f"{arm:<8} {variant:<10} {text:<13} {trigger:<18} {len(group):>3} " f"{_rate(group, 'searched'):>7} {_rate(group, 'proposed'):>8} {good:>5} " - f"{_rate(group, 'tests_pass'):>6}" + f"{_rate(group, 'tests_pass'):>6} {f'v{version}':>6}" ) for line in passing_tokens(rows): print(line) @@ -399,6 +401,7 @@ def effects(args: argparse.Namespace) -> int: """Pool a wording search by option and model: each session counts for every option it showed. Configs are ranked by their worse model, since the text has to work for every model. + Each grader version is pooled and ranked on its own, never with another. """ rows = [ row @@ -411,6 +414,14 @@ def effects(args: argparse.Namespace) -> int: if not rows: print(f"no wording trials for condition {args.condition}") return 0 + for version, graded in by_grader(rows).items(): + print(f"== {args.condition}: {len(graded)} sessions, grader v{version}") + _effects_of(graded) + return 0 + + +def _effects_of(rows: list[dict[str, Any]]) -> None: + """Print the per-option and per-config lines of one grader version's wording trials.""" models = sorted({str(row.get("model") or "sonnet") for row in rows}) print("per option (pooled over every config that showed it)") for key, size in agent_text.SURFACES.items(): @@ -427,7 +438,6 @@ def worst(config: str) -> float: for config in sorted(configs, key=worst, reverse=True): print(_effect_line(config, [row for row in rows if row["variant"] == config], models)) - return 0 def option_of(variant: str, key: str) -> int: diff --git a/tools/eval/runner.py b/tools/eval/runner.py index 7ce0a24..1e03751 100644 --- a/tools/eval/runner.py +++ b/tools/eval/runner.py @@ -136,7 +136,8 @@ # How proposals are graded (docs/decisions/2026-11-08-MIT-661-grader-v2.md). Every result # records it; a result without it was graded by version 1 (substring matching). GRADER_VERSION = 2 -WORD_CHARACTER = "[a-z0-9_]" # a word matches only where no such character touches it +# None may come just before a word, nor just after a word that ends in a digit. +WORD_CHARACTER = "[a-z0-9_]" # A cause word that names only a version, maybe after a library: "9.2", "v2", "polars 2". VERSION_WORD = re.compile(r"(?:[a-z][a-z0-9_.-]*\s+)?v?\d+(?:\.\d+)*") # The only hosts an experiment store may live on: scripts/test-db.sh's rule. @@ -904,9 +905,9 @@ def _text(content: object) -> str: def grade_proposal(row: tuple[Any, ...], discovery: Discovery | None) -> Proposal: """Judge a proposal row (symptom, cause, fix, environment) against the discovery. - Grader version 2: words match whole (`mentions`), and a version word never makes the - cause right. The cause and fix are kept, scrubbed and cut, so the proposal can be graded - again. + Grader version 2: words match from a word's start (`mentions`), and a version word never + makes the cause right. The cause and fix are kept, scrubbed and cut, so the proposal can be + graded again. """ symptom, cause, fix, environment = row kept_cause, kept_fix = excerpt(str(cause or "")), excerpt(str(fix or "")) @@ -938,16 +939,18 @@ def grade_proposal(row: tuple[Any, ...], discovery: Discovery | None) -> Proposa def mentions(text: str, word: str) -> bool: - """Whether `word` appears in `text` as a whole word, ignoring case. - - No letter, digit or underscore may touch it, so "utc" is not in "outcomes", "9.2" is not - in "9.21", and "polars 2" is in "polars 2.0" but not in "polars 2026". Words may hold - spaces, dots or hyphens. An edge of the word that is itself punctuation ("int(", ".prom") - needs no boundary, or "int(" would never match "int(x)". + """Whether `word` starts a word in `text`, ignoring case; a word ending in a digit ends one. + + No letter, digit or underscore may come just before it, so "utc" is not in "outcomes" and + "view" is not in "reviews". It may run on, so stems and plurals match: "truncat" is in + "truncated", "offset" in "offsets". A word ending in a digit is a whole number, so "9.2" + is not in "9.21", and "polars 2" is in "polars 2.0" but not in "polars 2026". Words may + hold spaces, dots or hyphens. A word starting with punctuation (".prom") needs nothing + before it, or it would never match "node.prom". """ word = word.lower() before = f"(? Date: Sun, 8 Nov 2026 15:49:07 -0500 Subject: [PATCH 4/4] fix(eval): a version word's prefix must be its library or python MIT-661 review. "pep 649" and "rfc 3339" are causes, not versions, so is_version_word takes the discovery's library. A grade_proposal test now shows symptom and cause words both match from a word's start, and another pins the skipped words. The ledger table names grader_version. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Qc1CQTb8r67J32HpkhS43y --- .../decisions/2026-11-08-MIT-661-grader-v2.md | 11 ++- docs/eval/README.md | 4 +- server/tests/test_eval_grader.py | 73 ++++++++++++++----- tools/eval/runner.py | 21 ++++-- 4 files changed, 78 insertions(+), 31 deletions(-) diff --git a/docs/decisions/2026-11-08-MIT-661-grader-v2.md b/docs/decisions/2026-11-08-MIT-661-grader-v2.md index bea2fac..b0f16ca 100644 --- a/docs/decisions/2026-11-08-MIT-661-grader-v2.md +++ b/docs/decisions/2026-11-08-MIT-661-grader-v2.md @@ -20,10 +20,13 @@ Decision: mid-word, so the left boundary alone stops them. A word starting with punctuation (".prom") needs nothing before it. Words may hold spaces, dots or hyphens. 2. **Version words never make the cause right.** A cause word that is only a version (digits - and dots, an optional leading `v`), maybe after one library name, is a version word: "9.2", - "2.14", "0.11", "polars 2", "python 3.14" (`runner.is_version_word`). The data keeps them; - the grader skips them for `cause_right`. A test checks that every measured scenario and every - GitHub task has at least one cause word that is not a version word. + and dots, an optional leading `v`), maybe after the discovery's `library` or "python", is a + version word: "9.2", "2.14", "0.11", "polars 2" (for polars), "python 3.14" + (`runner.is_version_word`). Any other prefix makes it part of a cause, not a version's + label: "pep 649", "rfc 3339". The data keeps version words; the grader skips them for + `cause_right`. Tests check that every measured scenario and every GitHub task has at least + one cause word that is not a version word, and that the skipped words are exactly 9.2, 2.14, + 0.0.45, 0.11, polars 2, 3.14, 3.0, 2.1 and 2.0. 3. **Every result records its grader.** `runner.GRADER_VERSION = 2`. Every new ledger line and per-session JSON carries `grader_version`; a result without it was graded by version 1. 4. **No figure crosses graders.** `agent_eval.py` `report`, `compare` and `effects` print a diff --git a/docs/eval/README.md b/docs/eval/README.md index eecb53b..e20da45 100644 --- a/docs/eval/README.md +++ b/docs/eval/README.md @@ -83,7 +83,8 @@ grader (`runner.GRADER_VERSION`): and plurals still match: "truncat" in "truncated", "round" in "rounding", "offset" in "offsets". A word starting with punctuation (".prom") needs nothing before it. - **Version words never make the cause right.** A cause word that is only a version, maybe after - a library name ("9.2", "2.14", "polars 2"), stays in the data and is skipped for + the discovery's `library` or "python" ("9.2", "2.14", "polars 2" for polars, "python 3.14"), + stays in the data and is skipped for `cause_right`. Every measured scenario and GitHub task must have another cause word (a test). - **Kept text.** Each proposal in a session's JSON keeps its `cause` and `fix`, scrubbed and cut like the patch excerpt, so a later grader can score it again. @@ -232,6 +233,7 @@ Each result and ledger line also records, in every arm and in `run` and `search` | `patch_truncated` | true when the diff is not all of the agent's change: a changed file was over 256 KiB, or the diff over 1 MiB | | `patch_hash` | 16 hex digits of sha256 over the whole diff of what was read, taken with whitespace removed and blank lines dropped, so whitespace-only differences share a hash | | `repeat` | which of `--runs` repeats of its task, arm and model the session was (1 to N); 0 outside `arms`. Unlike `run`, the same repeat pairs across arms | +| `grader_version` | the grader that judged the session's proposals (`runner.GRADER_VERSION`, see Grading); lines and results without it are version 1, and no report pools two versions | Test files (under `tests/`, `test/`, or `__tests__/`, or named `test_*`, `*_test.*`, `conftest.py`, `*.test.*`, `*.spec.*`) are left out of the patch, as is generated output: diff --git a/server/tests/test_eval_grader.py b/server/tests/test_eval_grader.py index 3834b1a..d8a0dc0 100644 --- a/server/tests/test_eval_grader.py +++ b/server/tests/test_eval_grader.py @@ -11,7 +11,7 @@ import pytest import tools.agent_eval -from tools.eval import github_tasks, repos, runner +from tools.eval import github_tasks, holdout, repos, runner from tools.eval.model import Discovery from tools.eval.runner import grade_proposal, is_version_word, mentions from tools.eval.scenarios import SCENARIOS @@ -82,36 +82,58 @@ def test_words_match_whole(text: str, word: str, *, found: bool) -> None: @pytest.mark.parametrize( - ("word", "version"), + ("word", "library", "version"), [ - ("9.2", True), - ("2.14", True), - ("polars 2", True), - ("0.11", True), - ("0.0.45", True), - ("v2", True), - ("python 3.14", True), - ("per node", False), - ("utc", False), - ("utf-8", False), - ("copy-on-write", False), - ("timezone=True", False), - ("text(", False), + ("9.2", "pytest", True), + ("pytest 9.2", "pytest", True), + ("2.14", "pydantic", True), + ("polars 2", "polars", True), + ("Polars 2", "polars", True), + ("0.11", "whenever", True), + ("0.0.45", "sqlmodel", True), + ("v2", "quickcache", True), + ("python 3.14", "python", True), + ("python 3.14", "sqlite3", True), # a runtime version, whatever the library + # A prefix that is no library is part of the cause, not a version's label. + ("pep 649", "python", False), + ("rfc 3339", "whenever", False), + ("utf 8", "csvkitlite", False), + ("polars 2", "pytest", False), + ("per node", "pytest", False), + ("utc", "sqlmodel", False), + ("utf-8", "csvkitlite", False), + ("copy-on-write", "pandas", False), + ("timezone=True", "sqlmodel", False), + ("text(", "sqlalchemy", False), ], ) def test_a_version_word_is_only_a_version_after_an_optional_library( - word: str, *, version: bool + word: str, library: str, *, version: bool ) -> None: - assert is_version_word(word) is version + assert is_version_word(word, library) is version PER_NODE = Discovery( library="pytest", symptom_words=("_arg2fixturedefs",), - cause_words=("9.2", "per node", "polars 2"), + cause_words=("9.2", "per node"), ) +def test_grading_matches_symptom_and_cause_words_from_a_words_start() -> None: + """Grader v1's substring hits ("view" in "reviews", "utc" in "outcomes") no longer count.""" + discovery = Discovery(library="sqlmodel", symptom_words=("view",), cause_words=("utc",)) + environment = {"libraries": {"sqlmodel": "0.0.45"}} + inside = ("the reviews page breaks", "outcomes are stored", "store the outcomes", environment) + graded = grade_proposal(inside, discovery) + assert not graded.on_topic + assert not graded.cause_right + whole = ("the view breaks", "times stored as UTC", "read them back as utc", environment) + graded = grade_proposal(whole, discovery) + assert graded.on_topic + assert graded.cause_right + + def test_a_proposal_whose_only_hit_is_a_version_word_has_the_cause_wrong() -> None: environment = {"libraries": {"pytest": "9.2.0"}} version_only = ("_arg2fixturedefs gone", "pytest 9.2 broke it", "pin pytest", environment) @@ -222,7 +244,20 @@ def test_every_measured_scenario_and_github_task_has_a_cause_word_that_is_no_ver discovery: Discovery, ) -> None: """Grader v2 skips version words for `cause_right`, so a task needs at least one other.""" - assert [word for word in discovery.cause_words if not is_version_word(word)] + assert [w for w in discovery.cause_words if not is_version_word(w, discovery.library)] + + +def test_the_cause_words_grader_v2_skips_are_exactly_the_versions() -> None: + """Measured and retired scenarios alike: a change here changes how some scenario grades.""" + retired = [holdout.WHENEVER_IGNORE_DST.discovery] + skipped = { + word + for discovery in [*MEASURED.values(), *retired] + if discovery is not None + for word in discovery.cause_words + if is_version_word(word, discovery.library) + } + assert skipped == {"9.2", "2.14", "0.0.45", "0.11", "polars 2", "3.14", "3.0", "2.1", "2.0"} # A recorded session (2026-10-25, hard-sqlalchemy-autoflush), trimmed to what grading reads. diff --git a/tools/eval/runner.py b/tools/eval/runner.py index 1e03751..9e8d926 100644 --- a/tools/eval/runner.py +++ b/tools/eval/runner.py @@ -138,8 +138,9 @@ GRADER_VERSION = 2 # None may come just before a word, nor just after a word that ends in a digit. WORD_CHARACTER = "[a-z0-9_]" -# A cause word that names only a version, maybe after a library: "9.2", "v2", "polars 2". -VERSION_WORD = re.compile(r"(?:[a-z][a-z0-9_.-]*\s+)?v?\d+(?:\.\d+)*") +# A cause word that names only a version, maybe after a prefix: "9.2", "v2", "polars 2". +# `is_version_word` accepts only the discovery's library or "python" as the prefix. +VERSION_WORD = re.compile(r"(?:(?P[a-z][a-z0-9_.-]*)\s+)?v?\d+(?:\.\d+)*") # The only hosts an experiment store may live on: scripts/test-db.sh's rule. LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1"}) _CLONING = threading.Lock() # one clone of the store at a time @@ -930,7 +931,9 @@ def grade_proposal(row: tuple[Any, ...], discovery: Discovery | None) -> Proposa symptom=symptom, on_topic=any(mentions(symptom, word) for word in discovery.symptom_words), cause_right=any( - mentions(text, word) for word in discovery.cause_words if not is_version_word(word) + mentions(text, word) + for word in discovery.cause_words + if not is_version_word(word, discovery.library) ), pinned=pinned, cause=kept_cause, @@ -954,12 +957,16 @@ def mentions(text: str, word: str) -> bool: return re.search(before + re.escape(word) + after, text.lower()) is not None -def is_version_word(word: str) -> bool: - """Whether a cause word is only a version, maybe after a library name ("9.2", "polars 2"). +def is_version_word(word: str, library: str) -> bool: + """Whether a cause word is only a version, maybe after `library` or "python". - A version says when the cause arrived, not what it is, so it never makes a cause right. + "9.2", "polars 2" (for polars) and "python 3.14" are; "pep 649" and "rfc 3339" are not, + since their prefix is no library. A version says when the cause arrived, not what it is, + so it never makes a cause right. """ - return VERSION_WORD.fullmatch(word.strip().lower()) is not None + match = VERSION_WORD.fullmatch(word.strip().lower()) + prefix = match.group("prefix") if match else None + return match is not None and prefix in {None, library.lower(), "python"} def best_grade(row: tuple[Any, ...], discoveries: list[Discovery | None]) -> Proposal: