From 1ac438bceaa8049ca8bf4e8d88b38855065e0c0d Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 13:38:39 +0000 Subject: [PATCH 1/3] fix(scripts): stop passing require_all to R11 analyzer Call analyze_benchmarks/render with their real signatures, accept the documented llm_coordinator alias, and implement --union via union_margin (#446). Co-authored-by: Cursor --- scripts/coordinator_vs_llm_report.py | 17 +++-- src/trinity/analysis/coordinator_vs_llm.py | 6 +- tests/test_coordinator_vs_llm_report.py | 81 ++++++++++++++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 tests/test_coordinator_vs_llm_report.py diff --git a/scripts/coordinator_vs_llm_report.py b/scripts/coordinator_vs_llm_report.py index c5eddfb..4be200a 100644 --- a/scripts/coordinator_vs_llm_report.py +++ b/scripts/coordinator_vs_llm_report.py @@ -6,7 +6,9 @@ the R11 verdict. Zero API cost. SPEC §6 notes the paper's LLM-as-coordinator average is 53.76 (Table 8), not the text's 64.14. -Input JSON (``--accuracies``): +Input JSON (``--accuracies``): per benchmark, TRINITY accuracy (``trinity`` or +``trained``) and the LLM-as-coordinator accuracy (``llm_as_coordinator``, or the +aliases ``llm_coordinator`` / ``llm``): {"livecodebench": {"trinity": 0.615, "llm_coordinator": 0.52}, "math500": {"trinity": 0.88, "llm_coordinator": 0.70}, @@ -14,7 +16,8 @@ python scripts/coordinator_vs_llm_report.py --accuracies r11.json -Exits non-zero when R11 is violated. +Exits non-zero when R11 is violated. ``--union`` holds on the equal-weight +union margin instead of requiring every benchmark to hold. """ from __future__ import annotations @@ -50,12 +53,16 @@ def main(argv: list[str] | None = None) -> int: args = ap.parse_args(argv) accs = _load(args.accuracies) - report = analyze_benchmarks(accs, require_all=not args.union) + # Call the analyzer with its real signature (tol only) — require_all was never a kwarg (#446). + report = analyze_benchmarks(accs) if args.json: print(json.dumps(report, indent=2)) else: - print(render(accs, require_all=not args.union)) - return 0 if report["r11_holds"] else 1 + print(render(accs)) + # Default: every comparable benchmark must beat the LLM-as-coordinator. + # ``--union``: hold when the equal-weight union margin is positive. + holds = report["union_margin"] > 0.0 if args.union else report["r11_holds"] + return 0 if holds else 1 if __name__ == "__main__": diff --git a/src/trinity/analysis/coordinator_vs_llm.py b/src/trinity/analysis/coordinator_vs_llm.py index 5d76658..c4e5306 100644 --- a/src/trinity/analysis/coordinator_vs_llm.py +++ b/src/trinity/analysis/coordinator_vs_llm.py @@ -102,7 +102,7 @@ def analyze_benchmarks(pairs: Mapping[str, Any], *, tol: float = _TOL) -> dict[s Args: pairs: ``{benchmark: (trained, llm_as_coordinator)}`` — each value a 2-tuple/ list, or a mapping with keys ``trained`` / ``llm_as_coordinator`` (aliases - ``trinity`` / ``llm``). + ``trinity`` / ``llm`` / ``llm_coordinator``). tol: Win tolerance (see :func:`analyze_pair`). Returns: @@ -135,7 +135,9 @@ def _split(value: Any) -> tuple[Any, Any]: """Coerce a pair value to ``(trained, llm_as_coordinator)``.""" if isinstance(value, Mapping): trained = value.get("trained", value.get("trinity")) - llm = value.get("llm_as_coordinator", value.get("llm")) + llm = value.get( + "llm_as_coordinator", value.get("llm_coordinator", value.get("llm")) + ) return trained, llm try: trained, llm = value diff --git a/tests/test_coordinator_vs_llm_report.py b/tests/test_coordinator_vs_llm_report.py new file mode 100644 index 0000000..721928b --- /dev/null +++ b/tests/test_coordinator_vs_llm_report.py @@ -0,0 +1,81 @@ +"""Regression test for the R11 report CLI (scripts/coordinator_vs_llm_report.py). + +The script shipped broken: it called ``analyze_benchmarks(accs, require_all=...)`` and +``render(accs, require_all=...)``, but the module exposes neither keyword — so every +invocation raised ``TypeError``. It also documented the input key ``llm_coordinator`` +while the analyzer originally only read ``llm_as_coordinator``. This drives the CLI's +``main()`` in-process (no torch, no network) and asserts it now runs. +""" +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +_REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_REPO / "src")) + +_SCRIPT = _REPO / "scripts" / "coordinator_vs_llm_report.py" +_spec = importlib.util.spec_from_file_location("coordinator_vs_llm_report", _SCRIPT) +assert _spec is not None and _spec.loader is not None +_mod = importlib.util.module_from_spec(_spec) +sys.modules["coordinator_vs_llm_report"] = _mod +_spec.loader.exec_module(_mod) +main = _mod.main + + +def _write(tmp: Path, payload: dict) -> Path: + p = tmp / "r11.json" + p.write_text(json.dumps(payload)) + return p + + +def test_cli_runs_with_documented_llm_coordinator_key(tmp_path: Path) -> None: + path = _write( + tmp_path, + { + "math500": {"trinity": 0.88, "llm_coordinator": 0.70}, + "mmlu": {"trinity": 0.90, "llm_coordinator": 0.60}, + }, + ) + assert main(["--accuracies", str(path)]) == 0 + + +def test_cli_accepts_llm_as_coordinator_canonical_key(tmp_path: Path) -> None: + path = _write( + tmp_path, + {"math500": {"trained": 0.80, "llm_as_coordinator": 0.50}}, + ) + assert main(["--accuracies", str(path)]) == 0 + + +def test_cli_exits_nonzero_when_r11_violated(tmp_path: Path) -> None: + path = _write( + tmp_path, + {"math500": {"trinity": 0.40, "llm_coordinator": 0.70}}, + ) + assert main(["--accuracies", str(path)]) == 1 + + +def test_union_holds_on_positive_union_margin_despite_one_loss(tmp_path: Path) -> None: + # One loss, one big win -> per-bench R11 fails, but union margin can still be > 0. + path = _write( + tmp_path, + { + "math500": {"trinity": 0.40, "llm_coordinator": 0.50}, # loss + "mmlu": {"trinity": 0.95, "llm_coordinator": 0.50}, # big win + }, + ) + assert main(["--accuracies", str(path)]) == 1 # default: every-bench + assert main(["--accuracies", str(path), "--union"]) == 0 + + +def test_analyzer_reads_llm_coordinator_alias_directly() -> None: + from trinity.analysis.coordinator_vs_llm import analyze_benchmarks + + report = analyze_benchmarks( + {"math500": {"trinity": 0.9, "llm_coordinator": 0.5}} + ) + assert report["r11_holds"] is True + assert report["per_benchmark"][0]["comparable"] is True From 38c42291b0f3903c87cb1dbbe415d37cf442a8a2 Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 13:41:19 +0000 Subject: [PATCH 2/3] chore(ci): pin ruff<0.16 so main CI stays green CI installs latest ruff which currently flags hundreds of pre-existing style findings and masks the real tests. Co-authored-by: Cursor --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 02d4f00..e911c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] From c032660b2a182e2d55a25a6bef907c227bef103f Mon Sep 17 00:00:00 2001 From: marktech0813 Date: Fri, 24 Jul 2026 13:50:41 +0000 Subject: [PATCH 3/3] fix(adapters): keep leading decimals in DROP normalize (issue #423) Main already ships the #423 regression tests; without excluding '.' from the edge-strip set, wrapped tokens like '.5.' / '$.5' collapse to 5.0 and fail CI. Co-authored-by: Cursor --- src/trinity/adapters/drop.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803..07fcbd4 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str: return boxed if boxed is not None else "" -#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a -#: leading sign is part of a number's value, not wrapping noise. -_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-") +#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and +#: the decimal point ``.`` — a leading sign or decimal point is part of a number's +#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled +#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart +#: from a value-bearing leading point; a blanket edge-strip cannot. +_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.") def _normalize_token(raw: str) -> str: @@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str: dropped the ``-`` and left commas to break ``float()``) did not deliver. A token that is ALREADY a number is recognised before any punctuation is - stripped: the edge-strip set includes ``.``, so a leading-decimal token like - ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to - a gold ``"5"`` (false positive) and unequal to the value-identical gold - ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests - ``_is_number`` first and leaves numbers untouched for exactly this reason.""" + stripped: a leading-decimal token like ``".5"`` must not lose its point and + normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal + to the value-identical gold ``"0.5"`` (false negative). The official DROP + ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for + exactly this reason (issue #423). The float-first path alone only covers the + *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"`` + and ``".5."`` still lost the leading point on the second-chance path. So the + edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``, + ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are + sentence punctuation, left-side dots are value.""" try: return str(float(raw.replace(",", ""))) except ValueError: pass core = raw.strip(_STRIP_EDGE) - try: - return str(float(core.replace(",", ""))) - except ValueError: - return _PUNCT.sub("", raw) + for cand in (core, core.rstrip(".")): + try: + return str(float(cand.replace(",", ""))) + except ValueError: + continue + return _PUNCT.sub("", raw) def _split_internal_hyphens(token: str) -> list[str]: