fix(scripts): stop passing require_all to R11 analyzer

pyproject.toml

@@ -18,7 +18,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"]
+dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"]
 
 [build-system]
 requires = ["hatchling"]

scripts/coordinator_vs_llm_report.py

@@ -6,15 +6,18 @@
 the R11 verdict. Zero API cost. SPEC §6 notes the paper's LLM-as-coordinator
 average is 53.76 (Table 8), not the text's 64.14.
 
-Input JSON (``--accuracies``):
+Input JSON (``--accuracies``): per benchmark, TRINITY accuracy (``trinity`` or
+``trained``) and the LLM-as-coordinator accuracy (``llm_as_coordinator``, or the
+aliases ``llm_coordinator`` / ``llm``):
 
     {"livecodebench": {"trinity": 0.615, "llm_coordinator": 0.52},
      "math500":       {"trinity": 0.88,  "llm_coordinator": 0.70},
      "mmlu":          {"trinity": 0.916, "llm_coordinator": 0.60}}
 
     python scripts/coordinator_vs_llm_report.py --accuracies r11.json
 
-Exits non-zero when R11 is violated.
+Exits non-zero when R11 is violated. ``--union`` holds on the equal-weight
+union margin instead of requiring every benchmark to hold.
 """
 from __future__ import annotations
 
@@ -50,12 +53,16 @@ def main(argv: list[str] | None = None) -> int:
     args = ap.parse_args(argv)
 
     accs = _load(args.accuracies)
-    report = analyze_benchmarks(accs, require_all=not args.union)
+    # Call the analyzer with its real signature (tol only) — require_all was never a kwarg (#446).
+    report = analyze_benchmarks(accs)
     if args.json:
         print(json.dumps(report, indent=2))
     else:
-        print(render(accs, require_all=not args.union))
-    return 0 if report["r11_holds"] else 1
+        print(render(accs))
+    # Default: every comparable benchmark must beat the LLM-as-coordinator.
+    # ``--union``: hold when the equal-weight union margin is positive.
+    holds = report["union_margin"] > 0.0 if args.union else report["r11_holds"]
+    return 0 if holds else 1
 
 
 if __name__ == "__main__":

src/trinity/adapters/drop.py

@@ -130,9 +130,12 @@ def _maybe_unbox(segment: str) -> str:
     return boxed if boxed is not None else ""
 
 
-#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` — a
-#: leading sign is part of a number's value, not wrapping noise.
-_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-")
+#: Surrounding punctuation stripped from a token, EXCLUDING the signs ``+``/``-`` and
+#: the decimal point ``.`` — a leading sign or decimal point is part of a number's
+#: value, not wrapping noise. A genuinely-trailing ``.`` (sentence period) is handled
+#: by the dedicated rstrip retry in :func:`_normalize_token`, which can tell it apart
+#: from a value-bearing leading point; a blanket edge-strip cannot.
+_STRIP_EDGE = "".join(c for c in string.punctuation if c not in "+-.")
 
 
 def _normalize_token(raw: str) -> str:
@@ -147,20 +150,27 @@ def _normalize_token(raw: str) -> str:
     dropped the ``-`` and left commas to break ``float()``) did not deliver.
 
     A token that is ALREADY a number is recognised before any punctuation is
-    stripped: the edge-strip set includes ``.``, so a leading-decimal token like
-    ``".5"`` would otherwise lose its point and normalize to ``"5.0"`` — equal to
-    a gold ``"5"`` (false positive) and unequal to the value-identical gold
-    ``"0.5"`` (false negative). The official DROP ``_remove_punc`` tests
-    ``_is_number`` first and leaves numbers untouched for exactly this reason."""
+    stripped: a leading-decimal token like ``".5"`` must not lose its point and
+    normalize to ``"5.0"`` — equal to a gold ``"5"`` (false positive) and unequal
+    to the value-identical gold ``"0.5"`` (false negative). The official DROP
+    ``_remove_punc`` tests ``_is_number`` first and leaves numbers untouched for
+    exactly this reason (issue #423). The float-first path alone only covers the
+    *bare* token: with ``.`` in the edge-strip set, wrapped forms like ``"$.5"``
+    and ``".5."`` still lost the leading point on the second-chance path. So the
+    edge strip excludes ``.`` entirely, and a genuinely-trailing period (``".5."``,
+    ``"16.."``) is retried with an explicit ``rstrip(".")`` — right-side dots are
+    sentence punctuation, left-side dots are value."""
     try:
         return str(float(raw.replace(",", "")))
     except ValueError:
         pass
     core = raw.strip(_STRIP_EDGE)
-    try:
-        return str(float(core.replace(",", "")))
-    except ValueError:
-        return _PUNCT.sub("", raw)
+    for cand in (core, core.rstrip(".")):
+        try:
+            return str(float(cand.replace(",", "")))
+        except ValueError:
+            continue
+    return _PUNCT.sub("", raw)
 
 
 def _split_internal_hyphens(token: str) -> list[str]:

src/trinity/analysis/coordinator_vs_llm.py

@@ -102,7 +102,7 @@ def analyze_benchmarks(pairs: Mapping[str, Any], *, tol: float = _TOL) -> dict[s
     Args:
         pairs: ``{benchmark: (trained, llm_as_coordinator)}`` — each value a 2-tuple/
             list, or a mapping with keys ``trained`` / ``llm_as_coordinator`` (aliases
-            ``trinity`` / ``llm``).
+            ``trinity`` / ``llm`` / ``llm_coordinator``).
         tol: Win tolerance (see :func:`analyze_pair`).
 
     Returns:
@@ -135,7 +135,9 @@ def _split(value: Any) -> tuple[Any, Any]:
     """Coerce a pair value to ``(trained, llm_as_coordinator)``."""
     if isinstance(value, Mapping):
         trained = value.get("trained", value.get("trinity"))
-        llm = value.get("llm_as_coordinator", value.get("llm"))
+        llm = value.get(
+            "llm_as_coordinator", value.get("llm_coordinator", value.get("llm"))
+        )
         return trained, llm
     try:
         trained, llm = value

tests/test_coordinator_vs_llm_report.py

@@ -0,0 +1,81 @@
+"""Regression test for the R11 report CLI (scripts/coordinator_vs_llm_report.py).
+
+The script shipped broken: it called ``analyze_benchmarks(accs, require_all=...)`` and
+``render(accs, require_all=...)``, but the module exposes neither keyword — so every
+invocation raised ``TypeError``. It also documented the input key ``llm_coordinator``
+while the analyzer originally only read ``llm_as_coordinator``. This drives the CLI's
+``main()`` in-process (no torch, no network) and asserts it now runs.
+"""
+from __future__ import annotations
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+_REPO = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(_REPO / "src"))
+
+_SCRIPT = _REPO / "scripts" / "coordinator_vs_llm_report.py"
+_spec = importlib.util.spec_from_file_location("coordinator_vs_llm_report", _SCRIPT)
+assert _spec is not None and _spec.loader is not None
+_mod = importlib.util.module_from_spec(_spec)
+sys.modules["coordinator_vs_llm_report"] = _mod
+_spec.loader.exec_module(_mod)
+main = _mod.main
+
+
+def _write(tmp: Path, payload: dict) -> Path:
+    p = tmp / "r11.json"
+    p.write_text(json.dumps(payload))
+    return p
+
+
+def test_cli_runs_with_documented_llm_coordinator_key(tmp_path: Path) -> None:
+    path = _write(
+        tmp_path,
+        {
+            "math500": {"trinity": 0.88, "llm_coordinator": 0.70},
+            "mmlu": {"trinity": 0.90, "llm_coordinator": 0.60},
+        },
+    )
+    assert main(["--accuracies", str(path)]) == 0
+
+
+def test_cli_accepts_llm_as_coordinator_canonical_key(tmp_path: Path) -> None:
+    path = _write(
+        tmp_path,
+        {"math500": {"trained": 0.80, "llm_as_coordinator": 0.50}},
+    )
+    assert main(["--accuracies", str(path)]) == 0
+
+
+def test_cli_exits_nonzero_when_r11_violated(tmp_path: Path) -> None:
+    path = _write(
+        tmp_path,
+        {"math500": {"trinity": 0.40, "llm_coordinator": 0.70}},
+    )
+    assert main(["--accuracies", str(path)]) == 1
+
+
+def test_union_holds_on_positive_union_margin_despite_one_loss(tmp_path: Path) -> None:
+    # One loss, one big win -> per-bench R11 fails, but union margin can still be > 0.
+    path = _write(
+        tmp_path,
+        {
+            "math500": {"trinity": 0.40, "llm_coordinator": 0.50},  # loss
+            "mmlu": {"trinity": 0.95, "llm_coordinator": 0.50},  # big win
+        },
+    )
+    assert main(["--accuracies", str(path)]) == 1  # default: every-bench
+    assert main(["--accuracies", str(path), "--union"]) == 0
+
+
+def test_analyzer_reads_llm_coordinator_alias_directly() -> None:
+    from trinity.analysis.coordinator_vs_llm import analyze_benchmarks
+
+    report = analyze_benchmarks(
+        {"math500": {"trinity": 0.9, "llm_coordinator": 0.5}}
+    )
+    assert report["r11_holds"] is True
+    assert report["per_benchmark"][0]["comparable"] is True