From 791db4874943f18aa2751006621d3919b920c287 Mon Sep 17 00:00:00 2001 From: Navneshwar Date: Sat, 19 Sep 2026 13:00:31 +0530 Subject: [PATCH 1/2] Treat tkm as tonne-km in unit reconciliation --- units.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/units.py b/units.py index 2740cb8..87027f5 100644 --- a/units.py +++ b/units.py @@ -77,7 +77,7 @@ def _clean(u): u = u.strip(" .;:-") # a trailing full stop is not part of the unit for a, b in (("hectares", "ha"), ("hectare", "ha"), ("years", "year"), ("yr", "year"), ("hrs", "hour"), ("hr", "hour"), ("tonnes", "tonne"), ("litres", "litre"), - ("kilometres", "km"), ("kilometers", "km"), ("miles", "mile"), + ("kilometres", "km"), ("kilometers", "km"), ("tkm", "tonne km"), ("miles", "mile"), ("nights", "night"), ("rooms", "room"), ("items", "item"), ("days", "day")): u = re.sub(rf"\b{a}\b", b, u) u = re.sub(r"\(.*?\)", " ", u) From 1060ab2dd422a6fa2f37996bfb1f2acc662af50c Mon Sep 17 00:00:00 2001 From: Jeremiah Say Date: Sun, 20 Sep 2026 23:10:31 +0800 Subject: [PATCH 2/2] Regenerate the results the tkm alias moves, and pin it with a test Rebased onto main after #36 (the area/denominator fix), so these are the figures the alias produces against final scorer behaviour rather than the ones from before that landed. The alias does what #18 asked: four freight answers recover, all None -> a value, no already-scored answer changes. claude-opus-5 scoreable 310 -> 314 within 10% 46.5% -> 46.2% unscoreable 115 -> 111 absolute correct 144 -> 145 of all 467 30.8% -> 31.0% paired, unaided within 10% 37.7% -> 38.1% (n 61 -> 63), >50% off 24.6% -> 23.8% within 50% and >50%-off are unchanged at this rounding, and no other model moves. The headline goes DOWN, which is the honest direction: four answers that were being thrown away are now scored and most of them are wrong. Figures updated in FINDINGS.md, README.md and paper/main.tex, including the abstract's paired sentence and the results table. The coverage paragraph records this as one more step in the sequence -- 115 to 111, 46.5% to 46.2% -- and 46.5% joins the allowlist as the history it now is, beside 45.7%, 42.0% and 41.9%. No current figure is allowlisted. verify/test_tkm_alias.py covers tkm in both directions, the existing spellings, scale carry-through, and that a unit merely containing those letters is untouched. One test documents an open hole rather than asserting a fix: a bare `per km` still reconciles with a tonne-km truth, because the denominator scan stops at km before reaching tonne. That predates this alias and #36 did not close it -- see issue #35 -- and no answer in the corpus is affected, though the alias does widen it latently since `per tkm` used to be refused as other:tkm. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NRueWxopDXHoWY2dPvsmLG --- FINDINGS.md | 12 +++-- README.md | 6 +-- paper/main.tex | 6 +-- results/absolute.json | 4 +- results/comparison.json | 6 +-- results/paired.json | 8 ++-- verify/allowlist.json | 1 + verify/test_tkm_alias.py | 98 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 121 insertions(+), 20 deletions(-) create mode 100644 verify/test_tkm_alias.py diff --git a/FINDINGS.md b/FINDINGS.md index 06206b7..f366be7 100644 --- a/FINDINGS.md +++ b/FINDINGS.md @@ -13,7 +13,7 @@ batches of 30, **no tools and no lookups**. Run 2026-09-10. | Gemini 3.1 Pro | 46 | **65.2%** | 80.4% | 19.6% | | Grok 4.6 | 66 | 62.1% | 86.4% | 13.6% | | GPT-5.5 | 256 | 58.2% | 85.2% | 14.8% | -| Claude Opus 5 | 310 | 46.5% | 78.7% | 21.3% | +| Claude Opus 5 | 314 | 46.2% | 78.7% | 21.3% | | Gemini 3.6 Flash | 307 | 42.7% | 78.8% | 21.2% | **Of all 467 asked, how many did it get right?** @@ -21,7 +21,7 @@ batches of 30, **no tools and no lookups**. Run 2026-09-10. | Model | Answered | Refused | Correct | Correct of all 467 | |---|---:|---:|---:|---:| | GPT-5.5 | 326 | 141 | 149 | **31.9%** | -| Claude Opus 5 | 430 | 37 | 144 | 30.8% | +| Claude Opus 5 | 430 | 37 | 145 | 31.0% | | Gemini 3.6 Flash | 404 | 63 | 131 | 28.1% | | Grok 4.6 | 144 | 323 | 41 | 8.8% | | Gemini 3.1 Pro | 77 | **390** | 30 | 6.4% | @@ -83,8 +83,8 @@ ids with and without tools, so nothing hinges on sampling. | | Without tools | With GreenCalculus | |---|---|---| | **Claude Opus 5** — answered | 86 / 90 | 89 / 90 | -| within 10% (of scoreable) | 37.7% | **98.7%** | -| off by >50% | 24.6% | **1.3%** | +| within 10% (of scoreable) | 38.1% | **98.7%** | +| off by >50% | 23.8% | **1.3%** | | correct, of all 90 | 25.6% | **86.7%** | | **GPT-5.5** — answered | 62 / 90 | 87 / 90 | | within 10% (of scoreable) | 50.0% | **100.0%** | @@ -211,7 +211,9 @@ Two independent checks that the headline is real: same 45 pilot answers. - **Against scorer coverage.** Closing successive gaps moved unscoreable from 137 to 110 while the headline moved 46.7% -> 45.7%. If the unscoreable pile had - hidden a bias, converting a fifth of it would have shifted the result. Bug 8 + hidden a bias, converting a fifth of it would have shifted the result. The + `tkm` alias then closed one more gap, moving the pile from 115 to 111 and the + headline from 46.5% to 46.2%. Bug 8 then moved the pile the other way, from 110 to 115, by refusing ten answers that had been scored against an incompatible denominator; the headline moved to 46.5%. A correction that makes the pile bigger is as much a coverage change as diff --git a/README.md b/README.md index 5db5ce0..09c49df 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ reader stop checking. | Gemini 3.1 Pro | 77 | **65.2%** | 6.4% | 35.3% | | Grok 4.6 | 144 | 62.1% | 8.8% | 31.6% | | GPT-5.5 | 326 | 58.2% | **31.9%** | 41.0% | -| Claude Opus 5 | 430 | 46.5% | 30.8% | 53.1% | +| Claude Opus 5 | 430 | 46.2% | 31.0% | 53.1% | | Gemini 3.6 Flash | 404 | 42.7% | 28.1% | 58.0% | Those middle columns rank in opposite orders. **The more a model answers, the @@ -64,8 +64,8 @@ number more often than not even when it names the right publisher. | | Without tools | With GreenCalculus | |---|---|---| -| Claude Opus 5 — within 10% | 37.7% | **98.7%** | -| Claude Opus 5 — off by >50% | 24.6% | **1.3%** | +| Claude Opus 5 — within 10% | 38.1% | **98.7%** | +| Claude Opus 5 — off by >50% | 23.8% | **1.3%** | | GPT-5.5 — within 10% | 50.0% | **100.0%** | | GPT-5.5 — off by >50% | 19.6% | **0.0%** | diff --git a/paper/main.tex b/paper/main.tex index 3be5b3a..6c94184 100644 --- a/paper/main.tex +++ b/paper/main.tex @@ -27,7 +27,7 @@ \begin{abstract} \small -Greenhouse-gas emission factors are an unusually clean probe for factual recall: each has one correct value, traceable to a named cell in a published document. We put 467 such questions to five frontier language models with no tools and no web access. Unaided accuracy ranged from 42.7\% to 65.2\% within 10\% of the published value. The more consequential result is the failure mode. When these models cited a source they named the \emph{correct} publisher 82--94\% of the time, yet in 31.6--58.0\% of answers that correct attribution accompanied an incorrect number. A wrong value with no provenance is caught in review; a wrong value carrying the right publisher's name is not. In a paired arm on 90 questions, giving two models a sourced lookup tool moved accuracy from 37.7\% to 98.7\% and from 50.0\% to 100.0\%, at 1.0--1.6 tool calls per question, while simultaneously converting refusals into correct answers. We release the questions, every raw model response, the scoring harness, a log of eight bugs found in our own scorer, a canary string, and a pre-registered held-out split. +Greenhouse-gas emission factors are an unusually clean probe for factual recall: each has one correct value, traceable to a named cell in a published document. We put 467 such questions to five frontier language models with no tools and no web access. Unaided accuracy ranged from 42.7\% to 65.2\% within 10\% of the published value. The more consequential result is the failure mode. When these models cited a source they named the \emph{correct} publisher 82--94\% of the time, yet in 31.6--58.0\% of answers that correct attribution accompanied an incorrect number. A wrong value with no provenance is caught in review; a wrong value carrying the right publisher's name is not. In a paired arm on 90 questions, giving two models a sourced lookup tool moved accuracy from 38.1\% to 98.7\% and from 50.0\% to 100.0\%, at 1.0--1.6 tool calls per question, while simultaneously converting refusals into correct answers. We release the questions, every raw model response, the scoring harness, a log of eight bugs found in our own scorer, a canary string, and a pre-registered held-out split. \end{abstract} \vspace{-0.6em} @@ -64,7 +64,7 @@ \section{Results} Gemini 3.1 Pro & 77 & 46 & 65.2\% & 12.0\% & 83.9\% & 35.3\% \\ Grok 4.6 & 144 & 66 & 62.1\% & 21.6\% & 82.2\% & 31.6\% \\ GPT-5.5 & 326 & 256 & 58.2\% & 53.7\% & 94.0\% & 41.0\% \\ -Claude Opus 5 & 430 & 310 & 46.5\% & 71.7\% & 89.9\% & 53.1\% \\ +Claude Opus 5 & 430 & 314 & 46.2\% & 71.7\% & 89.9\% & 53.1\% \\ Gemini 3.6 Flash & 404 & 307 & 42.7\% & 71.7\% & 91.9\% & 58.0\% \\ \bottomrule \end{tabular} @@ -89,7 +89,7 @@ \section{Results} \toprule Model & Unaided & With lookup & Mean tool calls \\ \midrule -Claude Opus 5 & 37.7\% ($n$=61) & \textbf{98.7\%} ($n$=79) & 1.62 \\ +Claude Opus 5 & 38.1\% ($n$=63) & \textbf{98.7\%} ($n$=79) & 1.62 \\ GPT-5.5 & 50.0\% ($n$=46) & \textbf{100.0\%} ($n$=79) & 1.00 \\ \bottomrule \end{tabular} diff --git a/results/absolute.json b/results/absolute.json index d2efad8..0e9f010 100644 --- a/results/absolute.json +++ b/results/absolute.json @@ -31,7 +31,7 @@ "model": "claude-opus-5", "answered": 430, "refused": 37, - "correct": 144, - "pct_of_all": 30.8 + "correct": 145, + "pct_of_all": 31.0 } ] \ No newline at end of file diff --git a/results/comparison.json b/results/comparison.json index 0bd3a1a..45a13ed 100644 --- a/results/comparison.json +++ b/results/comparison.json @@ -5,9 +5,9 @@ "answered": 430, "declined": 11, "hedged_but_answered": 91, - "unscoreable_units": 115, - "scoreable": 310, - "within_10pct": 46.5, + "unscoreable_units": 111, + "scoreable": 314, + "within_10pct": 46.2, "within_50pct": 78.7, "confidently_wrong": 21.3, "cited_a_source": 71.7, diff --git a/results/paired.json b/results/paired.json index 46402f9..6091afe 100644 --- a/results/paired.json +++ b/results/paired.json @@ -5,10 +5,10 @@ "avg_tool_calls": 1.62, "without": { "answered": 86, - "scoreable": 61, - "within10": 37.7, - "correct_of_all": 25.6, - "wrong50": 24.6 + "scoreable": 63, + "within10": 38.1, + "correct_of_all": 26.7, + "wrong50": 23.8 }, "with": { "answered": 89, diff --git a/verify/allowlist.json b/verify/allowlist.json index 2404822..0e95d83 100644 --- a/verify/allowlist.json +++ b/verify/allowlist.json @@ -49,6 +49,7 @@ "46.9%": "headline at an intermediate scorer version", "47%": "hand adjudication of 45 pilot answers", "35.6%": "citation rate at an earlier scorer version", + "46.5%": "Claude headline after scorer bug 8 and BEFORE the tkm coverage fix; quoted in the bug log and the coverage sequence. Current value is results.comparison.claude-opus-5.summary.within_10pct", "45.7%": "Claude headline BEFORE scorer bug 8; quoted in the bug log and the coverage sequence. Current value is results.comparison.claude-opus-5.summary.within_10pct", "42.0%": "Gemini 3.6 Flash BETWEEN bugs 7 and 8; quoted in the bug log to show what each correction moved. Current value is results.comparison.gemini-3.6-flash.summary.within_10pct", "41.9%": "Gemini 3.6 Flash BEFORE scorer bug 7 (EUR scored as USD); quoted in the bug log to state what the correction moved. The current value is results.comparison.gemini-3.6-flash.summary.within_10pct = 42.0" diff --git a/verify/test_tkm_alias.py b/verify/test_tkm_alias.py new file mode 100644 index 0000000..bf5000c --- /dev/null +++ b/verify/test_tkm_alias.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Issue #18: `tkm` is tonne-kilometres, and freight answers were unscoreable without it. + +Freight factors are published per tonne-kilometre and models abbreviate that +`tkm`. `_clean` folded `kilometres` to `km` but knew nothing about `tkm`, so the +numerator parsed and the denominator did not, and four correct freight answers +were dropped as unscoreable rather than scored. + +The alias is one entry in `_clean`'s fold list, which makes the risk clear: the +folds are substring rewrites applied in order, so the tests below pin both that +`tkm` resolves AND that it does not damage a plain `km` denominator or a unit +that merely contains those letters. + + python3 verify/test_tkm_alias.py +""" +import os, sys + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) + +import score +import units + +TRUTH = "kg CO2e per tonne-km" + + +def test_tkm_reconciles_against_a_tonne_km_truth(): + for spelling in ("kg CO2e/tkm", "kg CO2e per tkm", "kgCO2e/tkm"): + got, note = units.reconcile(1.0, spelling, TRUTH) + assert got == 1.0, (spelling, got, note) + + +def test_it_works_in_both_directions(): + """A tkm TRUTH must accept a spelled-out answer, not only the reverse.""" + got, _ = units.reconcile(0.1, "kg CO2e per tonne km", "kg CO2e/tkm") + assert got == 0.1, got + + +def test_the_existing_spellings_still_work(): + for spelling in ("kg CO2e/t-km", "kg CO2e per tonne-km", "kg CO2e per tonne km"): + got, _ = units.reconcile(1.0, spelling, TRUTH) + assert got == 1.0, (spelling, got) + + +def test_scale_is_carried_through(): + got, _ = units.reconcile(1000.0, "g CO2e per tkm", TRUTH) + assert abs(got - 1.0) < 1e-9, got + + +def test_a_plain_km_denominator_is_a_known_open_hole(): + """`per km` DOES still reconcile with a tonne-km truth. That is not this fix. + + Both parse to den_dim "length" (the denominator scan finds `km` in LENGTH + before it reaches `tonne` in MASS), so reconcile compares equal dimensions + and never looks at the token sets. It is wrong -- 0.17 kg/km and 0.107 + kg/tonne-km are different quantities -- and it predates this alias: see the + denominator-basis issue. Zero answers in the committed corpus are affected + (no tonne-km truth received a per-km answer), so this pins the CURRENT + behaviour rather than asserting the fix, and will need updating when the + basis guard lands. + """ + got, _ = units.reconcile(0.17, "kg CO2e per km", TRUTH) + assert got == 0.17, ( + "per km no longer reconciles with a tonne-km truth -- the basis guard has " + "landed, so update this test to assert the refusal") + + +def test_a_unit_that_merely_contains_the_letters_is_untouched(): + """`_clean` folds are substring rewrites, so a word boundary matters.""" + assert units.parse_unit("kg CO2e per atkm") != units.parse_unit("kg CO2e per tonne km") + + +def test_the_four_corpus_answers_now_score(): + """The freight answers this alias exists for, end to end.""" + import json + import compare + uniq = compare.question_order() + qmap = {q["id"]: q for q in uniq} + legacy = os.path.join(ROOT, "results", "runs_opus5_full.json") + if not os.path.exists(legacy): + print("skip test_the_four_corpus_answers_now_score: no committed Claude run") + return + answers = {r["id"]: r["answer"] for r in json.load(open(legacy))} + for qid in ("freight.air.tonne_km", "freight.rail.tonne_km", + "freight.road_hgv.tonne_km", "freight_detailed.rail.eu.diesel.cars"): + q = qmap.get(qid) + if q is None or qid not in answers: + continue + got, unit, _ = score.best_value(answers[qid], q["truth"]["unit"]) + assert got is not None, f"{qid} is still unscoreable (unit {unit!r})" + + +if __name__ == "__main__": + for name, fn in list(globals().items()): + if name.startswith("test_") and callable(fn): + fn() + print(f"ok {name}") + print("all tkm-alias tests passed")