From 4e63e26052a866e254c3b93a13691b11ce6f0f28 Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Tue, 2 Jun 2026 22:12:44 +1000 Subject: [PATCH 1/8] feat(experiment): add echo-judge-openai arm (GPT-4o-mini cross-family judge) Adds arm_echo_judge_openai which routes the agreement call to GPT-4o-mini instead of Haiku, removing same-family bias from the judge signal. Requires OPENAI_API_KEY in the environment. Co-Authored-By: Claude Sonnet 4.6 --- experiment/pyproject.toml | 1 + experiment/run_pilot.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/experiment/pyproject.toml b/experiment/pyproject.toml index cb1bc0b..e19e1de 100644 --- a/experiment/pyproject.toml +++ b/experiment/pyproject.toml @@ -8,4 +8,5 @@ dependencies = [ "langchain>=0.3,<0.4", "langchain-ollama>=0.2,<0.4", "datasets>=2.14", + "langchain-openai>=0.3", ] diff --git a/experiment/run_pilot.py b/experiment/run_pilot.py index d811104..fb4eb31 100644 --- a/experiment/run_pilot.py +++ b/experiment/run_pilot.py @@ -324,6 +324,28 @@ def arm_echo_small_judge(task: dict) -> tuple[str, int]: return call_with_persona(sonnet, PERSONA_A, task["prompt"]), 3 +def arm_echo_judge_openai(task: dict) -> tuple[str, int]: + """Echo with cross-family judge (GPT-4o-mini via OpenAI API). + + Same structure as echo-judge but the agreement call goes to OpenAI + instead of Haiku, removing same-family bias from the signal. + Requires OPENAI_API_KEY in the environment. + """ + try: + from langchain_openai import ChatOpenAI + except ImportError as exc: + raise RuntimeError( + "echo-judge-openai requires langchain-openai. " + "Run: pip install langchain-openai" + ) from exc + pair = _haiku_pair(task["prompt"]) + openai_judge = ChatOpenAI(model="gpt-4o-mini", temperature=0) + if judge_agree(pair["a"], pair["b"], task, judge=openai_judge): + return pair["a"], 3 + sonnet = ChatClaudeCode(model="sonnet") + return call_with_persona(sonnet, PERSONA_A, task["prompt"]), 4 + + def arm_echo_oracle(task: dict) -> tuple[str, int]: """Oracle agreement signal: ground-truth test pass/fail. @@ -352,6 +374,7 @@ def arm_echo_oracle(task: dict) -> tuple[str, int]: "echo-ast": arm_echo_ast, "echo-judge": arm_echo_judge, "echo-small-judge": arm_echo_small_judge, + "echo-judge-openai": arm_echo_judge_openai, "echo-oracle": arm_echo_oracle, } From 5bfa7103f502ecb88d9c52b0725b11dab8b069e6 Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Tue, 2 Jun 2026 22:49:02 +1000 Subject: [PATCH 2/8] fix(experiment): correct escalation threshold for judge arms echo-judge and echo-judge-openai use 3 calls minimum (pair + judge), so escalation (Sonnet called) is sub_calls > 3, not > 2. Co-Authored-By: Claude Sonnet 4.6 --- experiment/run_pilot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/experiment/run_pilot.py b/experiment/run_pilot.py index fb4eb31..f74db5b 100644 --- a/experiment/run_pilot.py +++ b/experiment/run_pilot.py @@ -414,10 +414,12 @@ def summarize(results: list[TaskResult]) -> dict: for arm, rs in by_arm.items(): n = len(rs) passed = sum(1 for r in rs if r.passed) - # "Escalation rate" for Echo arms: fraction of tasks that used >2 calls - # (oracle and lexical both escalate when sub_calls jumps from 2 to 3). - # For non-Echo arms this is just an extra metric that happens to be 0%. - escalated = sum(1 for r in rs if r.sub_calls > 2) + # Escalation = Sonnet was called. Threshold depends on arm: + # lexical/ast/oracle/small-judge: accept=2, escalate=3 → threshold >2 + # judge/judge-openai: accept=3 (pair+judge), escalate=4 → threshold >3 + judge_arms = {"echo-judge", "echo-judge-openai"} + threshold = 3 if arm in judge_arms else 2 + escalated = sum(1 for r in rs if r.sub_calls > threshold) summary[arm] = { "n": n, "pass_rate": round(passed / n, 3) if n else None, From dc2c2c06f84f05e3ca860f3d88b63f0f4379ab4a Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Tue, 2 Jun 2026 22:55:34 +1000 Subject: [PATCH 3/8] fix(experiment): carry prompt imports when model returns full def Haiku often returns a complete function without re-stating the typing imports from the prompt (e.g. List, Dict). Prepend those imports to the test program so NameError on List/Dict/etc no longer causes false failures. Co-Authored-By: Claude Sonnet 4.6 --- experiment/run_pilot.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/experiment/run_pilot.py b/experiment/run_pilot.py index f74db5b..ae212f5 100644 --- a/experiment/run_pilot.py +++ b/experiment/run_pilot.py @@ -132,7 +132,12 @@ def run_tests(implementation: str, task: dict) -> tuple[bool, str]: has_top_level_def = bool(re.search(r"^def\s", body, re.MULTILINE)) if has_top_level_def: - program = body + "\n" + task["test"] + f"\ncheck({task['entry_point']})\n" + prompt_imports = "\n".join( + ln for ln in task["prompt"].splitlines() + if ln.startswith("from ") or ln.startswith("import ") + ) + preamble = prompt_imports + "\n" if prompt_imports else "" + program = preamble + body + "\n" + task["test"] + f"\ncheck({task['entry_point']})\n" else: # Body is a function-body fragment. Two sub-cases via ast.parse: # - bare expression (e.g. `[x+1 for x in l]`): wrap as `return ` From e44477c69e6c3600dd4c3911c50f5d1ec66c5abc Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Wed, 3 Jun 2026 11:21:23 +1000 Subject: [PATCH 4/8] Add GPT-5.5 OpenAI judge --- experiment/README.md | 10 ++++++++-- experiment/run_pilot.py | 20 ++++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/experiment/README.md b/experiment/README.md index edf4366..68063dc 100644 --- a/experiment/README.md +++ b/experiment/README.md @@ -17,6 +17,11 @@ For sweep narratives and headline numbers, see [`results/README.md`](results/REA - [Ollama](https://ollama.com/) at `http://localhost:11434` - Model: `qwen2.5:7b-instruct-q4_K_M` (see `SMALL_JUDGE_MODEL` in `run_pilot.py`) +**Optional** — only for `echo-judge-openai`: + +- `OPENAI_API_KEY` in the environment +- OpenAI judge model: `gpt-5.5` (see `OPENAI_JUDGE_MODEL` in `run_pilot.py`) + ```bash claude --version # must work before running sweeps ollama pull qwen2.5:7b-instruct-q4_K_M # only if using echo-small-judge @@ -29,7 +34,7 @@ cd experiment python3 -m venv .venv source .venv/bin/activate -pip install "langchain-core>=0.3,<0.4" "langchain>=0.3,<0.4" "langchain-ollama>=0.2,<0.4" +pip install "langchain-core>=0.3,<0.4" "langchain>=0.3,<0.4" "langchain-ollama>=0.2,<0.4" "langchain-openai>=0.3" ``` ## Quick start (1 task) @@ -52,7 +57,7 @@ Tasks **100–163** (64 tasks), same range the team used for main results: python run_pilot.py --start 100 --n-tasks 64 ``` -Default runs **all 7 arms** → 448 model calls. Expect long runtime and Claude CLI usage. Confirm the 1-task run first. +Default runs **all 8 arms** → 512 arm runs. Expect long runtime and Claude CLI usage. Confirm the 1-task run first. Subset of arms: @@ -78,6 +83,7 @@ python run_pilot.py --start 100 --n-tasks 5 --arms haiku-only,sonnet-only,echo-s | `echo-ast` | Two Haiku personas; escalate if AST structure differs | | `echo-judge` | Two Haiku personas; Haiku judges equivalence | | `echo-small-judge` | Two Haiku personas; local Qwen 7B judge (Ollama) | +| `echo-judge-openai` | Two Haiku personas; GPT-5.5 judges equivalence via OpenAI | | `echo-oracle` | Two Haiku personas; escalate only if both fail tests (upper bound, not deployable) | ## Output format diff --git a/experiment/run_pilot.py b/experiment/run_pilot.py index ae212f5..7235378 100644 --- a/experiment/run_pilot.py +++ b/experiment/run_pilot.py @@ -20,8 +20,10 @@ import ast import json import re +import sys import textwrap import time +from collections import Counter from dataclasses import asdict, dataclass from datetime import datetime, timezone from pathlib import Path @@ -152,7 +154,7 @@ def run_tests(implementation: str, task: dict) -> tuple[bool, str]: try: result = _sp.run( - ["python3", "-c", full], + [sys.executable, "-c", full], capture_output=True, text=True, timeout=TEST_TIMEOUT_SECONDS, @@ -302,6 +304,7 @@ def arm_echo_judge(task: dict) -> tuple[str, int]: # qwen2.5:7b-instruct — middle ground we're now testing SMALL_JUDGE_MODEL = "qwen2.5:7b-instruct-q4_K_M" SMALL_JUDGE_BASE_URL = "http://localhost:11434" +OPENAI_JUDGE_MODEL = "gpt-5.5" def arm_echo_small_judge(task: dict) -> tuple[str, int]: @@ -330,7 +333,7 @@ def arm_echo_small_judge(task: dict) -> tuple[str, int]: def arm_echo_judge_openai(task: dict) -> tuple[str, int]: - """Echo with cross-family judge (GPT-4o-mini via OpenAI API). + """Echo with cross-family judge (GPT-5.5 via OpenAI API). Same structure as echo-judge but the agreement call goes to OpenAI instead of Haiku, removing same-family bias from the signal. @@ -344,7 +347,7 @@ def arm_echo_judge_openai(task: dict) -> tuple[str, int]: "Run: pip install langchain-openai" ) from exc pair = _haiku_pair(task["prompt"]) - openai_judge = ChatOpenAI(model="gpt-4o-mini", temperature=0) + openai_judge = ChatOpenAI(model=OPENAI_JUDGE_MODEL, temperature=0) if judge_agree(pair["a"], pair["b"], task, judge=openai_judge): return pair["a"], 3 sonnet = ChatClaudeCode(model="sonnet") @@ -402,12 +405,18 @@ def run_one(task: dict, arm_name: str, arm_fn: Callable[[dict], tuple[str, int]] t0 = time.perf_counter() try: output, sub_calls = arm_fn(task) - passed, detail = run_tests(output, task) except Exception as e: return TaskResult( task["task_id"], arm_name, False, f"{type(e).__name__}: {str(e)[:200]}", time.perf_counter() - t0, 0, ) + try: + passed, detail = run_tests(output, task) + except Exception as e: + return TaskResult( + task["task_id"], arm_name, False, f"test runner {type(e).__name__}: {str(e)[:200]}", + time.perf_counter() - t0, sub_calls, + ) return TaskResult(task["task_id"], arm_name, passed, detail, time.perf_counter() - t0, sub_calls) @@ -419,6 +428,7 @@ def summarize(results: list[TaskResult]) -> dict: for arm, rs in by_arm.items(): n = len(rs) passed = sum(1 for r in rs if r.passed) + failure_details = Counter(r.detail for r in rs if not r.passed) # Escalation = Sonnet was called. Threshold depends on arm: # lexical/ast/oracle/small-judge: accept=2, escalate=3 → threshold >2 # judge/judge-openai: accept=3 (pair+judge), escalate=4 → threshold >3 @@ -432,6 +442,8 @@ def summarize(results: list[TaskResult]) -> dict: "mean_wall_seconds": round(sum(r.wall_seconds for r in rs) / n, 2) if n else None, "total_sub_calls": sum(r.sub_calls for r in rs), "mean_sub_calls": round(sum(r.sub_calls for r in rs) / n, 2) if n else None, + "failures": n - passed, + "top_failure_details": dict(failure_details.most_common(5)), } return summary From f888a18e5d5528061bf09aeff54035ad7145f46f Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Wed, 3 Jun 2026 15:31:14 +1000 Subject: [PATCH 5/8] Add provider judge model arms --- experiment/README.md | 54 +++++++++++++++++++++++++-- experiment/pyproject.toml | 1 + experiment/run_pilot.py | 78 +++++++++++++++++++++++++++++++++++---- 3 files changed, 121 insertions(+), 12 deletions(-) diff --git a/experiment/README.md b/experiment/README.md index 68063dc..ddd999c 100644 --- a/experiment/README.md +++ b/experiment/README.md @@ -17,10 +17,15 @@ For sweep narratives and headline numbers, see [`results/README.md`](results/REA - [Ollama](https://ollama.com/) at `http://localhost:11434` - Model: `qwen2.5:7b-instruct-q4_K_M` (see `SMALL_JUDGE_MODEL` in `run_pilot.py`) -**Optional** — only for `echo-judge-openai`: +**Optional** — only for OpenAI judge arms: - `OPENAI_API_KEY` in the environment -- OpenAI judge model: `gpt-5.5` (see `OPENAI_JUDGE_MODEL` in `run_pilot.py`) +- OpenAI judge models: `gpt-5.5`, `gpt-5.4`, `gpt-5.4-mini`, `gpt-5.4-nano` (see `OPENAI_JUDGE_MODELS` in `run_pilot.py`) + +**Optional** — only for Gemini judge arms: + +- `GOOGLE_API_KEY` in the environment +- Gemini judge models: `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.5-flash-lite` (see `GEMINI_JUDGE_MODELS` in `run_pilot.py`) ```bash claude --version # must work before running sweeps @@ -34,7 +39,7 @@ cd experiment python3 -m venv .venv source .venv/bin/activate -pip install "langchain-core>=0.3,<0.4" "langchain>=0.3,<0.4" "langchain-ollama>=0.2,<0.4" "langchain-openai>=0.3" +pip install "langchain-core>=0.3,<0.4" "langchain>=0.3,<0.4" "langchain-ollama>=0.2,<0.4" "langchain-openai>=0.3" "langchain-google-genai>=2.0" ``` ## Quick start (1 task) @@ -57,7 +62,7 @@ Tasks **100–163** (64 tasks), same range the team used for main results: python run_pilot.py --start 100 --n-tasks 64 ``` -Default runs **all 8 arms** → 512 arm runs. Expect long runtime and Claude CLI usage. Confirm the 1-task run first. +Default runs all arms. Expect long runtime, Claude CLI usage, and provider API usage for the judge arms. Confirm the 1-task run first. Subset of arms: @@ -65,6 +70,24 @@ Subset of arms: python run_pilot.py --start 100 --n-tasks 5 --arms haiku-only,sonnet-only,echo-small-judge ``` +Compare OpenAI judges: + +```bash +python run_pilot.py --start 100 --n-tasks 10 --arms haiku-only,sonnet-only,echo-judge-openai,echo-judge-openai-gpt-5.4,echo-judge-openai-gpt-5.4-mini,echo-judge-openai-gpt-5.4-nano +``` + +Compare Gemini judges: + +```bash +python run_pilot.py --start 100 --n-tasks 10 --arms haiku-only,sonnet-only,echo-judge-gemini-pro,echo-judge-gemini-flash,echo-judge-gemini-flash-lite +``` + +Compare all provider judges: + +```bash +python run_pilot.py --start 100 --n-tasks 10 --arms echo-judge-openai,echo-judge-openai-gpt-5.4,echo-judge-openai-gpt-5.4-mini,echo-judge-openai-gpt-5.4-nano,echo-judge-gemini-pro,echo-judge-gemini-flash,echo-judge-gemini-flash-lite +``` + ## CLI reference | Flag | Default | Meaning | @@ -84,6 +107,12 @@ python run_pilot.py --start 100 --n-tasks 5 --arms haiku-only,sonnet-only,echo-s | `echo-judge` | Two Haiku personas; Haiku judges equivalence | | `echo-small-judge` | Two Haiku personas; local Qwen 7B judge (Ollama) | | `echo-judge-openai` | Two Haiku personas; GPT-5.5 judges equivalence via OpenAI | +| `echo-judge-openai-gpt-5.4` | Two Haiku personas; GPT-5.4 judges equivalence via OpenAI | +| `echo-judge-openai-gpt-5.4-mini` | Two Haiku personas; GPT-5.4 mini judges equivalence via OpenAI | +| `echo-judge-openai-gpt-5.4-nano` | Two Haiku personas; GPT-5.4 nano judges equivalence via OpenAI | +| `echo-judge-gemini-pro` | Two Haiku personas; Gemini 2.5 Pro judges equivalence | +| `echo-judge-gemini-flash` | Two Haiku personas; Gemini 2.5 Flash judges equivalence | +| `echo-judge-gemini-flash-lite` | Two Haiku personas; Gemini 2.5 Flash-Lite judges equivalence | | `echo-oracle` | Two Haiku personas; escalate only if both fail tests (upper bound, not deployable) | ## Output format @@ -97,6 +126,21 @@ Each sweep writes `results/_n.jsonl`. One line per run: - **`passed`** — automated HumanEval tests succeeded - **`sub_calls`** — model calls for that task (Echo escalate → typically 3) +Aggregate metrics: + +| Metric | Meaning | +|--------|---------| +| `n` | Number of tasks run for that arm | +| `pass_rate` | Fraction of final selected implementations that passed HumanEval tests | +| `escalation_rate` | Fraction of tasks where the arm called Sonnet after the cheap pair/judge disagreed | +| `mean_wall_seconds` | Average elapsed seconds per task for that arm | +| `total_sub_calls` | Total counted model calls for that arm | +| `mean_sub_calls` | Average counted model calls per task | +| `failures` | Number of tasks that failed tests or hit provider/harness errors | +| `top_failure_details` | Most common failure reasons, useful for spotting provider limits, auth errors, syntax errors, and timeouts | + +For provider judge arms, `3 calls` means two Haiku candidates plus one judge call. `4 calls` means the judge disagreed and the arm escalated to Sonnet. + ## View existing results (no run) Committed JSONL files are under `results/`. Summary and interpretation: [`results/README.md`](results/README.md). @@ -114,6 +158,8 @@ jq -s 'group_by(.arm) | map({arm: .[0].arm, n: length, passed: (map(select(.pass | `claude: command not found` | Install [Claude Code](https://docs.anthropic.com/en/docs/claude-code) and ensure `claude` is on your `PATH` | | `claude --print failed` | Log in to Claude Code; confirm Max/subscription access | | `echo-small-judge requires langchain-ollama` | `pip install "langchain-ollama>=0.2,<0.4"` or omit that arm | +| `echo-judge-openai requires langchain-openai` | `pip install "langchain-openai>=0.3"` and set `OPENAI_API_KEY` | +| `echo-judge-gemini requires langchain-google-genai` | `pip install "langchain-google-genai>=2.0"` and set `GOOGLE_API_KEY` | | Ollama connection errors | Start Ollama; `ollama pull qwen2.5:7b-instruct-q4_K_M`; check `SMALL_JUDGE_BASE_URL` in `run_pilot.py` | | Very slow runs | Expected — each call spawns `claude --print` (~seconds overhead per call) | diff --git a/experiment/pyproject.toml b/experiment/pyproject.toml index e19e1de..86052b0 100644 --- a/experiment/pyproject.toml +++ b/experiment/pyproject.toml @@ -9,4 +9,5 @@ dependencies = [ "langchain-ollama>=0.2,<0.4", "datasets>=2.14", "langchain-openai>=0.3", + "langchain-google-genai>=2.0", ] diff --git a/experiment/run_pilot.py b/experiment/run_pilot.py index 7235378..1c105af 100644 --- a/experiment/run_pilot.py +++ b/experiment/run_pilot.py @@ -304,7 +304,17 @@ def arm_echo_judge(task: dict) -> tuple[str, int]: # qwen2.5:7b-instruct — middle ground we're now testing SMALL_JUDGE_MODEL = "qwen2.5:7b-instruct-q4_K_M" SMALL_JUDGE_BASE_URL = "http://localhost:11434" -OPENAI_JUDGE_MODEL = "gpt-5.5" +OPENAI_JUDGE_MODELS = { + "gpt-5.5": "gpt-5.5", + "gpt-5.4": "gpt-5.4", + "gpt-5.4-mini": "gpt-5.4-mini", + "gpt-5.4-nano": "gpt-5.4-nano", +} +GEMINI_JUDGE_MODELS = { + "gemini-2.5-pro": "gemini-2.5-pro", + "gemini-2.5-flash": "gemini-2.5-flash", + "gemini-2.5-flash-lite": "gemini-2.5-flash-lite", +} def arm_echo_small_judge(task: dict) -> tuple[str, int]: @@ -332,8 +342,8 @@ def arm_echo_small_judge(task: dict) -> tuple[str, int]: return call_with_persona(sonnet, PERSONA_A, task["prompt"]), 3 -def arm_echo_judge_openai(task: dict) -> tuple[str, int]: - """Echo with cross-family judge (GPT-5.5 via OpenAI API). +def arm_echo_judge_openai_model(task: dict, model_name: str) -> tuple[str, int]: + """Echo with a cross-family OpenAI judge model. Same structure as echo-judge but the agreement call goes to OpenAI instead of Haiku, removing same-family bias from the signal. @@ -347,13 +357,59 @@ def arm_echo_judge_openai(task: dict) -> tuple[str, int]: "Run: pip install langchain-openai" ) from exc pair = _haiku_pair(task["prompt"]) - openai_judge = ChatOpenAI(model=OPENAI_JUDGE_MODEL, temperature=0) + openai_judge = ChatOpenAI(model=model_name, temperature=0) if judge_agree(pair["a"], pair["b"], task, judge=openai_judge): return pair["a"], 3 sonnet = ChatClaudeCode(model="sonnet") return call_with_persona(sonnet, PERSONA_A, task["prompt"]), 4 +def arm_echo_judge_openai(task: dict) -> tuple[str, int]: + """Default OpenAI judge arm, currently GPT-5.5.""" + return arm_echo_judge_openai_model(task, OPENAI_JUDGE_MODELS["gpt-5.5"]) + + +def arm_echo_judge_openai_gpt_5_4(task: dict) -> tuple[str, int]: + return arm_echo_judge_openai_model(task, OPENAI_JUDGE_MODELS["gpt-5.4"]) + + +def arm_echo_judge_openai_gpt_5_4_mini(task: dict) -> tuple[str, int]: + return arm_echo_judge_openai_model(task, OPENAI_JUDGE_MODELS["gpt-5.4-mini"]) + + +def arm_echo_judge_openai_gpt_5_4_nano(task: dict) -> tuple[str, int]: + return arm_echo_judge_openai_model(task, OPENAI_JUDGE_MODELS["gpt-5.4-nano"]) + + +def arm_echo_judge_gemini_model(task: dict, model_name: str) -> tuple[str, int]: + """Echo with a Gemini judge model via Google AI Studio/Gemini API.""" + try: + from langchain_google_genai import ChatGoogleGenerativeAI + except ImportError as exc: + raise RuntimeError( + "echo-judge-gemini requires langchain-google-genai. " + "Run: pip install langchain-google-genai" + ) from exc + pair = _haiku_pair(task["prompt"]) + gemini_judge = ChatGoogleGenerativeAI(model=model_name, temperature=0) + if judge_agree(pair["a"], pair["b"], task, judge=gemini_judge): + return pair["a"], 3 + sonnet = ChatClaudeCode(model="sonnet") + return call_with_persona(sonnet, PERSONA_A, task["prompt"]), 4 + + +def arm_echo_judge_gemini_pro(task: dict) -> tuple[str, int]: + return arm_echo_judge_gemini_model(task, GEMINI_JUDGE_MODELS["gemini-2.5-pro"]) + + +def arm_echo_judge_gemini_flash(task: dict) -> tuple[str, int]: + return arm_echo_judge_gemini_model(task, GEMINI_JUDGE_MODELS["gemini-2.5-flash"]) + + +def arm_echo_judge_gemini_flash_lite(task: dict) -> tuple[str, int]: + return arm_echo_judge_gemini_model(task, GEMINI_JUDGE_MODELS["gemini-2.5-flash-lite"]) + + def arm_echo_oracle(task: dict) -> tuple[str, int]: """Oracle agreement signal: ground-truth test pass/fail. @@ -383,6 +439,12 @@ def arm_echo_oracle(task: dict) -> tuple[str, int]: "echo-judge": arm_echo_judge, "echo-small-judge": arm_echo_small_judge, "echo-judge-openai": arm_echo_judge_openai, + "echo-judge-openai-gpt-5.4": arm_echo_judge_openai_gpt_5_4, + "echo-judge-openai-gpt-5.4-mini": arm_echo_judge_openai_gpt_5_4_mini, + "echo-judge-openai-gpt-5.4-nano": arm_echo_judge_openai_gpt_5_4_nano, + "echo-judge-gemini-pro": arm_echo_judge_gemini_pro, + "echo-judge-gemini-flash": arm_echo_judge_gemini_flash, + "echo-judge-gemini-flash-lite": arm_echo_judge_gemini_flash_lite, "echo-oracle": arm_echo_oracle, } @@ -430,10 +492,10 @@ def summarize(results: list[TaskResult]) -> dict: passed = sum(1 for r in rs if r.passed) failure_details = Counter(r.detail for r in rs if not r.passed) # Escalation = Sonnet was called. Threshold depends on arm: - # lexical/ast/oracle/small-judge: accept=2, escalate=3 → threshold >2 - # judge/judge-openai: accept=3 (pair+judge), escalate=4 → threshold >3 - judge_arms = {"echo-judge", "echo-judge-openai"} - threshold = 3 if arm in judge_arms else 2 + # lexical/ast/oracle/small-judge: accept=2, escalate=3 -> threshold >2 + # judge/provider-judge: accept=3 (pair+judge), escalate=4 -> threshold >3 + provider_judge_prefixes = ("echo-judge-openai", "echo-judge-gemini") + threshold = 3 if arm == "echo-judge" or arm.startswith(provider_judge_prefixes) else 2 escalated = sum(1 for r in rs if r.sub_calls > threshold) summary[arm] = { "n": n, From 4305fc9efcb7995d81369b0285b2cb4af9595645 Mon Sep 17 00:00:00 2001 From: meghanaganapa Date: Thu, 4 Jun 2026 00:07:04 +1000 Subject: [PATCH 6/8] Fix BBH answer parsing and binary scoring --- experiment/benchmarks/bbh.py | 89 +++++++++++++++++++++++++--- experiment/scripts/run_bbh_pilot.py | 12 +++- experiment/tests/test_bbh_scoring.py | 36 +++++++++++ 3 files changed, 129 insertions(+), 8 deletions(-) diff --git a/experiment/benchmarks/bbh.py b/experiment/benchmarks/bbh.py index 582329b..c9c68de 100644 --- a/experiment/benchmarks/bbh.py +++ b/experiment/benchmarks/bbh.py @@ -50,6 +50,27 @@ ] _CHOICE_LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +BINARY_CHOICE_TEXTS = { + "yes": ("Yes", "No"), + "no": ("Yes", "No"), + "true": ("True", "False"), + "false": ("True", "False"), +} + + +def _clean_label(label: str) -> str: + return str(label).strip().strip("()").upper() + + +def _clean_answer_text(text: str) -> str: + return re.sub(r"\s+", " ", str(text).strip().lower()) + + +def _synthetic_binary_choices(target: str) -> dict[str, list[str]] | None: + texts = BINARY_CHOICE_TEXTS.get(_clean_answer_text(target)) + if texts is None: + return None + return {"label": ["A", "B"], "text": list(texts)} def _format_choices(choices: dict[str, Any]) -> str: @@ -88,6 +109,26 @@ def normalize_gold(target: str) -> str: return letter +def normalize_gold_for_choices(target: str, choices: dict[str, Any]) -> str: + """Normalize a gold target against a concrete choice list. + + Some BBH configs have letter targets ("C"); binary configs can have text + targets ("Yes"/"No"). Convert either form to the matching choice label. + """ + labels = [_clean_label(label) for label in choices.get("label") or []] + texts = [_clean_answer_text(text) for text in choices.get("text") or []] + target_text = _clean_answer_text(target) + + if target_text in texts: + return labels[texts.index(target_text)] + + letter = extract_choice(str(target)) + if letter is not None and letter in labels: + return letter + + raise ValueError(f"Could not parse gold target: {target!r}") + + def extract_choice(text: str) -> str | None: """Parse a multiple-choice letter from model output. @@ -97,19 +138,21 @@ def extract_choice(text: str) -> str | None: if not text or not text.strip(): return None + tail = "\n".join(text.strip().splitlines()[-5:]) patterns = [ r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])\s*\)?\s*\.?\s*$", r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])\s*\)?", + r"(?i)\b(?:final\s+answer|answer|correct\s+answer|correct\s+choice)\s*(?:is|:)\s*\(?\s*([A-Z])\s*\)?", + r"(?i)\b(?:option|choice)\s+\(?\s*([A-Z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", + r"(?i)\b(?:therefore|so|thus),?\s+\(?\s*([A-Z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", + r"(?i)\b(?:therefore|so|thus),?\s+(?:the\s+)?(?:answer|correct\s+answer|choice)\s+is\s+\(?\s*([A-Z])\s*\)?", r"(?im)^\s*\(?\s*([A-Z])\s*\)\s*$", - r"(?im)correct\s+(?:answer\s+is|choice\s+is)\s*\(?\s*([A-Z])\s*\)?", - r"\(\s*([A-Z])\s*\)", ] for pat in patterns: - matches = re.findall(pat, text) + matches = re.findall(pat, tail) if matches: return matches[-1].upper() - tail = "\n".join(text.strip().splitlines()[-5:]) for pat in ( r"(?i)\b(?:option|choice|answer)\s+is\s+\(?\s*([A-Z])\s*\)?", r"(?i)\b([A-Z])\s*\.?\s*$", @@ -123,6 +166,9 @@ def extract_choice(text: str) -> str | None: def score_bbh(model_output: str, task: dict) -> tuple[bool, str]: """Grade a BBH response against task['gold'].""" pred = extract_choice(model_output) + valid_labels = set(task.get("choice_labels") or _CHOICE_LETTERS) + if pred not in valid_labels: + pred = extract_choice_text(model_output, task) if pred is None: return False, "unparseable" gold = task["gold"] @@ -131,17 +177,46 @@ def score_bbh(model_output: str, task: dict) -> tuple[bool, str]: return False, f"expected {gold} got {pred}" +def extract_choice_text(text: str, task: dict) -> str | None: + """Map answer text like 'Answer: No' to its choice label for binary tasks.""" + choices = task.get("choices") or {} + labels = [_clean_label(label) for label in choices.get("label") or []] + texts = [_clean_answer_text(choice_text) for choice_text in choices.get("text") or []] + if not labels or not texts: + return None + + tail = "\n".join(str(text).strip().splitlines()[-5:]) + patterns = [ + r"(?im)^\s*answer\s*:\s*(.+?)\s*\.?\s*$", + r"(?i)\b(?:final\s+answer|answer|correct\s+answer)\s*(?:is|:)\s*(.+?)(?:\.|\n|$)", + ] + for pat in patterns: + matches = re.findall(pat, tail) + for match in reversed(matches): + answer = _clean_answer_text(str(match).strip().strip("()")) + if answer in texts: + return labels[texts.index(answer)] + return None + + def _row_to_task(subtask: str, index: int, row: dict) -> dict: - gold = normalize_gold(row["target"]) + choices = row.get("choices") or _synthetic_binary_choices(row["target"]) + if choices is None: + raise ValueError( + f"BBH subtask {subtask!r} is not multiple-choice or binary; " + "this harness only supports choice-label scoring." + ) + gold = normalize_gold_for_choices(row["target"], choices) return { "task_id": f"bbh/{subtask}/{index}", - "prompt": format_prompt(row["question"], row["choices"]), + "prompt": format_prompt(row["question"], choices), "gold": gold, + "choice_labels": [_clean_label(label) for label in choices.get("label") or []], "benchmark": "bbh", "subtask": subtask, # Echo arms use task["prompt"]; keep raw fields for debugging. "question": row["question"], - "choices": row["choices"], + "choices": choices, } diff --git a/experiment/scripts/run_bbh_pilot.py b/experiment/scripts/run_bbh_pilot.py index 38ce526..ec14652 100644 --- a/experiment/scripts/run_bbh_pilot.py +++ b/experiment/scripts/run_bbh_pilot.py @@ -25,7 +25,6 @@ def run_one_bbh(task: dict, arm_name: str, arm_fn) -> TaskResult: t0 = time.perf_counter() try: output, sub_calls = arm_fn(task) - passed, detail = score_bbh(output, task) except Exception as e: return TaskResult( task["task_id"], @@ -35,6 +34,17 @@ def run_one_bbh(task: dict, arm_name: str, arm_fn) -> TaskResult: time.perf_counter() - t0, 0, ) + try: + passed, detail = score_bbh(output, task) + except Exception as e: + return TaskResult( + task["task_id"], + arm_name, + False, + f"scorer {type(e).__name__}: {str(e)[:200]}", + time.perf_counter() - t0, + sub_calls, + ) return TaskResult( task["task_id"], arm_name, passed, detail, time.perf_counter() - t0, sub_calls, diff --git a/experiment/tests/test_bbh_scoring.py b/experiment/tests/test_bbh_scoring.py index fb524af..381429d 100644 --- a/experiment/tests/test_bbh_scoring.py +++ b/experiment/tests/test_bbh_scoring.py @@ -9,6 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from benchmarks.bbh import extract_choice, format_prompt, normalize_gold, score_bbh +from benchmarks.bbh import _row_to_task class TestExtractChoice(unittest.TestCase): @@ -33,6 +34,21 @@ def test_last_line_letter(self) -> None: "D", ) + def test_final_answer_sentence_beats_reasoning_option_mentions(self) -> None: + self.assertEqual( + extract_choice("I considered (A) and then (B). Therefore the answer is C."), + "C", + ) + + def test_correct_answer_sentence_beats_reasoning_option_mentions(self) -> None: + self.assertEqual( + extract_choice("Option (A) is tempting, but the correct answer is (C)."), + "C", + ) + + def test_reasoning_option_mentions_alone_are_not_parseable(self) -> None: + self.assertIsNone(extract_choice("I considered (A), then (B), then (C).")) + class TestScoreBbh(unittest.TestCase): def _task(self, gold: str = "C") -> dict: @@ -57,6 +73,26 @@ def test_unparseable(self) -> None: self.assertFalse(ok) self.assertEqual(detail, "unparseable") + def test_binary_answer_text_scores_against_synthetic_choices(self) -> None: + task = _row_to_task( + "causal_judgement", + 0, + {"question": "Did X cause Y?", "target": "No"}, + ) + ok, detail = score_bbh("Reasoning...\nAnswer: No", task) + self.assertTrue(ok) + self.assertEqual(detail, "passed") + + def test_binary_answer_letter_scores_against_synthetic_choices(self) -> None: + task = _row_to_task( + "causal_judgement", + 0, + {"question": "Did X cause Y?", "target": "No"}, + ) + ok, detail = score_bbh("Answer: B", task) + self.assertTrue(ok) + self.assertEqual(detail, "passed") + class TestFormatPrompt(unittest.TestCase): def test_includes_choices(self) -> None: From 14565e630e4c930d86969a4e43cc38f9652972a7 Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Tue, 9 Jun 2026 10:03:01 +1000 Subject: [PATCH 7/8] fix(experiment): correct BBH answer extraction regressions from cage-match Two verified correctness bugs in the BBH answer parser, both of which silently corrupt pass_rate (the exact path under investigation #335): 1. extract_choice() / extract_choice_text() truncated to the last 5 lines BEFORE matching high-confidence patterns, so an answer stated early and followed by trailing reasoning ("The answer is C." + more lines) became unparseable -> scored as a failure. High-confidence patterns now run against the FULL text; only the weak positional fallbacks (lone "(A)" line, trailing single letter) stay tail-only. 2. Under re.I, [A-Z] also matches lowercase, so the broad "answer is X" pattern grabbed the first letter of the following word ("the answer is straightforward" -> "S"). Each capture now has a (?![A-Za-z]) boundary guard. This was bidirectional and silent: a valid bogus label was accepted by score_bbh without reaching the text-extraction safety net. Also from the cage-match: - normalize_gold_for_choices(): fall back to positional A,B,C... when a choice list carries text without explicit labels (was IndexError); and prefer label parsing first for single-letter targets so a decoy choice text can't remap the gold (Carnot). - BINARY_CHOICE_TEXTS: add valid/invalid so the formal_fallacies subtask can synthesize binary choices instead of raising (Kelvin). Adds regression tests for all of the above. Scoring suite: 20 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/benchmarks/bbh.py | 73 +++++++++++++++++++--------- experiment/tests/test_bbh_scoring.py | 36 +++++++++++++- 2 files changed, 85 insertions(+), 24 deletions(-) diff --git a/experiment/benchmarks/bbh.py b/experiment/benchmarks/bbh.py index c9c68de..71dca40 100644 --- a/experiment/benchmarks/bbh.py +++ b/experiment/benchmarks/bbh.py @@ -55,6 +55,8 @@ "no": ("Yes", "No"), "true": ("True", "False"), "false": ("True", "False"), + "valid": ("valid", "invalid"), + "invalid": ("valid", "invalid"), } @@ -117,49 +119,71 @@ def normalize_gold_for_choices(target: str, choices: dict[str, Any]) -> str: """ labels = [_clean_label(label) for label in choices.get("label") or []] texts = [_clean_answer_text(text) for text in choices.get("text") or []] + # A choice list may carry text without explicit labels; fall back to + # positional A, B, C... so the texts.index() lookups below can't raise + # IndexError (mirrors the _CHOICE_LETTERS fallback in score_bbh). + if not labels and texts: + labels = list(_CHOICE_LETTERS[: len(texts)]) target_text = _clean_answer_text(target) - if target_text in texts: - return labels[texts.index(target_text)] - + # Prefer label parsing first for single-letter targets ("C"), so a decoy + # choice whose *text* happens to be a bare letter can't remap the gold. + # Binary/textual targets ("Yes"/"No") extract_choice()-to-None and fall + # through to text matching below. letter = extract_choice(str(target)) if letter is not None and letter in labels: return letter + if target_text in texts: + return labels[texts.index(target_text)] + raise ValueError(f"Could not parse gold target: {target!r}") def extract_choice(text: str) -> str | None: """Parse a multiple-choice letter from model output. - Tries explicit patterns first, then the last standalone A–Z near the end. + High-confidence patterns (explicit "Answer: X" / "the answer is X") are + matched against the FULL text, so an answer stated early and followed by + trailing reasoning is still recovered. Weak positional fallbacks (a lone + "(A)" line, a trailing single letter) are matched only against the last + few lines, where a stray capital is least likely to be prose. + + Each capture is followed by a ``(?![A-Za-z])`` guard: under ``re.I`` the + class ``[A-Z]`` also matches lowercase, so without the guard a phrase like + "the answer is straightforward" would wrongly yield "S". + Returns uppercase A–Z or None if unparseable. """ if not text or not text.strip(): return None - tail = "\n".join(text.strip().splitlines()[-5:]) - patterns = [ - r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])\s*\)?\s*\.?\s*$", - r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])\s*\)?", - r"(?i)\b(?:final\s+answer|answer|correct\s+answer|correct\s+choice)\s*(?:is|:)\s*\(?\s*([A-Z])\s*\)?", - r"(?i)\b(?:option|choice)\s+\(?\s*([A-Z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", - r"(?i)\b(?:therefore|so|thus),?\s+\(?\s*([A-Z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", - r"(?i)\b(?:therefore|so|thus),?\s+(?:the\s+)?(?:answer|correct\s+answer|choice)\s+is\s+\(?\s*([A-Z])\s*\)?", + body = text.strip() + # High-confidence: explicit answer declarations, anywhere in the output. + high_confidence = [ + r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])(?![A-Za-z])\s*\)?\s*\.?\s*$", + r"(?im)^\s*answer\s*:\s*\(?\s*([A-Z])(?![A-Za-z])\s*\)?", + r"(?i)\b(?:final\s+answer|answer|correct\s+answer|correct\s+choice)\s*(?:is|:)\s*\(?\s*([A-Z])(?![A-Za-z])\s*\)?", + r"(?i)\b(?:option|choice)\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", + r"(?i)\b(?:therefore|so|thus),?\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", + r"(?i)\b(?:therefore|so|thus),?\s+(?:the\s+)?(?:answer|correct\s+answer|choice)\s+is\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?", + ] + for pat in high_confidence: + matches = re.findall(pat, body) + if matches: + return matches[-1].upper() + + # Weak positional fallbacks: only trusted near the end of the output. + tail = "\n".join(body.splitlines()[-5:]) + weak = [ r"(?im)^\s*\(?\s*([A-Z])\s*\)\s*$", + r"(?i)\b(?:option|choice|answer)\s+is\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?", + r"(?i)\b([A-Z])(?![A-Za-z])\s*\.?\s*$", ] - for pat in patterns: + for pat in weak: matches = re.findall(pat, tail) if matches: return matches[-1].upper() - - for pat in ( - r"(?i)\b(?:option|choice|answer)\s+is\s+\(?\s*([A-Z])\s*\)?", - r"(?i)\b([A-Z])\s*\.?\s*$", - ): - m = re.search(pat, tail.strip()) - if m: - return m.group(1).upper() return None @@ -185,13 +209,16 @@ def extract_choice_text(text: str, task: dict) -> str | None: if not labels or not texts: return None - tail = "\n".join(str(text).strip().splitlines()[-5:]) + # Matched against the FULL text (not just the tail): an answer stated + # early followed by trailing reasoning must still resolve. Over-matching + # is harmless here because a capture is only accepted if it is in `texts`. + body = str(text).strip() patterns = [ r"(?im)^\s*answer\s*:\s*(.+?)\s*\.?\s*$", r"(?i)\b(?:final\s+answer|answer|correct\s+answer)\s*(?:is|:)\s*(.+?)(?:\.|\n|$)", ] for pat in patterns: - matches = re.findall(pat, tail) + matches = re.findall(pat, body) for match in reversed(matches): answer = _clean_answer_text(str(match).strip().strip("()")) if answer in texts: diff --git a/experiment/tests/test_bbh_scoring.py b/experiment/tests/test_bbh_scoring.py index 381429d..9236f80 100644 --- a/experiment/tests/test_bbh_scoring.py +++ b/experiment/tests/test_bbh_scoring.py @@ -9,7 +9,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from benchmarks.bbh import extract_choice, format_prompt, normalize_gold, score_bbh -from benchmarks.bbh import _row_to_task +from benchmarks.bbh import _row_to_task, normalize_gold_for_choices class TestExtractChoice(unittest.TestCase): @@ -49,6 +49,40 @@ def test_correct_answer_sentence_beats_reasoning_option_mentions(self) -> None: def test_reasoning_option_mentions_alone_are_not_parseable(self) -> None: self.assertIsNone(extract_choice("I considered (A), then (B), then (C).")) + # Regression: an answer stated EARLY followed by trailing reasoning must + # still be recovered. Tail-only matching dropped these (cage-match #335). + def test_answer_stated_early_then_six_trailing_lines(self) -> None: + out = "The answer is C.\n\nl1\nl2\nl3\nl4\nl5\nl6" + self.assertEqual(extract_choice(out), "C") + + def test_answer_colon_early_then_trailing_lines(self) -> None: + out = "Answer: C\nthanks\nbye\n.\n-\n=" + self.assertEqual(extract_choice(out), "C") + + # Regression: under re.I, [A-Z] also matches lowercase, so a broad + # "answer is " pattern must NOT grab the first letter of prose. + def test_answer_is_lowercase_word_is_not_a_false_letter(self) -> None: + self.assertIsNone(extract_choice("After analysis, the answer is straightforward.")) + self.assertIsNone(extract_choice("The answer is dependent on the framing.")) + self.assertIsNone(extract_choice("So the final answer is best understood as follows.")) + + +class TestNormalizeGoldForChoices(unittest.TestCase): + # Regression: choices carrying text without an explicit "label" key must + # not raise IndexError (cage-match: Kelvin). + def test_text_choices_without_labels_fall_back_to_positional(self) -> None: + self.assertEqual( + normalize_gold_for_choices("No", {"text": ["Yes", "No"]}), + "B", + ) + + # Regression: a single-letter target must resolve to its own label, not be + # remapped by a decoy choice whose text is that same letter (cage-match: + # Carnot). + def test_single_letter_target_not_remapped_by_decoy_text(self) -> None: + choices = {"label": ["A", "B", "C"], "text": ["apple", "A", "cat"]} + self.assertEqual(normalize_gold_for_choices("A", choices), "A") + class TestScoreBbh(unittest.TestCase): def _task(self, gold: str = "C") -> dict: From 4868b87a210be912c00226c3ceb9a2c0ea3c886f Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Tue, 9 Jun 2026 10:59:55 +1000 Subject: [PATCH 8/8] fix(experiment): pick latest high-confidence BBH answer across pattern families MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-review (cage-match round 2, Carnot) found a latent ordering bug in extract_choice: the high-confidence loop returned on the first pattern *family* that matched anywhere in the text, ignoring position. So a chain of thought like "Answer: A ... therefore the answer is C" returned A — an early scratch declaration beat the final answer because the two used different pattern families. Fix: scan all high-confidence patterns with re.finditer and keep the match with the largest source offset (true recency across families), instead of returning from the first family with any match. Weak tail fallbacks are unchanged. Adds a regression test for the cross-family case. Scoring suite: 21 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/benchmarks/bbh.py | 16 +++++++++++++--- experiment/tests/test_bbh_scoring.py | 13 +++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/experiment/benchmarks/bbh.py b/experiment/benchmarks/bbh.py index 71dca40..d2aec32 100644 --- a/experiment/benchmarks/bbh.py +++ b/experiment/benchmarks/bbh.py @@ -153,6 +153,11 @@ def extract_choice(text: str) -> str | None: class ``[A-Z]`` also matches lowercase, so without the guard a phrase like "the answer is straightforward" would wrongly yield "S". + Among the high-confidence patterns the *latest* match in the text wins + (recency across ALL pattern families, by source offset) — so a chain of + thought like "Answer: A ... therefore the answer is C" resolves to C even + though the two declarations use different phrasings / different families. + Returns uppercase A–Z or None if unparseable. """ if not text or not text.strip(): @@ -168,10 +173,15 @@ class ``[A-Z]`` also matches lowercase, so without the guard a phrase like r"(?i)\b(?:therefore|so|thus),?\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?\s+(?:is\s+)?(?:correct|best|right)", r"(?i)\b(?:therefore|so|thus),?\s+(?:the\s+)?(?:answer|correct\s+answer|choice)\s+is\s+\(?\s*([A-Z])(?![A-Za-z])\s*\)?", ] + # Pick the high-confidence match with the largest source offset, so the + # final declaration wins regardless of which pattern family caught it. + best_pos, best_letter = -1, None for pat in high_confidence: - matches = re.findall(pat, body) - if matches: - return matches[-1].upper() + for m in re.finditer(pat, body): + if m.start(1) > best_pos: + best_pos, best_letter = m.start(1), m.group(1) + if best_letter is not None: + return best_letter.upper() # Weak positional fallbacks: only trusted near the end of the output. tail = "\n".join(body.splitlines()[-5:]) diff --git a/experiment/tests/test_bbh_scoring.py b/experiment/tests/test_bbh_scoring.py index 9236f80..04cad94 100644 --- a/experiment/tests/test_bbh_scoring.py +++ b/experiment/tests/test_bbh_scoring.py @@ -66,6 +66,19 @@ def test_answer_is_lowercase_word_is_not_a_false_letter(self) -> None: self.assertIsNone(extract_choice("The answer is dependent on the framing.")) self.assertIsNone(extract_choice("So the final answer is best understood as follows.")) + # Regression: the latest high-confidence declaration wins even when it + # uses a DIFFERENT pattern family than an earlier one (cage-match r2: + # Carnot). "Answer: A" early, "therefore the answer is C" late -> C. + def test_latest_high_confidence_wins_across_pattern_families(self) -> None: + self.assertEqual( + extract_choice("Answer: A\nLet me reconsider.\nTherefore the answer is C."), + "C", + ) + self.assertEqual( + extract_choice("Answer: A\nactually, the answer is C."), + "C", + ) + class TestNormalizeGoldForChoices(unittest.TestCase): # Regression: choices carrying text without an explicit "label" key must