From 5eaa384828f0546790faff890c0ecad049faf16e Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Thu, 25 Jun 2026 11:13:52 +1000 Subject: [PATCH 1/8] feat(bbh): stream results + abort-on-usage-exhaustion + --resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BBH sweep wrote results only once, at the end of the loop, from an in-memory list — so any process death mid-run (Max usage exhausted, kill, laptop sleep) lost the entire run. Worse, a usage-limit error was caught per-call and recorded as passed=False, so an exhausted run limped to completion writing hundreds of failure rows that summarize() averaged into pass rates as if they were wrong answers (silent corruption). Three fixes: - Stream each TaskResult to disk with a flush; partial progress now survives process death (OS page cache outlives the process). - Detect genuine Max-usage exhaustion (usage limit / credit / 429) and abort loudly with the exact resume command, instead of poisoning the dataset. Transient signals (timeout / 529) still record-and-continue. - Add --resume : skip already-completed (task_id, arm) pairs and append, so an interrupted sweep continues where it stopped. Output file format is byte-identical (meta line + one row per result); existing analyze_sweep.py / RUN_FOR_NICK.md commands are unaffected. Verified: unit tests for the classifier + resume parse, and a live n=1 end-to-end (fresh write lands on disk, resume skips the done pair). Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/scripts/run_bbh_pilot.py | 174 ++++++++++++++++++++++------ 1 file changed, 139 insertions(+), 35 deletions(-) diff --git a/experiment/scripts/run_bbh_pilot.py b/experiment/scripts/run_bbh_pilot.py index ec14652..0463ad7 100644 --- a/experiment/scripts/run_bbh_pilot.py +++ b/experiment/scripts/run_bbh_pilot.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 -"""BBH sweep — BBH personas, judge, oracle; same arm names as HumanEval.""" +"""BBH sweep — BBH personas, judge, oracle; same arm names as HumanEval. + +Resilience (added 2026-06-25): results stream to disk one row at a time with a +flush after each, so a process death mid-sweep (usage exhaustion, kill, sleep) +preserves everything completed so far instead of losing the in-memory list. +Genuine Max-usage exhaustion aborts loudly rather than being recorded as a pile +of ``passed=False`` rows (which would silently deflate pass rates — a usage +failure is not a wrong answer). ``--resume `` continues an interrupted +run, skipping ``(task_id, arm)`` pairs already present in the file. +""" from __future__ import annotations @@ -18,6 +27,33 @@ BENCHMARK = "bbh" +# Substrings (case-insensitive) in a failed-call's error that mean the Max +# subscription is exhausted — a stop-and-save signal, NOT a per-task wrong +# answer. Deliberately excludes transient signals (overloaded/529/timeout): +# those stay swallowed as single-task failures so one blip doesn't abort a run. +_USAGE_EXHAUSTION_SIGNATURES = ( + "usage limit", + "limit reached", + "limit will reset", + "rate limit", + "rate_limit", + "quota", + "credit balance", + "out of credits", + "no credits", + "insufficient", + "429", +) + + +class UsageLimitReached(RuntimeError): + """Raised when a model call fails due to Max-subscription exhaustion.""" + + +def _is_usage_exhaustion(text: str) -> bool: + low = text.lower() + return any(sig in low for sig in _USAGE_EXHAUSTION_SIGNATURES) + def run_one_bbh(task: dict, arm_name: str, arm_fn) -> TaskResult: import time @@ -26,13 +62,15 @@ def run_one_bbh(task: dict, arm_name: str, arm_fn) -> TaskResult: try: output, sub_calls = arm_fn(task) except Exception as e: + msg = f"{type(e).__name__}: {str(e)[:200]}" + # Usage exhaustion is account-level, not task-level: every subsequent + # call would fail the same way. Abort the sweep loudly so we don't + # pollute the dataset with hundreds of failure rows that look like + # wrong answers. Transient errors fall through and are recorded. + if _is_usage_exhaustion(str(e)): + raise UsageLimitReached(msg) from e return TaskResult( - task["task_id"], - arm_name, - False, - f"{type(e).__name__}: {str(e)[:200]}", - time.perf_counter() - t0, - 0, + task["task_id"], arm_name, False, msg, time.perf_counter() - t0, 0, ) try: passed, detail = score_bbh(output, task) @@ -51,6 +89,27 @@ def run_one_bbh(task: dict, arm_name: str, arm_fn) -> TaskResult: ) +def _load_prior(path: Path) -> tuple[list[TaskResult], set[tuple[str, str]]]: + """Read an existing results file: return (prior TaskResults, done pairs).""" + prior: list[TaskResult] = [] + done: set[tuple[str, str]] = set() + with path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + row = json.loads(line) + if row.get("_meta"): + continue + tr = TaskResult( + row["task_id"], row["arm"], row["passed"], + row["detail"], row["wall_seconds"], row["sub_calls"], + ) + prior.append(tr) + done.add((tr.task_id, tr.arm)) + return prior, done + + def main() -> None: default_subtasks = ",".join(PILOT_SUBTASKS) default_arms = ",".join(BBH_ARMS) @@ -60,6 +119,11 @@ def main() -> None: parser.add_argument("--n-per-subtask", type=int, default=5) parser.add_argument("--start", type=int, default=0) parser.add_argument("--arms", type=str, default=default_arms) + parser.add_argument( + "--resume", type=str, default=None, + help="Path to an existing results .jsonl to continue; completed " + "(task_id, arm) pairs are skipped and new rows appended.", + ) args = parser.parse_args() subtasks = [s.strip() for s in args.subtasks.split(",") if s.strip()] @@ -73,39 +137,79 @@ def main() -> None: tasks = load_bbh(subtasks, n_per_subtask=args.n_per_subtask, start=args.start) + # Resolve output path + prior state. + results: list[TaskResult] = [] + done: set[tuple[str, str]] = set() + RESULTS_DIR.mkdir(exist_ok=True) + if args.resume: + out_path = Path(args.resume) + if not out_path.exists(): + print(f"--resume target does not exist: {out_path}", file=sys.stderr) + sys.exit(1) + results, done = _load_prior(out_path) + print(f"Resuming {out_path}: {len(done)} pairs already done.") + out_f = out_path.open("a") + new_file = False + else: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + out_path = RESULTS_DIR / f"{ts}_bbh_n{len(tasks)}.jsonl" + out_f = out_path.open("w") + new_file = True + + total = len(tasks) * len(selected_arms) print(f"BBH sweep: {len(tasks)} tasks × {len(selected_arms)} arms = " - f"{len(tasks) * len(selected_arms)} runs\n") + f"{total} runs ({len(done)} skipped via resume)\n") - results: list[TaskResult] = [] - for task in tasks: - print(f"=== {task['task_id']} (gold={task['gold']}) ===") - for arm_name, arm_fn in selected_arms.items(): - r = run_one_bbh(task, arm_name, arm_fn) - results.append(r) - marker = "✓" if r.passed else "✗" - print(f" {marker} {arm_name:<18} {r.wall_seconds:>5.1f}s " - f"{r.sub_calls} calls {r.detail[:50]}") + def _write(row: dict) -> None: + out_f.write(json.dumps(row) + "\n") + out_f.flush() # survive process death (OS page cache outlives the proc) + + try: + if new_file: + _write({ + "_meta": True, + "benchmark": BENCHMARK, + "subtasks": subtasks, + "n_per_subtask": args.n_per_subtask, + "start": args.start, + "arms": list(selected_arms.keys()), + }) + + for task in tasks: + printed_header = False + for arm_name, arm_fn in selected_arms.items(): + if (task["task_id"], arm_name) in done: + continue + if not printed_header: + print(f"=== {task['task_id']} (gold={task['gold']}) ===") + printed_header = True + try: + r = run_one_bbh(task, arm_name, arm_fn) + except UsageLimitReached as e: + out_f.flush() + out_f.close() + print(f"\n!!! Max usage exhausted on {task['task_id']} / " + f"{arm_name}: {e}", file=sys.stderr) + print(f"Partial results saved to {out_path} " + f"({len(results)} rows completed).", file=sys.stderr) + print(f"Resume when usage resets with:\n" + f" python scripts/run_bbh_pilot.py --subtasks {args.subtasks} " + f"--n-per-subtask {args.n_per_subtask} --arms {args.arms} " + f"--resume {out_path}", file=sys.stderr) + sys.exit(2) + results.append(r) + row = asdict(r) + row["benchmark"] = BENCHMARK + _write(row) + marker = "✓" if r.passed else "✗" + print(f" {marker} {arm_name:<18} {r.wall_seconds:>5.1f}s " + f"{r.sub_calls} calls {r.detail[:50]}") + finally: + if not out_f.closed: + out_f.close() print("\n=== aggregate ===") print(json.dumps(summarize(results), indent=2)) - - RESULTS_DIR.mkdir(exist_ok=True) - ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - out_path = RESULTS_DIR / f"{ts}_bbh_n{len(tasks)}.jsonl" - with out_path.open("w") as f: - meta = { - "_meta": True, - "benchmark": BENCHMARK, - "subtasks": subtasks, - "n_per_subtask": args.n_per_subtask, - "start": args.start, - "arms": list(selected_arms.keys()), - } - f.write(json.dumps(meta) + "\n") - for r in results: - row = asdict(r) - row["benchmark"] = BENCHMARK - f.write(json.dumps(row) + "\n") print(f"\nResults written to {out_path}") print(f"Analyze: python scripts/analyze_sweep.py {out_path}") From b76a6c5f6e4241b0ddf77ba912dc20eff2e88453 Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Thu, 25 Jun 2026 11:22:35 +1000 Subject: [PATCH 2/8] feat(bbh): unattended auto-resume wrapper for the sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit run_bbh_resumable.sh wraps run_bbh_pilot.py: on a usage-exhaustion abort (exit 2) it extracts the partial-results path, waits BACKOFF_SECONDS for the Max window to recover, and re-invokes with --resume — looping up to MAX_RETRIES until the sweep completes or a hard error occurs. Lets a long n=100+ run survive hitting the 5-hour usage window with no human in the loop. Verified: syntax check, path-extraction unit, live n=1 run. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/scripts/run_bbh_resumable.sh | 71 +++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100755 experiment/scripts/run_bbh_resumable.sh diff --git a/experiment/scripts/run_bbh_resumable.sh b/experiment/scripts/run_bbh_resumable.sh new file mode 100755 index 0000000..1269c29 --- /dev/null +++ b/experiment/scripts/run_bbh_resumable.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Unattended auto-resume wrapper around run_bbh_pilot.py. +# +# run_bbh_pilot.py exits 2 when the Max subscription is exhausted, having +# flushed every completed row to disk and printed the partial-results path. +# This wrapper catches that exit, waits for the usage window to recover, and +# re-invokes with --resume so the sweep finishes without a human in the loop. +# +# Usage: +# scripts/run_bbh_resumable.sh --subtasks ... --n-per-subtask 33 --arms ... +# +# Env knobs: +# BACKOFF_SECONDS seconds to wait after an exhaustion abort (default 1800 = 30m) +# MAX_RETRIES max resume attempts before giving up (default 12 ≈ 6h) +# +# Exit codes: 0 = sweep complete; 1 = hard error / retries exhausted. +set -uo pipefail + +cd "$(dirname "$0")/.." || exit 1 # experiment/ dir + +BACKOFF_SECONDS="${BACKOFF_SECONDS:-1800}" +MAX_RETRIES="${MAX_RETRIES:-12}" +PY=(python scripts/run_bbh_pilot.py "$@") + +log() { printf '[resumable %s] %s\n' "$(date -u +%H:%M:%SZ)" "$*"; } + +# Pull the results path from a run's output (both the success line +# "Results written to " and the abort line "Partial results saved to +# " contain it). Last match wins. +extract_path() { + grep -oE 'results/[0-9TZ_]+_bbh_n[0-9]+\.jsonl' "$1" | tail -1 +} + +tmplog="$(mktemp)" +trap 'rm -f "$tmplog"' EXIT + +attempt=0 +resume_path="" +while :; do + if [ -z "$resume_path" ]; then + log "starting sweep: ${PY[*]}" + "${PY[@]}" 2>&1 | tee "$tmplog" + else + log "resuming $resume_path (attempt $attempt/$MAX_RETRIES)" + "${PY[@]}" --resume "$resume_path" 2>&1 | tee "$tmplog" + fi + rc=${PIPESTATUS[0]} + + if [ "$rc" -eq 0 ]; then + log "sweep complete." + exit 0 + elif [ "$rc" -eq 2 ]; then + # Usage exhausted — capture the partial-results path and back off. + p="$(extract_path "$tmplog")" + [ -n "$p" ] && resume_path="$p" + if [ -z "$resume_path" ]; then + log "exhaustion abort but no results path found in output; aborting." + exit 1 + fi + attempt=$((attempt + 1)) + if [ "$attempt" -gt "$MAX_RETRIES" ]; then + log "max retries ($MAX_RETRIES) reached; partial data at $resume_path." + exit 1 + fi + log "usage exhausted; partial data at $resume_path. Sleeping ${BACKOFF_SECONDS}s before resume." + sleep "$BACKOFF_SECONDS" + else + log "hard error (exit $rc); not a usage-exhaustion abort. See output above." + exit 1 + fi +done From 97dbcabe26e6fe7e0c4f35e43a0ba05ad293130d Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Thu, 25 Jun 2026 12:08:06 +1000 Subject: [PATCH 3/8] feat(bbh): live progress dashboard (gen_progress + publish loop) gen_progress.py renders the streaming sweep JSONL into a self-contained auto-refreshing HTML status page (overall + per-arm progress, pass rates, cost units, ETA). publish_progress.sh force-pushes it to the gh-pages branch every 5min (under GitHub Pages' ~10 builds/hr soft limit) from an isolated worktree until the run PID exits. Live at enspyrco.github.io/echo. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/scripts/gen_progress.py | 164 +++++++++++++++++++++++++ experiment/scripts/publish_progress.sh | 39 ++++++ 2 files changed, 203 insertions(+) create mode 100644 experiment/scripts/gen_progress.py create mode 100755 experiment/scripts/publish_progress.sh diff --git a/experiment/scripts/gen_progress.py b/experiment/scripts/gen_progress.py new file mode 100644 index 0000000..389beb4 --- /dev/null +++ b/experiment/scripts/gen_progress.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Render an in-progress BBH sweep JSONL into a self-contained live status page. + +Usage: gen_progress.py + +Reads the streaming results file (meta line + one row per completed run), +computes overall + per-arm progress, pass rates, cost units, and a wall-clock +ETA, and writes a single auto-refreshing HTML file (no external assets). +""" +from __future__ import annotations + +import json +import sys +from collections import defaultdict + +HAIKU_UNIT, SONNET_UNIT = 1, 3 + + +def cost_units(arm: str, sub_calls: int) -> int: + if arm == "haiku-only": + return sub_calls * HAIKU_UNIT + if arm == "sonnet-only": + return sub_calls * SONNET_UNIT + if arm == "echo-judge": + return 3 * HAIKU_UNIT + (SONNET_UNIT if sub_calls > 3 else 0) + if arm == "echo-oracle": + return 2 * HAIKU_UNIT + (SONNET_UNIT if sub_calls > 2 else 0) + return sub_calls * HAIKU_UNIT + + +def _fmt_dur(secs: float) -> str: + secs = max(0, int(secs)) + h, m = secs // 3600, (secs % 3600) // 60 + if h: + return f"{h}h {m}m" + return f"{m}m {secs % 60}s" + + +def main() -> None: + jsonl, out_html, start_epoch, now_epoch = sys.argv[1], sys.argv[2], float(sys.argv[3]), float(sys.argv[4]) + + meta = {} + rows = [] + try: + with open(jsonl) as f: + for line in f: + line = line.strip() + if not line: + continue + r = json.loads(line) + if r.get("_meta"): + meta = r + else: + rows.append(r) + except FileNotFoundError: + rows = [] + + arms = meta.get("arms", []) + subtasks = meta.get("subtasks", []) + n_per = meta.get("n_per_subtask", 0) + n_tasks = len(subtasks) * n_per if subtasks else 0 + total = n_tasks * len(arms) if arms else 0 + done = len(rows) + + by_arm = defaultdict(list) + for r in rows: + by_arm[r["arm"]].append(r) + + elapsed = now_epoch - start_epoch + rate = done / elapsed if elapsed > 0 and done else 0 # runs/sec + eta = (total - done) / rate if rate > 0 else 0 + pct = (100 * done / total) if total else 0 + + # Per-arm rows + arm_cards = [] + for arm in arms: + rs = by_arm.get(arm, []) + a_done = len(rs) + a_total = n_tasks + a_pass = sum(1 for r in rs if r["passed"]) + a_unp = sum(1 for r in rs if r["detail"] == "unparseable") + thr = 3 if arm == "echo-judge" else 2 + a_esc = sum(1 for r in rs if r["sub_calls"] > thr) + a_cost = sum(cost_units(arm, r["sub_calls"]) for r in rs) + rate_s = f"{a_pass}/{a_done} ({a_pass / a_done:.3f})" if a_done else "-" + a_pct = (100 * a_done / a_total) if a_total else 0 + arm_cards.append(f""" +
+
{arm}{a_done}/{a_total}
+
+
+ pass {rate_s} + esc {a_esc} + cost {a_cost}u + unparseable {a_unp} +
+
""") + + status = "RUNNING" if done < total else "COMPLETE" + status_cls = "running" if done < total else "complete" + eta_str = "done" if done >= total else (_fmt_dur(eta) if rate > 0 else "estimating...") + + html = f""" + + + + + +Echo BBH n={n_tasks} - live + + + +
+

Echo - cheap-routing on BBH {status}

+
n={n_tasks} tasks · {', '.join(subtasks)} · Claude Max (zero API spend)
+ +
+
{done} / {total} runs   {pct:.1f}%
+
+
+ elapsed {_fmt_dur(elapsed)} + eta {eta_str} + pace {rate * 60:.1f}/min +
+
+ + {''.join(arm_cards)} + +
auto-refreshes every 20s · streaming from the runner's JSONL · updated each cycle
+
+ +""" + + with open(out_html, "w") as f: + f.write(html) + print(f"wrote {out_html}: {done}/{total} ({pct:.1f}%)") + + +if __name__ == "__main__": + main() diff --git a/experiment/scripts/publish_progress.sh b/experiment/scripts/publish_progress.sh new file mode 100755 index 0000000..b2c26c4 --- /dev/null +++ b/experiment/scripts/publish_progress.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Live-publish loop: regenerate the BBH progress page from the streaming JSONL +# and force-push it to the gh-pages branch every INTERVAL seconds, until the +# tracked run PID exits. Operates entirely inside an isolated gh-pages worktree +# so the main working tree (where the run lives) is never touched. +# +# Usage: publish_progress.sh +set -uo pipefail + +RUN_PID="$1" +RESULTS_DIR="$2" +PAGES_DIR="$3" +START_EPOCH="$4" +INTERVAL="${INTERVAL:-300}" # GitHub Pages soft-limits ~10 builds/hr; 5min = 12/hr +REPO="/Users/nick/git/research/echo" +PY="$REPO/experiment/.venv/bin/python" +GEN="$REPO/experiment/scripts/gen_progress.py" + +publish() { + local f now + f="$(ls -t "$RESULTS_DIR"/*_bbh_n99.jsonl 2>/dev/null | head -1)" + [ -z "$f" ] && return 0 + now="$(date +%s)" + "$PY" "$GEN" "$f" "$PAGES_DIR/index.html" "$START_EPOCH" "$now" >/dev/null 2>&1 || return 0 + touch "$PAGES_DIR/.nojekyll" + ( cd "$PAGES_DIR" \ + && git add index.html .nojekyll \ + && git -c user.name="Nick Meinhold" -c user.email="langer.robin@gmail.com" \ + commit -q --amend --no-edit \ + && git push -q -f origin gh-pages ) >/dev/null 2>&1 +} + +# Update while the run is alive, then one final update to flip to COMPLETE. +while kill -0 "$RUN_PID" 2>/dev/null; do + publish + sleep "$INTERVAL" +done +publish +echo "publish loop done (run $RUN_PID exited)" From 23030a2c9187d6c9ad0792a73ebfea9f54a52738 Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Wed, 1 Jul 2026 09:09:20 +1000 Subject: [PATCH 4/8] fix(bbh): isolate model calls from project agent context (#1305) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The harness shells out to `claude --print` from inside the Echo project dir, where `claude -p` loads the project CLAUDE.md, runs SessionStart hooks, and ingests the task-restore injection. Every BBH model call was therefore answering AS the Echo agent ("All 8 tasks restored. Now to your question...") rather than the benchmark question. This — not a parser bug or "Sonnet burying the format line" — is the real cause of the n=30/n=99 Sonnet unparseables and the 98-125s wall times (loading a huge CLAUDE.md + hooks per call). Accuracy was confounded too. Fix: pass `--setting-sources ""` so none of {user,project,local} settings load (no hooks/CLAUDE.md/task-restore) while keeping the Max OAuth auth (--bare drops auth, so it's unusable here). Verified via scripts/repro_sonnet_unparse.py on the 3 known failing tasks: all now parse cleanly, wall times drop ~6-10x (99s->6s), and date_understanding/5 goes unparseable->correct. 25 parser/scoring unit tests still pass. IMPLICATION: all prior BBH sweeps (n=15/30/99) are contaminated and must be re-run with the fixed harness before #336/#619. Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/chat_claude_code.py | 13 +++++- experiment/scripts/repro_sonnet_unparse.py | 51 ++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 experiment/scripts/repro_sonnet_unparse.py diff --git a/experiment/chat_claude_code.py b/experiment/chat_claude_code.py index 472e2ff..4be9e47 100644 --- a/experiment/chat_claude_code.py +++ b/experiment/chat_claude_code.py @@ -67,7 +67,18 @@ def _generate( prompt = self._format(messages) try: result = subprocess.run( - [self.binary, "--print", "--model", self.model, prompt], + # --setting-sources "" loads NONE of {user, project, local} + # settings. Without it, `claude --print` run from inside a + # project dir inherits that project's CLAUDE.md, SessionStart + # hooks, and any task-restore injection — so the model answers + # AS the project agent ("All 8 tasks restored...") instead of + # the benchmark question. That contaminated every BBH sweep + # before 2026-07-01 (#1305): inflated wall times (loading a + # huge CLAUDE.md), unparseable agent-preamble outputs, and + # confounded accuracy. Empty sources keeps the Max OAuth auth + # (unlike --bare, which drops it) while stripping all priming. + [self.binary, "--print", "--setting-sources", "", + "--model", self.model, prompt], capture_output=True, text=True, timeout=self.timeout, diff --git a/experiment/scripts/repro_sonnet_unparse.py b/experiment/scripts/repro_sonnet_unparse.py new file mode 100644 index 0000000..b7adf54 --- /dev/null +++ b/experiment/scripts/repro_sonnet_unparse.py @@ -0,0 +1,51 @@ +"""Reproduce the Sonnet unparseable BBH outputs (#1305) and dump raw text. + +Loads the specific tasks that failed in the n=30 sweep, runs sonnet-only, +and prints the raw model output + what extract_choice / score_bbh return. +We capture WHY it's unparseable before touching the parser. +""" + +from __future__ import annotations + +import sys +import time + +from benchmarks.bbh import extract_choice, load_bbh, score_bbh +from benchmarks.bbh_arms import arm_sonnet_only + +# (subtask, index) pairs that came back unparseable from Sonnet in the n=30/n=99 sweeps. +TARGETS = [ + ("causal_judgement", 3), + ("causal_judgement", 7), + ("date_understanding", 5), +] + + +def main() -> None: + # Load enough of each subtask to cover the target indices. + subtasks = sorted({s for s, _ in TARGETS}) + tasks = load_bbh(subtasks, n_per_subtask=10, start=0) + by_id = {t["task_id"]: t for t in tasks} + + for subtask, idx in TARGETS: + tid = f"bbh/{subtask}/{idx}" + task = by_id.get(tid) + if task is None: + print(f"!! {tid} not loaded", flush=True) + continue + print("=" * 80, flush=True) + print(f"TASK {tid} gold={task['gold']} labels={task.get('choice_labels')}", flush=True) + t0 = time.time() + output, sub_calls = arm_sonnet_only(task) + wall = time.time() - t0 + pred = extract_choice(output) + passed, detail = score_bbh(output, task) + print(f"wall={wall:.1f}s extract_choice={pred!r} score={passed} ({detail})", flush=True) + print(f"--- raw output ({len(output)} chars) ---", flush=True) + print(output, flush=True) + print(f"--- last 300 chars ---\n{output[-300:]!r}", flush=True) + print(flush=True) + + +if __name__ == "__main__": + sys.exit(main()) From 3af774c7c067df10e686c70e522779f625d1e4f5 Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Wed, 1 Jul 2026 10:01:30 +1000 Subject: [PATCH 5/8] =?UTF-8?q?data(bbh):=20clean=20n=3D30=20sweep=20?= =?UTF-8?q?=E2=80=94=20decontaminated=20harness=20(#1305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First BBH sweep run with the --setting-sources "" fix (commit 23030a2), so model calls no longer inherit the project CLAUDE.md/hooks/task-restore. Clean n=30 (3 subtasks × 10): haiku-only 0.867 10.4s 0 unparse sonnet-only 0.900 10.3s 0 unparse echo-judge 0.867 17.4s 0 unparse echo-oracle 0.933 14.0s 0 unparse vs contaminated 20260625T013951Z_bbh_n30.jsonl: sonnet-only was 0.833 @ 40.7s with 3 unparseables — the contamination cost Sonnet ~7 pts and 4x latency, and had pushed it BELOW Haiku. Clean, Sonnet (0.900) edges Haiku (0.867) — direction restored — and zero unparseables across all arms. Separation is +1 task at n=30, so still needs a clean n=99 to be decisive; committing the datapoint. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../results/20260630T232105Z_bbh_n30.jsonl | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 experiment/results/20260630T232105Z_bbh_n30.jsonl diff --git a/experiment/results/20260630T232105Z_bbh_n30.jsonl b/experiment/results/20260630T232105Z_bbh_n30.jsonl new file mode 100644 index 0000000..3fb23c9 --- /dev/null +++ b/experiment/results/20260630T232105Z_bbh_n30.jsonl @@ -0,0 +1,121 @@ +{"_meta": true, "benchmark": "bbh", "subtasks": ["logical_deduction_three_objects", "causal_judgement", "date_understanding"], "n_per_subtask": 10, "start": 0, "arms": ["haiku-only", "sonnet-only", "echo-judge", "echo-oracle"]} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.133256500121206, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.03831462492235, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.01156829111278, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.8844416660722345, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.936752334004268, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.873426582897082, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.281210042070597, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.790916707832366, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 4.8973835001233965, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.8368842920754105, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.79972091712989, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.3208894999697804, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.325825041858479, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.837706417078152, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.843760707881302, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.388927624793723, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.947729042032734, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.481110916938633, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.119223541812971, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.573656709166244, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.2441512499935925, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.585093874949962, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.067356375046074, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.854345582891256, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.343141874996945, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.960824416950345, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.337839707965031, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.0494434589054435, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.03660004097037, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.470700416946784, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.779815332964063, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.759977249894291, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.887740624835715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 2.944666499970481, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.77205550018698, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.844611583976075, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.1592209581285715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.666819290956482, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.27446929202415, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.500312290852889, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.715282999910414, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.017668541986495, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.33730004215613, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.396783374948427, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 32.10707787517458, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 19.857557708164677, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 70.30251758289523, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 34.656666584080085, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 15.29057883308269, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.777149834204465, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 31.010495625203475, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 29.859363249968737, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 21.51052049989812, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 15.273428042186424, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "echo-judge", "passed": false, "detail": "expected B got A", "wall_seconds": 41.604730874998495, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 19.69896529102698, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.593052625190467, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 17.077932958956808, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.664838208816946, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 61.38340691616759, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.704659499926493, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.278195540886372, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 22.133517917012796, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.619719458045438, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 19.87051733303815, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.80458312504925, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.160859249997884, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.83299241703935, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.731589250033721, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 17.131971125025302, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.611289457883686, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.272828832967207, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 15.02112245792523, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 19.575943249976262, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 27.6146638749633, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 34.21082858298905, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.446576917078346, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 15.353303499985486, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 28.687631250126287, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.857115249847993, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.263688874896616, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.4302267499733716, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.938575916923583, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.238625708036125, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.390886374982074, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.841818500077352, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.526846208842471, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.98386566597037, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "haiku-only", "passed": false, "detail": "expected B got C", "wall_seconds": 7.934419583063573, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "sonnet-only", "passed": false, "detail": "expected B got C", "wall_seconds": 10.093535750173032, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "echo-judge", "passed": false, "detail": "expected B got C", "wall_seconds": 10.616874375147745, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "echo-oracle", "passed": false, "detail": "expected B got C", "wall_seconds": 11.194521958008409, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 19.132790209027007, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.385990124894306, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.84913829085417, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 18.501582500059158, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.49173133308068, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.7247482500970364, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.695192834129557, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.357433500001207, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 4.758404667023569, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.900991375092417, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.118695916142315, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.630096415989101, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "haiku-only", "passed": false, "detail": "expected A got D", "wall_seconds": 12.002490958897397, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "sonnet-only", "passed": false, "detail": "expected A got D", "wall_seconds": 14.90300887497142, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "echo-judge", "passed": false, "detail": "expected A got D", "wall_seconds": 20.47877095802687, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "echo-oracle", "passed": false, "detail": "expected A got D", "wall_seconds": 28.7409013339784, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 4.705210292013362, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.41999070905149, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.061663541942835, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 4.735660917125642, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.022053916938603, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.799415958113968, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.82291187508963, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.099794917041436, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.971698459004983, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.561875582905486, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.069442667067051, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.942228625062853, "sub_calls": 2, "benchmark": "bbh"} From 3974f21c6c2c7b1eb11f6a0e6a2beb0f3de7c98a Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Wed, 1 Jul 2026 11:40:40 +1000 Subject: [PATCH 6/8] =?UTF-8?q?data(bbh):=20clean=20n=3D99=20sweep=20?= =?UTF-8?q?=E2=80=94=20decontaminated=20harness=20(#1305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full n=99 (3 subtasks × 33) on the --setting-sources "" fixed harness. Clean n=99: haiku-only 0.848 11.3s 0 unparse sonnet-only 0.848 10.2s 0 unparse echo-judge 0.869 16.8s 0 unparse echo-oracle 0.899 14.3s 0 unparse vs contaminated 20260625T014218Z_bbh_n99.jsonl (sonnet-only 0.657 @ 53s, 20 unparseables): the contamination cost Sonnet ~19 pts and 5x latency. Findings: - Sonnet-vs-Haiku separation: NONE on this slice. Both 0.848, and identical per-subtask (causal 22/33, date 29/33, logical 33/33). These subtasks are near-ceiling and don't discriminate the two models. A decisive separation needs harder subtasks (logical_deduction_seven, etc). - Complementarity IS real despite the aggregate tie: echo-oracle 0.899 > both baselines because Haiku and Sonnet miss DIFFERENT items (oracle recovers causal_judgement 27/33 vs baselines' 22/33). That gap is the headroom Echo routing targets. - echo-judge 0.869 edges above both baselines but leaves ~3 pts of the oracle headroom uncaptured — the real judge isn't yet as good as the perfect router. This is where cross-family judges (#619) come in. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../results/20260701T000922Z_bbh_n99.jsonl | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 experiment/results/20260701T000922Z_bbh_n99.jsonl diff --git a/experiment/results/20260701T000922Z_bbh_n99.jsonl b/experiment/results/20260701T000922Z_bbh_n99.jsonl new file mode 100644 index 0000000..c7063d6 --- /dev/null +++ b/experiment/results/20260701T000922Z_bbh_n99.jsonl @@ -0,0 +1,397 @@ +{"_meta": true, "benchmark": "bbh", "subtasks": ["logical_deduction_three_objects", "causal_judgement", "date_understanding"], "n_per_subtask": 33, "start": 0, "arms": ["haiku-only", "sonnet-only", "echo-judge", "echo-oracle"]} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.316313208080828, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.849987749941647, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.295504208886996, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.102567457826808, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.937464584130794, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.725144374882802, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.958915167022496, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.355238707968965, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.44943245803006, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.180893041891977, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.5591838338878, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.889097791863605, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.6265948340296745, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.38088612514548, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.25422295788303, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.948591875145212, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.683167417068034, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.57320566684939, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.813669875031337, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.386151042068377, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.991383875021711, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.1881281659007072, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.217920124996454, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.644327291985974, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.764498499920592, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.741255666827783, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.92188633303158, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.088778333039954, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.762955624843016, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.023962541949004, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.814592292066664, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.957679792074487, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.76184329087846, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 2.9312865419778973, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.412404250120744, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.282524666981772, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.930794832995161, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.266014457913116, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.701914208941162, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.921354999998584, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/10", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.115651166066527, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/10", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.0185222909785807, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/10", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.610262250062078, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/10", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.0748129999265075, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.286909000016749, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.215950833167881, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.552975707920268, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.850315208081156, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.264800208155066, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.751056583132595, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.33009275002405, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.471472332952544, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.471863124985248, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.595667124958709, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.436739666154608, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.391826250124723, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/14", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.592561166035011, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/14", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.136535166064277, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/14", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.720119917066768, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/14", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.319167458917946, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.155447917059064, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.954087791033089, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.06160004180856, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.319559834199026, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/16", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.179873833898455, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/16", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.512218500021845, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/16", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.11954770819284, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/16", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.594211874995381, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.503814291907474, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 2.905533874873072, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.064536000136286, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.1592904578428715, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.52731812489219, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 2.7996552910190076, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.827872416004539, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.421220124932006, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.139527416788042, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.059321166016161, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.41395187494345, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.840540916891769, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/20", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.6135836250614375, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/20", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.097550750011578, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/20", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.512294458923861, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.85387225006707, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.592312375083566, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/21", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.675496374955401, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.40063300007023, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.050315790809691, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/22", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.2580013750121, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/22", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.5934527087956667, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/22", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.895853250054643, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/22", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.212365042185411, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/23", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.966494207968935, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/23", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.667026750044897, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/23", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.988251582952216, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/23", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 4.52180404192768, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.389966499991715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.959521082928404, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.230458792066202, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.304791833972558, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.054371833102778, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.441010749898851, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.419040374923497, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.7140864168759435, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 4.595363166881725, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.2354914580937475, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.334645374910906, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.529847999801859, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.119651542045176, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.614166208077222, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.334610000019893, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.130680249771103, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.297177624888718, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.73783279210329, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/28", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.711098584113643, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.059375791111961, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/29", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.750836624996737, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/29", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.023156750015914, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/29", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.902045208029449, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/29", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.461193665862083, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/30", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.911566334078088, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/30", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.6399731249548495, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/30", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.603513541864231, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/30", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.009519542101771, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.413686915999278, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/31", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.862728707958013, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.287493582814932, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.311912040924653, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/32", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.321892833802849, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/32", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.683459333144128, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/32", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.446946083102375, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_three_objects/32", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.698663624934852, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 21.313992875162512, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 20.37140812491998, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 24.661077124997973, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.531613166909665, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 19.329722583061084, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 24.62569187488407, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 95.87833620794117, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 48.23645116714761, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 33.04430437507108, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 20.896236083004624, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 45.84407191700302, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 31.07548183389008, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 19.362255375133827, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 15.211935458006337, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 29.802672875113785, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 21.05617812485434, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.89342400012538, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.770636708009988, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.58573374990374, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.077648874837905, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 13.788124290993437, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.402531208936125, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 21.623676457908005, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 31.399861624930054, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.630730208009481, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.044199167052284, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 43.81067970814183, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.47040820796974, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.594149790937081, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.31003383290954, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.835943832993507, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.776304582832381, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 24.729345582891256, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.33410312491469, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 25.722540416987613, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.172263333108276, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 23.46753945783712, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.856347208144143, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 21.53795670811087, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.915231833932921, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/10", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 13.231505084084347, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/10", "arm": "sonnet-only", "passed": false, "detail": "expected A got B", "wall_seconds": 11.650402708910406, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/10", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 16.464644459076226, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/10", "arm": "echo-oracle", "passed": false, "detail": "expected A got B", "wall_seconds": 22.71529566589743, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.536693124799058, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.38271062495187, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.730528957908973, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 21.984670749865472, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.580821332987398, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 15.022994541097432, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.687545791035518, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.110602875007316, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.445952625013888, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 21.538244040915743, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.178894500015303, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.605335916858166, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/14", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 13.475163917057216, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/14", "arm": "sonnet-only", "passed": false, "detail": "expected A got B", "wall_seconds": 14.232929499819875, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/14", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 17.79591237497516, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/14", "arm": "echo-oracle", "passed": false, "detail": "expected A got B", "wall_seconds": 27.655219292035326, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.962660166900605, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 15.47944224998355, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.07059504208155, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 20.55445229099132, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/16", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 16.343976999865845, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/16", "arm": "sonnet-only", "passed": false, "detail": "expected A got B", "wall_seconds": 27.252508250065148, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/16", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 26.100743958028033, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/16", "arm": "echo-oracle", "passed": false, "detail": "expected A got B", "wall_seconds": 54.314892292022705, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.866206042002887, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 26.429214084055275, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.029609374934807, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.754390666028485, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.706824334105477, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 22.576752708060667, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.157397333998233, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 42.61603912501596, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.796807333128527, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.542602832894772, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.849854624830186, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.746367417043075, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/20", "arm": "haiku-only", "passed": false, "detail": "expected B got A", "wall_seconds": 20.79487649980001, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/20", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 21.305280916858464, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/20", "arm": "echo-judge", "passed": false, "detail": "expected B got A", "wall_seconds": 16.693581541068852, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 59.566726291086525, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.802417750004679, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/21", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 17.805412708083168, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.492673333967105, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 20.935169666074216, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/22", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 14.72375691588968, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/22", "arm": "sonnet-only", "passed": false, "detail": "expected A got B", "wall_seconds": 17.34715470811352, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/22", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 16.759266584180295, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/22", "arm": "echo-oracle", "passed": false, "detail": "expected A got B", "wall_seconds": 30.274872374953702, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/23", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.417329292045906, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/23", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 15.559842250077054, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/23", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.165944291977212, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/23", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.407602457795292, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.227652916917577, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.57948033302091, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 36.342382584000006, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.913451583124697, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.271509292069823, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 17.877634833101183, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.409510541940108, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.676159708062187, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.779991290997714, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 13.383170000044629, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.41934566688724, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.848075417103246, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.80518408305943, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.448953166836873, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.919934374978766, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 21.110908041941002, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 19.005232708062977, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.959360874956474, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/28", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 39.13309937505983, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 20.49843145883642, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/29", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 58.14013308403082, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/29", "arm": "sonnet-only", "passed": false, "detail": "expected A got B", "wall_seconds": 18.248791000107303, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/29", "arm": "echo-judge", "passed": false, "detail": "expected A got B", "wall_seconds": 30.11282887496054, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/29", "arm": "echo-oracle", "passed": false, "detail": "expected A got B", "wall_seconds": 34.19621816696599, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/30", "arm": "haiku-only", "passed": false, "detail": "expected A got B", "wall_seconds": 54.24858050001785, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/30", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 30.741339500062168, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/30", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 46.91955741588026, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/30", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 34.80962033383548, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.246691041858867, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/31", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 17.49716891697608, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 22.8189043328166, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 22.569703875109553, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/32", "arm": "haiku-only", "passed": false, "detail": "expected B got A", "wall_seconds": 14.310736665967852, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/32", "arm": "sonnet-only", "passed": false, "detail": "expected B got A", "wall_seconds": 16.5534059999045, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/32", "arm": "echo-judge", "passed": false, "detail": "expected B got A", "wall_seconds": 43.97126245801337, "sub_calls": 4, "benchmark": "bbh"} +{"task_id": "bbh/causal_judgement/32", "arm": "echo-oracle", "passed": false, "detail": "expected B got A", "wall_seconds": 30.650843958137557, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.044546833029017, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.1854863329790533, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.675317500019446, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.093114332994446, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.053958915872499, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.49995783297345, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.801584582775831, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.357123041991144, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "haiku-only", "passed": false, "detail": "expected B got C", "wall_seconds": 4.8613827500958, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "sonnet-only", "passed": false, "detail": "expected B got C", "wall_seconds": 5.698599500115961, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "echo-judge", "passed": false, "detail": "expected B got C", "wall_seconds": 10.043859999859706, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/2", "arm": "echo-oracle", "passed": false, "detail": "expected B got C", "wall_seconds": 17.46874749986455, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.492860834114254, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.723362375050783, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.21530183404684, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.657284707995132, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.534594167023897, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.52542179194279, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.95973112503998, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.039476417005062, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.020659999921918, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.8046036248560995, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.36649270891212, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 4.9887536249589175, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "haiku-only", "passed": false, "detail": "expected A got D", "wall_seconds": 14.630997375119478, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "sonnet-only", "passed": false, "detail": "expected A got D", "wall_seconds": 18.28446620889008, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "echo-judge", "passed": false, "detail": "expected A got D", "wall_seconds": 13.845766792073846, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/6", "arm": "echo-oracle", "passed": false, "detail": "expected A got D", "wall_seconds": 33.4975946249906, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.275220833020285, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.505211791023612, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 8.595318875042722, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.327475250000134, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.121455500135198, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.320292916148901, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 9.024913165951148, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.483764708042145, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.724016832886264, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.83184695802629, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.344794541830197, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.556077417219058, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/10", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 4.766802542144433, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/10", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.7603706249501556, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/10", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.782143083866686, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/10", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.55077224993147, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.516799916047603, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.31208545784466, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.93327466584742, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.05262383306399, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.64779045782052, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.086727458052337, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.53945012507029, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.611534665804356, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.65514145907946, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.249139582971111, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.252571958117187, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.980295333079994, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/14", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.060294666793197, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/14", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.454785415902734, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/14", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.862115290947258, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/14", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.107136375037953, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.941008208086714, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.965928541962057, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.769661416998133, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.496617374941707, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/16", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.108305833069608, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/16", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.307362999999896, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/16", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.100104958983138, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/16", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.477110374951735, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.846763167064637, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 28.371449457947165, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 24.741839250084013, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 23.967344332952052, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.979590374976397, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.3383970418944955, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.160765167092904, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.8313665830064565, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.592914958950132, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.47795550012961, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.292820416856557, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.53985166689381, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/20", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.6660050000064075, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/20", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.829605124890804, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/20", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.168602249817923, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.09520179219544, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.167733459034935, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/21", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.218034124933183, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.532687874976546, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.798057708889246, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/22", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.6720506669953465, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/22", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.460356374969706, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/22", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.8713897918351, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/22", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.513512208126485, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/23", "arm": "haiku-only", "passed": false, "detail": "expected C got B", "wall_seconds": 9.135808916995302, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/23", "arm": "sonnet-only", "passed": false, "detail": "expected C got B", "wall_seconds": 8.234549666987732, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/23", "arm": "echo-judge", "passed": false, "detail": "expected C got B", "wall_seconds": 11.329074791865423, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/23", "arm": "echo-oracle", "passed": false, "detail": "expected C got B", "wall_seconds": 17.96945225005038, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.283770042005926, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.215210250113159, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.504489250015467, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.217754125129431, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.778665833175182, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.596697333035991, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.077823208877817, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.274052458116785, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.4171483751852065, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.784053707960993, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.462839582934976, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 5.186412000097334, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.590134042082354, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.7195081249810755, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.115993292070925, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 23.302590790903196, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.216262666974217, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.264693290926516, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/28", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.685828750021756, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.868758124997839, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/29", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.201574666891247, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/29", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.626526457956061, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/29", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.49447129201144, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/29", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.579630375141278, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/30", "arm": "haiku-only", "passed": false, "detail": "expected E got C", "wall_seconds": 12.56376766692847, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/30", "arm": "sonnet-only", "passed": false, "detail": "expected E got C", "wall_seconds": 20.144010417163372, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/30", "arm": "echo-judge", "passed": false, "detail": "expected E got C", "wall_seconds": 16.379267249954864, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/30", "arm": "echo-oracle", "passed": false, "detail": "expected E got C", "wall_seconds": 45.03564754198305, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.355691208969802, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/31", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.034291667165235, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 22.158228083048016, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.46936058299616, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/32", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.942319917026907, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/32", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.453196792164817, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/32", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.167246875120327, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/date_understanding/32", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.487180749885738, "sub_calls": 2, "benchmark": "bbh"} From 53340fb3a6d1f7c5ca8cced0abe564c521a560a5 Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Wed, 1 Jul 2026 16:35:28 +1000 Subject: [PATCH 7/8] =?UTF-8?q?data(bbh):=20hard-subtask=20sweep=20?= =?UTF-8?q?=E2=80=94=20MCQ=20BBH=20is=20SATURATED=20for=20current=20Claude?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Experiment A: tested whether HARDER BBH subtasks reveal a Sonnet-vs-Haiku separation that the near-ceiling easy slice couldn't (n=33 x 3, clean --setting-sources "" harness). Subtasks chosen as the reasoning-heavy "hard" siblings: logical_deduction_seven_objects, tracking_shuffled_objects_seven_objects, temporal_sequences Result — every subtask is a CEILING for BOTH model tiers: haiku-only 99/99 1.000 sonnet-only 99/99 1.000 echo-judge 98/99 0.990 (one routing miss) echo-oracle 99/99 1.000 Per-subtask: haiku & sonnet BOTH 33/33 on all three. KEY FINDING: current Haiku already aces the MCQ BBH subtasks that used to discriminate model tiers (incl. 7-object state-tracking). So: - No Sonnet-vs-Haiku accuracy separation exists to route on. - No oracle->echo-judge headroom (1-task noise gap) for a better judge to close, so the OpenAI cross-family judge run (Experiment B) was correctly NOT launched — nothing to capture. Implication: accuracy-based routing value cannot be demonstrated on MCQ BBH with 2026 Claude models. Forward paths: (1) extend harness to score free-form subtasks (word_sorting/dyck_languages/multistep_arithmetic — the genuinely hard ones, currently unscoreable), (2) move to a non-saturated benchmark (MMLU-Pro #620, GPQA), or (3) reframe Echo's value from accuracy to cost/latency-neutral routing ("route to cheap, lose nothing"). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../results/20260701T050041Z_bbh_n99.jsonl | 397 ++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 experiment/results/20260701T050041Z_bbh_n99.jsonl diff --git a/experiment/results/20260701T050041Z_bbh_n99.jsonl b/experiment/results/20260701T050041Z_bbh_n99.jsonl new file mode 100644 index 0000000..c9424de --- /dev/null +++ b/experiment/results/20260701T050041Z_bbh_n99.jsonl @@ -0,0 +1,397 @@ +{"_meta": true, "benchmark": "bbh", "subtasks": ["logical_deduction_seven_objects", "tracking_shuffled_objects_seven_objects", "temporal_sequences"], "n_per_subtask": 33, "start": 0, "arms": ["haiku-only", "sonnet-only", "echo-judge", "echo-oracle"]} +{"task_id": "bbh/logical_deduction_seven_objects/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 18.938317833002657, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 13.352589582791552, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 32.575990832876414, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.65974645782262, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 21.17677570786327, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 18.790194208966568, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 30.313567250035703, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 22.966045667184517, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/2", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.31998079107143, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.141241374891251, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.825250583933666, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.38063629087992, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.042738958960399, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.438195584109053, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.85631029191427, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.013894000090659, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.26206324994564, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.094485457986593, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.691501666791737, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.368035583058372, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.346694916952401, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.573085916927084, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.608133625006303, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.914662583963946, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.66191454185173, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.844628625083715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.33091749995947, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 17.2429865419399, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.309255250031129, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.865486291935667, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.57994300010614, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.841441167052835, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.970993292052299, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.734154541045427, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.825727167073637, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.991618457948789, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.656395709142089, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.4269322499167174, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.074797875015065, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.765514000086114, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/10", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.782236416824162, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/10", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.272647999925539, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/10", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.995723125059158, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/10", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 18.683790874900296, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.01869908394292, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.577572624897584, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.52223008289002, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.569691583048552, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.659880916820839, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.26959037501365, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.572507541859522, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.628382416907698, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.993203375022858, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.605027916841209, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.803110165987164, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.228417083853856, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/14", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.116884249961004, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/14", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.643616290995851, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/14", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.256994083058089, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/14", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.164123624796048, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.24442654196173, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.23504391615279, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.234342084033415, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.889402249827981, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/16", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 17.29972262494266, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/16", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 26.56362316594459, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/16", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 27.24542308296077, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/16", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 38.10982950008474, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.487621667096391, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.518423207802698, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.780238083098084, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.209293375024572, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.076131667010486, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 13.943388083018363, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 26.8820797088556, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.440626540919766, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.080293958075345, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.187371750129387, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.636249917093664, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.447255332954228, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/20", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.811942582949996, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/20", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.172384959179908, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/20", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.26317024999298, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.98875758284703, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.51413270784542, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/21", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.800495458068326, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.072612666990608, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.502266333904117, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/22", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.670969709055498, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/22", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 3.3655604580417275, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/22", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.946358958957717, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/22", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.727451334008947, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/23", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.207566249882802, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/23", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 16.378965583164245, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/23", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.300587208010256, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/23", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 23.186493125045672, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.654391292016953, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.491755832917988, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.384401499992236, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.679532709065825, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.7204885829705745, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.814982583047822, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.790575250051916, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.615455457940698, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 28.033320791786537, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 23.626313917106017, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 39.53266662498936, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 25.906134499935433, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.395156667102128, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.511561958119273, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.397082083160058, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.583188625052571, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.450398250017315, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.294858207926154, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/28", "arm": "echo-judge", "passed": false, "detail": "RuntimeError: claude --print timed out after 180s (model=haiku)", "wall_seconds": 180.0341895830352, "sub_calls": 0, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.98556112498045, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/29", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 25.68859054101631, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/29", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.174501582980156, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/29", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 28.388300333172083, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/29", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.998914040857926, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/30", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.158178915968165, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/30", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 23.770162540953606, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/30", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 25.45679450011812, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/30", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 20.688642709050328, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 23.18610162520781, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/31", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.395152000011876, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 23.46627916698344, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 45.60528091597371, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/32", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.507181541994214, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/32", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.332607166841626, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/32", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.89399908296764, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/logical_deduction_seven_objects/32", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.509515082929283, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.068467125063762, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.201169417006895, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.093540417030454, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.631635708967224, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.447756458073854, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.047892750008032, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.224328125128523, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.277342667104676, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/2", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.295417957939208, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.23502662498504, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.522889415966347, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.988068790873513, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.74248387501575, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.109681874979287, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.406816832954064, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.744654249865562, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.988004792016, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.334925125120208, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.341485708020627, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 19.731061875121668, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.519594208104536, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.386579292127863, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.34163716691546, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.790179583942518, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.562634625006467, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.894440790871158, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.23259445792064, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.068001375067979, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.502404541010037, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.012562958989292, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.67469908297062, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.161874999990687, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.877976457821205, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.742702666902915, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.453939334023744, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.767270917072892, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.104372333036736, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.92790775001049, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 21.03024937491864, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.9577391250059, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/10", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.882474707905203, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/10", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.581746625015512, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/10", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.134825375163928, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/10", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.780182667076588, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.634837541962042, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.15972129185684, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.143020916962996, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.322020207997411, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.372374791884795, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.652243084041402, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.668571916874498, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.856654333882034, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.655607874970883, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.904067499795929, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.201775833033025, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.357777500059456, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/14", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.405110582942143, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/14", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 13.064275041921064, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/14", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.530601208098233, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/14", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.548906665993854, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.628242459148169, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.591969166882336, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.91297645913437, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.677746208151802, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/16", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.07459379103966, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/16", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.41523129097186, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/16", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.378387541975826, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/16", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.472178374882787, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.932660165941343, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.226939207874238, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.688796958886087, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.941097290953621, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.678378124954179, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 11.221953667001799, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.74708562484011, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.77839033282362, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.700058500049636, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.291910292115062, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 19.482354416977614, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.107102332869545, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/20", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.292699583107606, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/20", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.430814459221438, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/20", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.448434208054096, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.940736583899707, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.640095457900316, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/21", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.997775541152805, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.868172208080068, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.321814374998212, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/22", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.877600707812235, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/22", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.55759954219684, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/22", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.44614045904018, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/22", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.36905249999836, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/23", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.526101915864274, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/23", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.121693208115175, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/23", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.279574583051726, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/23", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.620470667025074, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.478112500160933, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.879307499853894, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.878879207884893, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.035100667038932, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.902236749883741, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.305763915879652, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.554113124962896, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.491615749895573, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.394940208178014, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.981367415981367, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.92198183387518, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.69105983292684, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.457944791996852, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.301027208101004, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.09894633409567, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.978703750064597, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.2069878748152405, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 13.01931424997747, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/28", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.2602239579428, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.499825666891411, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/29", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.2851139169652, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/29", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 15.275409082882106, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/29", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 35.765605540946126, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/29", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 14.036489167017862, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/30", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.831086874939501, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/30", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.781137040816247, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/30", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.98756895796396, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/30", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 21.66752512520179, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.454553500050679, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/31", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 12.24552333401516, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.035656749969348, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 13.951147207990289, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/32", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.86665774998255, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/32", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 14.256339417072013, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/32", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.066450374899432, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/tracking_shuffled_objects_seven_objects/32", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.891825125087053, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/0", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.459644624963403, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/0", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.985433791065589, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/0", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.273229124955833, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/0", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.678333749994636, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/1", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.725089125102386, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/1", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.124495291849598, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/1", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.436515625100583, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/1", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.4710566671565175, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/2", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.996729834005237, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/2", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.974727415945381, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/2", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.581999042071402, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/2", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.78027604194358, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/3", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.032842958113179, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/3", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.285541209159419, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/3", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.457687583984807, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/3", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.98276041680947, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/4", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.977284415857866, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/4", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.883752791909501, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/4", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.151922957971692, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/4", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.835038041928783, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/5", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.578086792025715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/5", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.241887166863307, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/5", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.76940824999474, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/5", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.692644125083461, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/6", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.8428198751062155, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/6", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.921277374960482, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/6", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.702837916789576, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/6", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.290512459119782, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/7", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.734562332974747, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/7", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.713479584082961, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/7", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.177321959054098, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/7", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.60168333305046, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/8", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.243073541903868, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/8", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.23627383285202, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/8", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 16.173932624980807, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/8", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 10.8009478750173, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/9", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.350418375106528, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/9", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.587570833973587, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/9", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.242117083864287, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/9", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.34530054195784, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/10", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.093489875085652, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/10", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.282050041947514, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/10", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.8468199998606, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/10", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.774928249884397, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/11", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.682788457954302, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/11", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.738481791922823, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/11", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.487420208053663, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/11", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 15.787999833002687, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/12", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.016197165939957, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/12", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.217378959059715, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/12", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.861100458074361, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/12", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.817813458153978, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/13", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.523780707968399, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/13", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.02271741698496, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/13", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.331933416193351, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/13", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.340565458172932, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/14", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.627265541115776, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/14", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 4.324873457895592, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/14", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.716747040860355, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/14", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.328447207808495, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/15", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.69227149989456, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/15", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 10.257975999964401, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/15", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.112742082914338, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/15", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.603922750102356, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/16", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.984092582948506, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/16", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.258177291834727, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/16", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 14.215146249858662, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/16", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 16.934294833103195, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/17", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.290136833209544, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/17", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.950472667114809, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/17", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.827053166925907, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/17", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.776922999881208, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/18", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.681753542041406, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/18", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.797355249989778, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/18", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.425964625086635, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/18", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.32596812513657, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/19", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.956929040839896, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/19", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.547746459022164, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/19", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 18.88336891704239, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/19", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.463050958001986, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/20", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.598787208087742, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/20", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.5145538749638945, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/20", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.504801833070815, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/20", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.863392333034426, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/21", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.149987166980281, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/21", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.442000125069171, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/21", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.867692874977365, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/21", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.744475249899551, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/22", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.79565666615963, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/22", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.834368833107874, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/22", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.398001375142485, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/22", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.353511084103957, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/23", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.914371250197291, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/23", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.4545447919517756, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/23", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.272327499929816, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/23", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.179504459025338, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/24", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.609411957906559, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/24", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.101822958094999, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/24", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 13.100010499823838, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/24", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.821236292133108, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/25", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.241363666020334, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/25", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.238475332967937, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/25", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.575443249894306, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/25", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 11.285297292051837, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/26", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.939495041966438, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/26", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 9.725692332955077, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/26", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 17.46207904093899, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/26", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 12.30907449987717, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/27", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.177538082934916, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/27", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 7.628274207934737, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/27", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.073171708965674, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/27", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.850544458953664, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/28", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.56189095787704, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/28", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 5.070035791024566, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/28", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 12.039289542008191, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/28", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 6.36125212511979, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/29", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.738133875187486, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/29", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.099620875203982, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/29", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 10.868504459038377, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/29", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 9.283799208933488, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/30", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.982973208883777, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/30", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.522537709213793, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/30", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 20.120004875119776, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/30", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 7.978466792032123, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/31", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.95009870803915, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/31", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 6.357933208113536, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/31", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 11.082873583072796, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/31", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.775360457831994, "sub_calls": 2, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/32", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.888285791035742, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/32", "arm": "sonnet-only", "passed": true, "detail": "passed", "wall_seconds": 8.218812125036493, "sub_calls": 1, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/32", "arm": "echo-judge", "passed": true, "detail": "passed", "wall_seconds": 15.757697124965489, "sub_calls": 3, "benchmark": "bbh"} +{"task_id": "bbh/temporal_sequences/32", "arm": "echo-oracle", "passed": true, "detail": "passed", "wall_seconds": 8.347370957955718, "sub_calls": 2, "benchmark": "bbh"} From 1ed5770d54535e1042b0841f29255b1748101b4e Mon Sep 17 00:00:00 2001 From: Nick Meinhold Date: Thu, 2 Jul 2026 11:05:47 +1000 Subject: [PATCH 8/8] =?UTF-8?q?data(humaneval):=20clean-harness=20validati?= =?UTF-8?q?on=20=E2=80=94=20haiku-only=2063/64,=20matches=20blog?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rigorous check that the HumanEval headline predates/escaped the project-context contamination. Re-ran haiku-only over HumanEval 100-163 on the fixed harness (--setting-sources ""): 63/64 = 0.984, EXACTLY the blog's published baseline. An exact reproduction is only possible if the contamination never affected the HumanEval runs (the task-restore SessionStart hook postdated them; coherent high pass rates were incompatible with the contamination signature anyway). So the blog's HumanEval headline foundation is sound. (sonnet-only / echo-judge / echo-small-judge not yet re-run this pass — baseline match is the decisive contamination test.) Co-Authored-By: Claude Opus 4.8 (1M context) --- experiment/results/20260701T221948Z_n64.jsonl | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 experiment/results/20260701T221948Z_n64.jsonl diff --git a/experiment/results/20260701T221948Z_n64.jsonl b/experiment/results/20260701T221948Z_n64.jsonl new file mode 100644 index 0000000..cde6af5 --- /dev/null +++ b/experiment/results/20260701T221948Z_n64.jsonl @@ -0,0 +1,64 @@ +{"task_id": "HumanEval/100", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.243700708961114, "sub_calls": 1} +{"task_id": "HumanEval/101", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.539485833141953, "sub_calls": 1} +{"task_id": "HumanEval/102", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.03293270803988, "sub_calls": 1} +{"task_id": "HumanEval/103", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.53672012500465, "sub_calls": 1} +{"task_id": "HumanEval/104", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.408818915951997, "sub_calls": 1} +{"task_id": "HumanEval/105", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.911989874904975, "sub_calls": 1} +{"task_id": "HumanEval/106", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.756342833861709, "sub_calls": 1} +{"task_id": "HumanEval/107", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.621608417015523, "sub_calls": 1} +{"task_id": "HumanEval/108", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.273857625201344, "sub_calls": 1} +{"task_id": "HumanEval/109", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.007701166905463, "sub_calls": 1} +{"task_id": "HumanEval/110", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.055364083033055, "sub_calls": 1} +{"task_id": "HumanEval/111", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.319783915998414, "sub_calls": 1} +{"task_id": "HumanEval/112", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 133.42000912502408, "sub_calls": 1} +{"task_id": "HumanEval/113", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.838344416813925, "sub_calls": 1} +{"task_id": "HumanEval/114", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.241506582824513, "sub_calls": 1} +{"task_id": "HumanEval/115", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.517451124964282, "sub_calls": 1} +{"task_id": "HumanEval/116", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 24.631584583083168, "sub_calls": 1} +{"task_id": "HumanEval/117", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.787623165873811, "sub_calls": 1} +{"task_id": "HumanEval/118", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.26422179210931, "sub_calls": 1} +{"task_id": "HumanEval/119", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.859135332982987, "sub_calls": 1} +{"task_id": "HumanEval/120", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.950705457944423, "sub_calls": 1} +{"task_id": "HumanEval/121", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.8879195000045, "sub_calls": 1} +{"task_id": "HumanEval/122", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.340840166201815, "sub_calls": 1} +{"task_id": "HumanEval/123", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.236332707805559, "sub_calls": 1} +{"task_id": "HumanEval/124", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 20.33453462505713, "sub_calls": 1} +{"task_id": "HumanEval/125", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.525442374870181, "sub_calls": 1} +{"task_id": "HumanEval/126", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.662395334104076, "sub_calls": 1} +{"task_id": "HumanEval/127", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.183455666992813, "sub_calls": 1} +{"task_id": "HumanEval/128", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.543371124891564, "sub_calls": 1} +{"task_id": "HumanEval/129", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 38.034565082984045, "sub_calls": 1} +{"task_id": "HumanEval/130", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 20.914773917058483, "sub_calls": 1} +{"task_id": "HumanEval/131", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.313358583021909, "sub_calls": 1} +{"task_id": "HumanEval/132", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 43.435264457948506, "sub_calls": 1} +{"task_id": "HumanEval/133", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.833875084063038, "sub_calls": 1} +{"task_id": "HumanEval/134", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.732740124920383, "sub_calls": 1} +{"task_id": "HumanEval/135", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.712902458151802, "sub_calls": 1} +{"task_id": "HumanEval/136", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.986063041957095, "sub_calls": 1} +{"task_id": "HumanEval/137", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.668941458920017, "sub_calls": 1} +{"task_id": "HumanEval/138", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.399729375028983, "sub_calls": 1} +{"task_id": "HumanEval/139", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.810931750107557, "sub_calls": 1} +{"task_id": "HumanEval/140", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.120527041843161, "sub_calls": 1} +{"task_id": "HumanEval/141", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.800784458173439, "sub_calls": 1} +{"task_id": "HumanEval/142", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.908180417027324, "sub_calls": 1} +{"task_id": "HumanEval/143", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.829710042104125, "sub_calls": 1} +{"task_id": "HumanEval/144", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.2809189169202, "sub_calls": 1} +{"task_id": "HumanEval/145", "arm": "haiku-only", "passed": false, "detail": "AssertionError", "wall_seconds": 19.67869708291255, "sub_calls": 1} +{"task_id": "HumanEval/146", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.45333041716367, "sub_calls": 1} +{"task_id": "HumanEval/147", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 47.18500312510878, "sub_calls": 1} +{"task_id": "HumanEval/148", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.692784000188112, "sub_calls": 1} +{"task_id": "HumanEval/149", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.275010583922267, "sub_calls": 1} +{"task_id": "HumanEval/150", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.124026708072051, "sub_calls": 1} +{"task_id": "HumanEval/151", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 17.58009475003928, "sub_calls": 1} +{"task_id": "HumanEval/152", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.885027958080173, "sub_calls": 1} +{"task_id": "HumanEval/153", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.098198499996215, "sub_calls": 1} +{"task_id": "HumanEval/154", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.401176541112363, "sub_calls": 1} +{"task_id": "HumanEval/155", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.659992000088096, "sub_calls": 1} +{"task_id": "HumanEval/156", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.350211624987423, "sub_calls": 1} +{"task_id": "HumanEval/157", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.211205041967332, "sub_calls": 1} +{"task_id": "HumanEval/158", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.024867583997548, "sub_calls": 1} +{"task_id": "HumanEval/159", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.757393041159958, "sub_calls": 1} +{"task_id": "HumanEval/160", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.209359915927052, "sub_calls": 1} +{"task_id": "HumanEval/161", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.78537679114379, "sub_calls": 1} +{"task_id": "HumanEval/162", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.833883124869317, "sub_calls": 1} +{"task_id": "HumanEval/163", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.887018332956359, "sub_calls": 1}