feat(bbh): resilient sweep — stream results, abort-on-exhaustion, auto-resume

experiment/chat_claude_code.py

@@ -67,7 +67,18 @@ def _generate(
         prompt = self._format(messages)
         try:
             result = subprocess.run(
-                [self.binary, "--print", "--model", self.model, prompt],
+                # --setting-sources "" loads NONE of {user, project, local}
+                # settings. Without it, `claude --print` run from inside a
+                # project dir inherits that project's CLAUDE.md, SessionStart
+                # hooks, and any task-restore injection — so the model answers
+                # AS the project agent ("All 8 tasks restored...") instead of
+                # the benchmark question. That contaminated every BBH sweep
+                # before 2026-07-01 (#1305): inflated wall times (loading a
+                # huge CLAUDE.md), unparseable agent-preamble outputs, and
+                # confounded accuracy. Empty sources keeps the Max OAuth auth
+                # (unlike --bare, which drops it) while stripping all priming.
+                [self.binary, "--print", "--setting-sources", "",
+                 "--model", self.model, prompt],
                 capture_output=True,
                 text=True,
                 timeout=self.timeout,

experiment/results/20260701T221948Z_n64.jsonl

@@ -0,0 +1,64 @@
+{"task_id": "HumanEval/100", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.243700708961114, "sub_calls": 1}
+{"task_id": "HumanEval/101", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.539485833141953, "sub_calls": 1}
+{"task_id": "HumanEval/102", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.03293270803988, "sub_calls": 1}
+{"task_id": "HumanEval/103", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 14.53672012500465, "sub_calls": 1}
+{"task_id": "HumanEval/104", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.408818915951997, "sub_calls": 1}
+{"task_id": "HumanEval/105", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.911989874904975, "sub_calls": 1}
+{"task_id": "HumanEval/106", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.756342833861709, "sub_calls": 1}
+{"task_id": "HumanEval/107", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.621608417015523, "sub_calls": 1}
+{"task_id": "HumanEval/108", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.273857625201344, "sub_calls": 1}
+{"task_id": "HumanEval/109", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.007701166905463, "sub_calls": 1}
+{"task_id": "HumanEval/110", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.055364083033055, "sub_calls": 1}
+{"task_id": "HumanEval/111", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.319783915998414, "sub_calls": 1}
+{"task_id": "HumanEval/112", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 133.42000912502408, "sub_calls": 1}
+{"task_id": "HumanEval/113", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.838344416813925, "sub_calls": 1}
+{"task_id": "HumanEval/114", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.241506582824513, "sub_calls": 1}
+{"task_id": "HumanEval/115", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.517451124964282, "sub_calls": 1}
+{"task_id": "HumanEval/116", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 24.631584583083168, "sub_calls": 1}
+{"task_id": "HumanEval/117", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.787623165873811, "sub_calls": 1}
+{"task_id": "HumanEval/118", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.26422179210931, "sub_calls": 1}
+{"task_id": "HumanEval/119", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.859135332982987, "sub_calls": 1}
+{"task_id": "HumanEval/120", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.950705457944423, "sub_calls": 1}
+{"task_id": "HumanEval/121", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.8879195000045, "sub_calls": 1}
+{"task_id": "HumanEval/122", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.340840166201815, "sub_calls": 1}
+{"task_id": "HumanEval/123", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.236332707805559, "sub_calls": 1}
+{"task_id": "HumanEval/124", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 20.33453462505713, "sub_calls": 1}
+{"task_id": "HumanEval/125", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.525442374870181, "sub_calls": 1}
+{"task_id": "HumanEval/126", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.662395334104076, "sub_calls": 1}
+{"task_id": "HumanEval/127", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 15.183455666992813, "sub_calls": 1}
+{"task_id": "HumanEval/128", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 13.543371124891564, "sub_calls": 1}
+{"task_id": "HumanEval/129", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 38.034565082984045, "sub_calls": 1}
+{"task_id": "HumanEval/130", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 20.914773917058483, "sub_calls": 1}
+{"task_id": "HumanEval/131", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.313358583021909, "sub_calls": 1}
+{"task_id": "HumanEval/132", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 43.435264457948506, "sub_calls": 1}
+{"task_id": "HumanEval/133", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.833875084063038, "sub_calls": 1}
+{"task_id": "HumanEval/134", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.732740124920383, "sub_calls": 1}
+{"task_id": "HumanEval/135", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.712902458151802, "sub_calls": 1}
+{"task_id": "HumanEval/136", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.986063041957095, "sub_calls": 1}
+{"task_id": "HumanEval/137", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.668941458920017, "sub_calls": 1}
+{"task_id": "HumanEval/138", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.399729375028983, "sub_calls": 1}
+{"task_id": "HumanEval/139", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.810931750107557, "sub_calls": 1}
+{"task_id": "HumanEval/140", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.120527041843161, "sub_calls": 1}
+{"task_id": "HumanEval/141", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 12.800784458173439, "sub_calls": 1}
+{"task_id": "HumanEval/142", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 10.908180417027324, "sub_calls": 1}
+{"task_id": "HumanEval/143", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.829710042104125, "sub_calls": 1}
+{"task_id": "HumanEval/144", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.2809189169202, "sub_calls": 1}
+{"task_id": "HumanEval/145", "arm": "haiku-only", "passed": false, "detail": "AssertionError", "wall_seconds": 19.67869708291255, "sub_calls": 1}
+{"task_id": "HumanEval/146", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 16.45333041716367, "sub_calls": 1}
+{"task_id": "HumanEval/147", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 47.18500312510878, "sub_calls": 1}
+{"task_id": "HumanEval/148", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.692784000188112, "sub_calls": 1}
+{"task_id": "HumanEval/149", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.275010583922267, "sub_calls": 1}
+{"task_id": "HumanEval/150", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.124026708072051, "sub_calls": 1}
+{"task_id": "HumanEval/151", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 17.58009475003928, "sub_calls": 1}
+{"task_id": "HumanEval/152", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 8.885027958080173, "sub_calls": 1}
+{"task_id": "HumanEval/153", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.098198499996215, "sub_calls": 1}
+{"task_id": "HumanEval/154", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.401176541112363, "sub_calls": 1}
+{"task_id": "HumanEval/155", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.659992000088096, "sub_calls": 1}
+{"task_id": "HumanEval/156", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.350211624987423, "sub_calls": 1}
+{"task_id": "HumanEval/157", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 11.211205041967332, "sub_calls": 1}
+{"task_id": "HumanEval/158", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.024867583997548, "sub_calls": 1}
+{"task_id": "HumanEval/159", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 6.757393041159958, "sub_calls": 1}
+{"task_id": "HumanEval/160", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 9.209359915927052, "sub_calls": 1}
+{"task_id": "HumanEval/161", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.78537679114379, "sub_calls": 1}
+{"task_id": "HumanEval/162", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 5.833883124869317, "sub_calls": 1}
+{"task_id": "HumanEval/163", "arm": "haiku-only", "passed": true, "detail": "passed", "wall_seconds": 7.887018332956359, "sub_calls": 1}

experiment/scripts/gen_progress.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Render an in-progress BBH sweep JSONL into a self-contained live status page.
+
+Usage: gen_progress.py <jsonl> <out.html> <start_epoch> <now_epoch>
+
+Reads the streaming results file (meta line + one row per completed run),
+computes overall + per-arm progress, pass rates, cost units, and a wall-clock
+ETA, and writes a single auto-refreshing HTML file (no external assets).
+"""
+from __future__ import annotations
+
+import json
+import sys
+from collections import defaultdict
+
+HAIKU_UNIT, SONNET_UNIT = 1, 3
+
+
+def cost_units(arm: str, sub_calls: int) -> int:
+    if arm == "haiku-only":
+        return sub_calls * HAIKU_UNIT
+    if arm == "sonnet-only":
+        return sub_calls * SONNET_UNIT
+    if arm == "echo-judge":
+        return 3 * HAIKU_UNIT + (SONNET_UNIT if sub_calls > 3 else 0)
+    if arm == "echo-oracle":
+        return 2 * HAIKU_UNIT + (SONNET_UNIT if sub_calls > 2 else 0)
+    return sub_calls * HAIKU_UNIT
+
+
+def _fmt_dur(secs: float) -> str:
+    secs = max(0, int(secs))
+    h, m = secs // 3600, (secs % 3600) // 60
+    if h:
+        return f"{h}h {m}m"
+    return f"{m}m {secs % 60}s"
+
+
+def main() -> None:
+    jsonl, out_html, start_epoch, now_epoch = sys.argv[1], sys.argv[2], float(sys.argv[3]), float(sys.argv[4])
+
+    meta = {}
+    rows = []
+    try:
+        with open(jsonl) as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                r = json.loads(line)
+                if r.get("_meta"):
+                    meta = r
+                else:
+                    rows.append(r)
+    except FileNotFoundError:
+        rows = []
+
+    arms = meta.get("arms", [])
+    subtasks = meta.get("subtasks", [])
+    n_per = meta.get("n_per_subtask", 0)
+    n_tasks = len(subtasks) * n_per if subtasks else 0
+    total = n_tasks * len(arms) if arms else 0
+    done = len(rows)
+
+    by_arm = defaultdict(list)
+    for r in rows:
+        by_arm[r["arm"]].append(r)
+
+    elapsed = now_epoch - start_epoch
+    rate = done / elapsed if elapsed > 0 and done else 0  # runs/sec
+    eta = (total - done) / rate if rate > 0 else 0
+    pct = (100 * done / total) if total else 0
+
+    # Per-arm rows
+    arm_cards = []
+    for arm in arms:
+        rs = by_arm.get(arm, [])
+        a_done = len(rs)
+        a_total = n_tasks
+        a_pass = sum(1 for r in rs if r["passed"])
+        a_unp = sum(1 for r in rs if r["detail"] == "unparseable")
+        thr = 3 if arm == "echo-judge" else 2
+        a_esc = sum(1 for r in rs if r["sub_calls"] > thr)
+        a_cost = sum(cost_units(arm, r["sub_calls"]) for r in rs)
+        rate_s = f"{a_pass}/{a_done} ({a_pass / a_done:.3f})" if a_done else "-"
+        a_pct = (100 * a_done / a_total) if a_total else 0
+        arm_cards.append(f"""
+      <div class="arm">
+        <div class="arm-head"><span class="arm-name">{arm}</span><span class="arm-prog">{a_done}/{a_total}</span></div>
+        <div class="bar"><div class="fill" style="width:{a_pct:.1f}%"></div></div>
+        <div class="arm-stats">
+          <span>pass <b>{rate_s}</b></span>
+          <span>esc <b>{a_esc}</b></span>
+          <span>cost <b>{a_cost}u</b></span>
+          <span class="{'warn' if a_unp else ''}">unparseable <b>{a_unp}</b></span>
+        </div>
+      </div>""")
+
+    status = "RUNNING" if done < total else "COMPLETE"
+    status_cls = "running" if done < total else "complete"
+    eta_str = "done" if done >= total else (_fmt_dur(eta) if rate > 0 else "estimating...")
+
+    html = f"""<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<meta http-equiv="refresh" content="20">
+<title>Echo BBH n={n_tasks} - live</title>
+<style>
+  :root {{ color-scheme: dark; }}
+  body {{ font-family: ui-monospace, SFMono-Regular, Menlo, monospace; background:#0d1117; color:#e6edf3; margin:0; padding:2rem 1rem; }}
+  .wrap {{ max-width:760px; margin:0 auto; }}
+  h1 {{ font-size:1.15rem; font-weight:600; margin:0 0 .25rem; }}
+  .sub {{ color:#7d8590; font-size:.8rem; margin-bottom:1.25rem; }}
+  .badge {{ display:inline-block; padding:.15rem .55rem; border-radius:999px; font-size:.72rem; font-weight:700; letter-spacing:.04em; }}
+  .running {{ background:#1f6feb33; color:#58a6ff; }}
+  .complete {{ background:#23863633; color:#3fb950; }}
+  .overall {{ background:#161b22; border:1px solid #30363d; border-radius:10px; padding:1.1rem 1.2rem; margin-bottom:1.25rem; }}
+  .big {{ font-size:2rem; font-weight:700; }}
+  .big small {{ font-size:1rem; color:#7d8590; font-weight:400; }}
+  .bar {{ height:9px; background:#21262d; border-radius:6px; overflow:hidden; margin:.6rem 0; }}
+  .fill {{ height:100%; background:linear-gradient(90deg,#1f6feb,#3fb950); transition:width .4s; }}
+  .meta {{ display:flex; gap:1.5rem; flex-wrap:wrap; color:#7d8590; font-size:.8rem; }}
+  .meta b {{ color:#e6edf3; }}
+  .arm {{ background:#161b22; border:1px solid #30363d; border-radius:8px; padding:.7rem .9rem; margin-bottom:.6rem; }}
+  .arm-head {{ display:flex; justify-content:space-between; align-items:center; }}
+  .arm-name {{ font-weight:600; }}
+  .arm-prog {{ color:#7d8590; font-size:.8rem; }}
+  .arm-stats {{ display:flex; gap:1.1rem; flex-wrap:wrap; font-size:.78rem; color:#7d8590; margin-top:.35rem; }}
+  .arm-stats b {{ color:#e6edf3; }}
+  .warn b {{ color:#d29922; }}
+  .foot {{ color:#484f58; font-size:.72rem; margin-top:1.5rem; text-align:center; }}
+</style>
+</head>
+<body>
+<div class="wrap">
+  <h1>Echo - cheap-routing on BBH <span class="badge {status_cls}">{status}</span></h1>
+  <div class="sub">n={n_tasks} tasks &middot; {', '.join(subtasks)} &middot; Claude Max (zero API spend)</div>
+
+  <div class="overall">
+    <div class="big">{done}<small> / {total} runs</small> &nbsp; <small>{pct:.1f}%</small></div>
+    <div class="bar"><div class="fill" style="width:{pct:.1f}%"></div></div>
+    <div class="meta">
+      <span>elapsed <b>{_fmt_dur(elapsed)}</b></span>
+      <span>eta <b>{eta_str}</b></span>
+      <span>pace <b>{rate * 60:.1f}/min</b></span>
+    </div>
+  </div>
+
+  {''.join(arm_cards)}
+
+  <div class="foot">auto-refreshes every 20s &middot; streaming from the runner's JSONL &middot; updated each cycle</div>
+</div>
+</body>
+</html>"""
+
+    with open(out_html, "w") as f:
+        f.write(html)
+    print(f"wrote {out_html}: {done}/{total} ({pct:.1f}%)")
+
+
+if __name__ == "__main__":
+    main()

experiment/scripts/publish_progress.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Live-publish loop: regenerate the BBH progress page from the streaming JSONL
+# and force-push it to the gh-pages branch every INTERVAL seconds, until the
+# tracked run PID exits. Operates entirely inside an isolated gh-pages worktree
+# so the main working tree (where the run lives) is never touched.
+#
+# Usage: publish_progress.sh <run_pid> <results_glob_dir> <pages_worktree> <start_epoch>
+set -uo pipefail
+
+RUN_PID="$1"
+RESULTS_DIR="$2"
+PAGES_DIR="$3"
+START_EPOCH="$4"
+INTERVAL="${INTERVAL:-300}"  # GitHub Pages soft-limits ~10 builds/hr; 5min = 12/hr
+REPO="/Users/nick/git/research/echo"
+PY="$REPO/experiment/.venv/bin/python"
+GEN="$REPO/experiment/scripts/gen_progress.py"
+
+publish() {
+  local f now
+  f="$(ls -t "$RESULTS_DIR"/*_bbh_n99.jsonl 2>/dev/null | head -1)"
+  [ -z "$f" ] && return 0
+  now="$(date +%s)"
+  "$PY" "$GEN" "$f" "$PAGES_DIR/index.html" "$START_EPOCH" "$now" >/dev/null 2>&1 || return 0
+  touch "$PAGES_DIR/.nojekyll"
+  ( cd "$PAGES_DIR" \
+    && git add index.html .nojekyll \
+    && git -c user.name="Nick Meinhold" -c user.email="langer.robin@gmail.com" \
+         commit -q --amend --no-edit \
+    && git push -q -f origin gh-pages ) >/dev/null 2>&1
+}
+
+# Update while the run is alive, then one final update to flip to COMPLETE.
+while kill -0 "$RUN_PID" 2>/dev/null; do
+  publish
+  sleep "$INTERVAL"
+done
+publish
+echo "publish loop done (run $RUN_PID exited)"

experiment/scripts/repro_sonnet_unparse.py

@@ -0,0 +1,51 @@
+"""Reproduce the Sonnet unparseable BBH outputs (#1305) and dump raw text.
+
+Loads the specific tasks that failed in the n=30 sweep, runs sonnet-only,
+and prints the raw model output + what extract_choice / score_bbh return.
+We capture WHY it's unparseable before touching the parser.
+"""
+
+from __future__ import annotations
+
+import sys
+import time
+
+from benchmarks.bbh import extract_choice, load_bbh, score_bbh
+from benchmarks.bbh_arms import arm_sonnet_only
+
+# (subtask, index) pairs that came back unparseable from Sonnet in the n=30/n=99 sweeps.
+TARGETS = [
+    ("causal_judgement", 3),
+    ("causal_judgement", 7),
+    ("date_understanding", 5),
+]
+
+
+def main() -> None:
+    # Load enough of each subtask to cover the target indices.
+    subtasks = sorted({s for s, _ in TARGETS})
+    tasks = load_bbh(subtasks, n_per_subtask=10, start=0)
+    by_id = {t["task_id"]: t for t in tasks}
+
+    for subtask, idx in TARGETS:
+        tid = f"bbh/{subtask}/{idx}"
+        task = by_id.get(tid)
+        if task is None:
+            print(f"!! {tid} not loaded", flush=True)
+            continue
+        print("=" * 80, flush=True)
+        print(f"TASK {tid}  gold={task['gold']}  labels={task.get('choice_labels')}", flush=True)
+        t0 = time.time()
+        output, sub_calls = arm_sonnet_only(task)
+        wall = time.time() - t0
+        pred = extract_choice(output)
+        passed, detail = score_bbh(output, task)
+        print(f"wall={wall:.1f}s  extract_choice={pred!r}  score={passed} ({detail})", flush=True)
+        print(f"--- raw output ({len(output)} chars) ---", flush=True)
+        print(output, flush=True)
+        print(f"--- last 300 chars ---\n{output[-300:]!r}", flush=True)
+        print(flush=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())