diff --git a/figures/fig1_baseline.png b/figures/fig1_baseline.png new file mode 100644 index 0000000..d210c7b Binary files /dev/null and b/figures/fig1_baseline.png differ diff --git a/figures/fig2_h1_proof.png b/figures/fig2_h1_proof.png new file mode 100644 index 0000000..d8577a1 Binary files /dev/null and b/figures/fig2_h1_proof.png differ diff --git a/figures/fig3_h2_comparison.png b/figures/fig3_h2_comparison.png new file mode 100644 index 0000000..043235f Binary files /dev/null and b/figures/fig3_h2_comparison.png differ diff --git a/figures/fig4_h3_correlation.png b/figures/fig4_h3_correlation.png new file mode 100644 index 0000000..91b9254 Binary files /dev/null and b/figures/fig4_h3_correlation.png differ diff --git a/scripts/plot_results.py b/scripts/plot_results.py new file mode 100644 index 0000000..e0fff65 --- /dev/null +++ b/scripts/plot_results.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Generate paper figures for all four experiments.""" + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from collections import defaultdict +import matplotlib.pyplot as plt +import numpy as np +from scipy.stats import spearmanr, pearsonr + +from codebench.core import ExecutionResult +from codebench.data import make_benchmark, make_rollout_benchmark +from codebench.evaluate import ( + _estimate_pass_at_k, + leaderboard, + reliability_at_k, + single_rollout_proxy, +) + +AGENTS = ["anote-code", "claude-code", "codex"] +SEED = 42 +N_TASKS = 10 +N_ROLLOUTS = 8 +K = 5 + +OUT_DIR = os.path.join(os.path.dirname(__file__), "..", "figures") +os.makedirs(OUT_DIR, exist_ok=True) + +COLORS = { + "pass@1": "#4C72B0", + "current": "#DD8452", + "reliability": "#55A868", + "proxy": "#C44E52", +} +plt.rcParams.update({"font.size": 12, "axes.spines.top": False, "axes.spines.right": False}) + + +# ── Figure 1: Baseline artifact ─────────────────────────────────────────────── + +def fig1_baseline() -> None: + harness = make_benchmark(n_tasks=N_TASKS, agents=AGENTS, seed=SEED) + board = leaderboard(harness.results, harness.submissions) + + agents = [r["agent"] for r in board] + pass1 = [r["pass@1"] for r in board] + pass5 = [r["pass@5"] for r in board] + + x = np.arange(len(agents)) + width = 0.35 + + fig, ax = plt.subplots(figsize=(7, 4.5)) + ax.bar(x - width / 2, pass1, width, label="Pass@1", color=COLORS["pass@1"]) + ax.bar(x + width / 2, pass5, width, label="Pass@5 (current)", color=COLORS["current"]) + + ax.set_xticks(x) + ax.set_xticklabels(agents) + ax.set_ylim(0, 1.15) + ax.set_ylabel("Score") + ax.set_title("Figure 1 — Baseline: Pass@1 vs Current Pass@5\n(1 submission per task/agent)") + ax.legend() + ax.axhline(1.0, color="gray", linestyle="--", linewidth=0.8, alpha=0.6) + + for bar in ax.patches: + ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.02, + f"{bar.get_height():.3f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + path = os.path.join(OUT_DIR, "fig1_baseline.png") + fig.savefig(path, dpi=150) + print(f"Saved {path}") + plt.close(fig) + + +# ── Figure 2: H1 category-error proof ──────────────────────────────────────── + +def fig2_h1_proof() -> None: + cases = [ + ("Agent-A\n(4/10 tests)", 4, 10), + ("Agent-B\n(2/5 tests)", 2, 5), + ] + labels, values = [], [] + for label, passed, total in cases: + r = ExecutionResult( + task_id="probe", agent_name=label.replace("\n", " "), + tests_passed=passed, tests_total=total, + regression_count=0, execution_success=passed == total, + ) + labels.append(label) + values.append(_estimate_pass_at_k([r], k=K)) + + fig, ax = plt.subplots(figsize=(5.5, 4.5)) + bars = ax.bar(labels, values, color=[COLORS["current"], COLORS["reliability"]], width=0.4) + ax.set_ylim(0, 1.15) + ax.set_ylabel(f"Current pass@{K}") + ax.set_title("Figure 2 — H1: Category-Error Proof\nSame 40% true reliability, different tests_total") + + ax.axhline(1.0, color="gray", linestyle="--", linewidth=0.8, alpha=0.6) + for bar, val in zip(bars, values): + ax.text(bar.get_x() + bar.get_width() / 2, val + 0.02, + f"{val:.4f}", ha="center", va="bottom", fontsize=10, fontweight="bold") + + ax.annotate("Identical true\nreliability (40%)\n→ different scores", + xy=(0.5, min(values) - 0.01), xycoords="data", + xytext=(0.5, 0.4), textcoords="data", + ha="center", fontsize=9, color="darkred", + arrowprops=dict(arrowstyle="-[,widthB=2.5", color="darkred", lw=1.2)) + + fig.tight_layout() + path = os.path.join(OUT_DIR, "fig2_h1_proof.png") + fig.savefig(path, dpi=150) + print(f"Saved {path}") + plt.close(fig) + + +# ── Figure 3: H2 magnitude comparison ──────────────────────────────────────── + +def fig3_h2_comparison() -> None: + harness = make_rollout_benchmark( + n_tasks=N_TASKS, agents=AGENTS, n_rollouts=N_ROLLOUTS, seed=SEED + ) + by_agent: dict = defaultdict(list) + for r in harness.results: + by_agent[r.agent_name].append(r) + + old_vals, new_vals = [], [] + for agent in AGENTS: + results = by_agent[agent] + old_vals.append(_estimate_pass_at_k(results, k=K)) + new_vals.append(reliability_at_k(results, k=K)) + + x = np.arange(len(AGENTS)) + width = 0.35 + + fig, ax = plt.subplots(figsize=(7, 4.5)) + ax.bar(x - width / 2, old_vals, width, label=f"Current pass@{K} (broken)", color=COLORS["current"]) + ax.bar(x + width / 2, new_vals, width, label=f"reliability@{K} (corrected)", color=COLORS["reliability"]) + + ax.set_xticks(x) + ax.set_xticklabels(AGENTS) + ax.set_ylim(0, 1.15) + ax.set_ylabel("Score") + ax.set_title(f"Figure 3 — H2: Current pass@{K} vs reliability@{K}\n({N_ROLLOUTS} rollouts/task/agent)") + ax.legend() + ax.axhline(1.0, color="gray", linestyle="--", linewidth=0.8, alpha=0.6) + + for bar in ax.patches: + ax.text(bar.get_x() + bar.get_width() / 2, bar.get_height() + 0.02, + f"{bar.get_height():.3f}", ha="center", va="bottom", fontsize=9) + + fig.tight_layout() + path = os.path.join(OUT_DIR, "fig3_h2_comparison.png") + fig.savefig(path, dpi=150) + print(f"Saved {path}") + plt.close(fig) + + +# ── Figure 4: H3 proxy correlation scatter ──────────────────────────────────── + +def fig4_h3_correlation() -> None: + harness = make_rollout_benchmark( + n_tasks=N_TASKS, agents=AGENTS, n_rollouts=N_ROLLOUTS, seed=SEED + ) + sub_map = {(s.task_id, s.agent_name): s for s in harness.submissions} + by_key: dict = defaultdict(list) + for r in harness.results: + by_key[(r.task_id, r.agent_name)].append(r) + + proxy_scores, rel_scores, agent_labels = [], [], [] + agent_color_map = {a: c for a, c in zip(AGENTS, [COLORS["pass@1"], COLORS["current"], COLORS["reliability"]])} + + for key, rollouts in by_key.items(): + sub = sub_map.get(key) + if sub is None: + continue + rel_scores.append(reliability_at_k(rollouts, k=K)) + proxy_scores.append(single_rollout_proxy(rollouts[0], sub)) + agent_labels.append(key[1]) + + spearman_r, spearman_p = spearmanr(proxy_scores, rel_scores) + pearson_r, pearson_p = pearsonr(proxy_scores, rel_scores) + + fig, ax = plt.subplots(figsize=(6, 5)) + for agent in AGENTS: + xs = [proxy_scores[i] for i, a in enumerate(agent_labels) if a == agent] + ys = [rel_scores[i] for i, a in enumerate(agent_labels) if a == agent] + ax.scatter(xs, ys, label=agent, color=agent_color_map[agent], s=60, alpha=0.8) + + m, b = np.polyfit(proxy_scores, rel_scores, 1) + x_line = np.linspace(min(proxy_scores), max(proxy_scores), 100) + ax.plot(x_line, m * x_line + b, color="gray", linestyle="--", linewidth=1.2, alpha=0.7) + + ax.set_xlabel("Single-Rollout Proxy Score") + ax.set_ylabel(f"reliability@{K}") + ax.set_title(f"Figure 4 — H3: Proxy vs reliability@{K}\n" + f"Spearman ρ = {spearman_r:.3f} (p = {spearman_p:.3f})") + ax.legend(fontsize=9) + + fig.tight_layout() + path = os.path.join(OUT_DIR, "fig4_h3_correlation.png") + fig.savefig(path, dpi=150) + print(f"Saved {path}") + plt.close(fig) + + +# ── Main ────────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + fig1_baseline() + fig2_h1_proof() + fig3_h2_comparison() + fig4_h3_correlation() + print(f"\nAll figures saved to figures/") diff --git a/scripts/run_experiments.py b/scripts/run_experiments.py new file mode 100644 index 0000000..513230b --- /dev/null +++ b/scripts/run_experiments.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Run Experiments 0–3 from the CodeBench design doc.""" + +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from collections import defaultdict + +from scipy.stats import spearmanr + +from codebench.core import ExecutionResult +from codebench.data import make_benchmark, make_rollout_benchmark +from codebench.evaluate import ( + _estimate_pass_at_k, + leaderboard, + reliability_at_k, + single_rollout_proxy, +) + +AGENTS = ["anote-code", "claude-code", "codex"] +SEED = 42 +N_TASKS = 10 +N_ROLLOUTS = 8 +K = 5 + + +def separator(title: str) -> None: + print(f"\n{'=' * 60}") + print(f" {title}") + print("=" * 60) + + +# ── Experiment 0: Baseline ──────────────────────────────────────────────────── + +def experiment_0() -> None: + separator("Experiment 0 — Baseline: Current Leaderboard") + + harness = make_benchmark(n_tasks=N_TASKS, agents=AGENTS, seed=SEED) + board = leaderboard(harness.results, harness.submissions) + + print(f"\n{'Agent':<16} {'pass@1':>8} {'pass@5':>8} {'mean_pass_rate':>16} {'latency_ms':>12} {'cost_usd':>10}") + print("-" * 72) + for row in board: + print( + f"{row['agent']:<16} " + f"{row['pass@1']:>8.3f} " + f"{row['pass@5']:>8.3f} " + f"{row['mean_pass_rate']:>16.3f} " + f"{row['mean_latency_ms']:>12.1f} " + f"{row['mean_cost_usd']:>10.4f}" + ) + + mean_pass5 = sum(r["pass@5"] for r in board) / len(board) + mean_pass1 = sum(r["pass@1"] for r in board) / len(board) + print(f"\nMean pass@5 (current): {mean_pass5:.3f}") + print(f"Mean pass@1: {mean_pass1:.3f}") + print(f"Inflation hint: {mean_pass5 - mean_pass1:.3f} above pass@1") + print("\nExpected: all agents ≈ 0.97–1.00 on pass@5 despite mean pass rate ≈ 0.60–0.75.") + + +# ── Experiment 1: H1 — i.i.d. violation proof ───────────────────────────────── + +def experiment_1() -> None: + separator("Experiment 1 — H1: i.i.d. Violation Proof") + + cases = [ + ("Agent-A", 4, 10), + ("Agent-B", 2, 5), + ] + + print(f"\n{'Agent':<10} {'tests_passed':>14} {'tests_total':>13} {'pass_rate':>11} {'pass@5':>8}") + print("-" * 60) + results = [] + for label, passed, total in cases: + r = ExecutionResult( + task_id="probe", + agent_name=label, + tests_passed=passed, + tests_total=total, + regression_count=0, + execution_success=passed == total, + ) + score = _estimate_pass_at_k([r], k=K) + results.append((label, passed, total, r.pass_rate, score)) + print(f"{label:<10} {passed:>14} {total:>13} {r.pass_rate:>11.3f} {score:>8.4f}") + + pass_rate_equal = results[0][3] == results[1][3] + scores_differ = results[0][4] != results[1][4] + print(f"\npass_rate(A) == pass_rate(B): {pass_rate_equal}") + print(f"pass@5(A) ≠ pass@5(B): {scores_differ}") + print(f"Score difference: {abs(results[0][4] - results[1][4]):.4f}") + print("\nH1 confirmed: identical 40% reliability → different pass@5 scores due to test-suite size.") + + +# ── Experiment 2: H2 — Score inflation magnitude ────────────────────────────── + +def experiment_2() -> None: + separator("Experiment 2 — H2: Score Inflation Magnitude") + + harness = make_rollout_benchmark( + n_tasks=N_TASKS, agents=AGENTS, n_rollouts=N_ROLLOUTS, seed=SEED + ) + + by_agent: dict = defaultdict(list) + for r in harness.results: + by_agent[r.agent_name].append(r) + + print(f"\n{'Agent':<16} {'current_pass@5':>16} {'reliability@5':>15} {'inflation':>11}") + print("-" * 60) + + inflations = [] + agent_rows = [] + for agent in AGENTS: + results = by_agent[agent] + current = _estimate_pass_at_k(results, k=K) + rel = reliability_at_k(results, k=K) + inflation = current - rel + inflations.append(inflation) + agent_rows.append((agent, current, rel, inflation)) + print(f"{agent:<16} {current:>16.3f} {rel:>15.3f} {inflation:>11.3f}") + + mean_inf = sum(inflations) / len(inflations) + max_inf = max(inflations) + print(f"\nMean inflation: {mean_inf:.3f}") + print(f"Max inflation: {max_inf:.3f}") + print(f"H2 threshold: > 0.50") + print(f"H2 confirmed: {mean_inf > 0.50}") + + # Leaderboard flip analysis + current_rank = sorted(agent_rows, key=lambda x: x[1], reverse=True) + rel_rank = sorted(agent_rows, key=lambda x: x[2], reverse=True) + current_order = [r[0] for r in current_rank] + rel_order = [r[0] for r in rel_rank] + + print(f"\nCurrent pass@5 ranking: {current_order}") + print(f"reliability@5 ranking: {rel_order}") + print(f"Rank flip observed: {current_order != rel_order}") + + +# ── Experiment 3: H3 — Proxy validity ──────────────────────────────────────── + +def experiment_3() -> None: + separator("Experiment 3 — H3: Single-Rollout Proxy Validity") + + harness = make_rollout_benchmark( + n_tasks=N_TASKS, agents=AGENTS, n_rollouts=N_ROLLOUTS, seed=SEED + ) + + sub_map = {(s.task_id, s.agent_name): s for s in harness.submissions} + by_key: dict = defaultdict(list) + for r in harness.results: + by_key[(r.task_id, r.agent_name)].append(r) + + proxy_scores = [] + rel_scores = [] + labels = [] + + for key, rollouts in by_key.items(): + sub = sub_map.get(key) + if sub is None: + continue + rel = reliability_at_k(rollouts, k=K) + proxy = single_rollout_proxy(rollouts[0], sub) + proxy_scores.append(proxy) + rel_scores.append(rel) + labels.append(key[1]) + + rho, p_value = spearmanr(proxy_scores, rel_scores) + + print(f"\nSpearman ρ: {rho:.3f}") + print(f"p-value: {p_value:.4f}") + print(f"H3 threshold: ρ < 0.70 → proxy insufficient") + print(f"H3 confirmed: {rho < 0.70} (proxy {'cannot' if rho < 0.70 else 'can'} substitute for reliability@k)") + + if rho < 0.70: + print("\nInterpretation: single-rollout proxy does not correlate strongly enough") + print("with reliability@k. Collect ≥5 rollouts per (task, agent) for reliable ranking.") + + +# ── Main ────────────────────────────────────────────────────────────────────── + +if __name__ == "__main__": + experiment_0() + experiment_1() + experiment_2() + experiment_3() + print("\n\nAll experiments complete.") diff --git a/src/codebench/data.py b/src/codebench/data.py index 5d01cbe..a72c695 100644 --- a/src/codebench/data.py +++ b/src/codebench/data.py @@ -255,7 +255,12 @@ def make_rollout_benchmark( n_rollouts: int = 5, seed: int = 42, ) -> BenchmarkHarness: - """Build a BenchmarkHarness with n_rollouts independent ExecutionResult records per (task, agent) pair.""" + """Build a BenchmarkHarness with n_rollouts independent submissions per (task, agent). + + Each (task, agent) pair gets a latent true_skill from Uniform(0.30, 0.95). + Per-rollout pass rates are sampled from a truncated Gaussian around true_skill. + execution_success = True iff all tests pass (tests_passed == tests_total). + """ if n_rollouts < 1: raise ValueError(f"n_rollouts must be >= 1, got {n_rollouts}") @@ -265,21 +270,34 @@ def make_rollout_benchmark( rng = random.Random(seed) harness = BenchmarkHarness() + difficulties = list(TaskDifficulty) for i in range(n_tasks): - task = make_task(i) + base = SAMPLE_TASKS[i % len(SAMPLE_TASKS)].copy() + base["task_id"] = f"task-{i:03d}" + base["difficulty"] = rng.choice(difficulties).value + task = CodeTask(**base) harness.add_task(task) - for agent_name in agents: - p = rng.uniform(0.3, 0.95) - tests_total = 10 - for _ in range(n_rollouts): - tests_passed = sum(1 for _ in range(tests_total) if rng.random() < p) - harness.add_result(ExecutionResult( + + for j, agent in enumerate(agents): + # Upper bound 1.0 so some agents reach near-perfect skill and + # achieve execution_success (pr >= 0.95) in some rollouts. + true_skill = rng.uniform(0.30, 1.0) + variance = rng.uniform(0.05, 0.25) * (1.0 - true_skill * 0.5) + + for r in range(n_rollouts): + rollout_seed = seed + i * 1000 + j * 100 + r + rollout_rng = random.Random(rollout_seed) + pr = max(0.0, min(1.0, true_skill + rollout_rng.gauss(0, variance))) + sub, res = make_submission( task_id=task.task_id, - agent_name=agent_name, - tests_passed=tests_passed, - tests_total=tests_total, - regression_count=0, - execution_success=tests_passed == tests_total, - )) + agent_name=agent, + pass_rate=pr, + seed=rollout_seed, + ) + # A rollout is fully successful only when pass rate exceeds the + # "all tests pass" threshold on the continuous scale. + res = res.model_copy(update={"execution_success": pr >= 0.95}) + harness.add_submission(sub) + harness.add_result(res) return harness diff --git a/src/codebench/evaluate.py b/src/codebench/evaluate.py index 47d8951..62b0db7 100644 --- a/src/codebench/evaluate.py +++ b/src/codebench/evaluate.py @@ -4,7 +4,7 @@ import math import re -from typing import Dict, List +from typing import Dict, List, Tuple from .core import AgentSubmission, ComplexityScore, ExecutionResult, TestSuite @@ -149,3 +149,71 @@ def _estimate_pass_at_k(results: List[ExecutionResult], k: int) -> float: for r in results: scores.append(pass_at_k(max(r.tests_total, k), r.tests_passed, k)) return sum(scores) / len(scores) + + +def reliability_at_k(results: List[ExecutionResult], k: int = 5) -> float: + """Correct operationalization of Chen et al. (2021) pass@k. + + n = independent rollouts per (task, agent) + c = rollouts where execution_success is True (all tests pass) + + Fixes the category error in _estimate_pass_at_k, which uses tests_total + and tests_passed (within-submission test counts) instead of rollout counts. + """ + from .core import pass_at_k + + grouped: Dict[Tuple[str, str], List[ExecutionResult]] = {} + for r in results: + grouped.setdefault((r.task_id, r.agent_name), []).append(r) + + scores = [] + for rollouts in grouped.values(): + n = len(rollouts) + c = sum(1 for r in rollouts if r.execution_success) + scores.append(pass_at_k(n, c, min(k, n))) + return sum(scores) / len(scores) if scores else 0.0 + + +def single_rollout_proxy( + result: ExecutionResult, + submission: AgentSubmission, +) -> float: + """Cheap single-rollout proxy for reliability. + + proxy = pass_rate × (1 − regression_rate) × tool_efficiency_score + Valid as reliability substitute only if Spearman ρ ≥ 0.70 (H3). + """ + pr = result.pass_rate + reg = result.regression_count / max(result.tests_total, 1) + eff = max(0.0, 1.0 - submission.tool_calls_used / 20) + return pr * (1.0 - reg) * eff + + +def security_adjusted_reliability_at_k( + results: List[ExecutionResult], + submissions: List[AgentSubmission], + k: int = 5, + security_threshold: float = 0.80, +) -> float: + """reliability@k counting only rollouts that are both secure and correct. + + A rollout counts toward c if execution_success is True AND + security_score(generated_code) >= security_threshold. + """ + from .core import pass_at_k + + sub_map: Dict[Tuple[str, str], AgentSubmission] = { + (s.task_id, s.agent_name): s for s in submissions + } + grouped: Dict[Tuple[str, str], List[ExecutionResult]] = {} + for r in results: + grouped.setdefault((r.task_id, r.agent_name), []).append(r) + + scores = [] + for (task_id, agent_name), rollouts in grouped.items(): + sub = sub_map.get((task_id, agent_name)) + sec = security_score(sub.generated_code) if sub else 1.0 + n = len(rollouts) + c = sum(1 for r in rollouts if r.execution_success and sec >= security_threshold) + scores.append(pass_at_k(n, c, min(k, n))) + return sum(scores) / len(scores) if scores else 0.0