From 29b3590b2978001dae71c0e864bd61660d639d9c Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:13:22 -0700 Subject: [PATCH 1/5] Preserve scored terminal model failures --- docs/prime-native-benchmarks.md | 24 ++- .../prime-scorecard/build-scorecard.mjs | 64 ++++++-- src/prime-benchmark-import.ts | 43 ++++-- src/prime-benchmark-runner.ts | 18 ++- src/prime-trace-contract.ts | 107 +++++++++++++ tests/prime-benchmark-import.test.mjs | 143 +++++++++++++++++- 6 files changed, 353 insertions(+), 46 deletions(-) create mode 100644 src/prime-trace-contract.ts diff --git a/docs/prime-native-benchmarks.md b/docs/prime-native-benchmarks.md index 31bf7ad0..6b2de5b2 100644 --- a/docs/prime-native-benchmarks.md +++ b/docs/prime-native-benchmarks.md @@ -145,13 +145,23 @@ Prime verifier version, model, run, and task. Resume runs only missing or invali tasks. Provider and deployment must be resolved in advance and included in the approved allowlist; required ZDR must be explicitly confirmed. -Every provider attempt writes to a private staging directory. Only rows with -`is_completed=true`, `stop_condition=agent_completed`, no errors, the pinned -verifier, matching task/model identity, native nodes and calls, usage and timing, -outcome contract, reward, and partial credit enter `source_dir`. Failed or -incomplete attempts move append-only to `rejected_dir` with provider/request -metadata. A nonempty `traces.jsonl` is therefore never treated as completion. -Only 429/503-style transient failures receive exponential backoff. +Every provider attempt writes to a private staging directory. Accepted rows +must be terminal, pinned to the reviewed verifier, match task/model identity, +and carry the required native evidence described below. Failed or incomplete +attempts move append-only to `rejected_dir` with provider/request metadata. A +nonempty `traces.jsonl` is therefore never treated as completion. Only +429/503-style transient failures receive exponential backoff. + +Scored terminal model failures are evidence, not missing rows. Prime +`context_length` and `max_turns` rows are importable when they are error-free and +carry final reward plus partial credit. A narrower normalization also accepts a +terminal ProviderError 400 only when every recorded error explicitly identifies +a context-window overflow (for example, `ContextWindowExceededError` or +`prompt is too long: N tokens > M maximum`). The raw trace remains unchanged; +the aggregate row records score `0`, `terminal_outcome=model_failure`, +`stop_condition=context_window_exceeded`, and an explicit normalization marker. +Generic 400s, invalid JSON, 429s, 5xx responses, network errors, and rows without +recognized terminal evidence remain rejected. ## Production replication contract diff --git a/runtime-assets/prime-scorecard/build-scorecard.mjs b/runtime-assets/prime-scorecard/build-scorecard.mjs index 2f2c85f5..3c558cf1 100644 --- a/runtime-assets/prime-scorecard/build-scorecard.mjs +++ b/runtime-assets/prime-scorecard/build-scorecard.mjs @@ -29,6 +29,39 @@ const discoveredPrimeTraces = modelIds.flatMap((model) => .map(JSON.parse), ); const tracesById = new Map(); +const scoredTerminalStopConditions = new Set(["agent_completed", "context_length", "max_turns"]); +const contextWindowMessage = /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum/i; +const isContextWindowError = (error) => + error?.type === "ProviderError" && + Number(error?.status_code) === 400 && + contextWindowMessage.test(String(error?.message ?? "")); +const isNormalizedContextWindowFailure = (trace) => { + const errors = Array.isArray(trace.errors) ? trace.errors : []; + const callErrors = (trace.calls ?? []).map((call) => call?.error).filter(Boolean); + const allErrors = [...errors, ...callErrors]; + return trace.is_completed === true && trace.stop_condition === "error" && + allErrors.length > 0 && allErrors.every(isContextWindowError); +}; +const traceDisposition = (trace) => { + if (trace.verifiers?.version !== config.verifier_version || trace.is_completed !== true) return null; + if (isNormalizedContextWindowFailure(trace)) { + return { score: 0, partialCredit: 0, stopReason: "context_window_exceeded", normalized: true }; + } + if ( + !scoredTerminalStopConditions.has(String(trace.stop_condition ?? "")) || + !Array.isArray(trace.errors) || + trace.errors.length > 0 || + (trace.calls ?? []).some((call) => call?.error) || + !Number.isFinite(trace.rewards?.final_state) || + !Number.isFinite(trace.metrics?.final_state_partial_credit) + ) return null; + return { + score: Number(trace.rewards.final_state), + partialCredit: Number(trace.metrics.final_state_partial_credit), + stopReason: String(trace.stop_condition), + normalized: false, + }; +}; for (const trace of discoveredPrimeTraces) { const id = String(trace.id ?? ""); if (!id) throw new Error("Prime trace is missing its stable id"); @@ -42,15 +75,9 @@ for (const trace of discoveredPrimeTraces) { const primeTraces = [...tracesById.values()].map(({ trace }) => trace); if ( primeTraces.length === 0 || - primeTraces.some( - (trace) => - trace.verifiers?.version !== config.verifier_version || - !trace.is_completed || - trace.stop_condition !== "agent_completed" || - (trace.errors?.length ?? 0) > 0, - ) + primeTraces.some((trace) => !traceDisposition(trace)) ) { - throw new Error(`Refusing to build: every discovered trace must be a completed, error-free Prime Verifiers ${config.verifier_version} run`); + throw new Error(`Refusing to build: every discovered trace must be an error-free, scored terminal Prime Verifiers ${config.verifier_version} row`); } const pricing = config.pricing; const contentText = (content) => (Array.isArray(content) @@ -141,6 +168,7 @@ const rollouts = primeTraces.map((prime) => { const calls = prime.calls.filter((call) => !isHarnessTitleCall(call)); const usage = prime.calls.map((call) => call.usage ?? {}); const rate = pricing[model]; + const disposition = traceDisposition(prime); if (!rate) throw new Error(`Missing Understudy customer rate card for ${model}`); const cost = usage.reduce( (sum, row) => @@ -160,8 +188,9 @@ const rollouts = primeTraces.map((prime) => { : block.name, ); const expected = expectedToolPath(prime); - const started = Math.min(...calls.map((call) => call.time.start)); - const ended = Math.max(...calls.map((call) => call.time.end)); + const timedCalls = calls.filter((call) => Number.isFinite(call.time?.start) && Number.isFinite(call.time?.end)); + const started = timedCalls.length ? Math.min(...timedCalls.map((call) => call.time.start)) : 0; + const ended = timedCalls.length ? Math.max(...timedCalls.map((call) => call.time.end)) : started; const generationStarted = Number(prime.timing?.generation?.start); const generationEnded = Number(prime.timing?.generation?.end); const latencyMs = Number.isFinite(generationStarted) && Number.isFinite(generationEnded) @@ -173,14 +202,17 @@ const rollouts = primeTraces.map((prime) => { task_id: taskId, prompt: taskDetails(prime).label, task_summary: taskDetails(prime).summary, - strict_pass: prime.rewards.final_state === 1, - reward: prime.rewards.final_state, - partial_credit: prime.metrics.final_state_partial_credit, + strict_pass: disposition.score === 1, + reward: disposition.score, + partial_credit: disposition.partialCredit, turns: calls.length, latency_ms: latencyMs, tokens: usage.reduce((sum, row) => sum + (row.prompt_tokens ?? 0) + (row.cached_input_tokens ?? 0) + (row.completion_tokens ?? 0), 0), cost_usd: cost, - terminal_reason: prime.stop_condition, + terminal_reason: disposition.stopReason, + native_terminal_reason: prime.stop_condition, + terminal_outcome: prime.stop_condition === "agent_completed" ? "completed" : "model_failure", + score_normalization: disposition.normalized ? "recognized_context_window_failure_zero" : null, grading_method: `Prime Verifiers ${config.verifier_version} deterministic final-state contract (no LLM judge or regex)`, tool_path: observed, expected_tool_path: expected, @@ -389,7 +421,7 @@ function render(){ document.querySelector(".layout").classList.toggle("summary-mode",viewMode==="summary"); document.querySelector("#count").textContent=D.rollouts.length; const models=[...new Set(D.rollouts.map(r=>r.model))]; - document.querySelector("#list").innerHTML=''+models.map((model,modelIndex)=>{const rows=D.rollouts.filter(r=>r.model===model);return '
'+esc(model.replace("claude-",""))+''+rows.filter(row=>row.strict_pass).length+'/'+rows.length+' pass'+rows.map((r,i)=>'').join("")+'
'}).join(""); + document.querySelector("#list").innerHTML=''+models.map((model,modelIndex)=>{const rows=D.rollouts.filter(r=>r.model===model);return '
'+esc(model.replace("claude-",""))+''+rows.filter(row=>row.strict_pass).length+'/'+rows.length+' pass'+rows.map((r,i)=>'').join("")+'
'}).join(""); if(viewMode==="summary"){ document.querySelector("#center-title").textContent="Summary"; document.querySelector("#history").innerHTML=summaryView(models); @@ -404,7 +436,7 @@ function render(){ document.querySelector("#history").innerHTML=taskSummary+conversation+verifier; document.querySelector("#reward").className="reward "+(selected.strict_pass?"green":"red"); document.querySelector("#reward").textContent=selected.partial_credit.toFixed(3); - document.querySelector("#overview").innerHTML='
'+(selected.strict_pass?"PASS":"FAIL")+'
Prime metrics
reward '+selected.reward.toFixed(3)+'
partial_credit '+selected.partial_credit.toFixed(3)+'
cost_per_task '+usd(selected.cost_usd)+'
model calls '+selected.turns+'
latency '+ms(selected.latency_ms)+'
tokens '+tok(selected.tokens)+'
Native run
model '+esc(selected.model)+'
verifiers '+esc(selected.verifier_version)+'
harness '+esc(selected.harness)+'
task '+esc(selected.task_id.slice(-8))+'
run '+esc(selected.run_id.slice(0,8))+'
Tool path
'+esc(selected.tool_path.join(" → ")||"none")+'
'+selected.misses.map(x=>'
'+esc(x)+'
').join(""); + document.querySelector("#overview").innerHTML='
'+(selected.strict_pass?"PASS":"FAIL")+'
Prime metrics
reward '+selected.reward.toFixed(3)+'
partial_credit '+selected.partial_credit.toFixed(3)+'
stop reason '+esc(selected.terminal_reason)+'
cost_per_task '+usd(selected.cost_usd)+'
model calls '+selected.turns+'
latency '+ms(selected.latency_ms)+'
tokens '+tok(selected.tokens)+'
Native run
model '+esc(selected.model)+'
verifiers '+esc(selected.verifier_version)+'
harness '+esc(selected.harness)+'
task '+esc(selected.task_id.slice(-8))+'
run '+esc(selected.run_id.slice(0,8))+'
Tool path
'+esc(selected.tool_path.join(" → ")||"none")+'
'+selected.misses.map(x=>'
'+esc(x)+'
').join(""); } document.querySelector("#view-summary").addEventListener("click",()=>{viewMode="summary";render()}); document.querySelectorAll(".sort-button").forEach(button=>button.addEventListener("click",()=>{ diff --git a/src/prime-benchmark-import.ts b/src/prime-benchmark-import.ts index fc5142e9..296fe6c9 100644 --- a/src/prime-benchmark-import.ts +++ b/src/prime-benchmark-import.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs"; import { basename, dirname, isAbsolute, join, resolve } from "node:path"; +import { primeTraceDisposition } from "./prime-trace-contract.js"; type Price = { input: number; cache_read: number; output: number; source: string }; type TaskMetadata = { @@ -167,13 +168,9 @@ export function importPrimeBenchmark(configPath: string): { if (traces.length === 0) throw new Error(`no Prime traces found under ${sourceDir}`); for (const trace of traces) { - if ( - trace.verifiers?.version !== config.verifier_version || - !trace.is_completed || - trace.stop_condition !== "agent_completed" || - (trace.errors?.length ?? 0) > 0 - ) { - throw new Error(`trace ${trace.id ?? "unknown"} is not a completed, error-free Prime ${config.verifier_version} run`); + const disposition = primeTraceDisposition(trace, config.verifier_version); + if (!disposition.accepted) { + throw new Error(`trace ${trace.id ?? "unknown"} is not an importable Prime ${config.verifier_version} scored terminal row: ${disposition.issue}`); } const taskId = trace.task?.data?.task_id; const model = trace.agent?.model; @@ -228,6 +225,7 @@ export function importPrimeBenchmark(configPath: string): { const taskId = String(trace.task.data.task_id); const tokenUsage = tokensFor(trace); const price = config.pricing[model]; + const disposition = primeTraceDisposition(trace, config.verifier_version); return { schema_version: "understudy.eval_result.v1", run_id: String(trace.run?.id ?? trace.id), @@ -235,9 +233,13 @@ export function importPrimeBenchmark(configPath: string): { runtime_backend: "prime-verifiers", task_id: taskId, split: config.tasks[taskId].split ?? "none", - score: Number(trace.rewards?.final_state ?? 0), - subscores: { final_state_partial_credit: Number(trace.metrics?.final_state_partial_credit ?? 0) }, + score: disposition.score, + subscores: { final_state_partial_credit: disposition.partial_credit }, status: "ok", + stop_condition: disposition.display_stop_reason, + native_stop_condition: disposition.stop_condition, + terminal_outcome: disposition.terminal_outcome, + score_normalization: disposition.normalized ? "recognized_context_window_failure_zero" : null, model, route: "byo-provider", cost: { usd: costFor(trace, price), basis: price.source }, @@ -298,13 +300,7 @@ export function inspectPrimeBenchmark(configPath: string): Record - trace.verifiers?.version !== config.verifier_version || - !trace.is_completed || - trace.stop_condition !== "agent_completed" || - (trace.errors?.length ?? 0) > 0, - ); + const invalid = traces.filter((trace) => !primeTraceDisposition(trace, config.verifier_version).accepted); const models = [...new Set(traces.map((trace) => String(trace.agent?.model ?? "unknown")))].sort(); const tasks = [...new Set(traces.map((trace) => String(trace.task?.data?.task_id ?? "unknown")))].sort(); const expected = Object.keys(config.tasks).length * Math.max(models.length, 1); @@ -319,6 +315,21 @@ export function inspectPrimeBenchmark(configPath: string): Record { + const disposition = primeTraceDisposition(trace, config.verifier_version); + return disposition.accepted && disposition.terminal_outcome === "model_failure"; + }, + ).map((trace) => { + const disposition = primeTraceDisposition(trace, config.verifier_version); + return { + trace_id: trace.id ?? "unknown", + stop_condition: disposition.display_stop_reason, + native_stop_condition: disposition.stop_condition, + normalized: disposition.normalized, + }; + }), invalid_traces: invalid.map((trace) => trace.id ?? "unknown"), ready_to_import: traces.length > 0 && diff --git a/src/prime-benchmark-runner.ts b/src/prime-benchmark-runner.ts index 547d31ee..f11a8102 100644 --- a/src/prime-benchmark-runner.ts +++ b/src/prime-benchmark-runner.ts @@ -12,6 +12,7 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { inspectPrimeBenchmark } from "./prime-benchmark-import.js"; +import { primeTraceDisposition } from "./prime-trace-contract.js"; export type PrimeProvider = "openai" | "anthropic" | "openai-compatible" | "understudy"; export type PrimeSampling = { @@ -168,9 +169,14 @@ export function validatePrimeTrace( const taskId = typeof trace.task?.data?.task_id === "string" ? trace.task.data.task_id : null; const model = typeof trace.agent?.model === "string" ? trace.agent.model : null; const calls = Array.isArray(trace.calls) ? trace.calls.filter((call: any) => !call?.sampling?.output_config) : []; + const disposition = primeTraceDisposition(trace, expected.verifierVersion); if (trace.is_completed !== true) reasons.push("is_completed_not_true"); - if (trace.stop_condition !== "agent_completed") reasons.push("stop_condition_not_agent_completed"); - if (!Array.isArray(trace.errors) || trace.errors.length > 0) reasons.push("errors_not_empty"); + if (!disposition.accepted && disposition.issue?.startsWith("unscored stop condition")) { + reasons.push("stop_condition_not_scored_terminal"); + } + if (!disposition.normalized && (!Array.isArray(trace.errors) || trace.errors.length > 0)) { + reasons.push("errors_not_empty"); + } if (trace.verifiers?.version !== expected.verifierVersion) reasons.push("verifier_version_mismatch"); if (!trace.run?.id) reasons.push("missing_run_id"); if (!model) reasons.push("missing_model"); @@ -180,15 +186,15 @@ export function validatePrimeTrace( if (!Array.isArray(trace.task?.data?.outcome_contract?.required)) reasons.push("missing_outcome_contract"); if (!Array.isArray(trace.nodes) || trace.nodes.length === 0) reasons.push("missing_nodes"); if (calls.length === 0) reasons.push("missing_model_calls"); - if (calls.some((call: any) => call.error)) reasons.push("provider_call_error"); - if (calls.some((call: any) => !call.usage || !call.time || !Number.isFinite(call.time.start) || !Number.isFinite(call.time.end))) { + if (!disposition.normalized && calls.some((call: any) => call.error)) reasons.push("provider_call_error"); + if (!disposition.normalized && calls.some((call: any) => !call.usage || !call.time || !Number.isFinite(call.time.start) || !Number.isFinite(call.time.end))) { reasons.push("missing_call_usage_or_timing"); } if (!Number.isFinite(trace.timing?.generation?.start) || !Number.isFinite(trace.timing?.generation?.end)) { reasons.push("missing_generation_timing"); } - if (!Number.isFinite(trace.rewards?.final_state)) reasons.push("missing_final_reward"); - if (!Number.isFinite(trace.metrics?.final_state_partial_credit)) reasons.push("missing_partial_credit"); + if (!disposition.normalized && !Number.isFinite(trace.rewards?.final_state)) reasons.push("missing_final_reward"); + if (!disposition.normalized && !Number.isFinite(trace.metrics?.final_state_partial_credit)) reasons.push("missing_partial_credit"); return { accepted: reasons.length === 0, trace_id: String(trace.id ?? "unknown"), diff --git a/src/prime-trace-contract.ts b/src/prime-trace-contract.ts new file mode 100644 index 00000000..57d19d7b --- /dev/null +++ b/src/prime-trace-contract.ts @@ -0,0 +1,107 @@ +export type PrimeTraceDisposition = { + accepted: boolean; + normalized: boolean; + stop_condition: string; + display_stop_reason: string; + terminal_outcome: "completed" | "model_failure" | "rejected"; + score: number | null; + partial_credit: number | null; + issue: string | null; +}; + +export const SCORED_TERMINAL_STOP_CONDITIONS = new Set([ + "agent_completed", + "context_length", + "max_turns", +]); + +const CONTEXT_WINDOW_MESSAGE = + /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum/i; + +function isRecognizedContextWindowError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const row = error as Record; + return ( + row.type === "ProviderError" && + Number(row.status_code) === 400 && + typeof row.message === "string" && + CONTEXT_WINDOW_MESSAGE.test(row.message) + ); +} + +export function isNormalizedContextWindowFailure(trace: Record): boolean { + const errors = Array.isArray(trace.errors) ? trace.errors : []; + const callErrors = (Array.isArray(trace.calls) ? trace.calls : []) + .map((call: any) => call?.error) + .filter(Boolean); + const allErrors = [...errors, ...callErrors]; + return ( + trace.is_completed === true && + trace.stop_condition === "error" && + allErrors.length > 0 && + allErrors.every(isRecognizedContextWindowError) + ); +} + +export function primeTraceDisposition( + trace: Record, + verifierVersion: string, +): PrimeTraceDisposition { + const stopCondition = String(trace.stop_condition ?? ""); + if (trace.verifiers?.version !== verifierVersion) { + return rejected(stopCondition, "verifier version mismatch"); + } + if (trace.is_completed !== true) return rejected(stopCondition, "run is not terminal"); + + if (isNormalizedContextWindowFailure(trace)) { + return { + accepted: true, + normalized: true, + stop_condition: stopCondition, + display_stop_reason: "context_window_exceeded", + terminal_outcome: "model_failure", + score: 0, + partial_credit: 0, + issue: null, + }; + } + + if (!SCORED_TERMINAL_STOP_CONDITIONS.has(stopCondition)) { + return rejected(stopCondition, `unscored stop condition ${stopCondition || "missing"}`); + } + if (!Array.isArray(trace.errors) || trace.errors.length > 0) { + return rejected(stopCondition, "runtime/provider errors are present"); + } + if ((trace.calls ?? []).some((call: any) => call?.error)) { + return rejected(stopCondition, "provider call errors are present"); + } + if (!Number.isFinite(trace.rewards?.final_state)) { + return rejected(stopCondition, "final reward is missing"); + } + if (!Number.isFinite(trace.metrics?.final_state_partial_credit)) { + return rejected(stopCondition, "partial credit is missing"); + } + return { + accepted: true, + normalized: false, + stop_condition: stopCondition, + display_stop_reason: stopCondition, + terminal_outcome: stopCondition === "agent_completed" ? "completed" : "model_failure", + score: Number(trace.rewards.final_state), + partial_credit: Number(trace.metrics.final_state_partial_credit), + issue: null, + }; +} + +function rejected(stopCondition: string, issue: string): PrimeTraceDisposition { + return { + accepted: false, + normalized: false, + stop_condition: stopCondition, + display_stop_reason: stopCondition || "unknown", + terminal_outcome: "rejected", + score: null, + partial_credit: null, + issue, + }; +} diff --git a/tests/prime-benchmark-import.test.mjs b/tests/prime-benchmark-import.test.mjs index c1531b8a..b0913495 100644 --- a/tests/prime-benchmark-import.test.mjs +++ b/tests/prime-benchmark-import.test.mjs @@ -9,12 +9,14 @@ import { importPrimeBenchmark, inspectPrimeBenchmark } from "../dist/prime-bench import { appendPrimeBenchmarkReview, freezePrimeBenchmark } from "../dist/prime-benchmark-lifecycle.js"; import { comparePrimeModels } from "../dist/prime-benchmark-compare.js"; import { discoverPrimeScorecards, renderScorecardGallery } from "../dist/prime-scorecard-server.js"; +import { primeTraceDisposition } from "../dist/prime-trace-contract.js"; import { normalizePrimeSampling, planPrimeRun, primeExecutionCoverage, renderProviderAwarePrimeConfig, runPrimeEvaluation, + validatePrimeTrace, watchPrimeBenchmark, } from "../dist/prime-benchmark-runner.js"; @@ -305,7 +307,146 @@ test("a nonempty all-error traces.jsonl is rejected and remains resumable", () = assert.equal(coverage.missing, 1); assert.equal(coverage.complete, false); assert.deepEqual(coverage.missing_task_ids, ["task-synthetic-1"]); - assert.ok(coverage.rejected_rows[0].reasons.includes("stop_condition_not_agent_completed")); + assert.ok(coverage.rejected_rows[0].reasons.includes("stop_condition_not_scored_terminal")); assert.ok(coverage.rejected_rows[0].reasons.includes("provider_call_error")); assert.ok(coverage.rejected_rows[0].reasons.includes("missing_final_reward")); }); + +test("imports scored context-fit failures and renders their terminal stop reason", () => { + const root = mkdtempSync(join(tmpdir(), "understudy-prime-context-fit-")); + const source = join(root, "runs"); + mkdirSync(join(source, "incumbent"), { recursive: true }); + mkdirSync(join(source, "candidate"), { recursive: true }); + mkdirSync(join(source, "haiku"), { recursive: true }); + const incumbent = syntheticTrace("incumbent"); + const contextFailure = syntheticTrace("gpt-5.5", 0); + contextFailure.id = "trace-gpt-5.5-context-length"; + contextFailure.stop_condition = "context_length"; + contextFailure.metrics.final_state_partial_credit = 0; + contextFailure.timing = { generation: { start: 100, end: 101.5 } }; + const providerContextFailure = syntheticTrace("claude-haiku-4-5", 0); + providerContextFailure.id = "trace-haiku-provider-context-window"; + providerContextFailure.stop_condition = "error"; + providerContextFailure.errors = [{ + type: "ProviderError", + status_code: 400, + message: 'upstream 400: litellm.ContextWindowExceededError: AnthropicError - {"message":"prompt is too long: 287580 tokens > 200000 maximum"}', + }]; + providerContextFailure.calls.push({ + endpoint: "/chat/completions", + error: { + type: "ProviderError", + status_code: 400, + message: 'AnthropicError - {"message":"prompt is too long: 287580 tokens > 200000 maximum"}', + }, + }); + delete providerContextFailure.rewards; + delete providerContextFailure.metrics; + providerContextFailure.timing = { generation: { start: 100, end: 101.5 } }; + writeFileSync(join(source, "incumbent", "traces.jsonl"), `${JSON.stringify(incumbent)}\n`); + writeFileSync(join(source, "candidate", "traces.jsonl"), `${JSON.stringify(contextFailure)}\n`); + writeFileSync(join(source, "haiku", "traces.jsonl"), `${JSON.stringify(providerContextFailure)}\n`); + const configPath = join(root, "config.json"); + const output = join(root, "aggregate"); + const scorecard = join(root, "scorecard"); + writeFileSync(configPath, JSON.stringify({ + schema_version: "understudy.prime_benchmark_import.v1", + benchmark_id: "context-fit-v1", + name: "Context Fit", + source_dir: source, + output_dir: output, + scorecard_output_dir: scorecard, + verifier_version: "0.2.1", + incumbent_model: "incumbent", + anonymized: true, + environment: { package_ref: "synthetic:context-fit" }, + tasks: { + "task-synthetic-1": { + label: "Long-context automation", + category_id: "context-fit", + summary: ["Exercises a long context.", "Requires tool completion.", "Measures terminal context fit."], + }, + }, + pricing: { + incumbent: { input: 1, cache_read: 0, output: 1, source: "synthetic" }, + "gpt-5.5": { input: 1, cache_read: 0, output: 1, source: "synthetic" }, + "claude-haiku-4-5": { input: 1, cache_read: 0, output: 1, source: "synthetic" }, + }, + })); + + const validation = validatePrimeTrace(contextFailure, { + verifierVersion: "0.2.1", + model: "gpt-5.5", + taskIds: ["task-synthetic-1"], + }); + assert.equal(validation.accepted, true); + const normalizedValidation = validatePrimeTrace(providerContextFailure, { + verifierVersion: "0.2.1", + model: "claude-haiku-4-5", + taskIds: ["task-synthetic-1"], + }); + assert.equal(normalizedValidation.accepted, true); + const status = inspectPrimeBenchmark(configPath); + assert.equal(status.ready_to_import, true); + assert.deepEqual( + status.terminal_model_failures, + [ + { + trace_id: "trace-gpt-5.5-context-length", + stop_condition: "context_length", + native_stop_condition: "context_length", + normalized: false, + }, + { + trace_id: "trace-haiku-provider-context-window", + stop_condition: "context_window_exceeded", + native_stop_condition: "error", + normalized: true, + }, + ], + ); + importPrimeBenchmark(configPath); + const rows = readFileSync(join(output, "rows-prime.jsonl"), "utf8") + .trim() + .split("\n") + .map(JSON.parse); + const failed = rows.find((row) => row.model === "gpt-5.5"); + assert.equal(failed.status, "ok"); + assert.equal(failed.score, 0); + assert.equal(failed.stop_condition, "context_length"); + assert.equal(failed.terminal_outcome, "model_failure"); + const normalized = rows.find((row) => row.model === "claude-haiku-4-5"); + assert.equal(normalized.status, "ok"); + assert.equal(normalized.score, 0); + assert.equal(normalized.stop_condition, "context_window_exceeded"); + assert.equal(normalized.native_stop_condition, "error"); + assert.equal(normalized.score_normalization, "recognized_context_window_failure_zero"); + + const renderer = resolve("runtime-assets/prime-scorecard/build-scorecard.mjs"); + const rendered = spawnSync(process.execPath, [renderer, configPath], { encoding: "utf8" }); + assert.equal(rendered.status, 0, rendered.stderr); + const viewer = readFileSync(join(scorecard, "viewer", "index.html"), "utf8"); + assert.match(viewer, /stop reason/); + assert.match(viewer, /context_length/); + assert.match(viewer, /context_window_exceeded/); +}); + +test("context-window normalization rejects transport and unrelated provider failures", () => { + const cases = [ + { type: "ProviderError", status_code: 429, message: "rate limit exceeded" }, + { type: "ProviderError", status_code: 503, message: "service unavailable" }, + { type: "ProviderError", status_code: 400, message: "invalid JSON tool arguments" }, + { type: "NetworkError", status_code: 400, message: "prompt is too long: 300000 tokens > 200000 maximum" }, + ]; + for (const error of cases) { + const trace = syntheticTrace("candidate", 0); + trace.stop_condition = "error"; + trace.errors = [error]; + trace.calls.push({ error }); + delete trace.rewards; + delete trace.metrics; + const disposition = primeTraceDisposition(trace, "0.2.1"); + assert.equal(disposition.accepted, false, JSON.stringify(error)); + assert.equal(disposition.normalized, false, JSON.stringify(error)); + } +}); From 22996e931a0e3aeb46c88ef6d5f5c355f6153d17 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:28:16 -0700 Subject: [PATCH 2/5] Add non-scoring provider availability lane --- docs/prime-native-benchmarks.md | 23 ++++++++--- .../prime-scorecard/build-scorecard.mjs | 28 +++++++++++++- ...tudy.prime_benchmark_import.v1.schema.json | 17 +++++++++ src/prime-benchmark-import.ts | 38 +++++++++++++++++++ tests/prime-benchmark-import.test.mjs | 26 +++++++++++++ 5 files changed, 124 insertions(+), 8 deletions(-) diff --git a/docs/prime-native-benchmarks.md b/docs/prime-native-benchmarks.md index 6b2de5b2..952ca069 100644 --- a/docs/prime-native-benchmarks.md +++ b/docs/prime-native-benchmarks.md @@ -28,8 +28,10 @@ Never commit the scorecard merely because the aggregate package is safe. Every discovered trace must: -- be complete and error-free; -- stop with `agent_completed`; +- be complete and either error-free or a narrowly recognized deterministic + context-window failure; +- stop with `agent_completed`, a scored terminal model stop such as + `context_length` or `max_turns`, or a recognized context-window error; - carry the exact configured `verifiers.version`; - identify `agent.model`, `run.id`, and `task.data.task_id`; - retain `task.data.outcome_contract.required`; @@ -42,6 +44,13 @@ The importer rejects a corpus when any model lacks reviewed pricing or any task lacks explicit anonymized metadata. It also rejects calibration unless the declared incumbent strictly passes every frozen task. +Models with incomplete evidence caused by repeatable upstream availability +failures must not receive a score. An optional `availability_annotations` entry +may retain their reviewed receipt as `provider_unavailable`; those models are +shown in a non-scoring lane and are excluded from rows, pass rates, rankings, +cost comparisons, and the Pareto frontier. An annotated model may not also +appear in the canonical trace source. + ## Build and reopen ```sh @@ -70,10 +79,12 @@ understudy benchmarks reopen-prime config.json ``` `run-prime --dry-run` validates and prints the exact invocation without provider -execution. `watch-prime` exits only when every discovered native trace is -complete, error-free, stopped with `agent_completed`, and pinned to the -configured verifier version. `reopen-prime` serves the scorecard through the -loopback gallery instead of relying on `file://`. +execution. `watch-prime` exits only when every discovered native trace has an +accepted terminal disposition and is pinned to the configured verifier version. +Provider/transport failures remain rejected, except for narrowly recognized +deterministic context-window failures that are preserved as score-zero model +outcomes. `reopen-prime` serves the scorecard through the loopback gallery +instead of relying on `file://`. Agents can call the MCP `plan_prime_run` tool to inspect the exact native command without spend or data transfer, then use `prime_status` while execution proceeds. diff --git a/runtime-assets/prime-scorecard/build-scorecard.mjs b/runtime-assets/prime-scorecard/build-scorecard.mjs index 3c558cf1..5831a1be 100644 --- a/runtime-assets/prime-scorecard/build-scorecard.mjs +++ b/runtime-assets/prime-scorecard/build-scorecard.mjs @@ -73,6 +73,23 @@ for (const trace of discoveredPrimeTraces) { if (!prior) tracesById.set(id, { canonical, trace }); } const primeTraces = [...tracesById.values()].map(({ trace }) => trace); +const availabilityAnnotations = Object.entries(config.availability_annotations ?? {}).map(([model, annotation]) => ({ + model, + status: annotation.status, + reason: annotation.reason, + receipt_ref: annotation.receipt_ref, + attempt_rows: annotation.attempt_rows, + clean_tasks: annotation.clean_tasks, + required_tasks: annotation.required_tasks, +})); +for (const annotation of availabilityAnnotations) { + if (annotation.status !== "provider_unavailable") { + throw new Error(`Unsupported availability status for ${annotation.model}: ${annotation.status}`); + } + if (primeTraces.some((trace) => trace.agent?.model === annotation.model)) { + throw new Error(`Availability annotation for ${annotation.model} conflicts with discovered scored traces`); + } +} if ( primeTraces.length === 0 || primeTraces.some((trace) => !traceDisposition(trace)) @@ -233,6 +250,7 @@ const data = JSON.stringify({ incumbent_model: config.incumbent_model, verifier_version: config.verifier_version, source: `Prime Verifiers ${config.verifier_version} native traces only`, + availability_annotations: availabilityAnnotations, rollouts, }).replaceAll("aside{display:none} .summary-highlights{display:grid;grid-template-columns:repeat(3,minmax(0,1fr));gap:12px} .summary-highlights .winner{margin:0;padding:12px 14px} +.availability-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:10px} +.availability-card{border:1px solid #56421f;border-radius:9px;background:#0d0a05;padding:13px} +.availability-card b,.availability-card span,.availability-card small{display:block} +.availability-card b{color:var(--amber);margin-bottom:5px} +.availability-card span{color:#bbb}.availability-card small{color:#777;margin-top:7px;line-height:1.5} .model-toggle{all:unset;display:inline-flex;align-items:center;gap:8px;cursor:pointer} .model-toggle:before{content:"▸";color:#666;font-size:10px} .model-toggle.open:before{content:"▾";color:var(--green)} @@ -400,6 +423,7 @@ function summaryView(models){ const cheapest=[...qualified].filter(stat=>Number.isFinite(stat.cost)).sort((a,b)=>a.cost-b.cost)[0]; const callout=(label,stat,value)=>'
'+label+''+esc(shortModel(stat?.model??"—"))+''+value+'
'; const highlights='
'+callout("highest score",best,best?best.score.toFixed(3):"—")+callout("fastest passing",fastest,fastest?ms(fastest.time):"—")+callout("cheapest passing",cheapest,cheapest?usd(cheapest.cost):"—")+'
'; + const availability=D.availability_annotations.length?'

Provider availability · non-scoring

'+D.availability_annotations.map(row=>'
'+esc(shortModel(row.model))+''+esc(row.status)+' · '+row.clean_tasks+'/'+row.required_tasks+' clean tasks'+esc(row.reason)+''+row.attempt_rows+' attempt rows · receipt '+esc(row.receipt_ref)+'
').join("")+'
':""; const leaderboard=stats.map((stat,index)=>{ const open=expandedModel===stat.model; const rollouts=stat.rows.map((row,taskIndex)=>'').join(""); @@ -407,7 +431,7 @@ function summaryView(models){ return ''+(index+1)+''+statusPill(stat)+baseline+''+stat.score.toFixed(3)+(incumbent&&stat!==incumbent?delta(stat.score,incumbent.score):"")+''+Math.round(stat.pass*stat.rows.length)+'/'+stat.rows.length+(incumbent&&stat!==incumbent?delta(stat.pass,incumbent.pass):"")+''+ms(stat.time)+(incumbent&&stat!==incumbent?delta(stat.time,incumbent.time,true):"")+''+usd(stat.cost)+(incumbent&&stat!==incumbent?delta(stat.cost,incumbent.cost,true):"")+''+stat.calls.toFixed(1)+(incumbent&&stat!==incumbent?delta(stat.calls,incumbent.calls,true):"")+''+(open?'
'+rollouts+'
':""); }).join(""); const taskCards=tasks.map((task,index)=>{const rows=stats.map(stat=>D.rollouts.find(row=>row.model===stat.model&&row.task_id===task.task_id)).filter(Boolean);const incumbentRow=rows.find(row=>row.model===D.incumbent_model);return '
Task '+(index+1)+''+esc(task.prompt)+'
'+rows.filter(row=>row.strict_pass).length+'/'+rows.length+' pass
'+rows.map(row=>'').join("")+'
ModelScoreTimeCostCalls
'+esc(shortModel(row.model))+(row.model===D.incumbent_model?'incumbent':"")+''+row.partial_credit.toFixed(2)+(incumbentRow&&row!==incumbentRow?delta(row.partial_credit,incumbentRow.partial_credit):"")+''+ms(row.latency_ms)+(incumbentRow&&row!==incumbentRow?delta(row.latency_ms,incumbentRow.latency_ms,true):"")+''+usd(row.cost_usd)+(incumbentRow&&row!==incumbentRow?delta(row.cost_usd,incumbentRow.cost_usd,true):"")+''+row.turns+(incumbentRow&&row!==incumbentRow?delta(row.turns,incumbentRow.turns,true):"")+'
'}).join(""); - return '
'+highlights+'

Verifier score vs cost per task

'+paretoPlot(stats)+'

Model leaderboard

'+sortHead("model","Model")+sortHead("score","Score")+sortHead("pass","Pass")+sortHead("time","Time / task")+sortHead("cost","Cost / task")+sortHead("calls","Calls")+''+leaderboard+'
#

Production task comparison

'+taskCards+'
'; + return '
'+highlights+availability+'

Verifier score vs cost per task

'+paretoPlot(stats)+'

Model leaderboard

'+sortHead("model","Model")+sortHead("score","Score")+sortHead("pass","Pass")+sortHead("time","Time / task")+sortHead("cost","Cost / task")+sortHead("calls","Calls")+''+leaderboard+'
#

Production task comparison

'+taskCards+'
'; } function summaryOverview(models){ const stats=modelStats(models),qualified=stats.filter(stat=>stat.pass===1),priced=stats.filter(stat=>Number.isFinite(stat.cost)); @@ -415,7 +439,7 @@ function summaryOverview(models){ const cheapest=[...qualified].filter(stat=>Number.isFinite(stat.cost)).sort((a,b)=>a.cost-b.cost)[0]; const best=stats[0]; const callout=(label,stat,value)=>'
'+label+''+esc(shortModel(stat?.model??"—"))+''+value+'
'; - return '
MODEL MATRIX
Prime Verifiers '+esc(D.verifier_version)+' only
'+callout("highest score",best,best?best.score.toFixed(3):"—")+callout("fastest passing",fastest,fastest?ms(fastest.time):"—")+callout("cheapest passing",cheapest,cheapest?usd(cheapest.cost):"—")+'
Coverage
models '+stats.length+'
rollouts '+D.rollouts.length+'
strict passes '+D.rollouts.filter(row=>row.strict_pass).length+'/'+D.rollouts.length+'
priced models '+priced.length+'/'+stats.length+'
Models are rows, so new candidates add vertically without clipping. Missing cost stays unavailable instead of using an unreviewed estimate.
'; + return '
MODEL MATRIX
Prime Verifiers '+esc(D.verifier_version)+' only
'+callout("highest score",best,best?best.score.toFixed(3):"—")+callout("fastest passing",fastest,fastest?ms(fastest.time):"—")+callout("cheapest passing",cheapest,cheapest?usd(cheapest.cost):"—")+'
Coverage
scored models '+stats.length+'
provider unavailable '+D.availability_annotations.length+'
rollouts '+D.rollouts.length+'
strict passes '+D.rollouts.filter(row=>row.strict_pass).length+'/'+D.rollouts.length+'
priced models '+priced.length+'/'+stats.length+'
Availability annotations are evidence-only and excluded from scores, rankings, and Pareto. Missing cost stays unavailable instead of using an unreviewed estimate.
'; } function render(){ document.querySelector(".layout").classList.toggle("summary-mode",viewMode==="summary"); diff --git a/schemas/understudy.prime_benchmark_import.v1.schema.json b/schemas/understudy.prime_benchmark_import.v1.schema.json index 935f8497..da465108 100644 --- a/schemas/understudy.prime_benchmark_import.v1.schema.json +++ b/schemas/understudy.prime_benchmark_import.v1.schema.json @@ -60,6 +60,23 @@ "source": {"type": "string", "minLength": 1} } } + }, + "availability_annotations": { + "type": "object", + "description": "Non-scoring provider availability evidence. Annotated models are excluded from leaderboard and Pareto inputs.", + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["status", "reason", "receipt_ref", "attempt_rows", "clean_tasks", "required_tasks"], + "properties": { + "status": {"const": "provider_unavailable"}, + "reason": {"type": "string", "minLength": 1}, + "receipt_ref": {"type": "string", "minLength": 1}, + "attempt_rows": {"type": "integer", "minimum": 1}, + "clean_tasks": {"type": "integer", "minimum": 0}, + "required_tasks": {"type": "integer", "minimum": 1} + } + } } } } diff --git a/src/prime-benchmark-import.ts b/src/prime-benchmark-import.ts index 296fe6c9..c2819f26 100644 --- a/src/prime-benchmark-import.ts +++ b/src/prime-benchmark-import.ts @@ -10,6 +10,14 @@ type TaskMetadata = { summary?: string[]; split?: "train" | "dev" | "holdout" | "none"; }; +type AvailabilityAnnotation = { + status: "provider_unavailable"; + reason: string; + receipt_ref: string; + attempt_rows: number; + clean_tasks: number; + required_tasks: number; +}; type ImportConfig = { schema_version: "understudy.prime_benchmark_import.v1"; benchmark_id: string; @@ -28,6 +36,7 @@ type ImportConfig = { }; pricing: Record; tasks: Record; + availability_annotations?: Record; }; type PrimeTrace = Record; @@ -52,6 +61,24 @@ function assertConfig(value: unknown): asserts value is ImportConfig { throw new Error("config.environment.package_ref is required"); } if (!config.pricing || !config.tasks) throw new Error("config.pricing and config.tasks are required"); + for (const [model, annotation] of Object.entries(config.availability_annotations ?? {})) { + if ( + annotation.status !== "provider_unavailable" || + typeof annotation.reason !== "string" || + !annotation.reason || + typeof annotation.receipt_ref !== "string" || + !annotation.receipt_ref || + !Number.isInteger(annotation.attempt_rows) || + annotation.attempt_rows < 1 || + !Number.isInteger(annotation.clean_tasks) || + annotation.clean_tasks < 0 || + !Number.isInteger(annotation.required_tasks) || + annotation.required_tasks < 1 || + annotation.clean_tasks >= annotation.required_tasks + ) { + throw new Error(`config.availability_annotations.${model} must describe incomplete provider_unavailable coverage`); + } + } } function listTraceFiles(sourceDir: string): string[] { @@ -166,6 +193,11 @@ export function importPrimeBenchmark(configPath: string): { const outputDir = fromConfig(config.output_dir); const traces = loadPrimeTraces(sourceDir); if (traces.length === 0) throw new Error(`no Prime traces found under ${sourceDir}`); + for (const model of Object.keys(config.availability_annotations ?? {})) { + if (traces.some((trace) => trace.agent?.model === model)) { + throw new Error(`availability annotation for ${model} conflicts with discovered scored traces`); + } + } for (const trace of traces) { const disposition = primeTraceDisposition(trace, config.verifier_version); @@ -218,6 +250,12 @@ export function importPrimeBenchmark(configPath: string): { dense_metric: "final_state_partial_credit", replayable: false, }, + availability_annotations: Object.fromEntries( + Object.entries(config.availability_annotations ?? {}).map(([model, annotation]) => [ + model, + { ...annotation, canonical_score: null, scoring_policy: "excluded_from_leaderboard_and_pareto" }, + ]), + ), }; const rows = traces.map((trace) => { diff --git a/tests/prime-benchmark-import.test.mjs b/tests/prime-benchmark-import.test.mjs index b0913495..68db1a04 100644 --- a/tests/prime-benchmark-import.test.mjs +++ b/tests/prime-benchmark-import.test.mjs @@ -113,6 +113,16 @@ test("imports Prime-native traces into an anonymized benchmark and renders the r "synthetic-model": { input: 1, cache_read: 0.1, output: 5, source: "synthetic reviewed fixture price" }, "synthetic-candidate": { input: 0.5, cache_read: 0.05, output: 2.5, source: "synthetic reviewed fixture price" }, }, + availability_annotations: { + "synthetic-unavailable": { + status: "provider_unavailable", + reason: "Repeated explicit upstream 503 overloads left incomplete frozen-task coverage.", + receipt_ref: "corrected-v3/provider-availability.json", + attempt_rows: 135, + clean_tasks: 8, + required_tasks: 50, + }, + }, }; const configPath = join(root, "config.json"); writeFileSync(configPath, JSON.stringify(config, null, 2)); @@ -128,8 +138,19 @@ test("imports Prime-native traces into an anonymized benchmark and renders the r const manifest = JSON.parse(readFileSync(join(aggregateOutput, "benchmark.json"), "utf8")); assert.equal(manifest.schema_version, "understudy.benchmark.v1"); assert.equal(manifest.benchmark_id, "synthetic-prime-benchmark-v1"); + assert.deepEqual(manifest.availability_annotations["synthetic-unavailable"], { + status: "provider_unavailable", + reason: "Repeated explicit upstream 503 overloads left incomplete frozen-task coverage.", + receipt_ref: "corrected-v3/provider-availability.json", + attempt_rows: 135, + clean_tasks: 8, + required_tasks: 50, + canonical_score: null, + scoring_policy: "excluded_from_leaderboard_and_pareto", + }); const aggregateText = readFileSync(join(aggregateOutput, "rows-prime.jsonl"), "utf8"); assert.doesNotMatch(aggregateText, /Synthetic private prompt/); + assert.doesNotMatch(aggregateText, /synthetic-unavailable/); assert.match(aggregateText, /"usd":0\.00155/); const comparison = comparePrimeModels(aggregateOutput, "synthetic-model", "synthetic-candidate"); assert.equal(comparison.comparable_task_count, 1); @@ -146,6 +167,11 @@ test("imports Prime-native traces into an anonymized benchmark and renders the r assert.match(viewer, /Conversation history/); assert.match(viewer, /production incumbent/); assert.match(viewer, /% vs incumbent/); + assert.match(viewer, /provider_unavailable/); + assert.match(viewer, /"clean_tasks":8/); + assert.match(viewer, /"required_tasks":50/); + assert.match(viewer, /clean tasks/); + assert.match(viewer, /corrected-v3\/provider-availability\.json/); const entries = discoverPrimeScorecards(join(root, "private-scorecards")); assert.equal(entries.length, 1); assert.equal(entries[0].benchmark_id, "synthetic-prime-benchmark-v1"); From 588572e345c665287c63d0567a3d1133e13f410c Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:05:23 -0700 Subject: [PATCH 3/5] Recognize Poolside context-window failures --- src/prime-trace-contract.ts | 2 +- tests/prime-benchmark-import.test.mjs | 34 +++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/prime-trace-contract.ts b/src/prime-trace-contract.ts index 57d19d7b..4013fcf8 100644 --- a/src/prime-trace-contract.ts +++ b/src/prime-trace-contract.ts @@ -16,7 +16,7 @@ export const SCORED_TERMINAL_STOP_CONDITIONS = new Set([ ]); const CONTEXT_WINDOW_MESSAGE = - /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum/i; + /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum|Input length \d+ exceeds the maximum allowed input length of \d+ tokens\./i; function isRecognizedContextWindowError(error: unknown): boolean { if (!error || typeof error !== "object") return false; diff --git a/tests/prime-benchmark-import.test.mjs b/tests/prime-benchmark-import.test.mjs index 68db1a04..a25c0ff0 100644 --- a/tests/prime-benchmark-import.test.mjs +++ b/tests/prime-benchmark-import.test.mjs @@ -476,3 +476,37 @@ test("context-window normalization rejects transport and unrelated provider fail assert.equal(disposition.normalized, false, JSON.stringify(error)); } }); + +test("normalizes Poolside's exact input-length context-window failure shape", () => { + const trace = syntheticTrace("laguna-xs-2.1"); + trace.stop_condition = "error"; + delete trace.rewards; + delete trace.metrics; + trace.errors = [{ + type: "ProviderError", + status_code: 400, + message: 'upstream 400: {"error":{"code":400,"message":"Input length 315811 exceeds the maximum allowed input length of 262112 tokens.","type":"Bad Request"}}', + }]; + trace.calls[0].error = { + type: "ProviderError", + status_code: 400, + message: 'upstream 400: {"error":{"code":400,"message":"Input length 315797 exceeds the maximum allowed input length of 262112 tokens.","type":"Bad Request"}}', + }; + + assert.deepEqual( + primeTraceDisposition(trace, "0.2.1"), + { + accepted: true, + normalized: true, + stop_condition: "error", + display_stop_reason: "context_window_exceeded", + terminal_outcome: "model_failure", + score: 0, + partial_credit: 0, + issue: null, + }, + ); + + trace.errors[0].message = "Input length is invalid."; + assert.equal(primeTraceDisposition(trace, "0.2.1").accepted, false); +}); From 816c64328f84c6194f6711698e0667a3a0260e7f Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Thu, 30 Jul 2026 23:24:08 -0700 Subject: [PATCH 4/5] Separate diagnostic audits from parity imports --- docs/prime-native-benchmarks.md | 13 ++++ .../prime-scorecard/build-scorecard.mjs | 33 ++++++++- ...tudy.prime_benchmark_import.v1.schema.json | 4 + src/prime-benchmark-import.ts | 7 ++ src/prime-benchmark-runner.ts | 11 ++- src/prime-trace-contract.ts | 38 +++++++++- tests/prime-benchmark-import.test.mjs | 73 ++++++++++++++++++- 7 files changed, 166 insertions(+), 13 deletions(-) diff --git a/docs/prime-native-benchmarks.md b/docs/prime-native-benchmarks.md index 952ca069..be30336f 100644 --- a/docs/prime-native-benchmarks.md +++ b/docs/prime-native-benchmarks.md @@ -51,6 +51,19 @@ shown in a non-scoring lane and are excluded from rows, pass rates, rankings, cost comparisons, and the Pareto frontier. An annotated model may not also appear in the canonical trace source. +Set `benchmark_mode` to `diagnostic` for a full audit view that deliberately +preserves incumbent replay misses. Diagnostic configs may build a private +scorecard, which is visibly labeled non-authoritative, but `import-prime` +refuses to create an aggregate package from them. Authoritative parity configs +use `benchmark_mode: "authoritative"` (the default) and retain the strict +all-frozen-tasks incumbent calibration gate. + +A scored terminal trace may retain a recognized transient 429/5xx call attempt +only when the top-level error list is empty and a later usage-bearing call +proves recovery inside the same trace. Terminal provider failures, unknown +transport messages, and retries without a later successful call remain +rejected. + ## Build and reopen ```sh diff --git a/runtime-assets/prime-scorecard/build-scorecard.mjs b/runtime-assets/prime-scorecard/build-scorecard.mjs index 5831a1be..b6214710 100644 --- a/runtime-assets/prime-scorecard/build-scorecard.mjs +++ b/runtime-assets/prime-scorecard/build-scorecard.mjs @@ -11,6 +11,9 @@ if (!["understudy.prime_scorecard.v1", "understudy.prime_benchmark_import.v1"].i if (config.anonymized !== true) { throw new Error("config.anonymized must be true before building a durable scorecard"); } +if (config.benchmark_mode !== undefined && !["authoritative", "diagnostic"].includes(config.benchmark_mode)) { + throw new Error("config.benchmark_mode must be authoritative or diagnostic"); +} const configDir = dirname(resolvedConfigPath); const fromConfig = (value) => isAbsolute(value) ? value : resolve(configDir, value); const primeRuns = fromConfig(config.source_dir); @@ -30,11 +33,31 @@ const discoveredPrimeTraces = modelIds.flatMap((model) => ); const tracesById = new Map(); const scoredTerminalStopConditions = new Set(["agent_completed", "context_length", "max_turns"]); -const contextWindowMessage = /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum/i; +const contextWindowMessage = /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum|Input length \d+ exceeds the maximum allowed input length of \d+ tokens\.|Input tokens exceed the configured limit of \d+ tokens\.\s*(?:Your input contained|Your messages resulted in) \d+ tokens\.(?:\s*Please reduce the length of the messages\.)?/i; const isContextWindowError = (error) => - error?.type === "ProviderError" && + ["ProviderError", "OverlongPromptError"].includes(error?.type) && Number(error?.status_code) === 400 && contextWindowMessage.test(String(error?.message ?? "")); +const retryableTransportMessages = { + 429: /upstream 429:.*rate limit exceeded/is, + 500: /upstream 500:.*internal server error/is, + 502: /upstream 502:.*provider connection error:.*upstream response stream/is, + 503: /upstream 503:.*overloaded_error.*provider is temporarily unavailable/is, +}; +const isRetryableTransportError = (error) => + error?.type === "ProviderError" && + retryableTransportMessages[Number(error?.status_code)]?.test(String(error?.message ?? "")) === true; +const hasOnlyAcceptedCallErrors = (trace, stopCondition) => { + const calls = (trace.calls ?? []).filter((call) => !call?.sampling?.output_config); + const errorIndexes = calls.flatMap((call, index) => call?.error ? [index] : []); + if (!errorIndexes.length) return true; + if (stopCondition === "context_length") return errorIndexes.every((index) => isContextWindowError(calls[index].error)); + if (stopCondition !== "agent_completed") return false; + return errorIndexes.every((index) => + isRetryableTransportError(calls[index].error) && + calls.slice(index + 1).some((later) => !later?.error && later?.usage), + ); +}; const isNormalizedContextWindowFailure = (trace) => { const errors = Array.isArray(trace.errors) ? trace.errors : []; const callErrors = (trace.calls ?? []).map((call) => call?.error).filter(Boolean); @@ -51,7 +74,7 @@ const traceDisposition = (trace) => { !scoredTerminalStopConditions.has(String(trace.stop_condition ?? "")) || !Array.isArray(trace.errors) || trace.errors.length > 0 || - (trace.calls ?? []).some((call) => call?.error) || + !hasOnlyAcceptedCallErrors(trace, String(trace.stop_condition ?? "")) || !Number.isFinite(trace.rewards?.final_state) || !Number.isFinite(trace.metrics?.final_state_partial_credit) ) return null; @@ -247,6 +270,7 @@ const data = JSON.stringify({ created_at: new Date().toISOString(), benchmark_id: config.benchmark_id, name: config.name, + benchmark_mode: config.benchmark_mode ?? "authoritative", incumbent_model: config.incumbent_model, verifier_version: config.verifier_version, source: `Prime Verifiers ${config.verifier_version} native traces only`, @@ -439,7 +463,8 @@ function summaryOverview(models){ const cheapest=[...qualified].filter(stat=>Number.isFinite(stat.cost)).sort((a,b)=>a.cost-b.cost)[0]; const best=stats[0]; const callout=(label,stat,value)=>'
'+label+''+esc(shortModel(stat?.model??"—"))+''+value+'
'; - return '
MODEL MATRIX
Prime Verifiers '+esc(D.verifier_version)+' only
'+callout("highest score",best,best?best.score.toFixed(3):"—")+callout("fastest passing",fastest,fastest?ms(fastest.time):"—")+callout("cheapest passing",cheapest,cheapest?usd(cheapest.cost):"—")+'
Coverage
scored models '+stats.length+'
provider unavailable '+D.availability_annotations.length+'
rollouts '+D.rollouts.length+'
strict passes '+D.rollouts.filter(row=>row.strict_pass).length+'/'+D.rollouts.length+'
priced models '+priced.length+'/'+stats.length+'
Availability annotations are evidence-only and excluded from scores, rankings, and Pareto. Missing cost stays unavailable instead of using an unreviewed estimate.
'; + const mode=D.benchmark_mode==="diagnostic"?'
DIAGNOSTIC AUDIT
Non-authoritative private review; incumbent replay misses are preserved and this config cannot be aggregate-imported.
':'
MODEL MATRIX
'; + return mode+'
Prime Verifiers '+esc(D.verifier_version)+' only
'+callout("highest score",best,best?best.score.toFixed(3):"—")+callout("fastest passing",fastest,fastest?ms(fastest.time):"—")+callout("cheapest passing",cheapest,cheapest?usd(cheapest.cost):"—")+'
Coverage
scored models '+stats.length+'
provider unavailable '+D.availability_annotations.length+'
rollouts '+D.rollouts.length+'
strict passes '+D.rollouts.filter(row=>row.strict_pass).length+'/'+D.rollouts.length+'
priced models '+priced.length+'/'+stats.length+'
Availability annotations are evidence-only and excluded from scores, rankings, and Pareto. Missing cost stays unavailable instead of using an unreviewed estimate.
'; } function render(){ document.querySelector(".layout").classList.toggle("summary-mode",viewMode==="summary"); diff --git a/schemas/understudy.prime_benchmark_import.v1.schema.json b/schemas/understudy.prime_benchmark_import.v1.schema.json index da465108..aac7885c 100644 --- a/schemas/understudy.prime_benchmark_import.v1.schema.json +++ b/schemas/understudy.prime_benchmark_import.v1.schema.json @@ -16,6 +16,10 @@ "scorecard_output_dir": {"type": "string", "minLength": 1}, "verifier_version": {"type": "string", "minLength": 1}, "incumbent_model": {"type": "string", "minLength": 1}, + "benchmark_mode": { + "enum": ["authoritative", "diagnostic"], + "description": "Defaults to authoritative. Diagnostic configs may render private scorecards but the aggregate importer rejects them." + }, "anonymized": { "const": true, "description": "Fail-closed acknowledgement that durable labels and metadata contain no customer identity. Raw traces remain local." diff --git a/src/prime-benchmark-import.ts b/src/prime-benchmark-import.ts index c2819f26..509ed45e 100644 --- a/src/prime-benchmark-import.ts +++ b/src/prime-benchmark-import.ts @@ -27,6 +27,7 @@ type ImportConfig = { output_dir: string; verifier_version: string; incumbent_model: string; + benchmark_mode?: "authoritative" | "diagnostic"; anonymized: boolean; environment: { package_ref: string; @@ -61,6 +62,9 @@ function assertConfig(value: unknown): asserts value is ImportConfig { throw new Error("config.environment.package_ref is required"); } if (!config.pricing || !config.tasks) throw new Error("config.pricing and config.tasks are required"); + if (config.benchmark_mode !== undefined && !["authoritative", "diagnostic"].includes(config.benchmark_mode)) { + throw new Error("config.benchmark_mode must be authoritative or diagnostic"); + } for (const [model, annotation] of Object.entries(config.availability_annotations ?? {})) { if ( annotation.status !== "provider_unavailable" || @@ -187,6 +191,9 @@ export function importPrimeBenchmark(configPath: string): { const configValue = readJson(configFile); assertConfig(configValue); const config = configValue; + if (config.benchmark_mode === "diagnostic") { + throw new Error("diagnostic benchmark configs may build private scorecards but cannot be aggregate-imported"); + } const configDir = dirname(configFile); const fromConfig = (value: string) => isAbsolute(value) ? value : resolve(configDir, value); const sourceDir = fromConfig(config.source_dir); diff --git a/src/prime-benchmark-runner.ts b/src/prime-benchmark-runner.ts index f11a8102..f99e3172 100644 --- a/src/prime-benchmark-runner.ts +++ b/src/prime-benchmark-runner.ts @@ -12,7 +12,7 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { inspectPrimeBenchmark } from "./prime-benchmark-import.js"; -import { primeTraceDisposition } from "./prime-trace-contract.js"; +import { hasOnlyAcceptedCallErrors, primeTraceDisposition } from "./prime-trace-contract.js"; export type PrimeProvider = "openai" | "anthropic" | "openai-compatible" | "understudy"; export type PrimeSampling = { @@ -186,8 +186,13 @@ export function validatePrimeTrace( if (!Array.isArray(trace.task?.data?.outcome_contract?.required)) reasons.push("missing_outcome_contract"); if (!Array.isArray(trace.nodes) || trace.nodes.length === 0) reasons.push("missing_nodes"); if (calls.length === 0) reasons.push("missing_model_calls"); - if (!disposition.normalized && calls.some((call: any) => call.error)) reasons.push("provider_call_error"); - if (!disposition.normalized && calls.some((call: any) => !call.usage || !call.time || !Number.isFinite(call.time.start) || !Number.isFinite(call.time.end))) { + if (!disposition.normalized && !hasOnlyAcceptedCallErrors(trace, String(trace.stop_condition ?? ""))) reasons.push("provider_call_error"); + if (!disposition.normalized && calls.some((call: any) => + !call.time || + !Number.isFinite(call.time.start) || + !Number.isFinite(call.time.end) || + (!call.error && !call.usage) + )) { reasons.push("missing_call_usage_or_timing"); } if (!Number.isFinite(trace.timing?.generation?.start) || !Number.isFinite(trace.timing?.generation?.end)) { diff --git a/src/prime-trace-contract.ts b/src/prime-trace-contract.ts index 4013fcf8..522ae8e1 100644 --- a/src/prime-trace-contract.ts +++ b/src/prime-trace-contract.ts @@ -16,19 +16,51 @@ export const SCORED_TERMINAL_STOP_CONDITIONS = new Set([ ]); const CONTEXT_WINDOW_MESSAGE = - /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum|Input length \d+ exceeds the maximum allowed input length of \d+ tokens\./i; + /ContextWindowExceededError|prompt is too long:\s*\d+\s*tokens?\s*>\s*\d+\s*maximum|Input length \d+ exceeds the maximum allowed input length of \d+ tokens\.|Input tokens exceed the configured limit of \d+ tokens\.\s*(?:Your input contained|Your messages resulted in) \d+ tokens\.(?:\s*Please reduce the length of the messages\.)?/i; function isRecognizedContextWindowError(error: unknown): boolean { if (!error || typeof error !== "object") return false; const row = error as Record; return ( - row.type === "ProviderError" && + (row.type === "ProviderError" || row.type === "OverlongPromptError") && Number(row.status_code) === 400 && typeof row.message === "string" && CONTEXT_WINDOW_MESSAGE.test(row.message) ); } +const RETRYABLE_TRANSPORT_MESSAGES: Record = { + 429: /upstream 429:.*rate limit exceeded/is, + 500: /upstream 500:.*internal server error/is, + 502: /upstream 502:.*provider connection error:.*upstream response stream/is, + 503: /upstream 503:.*overloaded_error.*provider is temporarily unavailable/is, +}; + +function isRecognizedRetryableTransportError(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const row = error as Record; + const status = Number(row.status_code); + return ( + row.type === "ProviderError" && + typeof row.message === "string" && + RETRYABLE_TRANSPORT_MESSAGES[status]?.test(row.message) === true + ); +} + +export function hasOnlyAcceptedCallErrors(trace: Record, stopCondition: string): boolean { + const calls = Array.isArray(trace.calls) ? trace.calls.filter((call: any) => !call?.sampling?.output_config) : []; + const errorIndexes = calls.flatMap((call: any, index: number) => call?.error ? [index] : []); + if (errorIndexes.length === 0) return true; + if (stopCondition === "context_length") { + return errorIndexes.every((index) => isRecognizedContextWindowError(calls[index].error)); + } + if (stopCondition !== "agent_completed") return false; + return errorIndexes.every((index) => + isRecognizedRetryableTransportError(calls[index].error) && + calls.slice(index + 1).some((later: any) => !later?.error && later?.usage), + ); +} + export function isNormalizedContextWindowFailure(trace: Record): boolean { const errors = Array.isArray(trace.errors) ? trace.errors : []; const callErrors = (Array.isArray(trace.calls) ? trace.calls : []) @@ -72,7 +104,7 @@ export function primeTraceDisposition( if (!Array.isArray(trace.errors) || trace.errors.length > 0) { return rejected(stopCondition, "runtime/provider errors are present"); } - if ((trace.calls ?? []).some((call: any) => call?.error)) { + if (!hasOnlyAcceptedCallErrors(trace, stopCondition)) { return rejected(stopCondition, "provider call errors are present"); } if (!Number.isFinite(trace.rewards?.final_state)) { diff --git a/tests/prime-benchmark-import.test.mjs b/tests/prime-benchmark-import.test.mjs index a25c0ff0..29618e4f 100644 --- a/tests/prime-benchmark-import.test.mjs +++ b/tests/prime-benchmark-import.test.mjs @@ -172,10 +172,27 @@ test("imports Prime-native traces into an anonymized benchmark and renders the r assert.match(viewer, /"required_tasks":50/); assert.match(viewer, /clean tasks/); assert.match(viewer, /corrected-v3\/provider-availability\.json/); + const diagnosticConfigPath = join(root, "diagnostic-config.json"); + writeFileSync(diagnosticConfigPath, JSON.stringify({ + ...config, + benchmark_id: "synthetic-prime-diagnostic-v1", + name: "Synthetic Prime Diagnostic", + benchmark_mode: "diagnostic", + scorecard_output_dir: join(root, "private-scorecards", "synthetic-diagnostic"), + }, null, 2)); + assert.throws( + () => importPrimeBenchmark(diagnosticConfigPath), + /diagnostic benchmark configs may build private scorecards but cannot be aggregate-imported/, + ); + const diagnosticRendered = spawnSync(process.execPath, [renderer, diagnosticConfigPath], { encoding: "utf8" }); + assert.equal(diagnosticRendered.status, 0, diagnosticRendered.stderr); + const diagnosticViewer = readFileSync(join(root, "private-scorecards", "synthetic-diagnostic", "viewer", "index.html"), "utf8"); + assert.match(diagnosticViewer, /DIAGNOSTIC AUDIT/); + assert.match(diagnosticViewer, /Non-authoritative private review/); const entries = discoverPrimeScorecards(join(root, "private-scorecards")); - assert.equal(entries.length, 1); - assert.equal(entries[0].benchmark_id, "synthetic-prime-benchmark-v1"); - assert.equal(entries[0].rollouts, 2); + assert.equal(entries.length, 2); + const authoritativeEntry = entries.find((entry) => entry.benchmark_id === "synthetic-prime-benchmark-v1"); + assert.equal(authoritativeEntry.rollouts, 2); const gallery = renderScorecardGallery(entries); assert.match(gallery, /Synthetic Prime Benchmark/); assert.match(gallery, /\/b\/synthetic-prime-benchmark\//); @@ -510,3 +527,53 @@ test("normalizes Poolside's exact input-length context-window failure shape", () trace.errors[0].message = "Input length is invalid."; assert.equal(primeTraceDisposition(trace, "0.2.1").accepted, false); }); + +test("accepts only recovered, recognized retryable call errors inside a scored completed trace", () => { + const trace = syntheticTrace("poolside/laguna-s-2.1"); + trace.timing = { generation: { start: 100, end: 103 } }; + trace.calls.splice(1, 0, { + error: { + type: "ProviderError", + status_code: 429, + message: 'upstream 429: {"error":"Rate limit exceeded"}', + }, + usage: null, + time: { start: 101.5, end: 102 }, + }); + trace.calls.push({ + usage: { prompt_tokens: 1100, cached_input_tokens: 0, completion_tokens: 50 }, + time: { start: 102, end: 103 }, + }); + + assert.equal(primeTraceDisposition(trace, "0.2.1").accepted, true); + assert.equal(validatePrimeTrace(trace, { verifierVersion: "0.2.1" }).accepted, true); + + const terminalTransportFailure = structuredClone(trace); + terminalTransportFailure.calls.pop(); + assert.equal(primeTraceDisposition(terminalTransportFailure, "0.2.1").accepted, false); + + const unrecognized = structuredClone(trace); + unrecognized.calls[1].error.message = "upstream 429: unrelated bad request"; + assert.equal(primeTraceDisposition(unrecognized, "0.2.1").accepted, false); +}); + +test("accepts the exact scored OpenAI overlong-prompt call shape", () => { + const trace = syntheticTrace("gpt-5.5", 0); + trace.stop_condition = "context_length"; + trace.metrics.final_state_partial_credit = 0; + trace.timing = { generation: { start: 100, end: 102 } }; + trace.calls.push({ + error: { + type: "OverlongPromptError", + status_code: 400, + message: "upstream 400: OpenAI upstream error: Input tokens exceed the configured limit of 922000 tokens. Your messages resulted in 923456 tokens. Please reduce the length of the messages.", + }, + usage: null, + time: { start: 101.5, end: 102 }, + }); + assert.equal(primeTraceDisposition(trace, "0.2.1").accepted, true); + assert.equal(validatePrimeTrace(trace, { verifierVersion: "0.2.1" }).accepted, true); + + trace.calls[1].error.message = "input was too large"; + assert.equal(primeTraceDisposition(trace, "0.2.1").accepted, false); +}); From 1667794744aedeb23f4bd9ab06b107a875903740 Mon Sep 17 00:00:00 2001 From: luis manrique <166242911+lluisinthedesert@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:15:32 -0700 Subject: [PATCH 5/5] Move scorecard graph below leaderboard --- runtime-assets/prime-scorecard/build-scorecard.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime-assets/prime-scorecard/build-scorecard.mjs b/runtime-assets/prime-scorecard/build-scorecard.mjs index b6214710..c9c316e0 100644 --- a/runtime-assets/prime-scorecard/build-scorecard.mjs +++ b/runtime-assets/prime-scorecard/build-scorecard.mjs @@ -455,7 +455,7 @@ function summaryView(models){ return ''+(index+1)+''+statusPill(stat)+baseline+''+stat.score.toFixed(3)+(incumbent&&stat!==incumbent?delta(stat.score,incumbent.score):"")+''+Math.round(stat.pass*stat.rows.length)+'/'+stat.rows.length+(incumbent&&stat!==incumbent?delta(stat.pass,incumbent.pass):"")+''+ms(stat.time)+(incumbent&&stat!==incumbent?delta(stat.time,incumbent.time,true):"")+''+usd(stat.cost)+(incumbent&&stat!==incumbent?delta(stat.cost,incumbent.cost,true):"")+''+stat.calls.toFixed(1)+(incumbent&&stat!==incumbent?delta(stat.calls,incumbent.calls,true):"")+''+(open?'
'+rollouts+'
':""); }).join(""); const taskCards=tasks.map((task,index)=>{const rows=stats.map(stat=>D.rollouts.find(row=>row.model===stat.model&&row.task_id===task.task_id)).filter(Boolean);const incumbentRow=rows.find(row=>row.model===D.incumbent_model);return '
Task '+(index+1)+''+esc(task.prompt)+'
'+rows.filter(row=>row.strict_pass).length+'/'+rows.length+' pass
'+rows.map(row=>'').join("")+'
ModelScoreTimeCostCalls
'+esc(shortModel(row.model))+(row.model===D.incumbent_model?'incumbent':"")+''+row.partial_credit.toFixed(2)+(incumbentRow&&row!==incumbentRow?delta(row.partial_credit,incumbentRow.partial_credit):"")+''+ms(row.latency_ms)+(incumbentRow&&row!==incumbentRow?delta(row.latency_ms,incumbentRow.latency_ms,true):"")+''+usd(row.cost_usd)+(incumbentRow&&row!==incumbentRow?delta(row.cost_usd,incumbentRow.cost_usd,true):"")+''+row.turns+(incumbentRow&&row!==incumbentRow?delta(row.turns,incumbentRow.turns,true):"")+'
'}).join(""); - return '
'+highlights+availability+'

Verifier score vs cost per task

'+paretoPlot(stats)+'

Model leaderboard

'+sortHead("model","Model")+sortHead("score","Score")+sortHead("pass","Pass")+sortHead("time","Time / task")+sortHead("cost","Cost / task")+sortHead("calls","Calls")+''+leaderboard+'
#

Production task comparison

'+taskCards+'
'; + return '
'+highlights+availability+'

Model leaderboard

'+sortHead("model","Model")+sortHead("score","Score")+sortHead("pass","Pass")+sortHead("time","Time / task")+sortHead("cost","Cost / task")+sortHead("calls","Calls")+''+leaderboard+'
#

Verifier score vs cost per task

'+paretoPlot(stats)+'

Production task comparison

'+taskCards+'
'; } function summaryOverview(models){ const stats=modelStats(models),qualified=stats.filter(stat=>stat.pass===1),priced=stats.filter(stat=>Number.isFinite(stat.cost));