diff --git a/docs/developer_overview.md b/docs/developer_overview.md index aa15587..3ed3a29 100644 --- a/docs/developer_overview.md +++ b/docs/developer_overview.md @@ -26,9 +26,9 @@ focused internal notes instead of duplicating their details. 4. `eval-magic ingest` reads the harness outputs, transcript evidence, guard denials, and final task state. Runner-owned deterministic checks and diff-scope evidence are collected here. 5. `eval-magic grade` evaluates runner-owned assertions, writes one bounded `judge-evidence.md` - per recorded run, and emits tasks for assertions that require an LLM. Each task inlines the - exact bundle for its run. `eval-magic dispatch --judges` runs those judge tasks through the - selected harness. + per recorded run, and emits one or more tasks for assertions that require an LLM. Every sample + inlines the exact same bundle for its run. `eval-magic dispatch --judges` runs those judge tasks + through the selected harness. 6. `eval-magic finalize` checks that required work is complete and writes the final per-run and benchmark artifacts. `eval-magic aggregate` combines campaigns when a larger comparison is needed. diff --git a/docs/guides/judging.md b/docs/guides/judging.md index 9ee7877..6d0ff54 100644 --- a/docs/guides/judging.md +++ b/docs/guides/judging.md @@ -29,6 +29,59 @@ files are injected, and keeping the bundle at that boundary prevents a judge fro runner-owned mutations with agent work. Mechanical assertion results remain runner-owned and are merged during `finalize`. +## Sample an LLM judge + +An authored `llm_judge` assertion can request several independent verdicts for the same run: + +```json +{ + "id": "clear-review", + "type": "llm_judge", + "rubric": "The review identifies the most important defect and explains its impact.", + "samples": 10 +} +``` + +Use `run --judge-samples N` to set a campaign-wide default. An assertion's `samples` field takes +precedence over that default. The effective count must be at least one. The framework-injected +`__skill_invoked` meta-check is not substantive grading and remains single-shot. + +Each sample is a separate judge task and response, but every sample for a run receives the exact +same bounded `judge-evidence.md`. The agent is not rerun, and eval-magic does not rebuild or expand +the evidence between samples. This measures agreement among repeated judgments of one execution; +it does not estimate how reliably the agent would succeed across repeated executions. + +For a sampled assertion with `N` verdicts, `grading.json` reports: + +- each verdict in order, including its evidence and confidence +- vote counts and the pass proportion `p = passed / N` +- `pass_power_k = p^N`, the estimated probability that all `N` judgments pass under an + independent-draw assumption + +For example, 6 / 10 passing verdicts produce a vote proportion of `0.6` and pass^k of +`0.6^10`, approximately `0.006047`. This is a stricter judge-consistency endpoint than majority +vote. It is not a statistical significance test. Anthropic's +[eval overview](https://www.anthropic.com/engineering/demystifying-evals-for-ai-agents) explains +the pass^k interpretation in the broader agent-evaluation context. Correlated judge behavior means +`p^N` is a consistency score rather than a calibrated probability, so retain and inspect the +individual verdicts. + +Multi-sample prompts and responses add `__sample-N` to the assertion id in their filenames, and +`judge-tasks.json` records `sample_index` and `sample_count`. `dispatch --judges` skips every +nonempty response independently, so rerunning fills only missing samples. During `finalize`, a +missing response fails that sample and leaves the other samples intact. + +When a run mixes sampled and binary assertions, each authored assertion has equal weight in the +run summary. A binary assertion contributes either 0 or 1; a sampled assertion contributes its +vote proportion to `vote_proportion` and its `p^N` value to `pass_power_k`. `benchmark.json` +reports both endpoints by condition, their deltas, and pooled per-assertion vote counts. The run +plan prints these non-binary endpoints instead of a Fisher exact floor. Fully binary campaigns +retain the Fisher sample-size line. + +An effective sample count of one preserves the binary artifact contract: the legacy response +filename, assertion-level `passed`, `evidence`, and `confidence`, binary grading summary, and +per-assertion `passed` / `n` benchmark rollup remain unchanged. + ## How the bounds work Each evidence bundle is at most 98,304 bytes (96 KiB). The complete judge prompt, including its diff --git a/schema/benchmark.schema.json b/schema/benchmark.schema.json index 00fcbb0..90e5ecb 100644 --- a/schema/benchmark.schema.json +++ b/schema/benchmark.schema.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://slow-powers.dev/schemas/benchmark.schema.json", "title": "Benchmark", - "description": "Output of evals:aggregate. The before/after comparison across the two conditions, with per-condition stats, per-assertion pass counts, the a-b delta, and validity warnings. Lives at /iteration-N/benchmark.json.", + "description": "Output of evals:aggregate. The before/after comparison across the two conditions, with per-condition grading stats, per-assertion binary pass or sampled vote counts, the a-b delta, and validity warnings. Lives at /iteration-N/benchmark.json.", "type": "object", "required": [ "generated", @@ -39,12 +39,17 @@ }, "assertions": { "type": "object", - "description": "Observed substantive assertion pass counts, keyed by eval id, assertion id, then condition. Omitted from historical benchmarks generated before this rollup was available.", + "description": "Observed substantive assertion results, keyed by eval id, assertion id, then condition. Binary assertions carry passed/n; sampled assertions carry pooled votes, run count, samples per run, and pass^k. Omitted from historical benchmarks generated before this rollup was available.", "additionalProperties": { "type": "object", "additionalProperties": { "type": "object", - "additionalProperties": { "$ref": "#/definitions/assertionCount" } + "additionalProperties": { + "oneOf": [ + { "$ref": "#/definitions/assertionCount" }, + { "$ref": "#/definitions/sampledAssertionCount" } + ] + } } } }, @@ -72,6 +77,8 @@ "properties": { "direction": { "type": "string" }, "pass_rate": { "type": "number" }, + "vote_proportion": { "type": "number", "description": "Condition A mean vote proportion minus condition B mean vote proportion." }, + "pass_power_k": { "type": "number", "description": "Condition A mean pass^k minus condition B mean pass^k." }, "duration_ms": { "type": "number" }, "total_tokens": { "type": "number" } } @@ -175,6 +182,28 @@ } } }, + "sampledAssertionCount": { + "type": "object", + "required": ["votes", "samples_per_run", "run_count", "pass_power_k"], + "additionalProperties": false, + "properties": { + "votes": { "$ref": "#/definitions/voteCount" }, + "samples_per_run": { "type": "integer", "minimum": 2 }, + "run_count": { "type": "integer", "minimum": 1 }, + "pass_power_k": { "type": "number", "minimum": 0, "maximum": 1, "description": "Pooled vote proportion raised to samples_per_run." } + } + }, + "voteCount": { + "type": "object", + "required": ["passed", "failed", "total", "proportion"], + "additionalProperties": false, + "properties": { + "passed": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 2 }, + "proportion": { "type": "number", "minimum": 0, "maximum": 1, "description": "Passed votes divided by total votes across all runs in this cell." } + } + }, "stats": { "type": "object", "required": ["mean", "stddev", "n"], @@ -200,6 +229,8 @@ "additionalProperties": false, "properties": { "pass_rate": { "$ref": "#/definitions/stats" }, + "vote_proportion": { "$ref": "#/definitions/stats", "description": "Per-run equal-weight mean of substantive assertion vote proportions; present when the campaign contains sampled grading." }, + "pass_power_k": { "$ref": "#/definitions/stats", "description": "Per-run equal-weight mean of substantive assertion pass^k values; present when the campaign contains sampled grading." }, "duration_ms": { "$ref": "#/definitions/stats" }, "total_tokens": { "$ref": "#/definitions/stats" }, "skill_invocation_n": { "type": "integer" }, diff --git a/schema/evals.schema.json b/schema/evals.schema.json index 54d287a..4759fe7 100644 --- a/schema/evals.schema.json +++ b/schema/evals.schema.json @@ -270,6 +270,11 @@ "model": { "type": "string", "description": "Optional judge model override. When absent, defaults to the run-level judge model recorded in conditions.json, or the harness default when no run-level model was selected." + }, + "samples": { + "type": "integer", + "minimum": 1, + "description": "Independent judge verdicts requested for this assertion. Overrides run --judge-samples and defaults to 1." } } }, diff --git a/schema/grading.schema.json b/schema/grading.schema.json index 5a3c2f3..34d785a 100644 --- a/schema/grading.schema.json +++ b/schema/grading.schema.json @@ -9,75 +9,129 @@ "properties": { "assertion_results": { "type": "array", - "items": { - "type": "object", - "required": ["id", "passed", "evidence"], - "additionalProperties": false, - "properties": { - "id": { - "type": "string", - "description": "Matches the assertion id in evals.json." - }, - "passed": { "type": "boolean" }, - "evidence": { - "type": "string", - "description": "Direct quote or specific reference from the run record. Vague summaries are not evidence." - }, - "confidence": { - "type": "number", - "minimum": 0, - "maximum": 1, - "description": "Judge confidence. Low confidence (< 0.7) flags this result for human review. Always 1.0 for transcript_check results." - }, - "grader": { - "type": "string", - "enum": ["transcript_check", "llm_judge", "command_check", "diff_scope"], - "description": "Which grader produced this result." - } - } - } + "items": { "$ref": "#/definitions/assertionResult" } }, "summary": { + "$ref": "#/definitions/gradingSummary" + }, + "meta_results": { + "type": "array", + "description": "Framework-injected meta-assertions (e.g. skill-invocation check). Reserved id prefix: __ (double underscore). Tracked separately from substantive assertion_results so they do not pollute the skill effectiveness pass_rate.", + "items": { "$ref": "#/definitions/binaryAssertionResult" } + }, + "meta_summary": { "type": "object", - "required": ["passed", "failed", "total", "pass_rate"], "additionalProperties": false, "properties": { "passed": { "type": "integer", "minimum": 0 }, "failed": { "type": "integer", "minimum": 0 }, "total": { "type": "integer", "minimum": 0 }, - "pass_rate": { "type": "number", "minimum": 0, "maximum": 1 } + "skill_invoked": { + "description": "True when the skill-invocation meta-check passed; false when the judge found no evidence the skill influenced behavior; null when no skill was loaded for this run.", + "type": ["boolean", "null"] + } } + } + }, + "definitions": { + "assertionResult": { + "oneOf": [ + { "$ref": "#/definitions/binaryAssertionResult" }, + { "$ref": "#/definitions/sampledAssertionResult" } + ] }, - "meta_results": { - "type": "array", - "description": "Framework-injected meta-assertions (e.g. skill-invocation check). Reserved id prefix: __ (double underscore). Tracked separately from substantive assertion_results so they do not pollute the skill effectiveness pass_rate.", - "items": { - "type": "object", - "required": ["id", "passed", "evidence"], - "additionalProperties": false, - "properties": { - "id": { "type": "string" }, - "passed": { "type": "boolean" }, - "evidence": { "type": "string" }, - "confidence": { "type": "number", "minimum": 0, "maximum": 1 }, - "grader": { - "type": "string", - "enum": ["transcript_check", "llm_judge", "command_check", "diff_scope"] - } + "binaryAssertionResult": { + "type": "object", + "required": ["id", "passed", "evidence"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Matches the assertion id in evals.json." + }, + "passed": { "type": "boolean" }, + "evidence": { + "type": "string", + "description": "Direct quote or specific reference from the run record. Vague summaries are not evidence." + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1, + "description": "Judge confidence. Low confidence (< 0.7) flags this result for human review. Always 1.0 for transcript_check results." + }, + "grader": { + "type": "string", + "enum": ["transcript_check", "llm_judge", "command_check", "diff_scope"], + "description": "Which grader produced this result." } } }, - "meta_summary": { + "sampledAssertionResult": { "type": "object", + "required": ["id", "grader", "votes", "judge_samples"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "description": "Matches the authored llm_judge assertion id in evals.json." }, + "grader": { "const": "llm_judge", "description": "Sampled results are produced only by authored LLM-judge assertions." }, + "votes": { "$ref": "#/definitions/judgeVotes" }, + "judge_samples": { + "type": "array", + "minItems": 2, + "description": "Every requested verdict in sample-index order. A missing response is retained as a failed sample rather than failing the whole assertion.", + "items": { "$ref": "#/definitions/judgeSample" } + } + } + }, + "judgeVotes": { + "type": "object", + "required": ["passed", "failed", "total", "proportion", "pass_power_k"], + "additionalProperties": false, + "properties": { + "passed": { "type": "integer", "minimum": 0 }, + "failed": { "type": "integer", "minimum": 0 }, + "total": { "type": "integer", "minimum": 2 }, + "proportion": { "type": "number", "minimum": 0, "maximum": 1, "description": "Passed divided by total." }, + "pass_power_k": { "type": "number", "minimum": 0, "maximum": 1, "description": "proportion raised to total: the estimated probability every requested judgment passes." } + } + }, + "judgeSample": { + "type": "object", + "required": ["sample_index", "passed", "evidence", "confidence"], + "additionalProperties": false, + "properties": { + "sample_index": { "type": "integer", "minimum": 1 }, + "passed": { "type": "boolean" }, + "evidence": { "type": "string" }, + "confidence": { "type": "number", "minimum": 0, "maximum": 1 } + } + }, + "gradingSummary": { + "oneOf": [ + { "$ref": "#/definitions/binaryGradingSummary" }, + { "$ref": "#/definitions/sampledGradingSummary" } + ] + }, + "binaryGradingSummary": { + "type": "object", + "required": ["passed", "failed", "total", "pass_rate"], "additionalProperties": false, "properties": { "passed": { "type": "integer", "minimum": 0 }, "failed": { "type": "integer", "minimum": 0 }, "total": { "type": "integer", "minimum": 0 }, - "skill_invoked": { - "description": "True when the skill-invocation meta-check passed; false when the judge found no evidence the skill influenced behavior; null when no skill was loaded for this run.", - "type": ["boolean", "null"] - } + "pass_rate": { "type": "number", "minimum": 0, "maximum": 1 } + } + }, + "sampledGradingSummary": { + "type": "object", + "required": ["total", "pass_rate", "vote_proportion", "pass_power_k"], + "additionalProperties": false, + "properties": { + "total": { "type": "integer", "minimum": 0 }, + "pass_rate": { "type": "number", "minimum": 0, "maximum": 1, "description": "Compatibility alias for vote_proportion in a sampled grading." }, + "vote_proportion": { "type": "number", "minimum": 0, "maximum": 1, "description": "Equal-weight mean of each substantive assertion's binary result or sampled vote proportion." }, + "pass_power_k": { "type": "number", "minimum": 0, "maximum": 1, "description": "Equal-weight mean of each substantive assertion's binary result or sampled pass^k value." } } } } diff --git a/schema/judge-tasks.schema.json b/schema/judge-tasks.schema.json index 51eabf3..2557604 100644 --- a/schema/judge-tasks.schema.json +++ b/schema/judge-tasks.schema.json @@ -40,6 +40,10 @@ "dispatch_prompt_bytes", "dispatch_prompt_byte_limit" ], + "dependencies": { + "sample_index": ["sample_count"], + "sample_count": ["sample_index"] + }, "additionalProperties": false, "properties": { "eval_id": { "type": "string" }, @@ -50,6 +54,16 @@ "description": "1-based run index within a multi-run (eval, condition) cell; absent for single-run cells." }, "assertion_id": { "type": "string" }, + "sample_index": { + "type": "integer", + "minimum": 1, + "description": "1-based verdict index for an assertion requesting more than one judge sample; absent for the legacy single-verdict shape." + }, + "sample_count": { + "type": "integer", + "minimum": 2, + "description": "Total verdicts requested for this sampled assertion; absent together with sample_index when the effective count is 1." + }, "rubric": { "type": "string" }, "model": { "type": ["string", "null"], diff --git a/src/cli/args.rs b/src/cli/args.rs index 9660a47..f86d937 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -520,11 +520,12 @@ pub struct RunArgs { /// unchanged (artifacts sit directly in the condition directory). The /// benchmark's per-condition `mean`/`stddev`/`n` then reflect all runs. A /// per-eval `runs` field in evals.json overrides this flag for that eval. - /// Before staging, the run summary prints the minimum attainable two-sided - /// Fisher exact p-value for each effective run count, assuming a binary - /// endpoint and perfect separation between the two conditions. This is a - /// sample-size bound only: eval-magic does not calculate observed p-values or - /// apply a significance threshold. + /// Before staging, a fully binary run summary prints the minimum attainable + /// two-sided Fisher exact p-value for each effective run count, assuming + /// perfect separation between the two conditions. A run with sampled LLM + /// assertions instead identifies vote proportion and pass^k as non-binary + /// endpoints. eval-magic does not calculate observed p-values or apply a + /// significance threshold. #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] pub runs: u32, /// Agent-under-test model for CLI dispatches; otherwise recorded as @@ -554,6 +555,15 @@ pub struct RunArgs { /// `conditions.json` for `promote-baseline`. #[arg(long)] pub judge_model: Option, + /// Default verdict count for authored LLM-judge assertions (default: 1). + /// + /// An assertion-level `samples` value overrides this option. Counts above one + /// dispatch independent judges over the same bounded evidence bundle and are + /// reported as vote proportion p plus pass^k = p^N. The framework-injected + /// skill-invocation meta-check remains single-shot. See + /// `eval-magic docs judging`. + #[arg(long, default_value_t = 1, value_parser = clap::value_parser!(u32).range(1..))] + pub judge_samples: u32, /// Model that answers the agent for evals declaring a `responder`. /// /// `dispatch` consults it once after every round, through the same harness @@ -613,9 +623,9 @@ pub(crate) enum Commands { /// per condition and repetition. Scripted follow-ups add up to `2R × F` model /// turns for `F` declared follow-ups. A `responder` case instead adds one /// agent turn and one small responder dispatch per round, up to its - /// `max_turns` bound. Each `llm_judge` assertion creates a judge task per - /// condition and repetition. Review the printed run summary and obtain - /// confirmation before spending model usage. + /// `max_turns` bound. Each `llm_judge` assertion creates its effective sample + /// count of judge tasks per condition and repetition. Review the printed run + /// summary and obtain confirmation before spending model usage. /// /// Git is required. Every task environment is initialized as an independent, /// clean repository on branch `work` with a deterministic baseline commit and @@ -693,11 +703,12 @@ pub(crate) enum Commands { /// environment overrides and running every environment matrix cell. Diff /// scope is captured before held-out files are injected. Then stops at the /// judge hand-off, writing one bounded `judge-evidence.md` per recorded run - /// and listing a judge task per `llm_judge` assertion. The exact evidence - /// bundle is shared by that run's tasks and inlined into their prompts. Requires - /// `--iteration`; reads each task's `outputs/-events.jsonl` when the - /// harness exposes transcripts, under `outputs/turn-/`. Dispatch the judge - /// tasks it lists with `eval-magic dispatch --judges`. + /// and listing the effective sample count of judge tasks per `llm_judge` + /// assertion. The exact evidence bundle is shared by that run's tasks and + /// inlined into their prompts. Requires `--iteration`; reads each task's + /// `outputs/-events.jsonl` when the harness exposes transcripts, + /// under `outputs/turn-/`. Dispatch the judge tasks it lists with + /// `eval-magic dispatch --judges`. /// Re-running after a fix is safe — every sub-step skips work already done. Ingest(CommonArgs), /// Finalize grading after judge responses are in. @@ -705,9 +716,11 @@ pub(crate) enum Commands { /// Fixed-order chain: grade `--finalize` → aggregate. Merges judge verdicts, /// runner-owned `command_check` results, and deterministic `diff_scope` /// files/lines thresholds into normal `grading.json` files, then writes - /// `benchmark.json` with a per-assertion `passed`/`n` rollup from observed - /// assertion results and raw per-run metrics from `diff-scope.json`. The - /// per-run changed-file list and `diff.patch` stay beside each run rather + /// `benchmark.json` with per-assertion rollups from observed assertion + /// results. Binary assertions keep their `passed`/`n` rollup; sampled LLM + /// assertions retain every verdict, pooled vote counts, vote proportion, and + /// pass^k. Raw per-run metrics come from `diff-scope.json`. + /// The per-run changed-file list and `diff.patch` stay beside each run rather /// than being rolled up. If a live /// guard remains armed — the cwd guard, or any per-task Cli env guard — prints /// a `teardown` reminder before source edits. Requires `--iteration`. @@ -785,6 +798,12 @@ pub(crate) enum Commands { /// `eval-magic docs judging`. With `--finalize`, merges every result into /// per-run `grading.json`. /// + /// An authored `llm_judge.samples` count overrides `run --judge-samples`. + /// Counts above one emit independent `__sample-N` tasks over the shared + /// evidence bundle. Finalization retains each verdict and reports vote + /// proportion plus pass^k; one missing response fails only that sample. An + /// effective count of one preserves the binary grading artifact. + /// /// Injects the `__skill_invoked` meta-check — did the skill actually influence /// behavior? It has two tiers, chosen automatically per run: code-based (where /// the staged slug + transcript are available, as on Claude Code, it checks the @@ -797,8 +816,9 @@ pub(crate) enum Commands { /// Aggregate before/after benchmark deltas. /// /// Reads grading + timing from an iteration and writes `benchmark.json` with - /// pass-rate / duration / token stats per condition, a per-assertion - /// `passed`/`n` rollup from observed assertion results, the delta, + /// grading / duration / token stats per condition, a per-assertion binary + /// `passed`/`n` or sampled-vote rollup from observed assertion results, the + /// delta, /// `validity_warnings` (including incomplete timing sample counts, one per /// task in `guard-denials.json`, and one per task in /// `permission-denials.json` whose refusals were not the guard's own, plus diff --git a/src/cli/commands/run.rs b/src/cli/commands/run.rs index 850fe25..f02f964 100644 --- a/src/cli/commands/run.rs +++ b/src/cli/commands/run.rs @@ -52,6 +52,7 @@ pub(crate) fn run_run(args: RunArgs) -> anyhow::Result<()> { agent_model: args.agent_model.as_deref(), agent_env, judge_model: args.judge_model.as_deref(), + judge_samples: (args.judge_samples != 1).then_some(args.judge_samples), responder_model: args.responder_model.as_deref(), label: args.label.as_deref(), }, diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 1113890..798b31a 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -98,6 +98,7 @@ fn dispatch(command: Option, harness_file: Option<&str>) -> anyhow::Re agent_model: None, agent_env: Vec::new(), judge_model: None, + judge_samples: 1, responder_model: None, label: None, })); diff --git a/src/cli/run/drive/judges.rs b/src/cli/run/drive/judges.rs index 11924c7..dd2e9c8 100644 --- a/src/cli/run/drive/judges.rs +++ b/src/cli/run/drive/judges.rs @@ -38,11 +38,19 @@ struct JudgeTask { model: Option, response_path: String, dispatch_prompt_path: String, + #[serde(default)] + sample_index: Option, + #[serde(default)] + sample_count: Option, } impl JudgeTask { fn description(&self) -> String { - format!("{}:{}:{}", self.eval_id, self.condition, self.assertion_id) + let base = format!("{}:{}:{}", self.eval_id, self.condition, self.assertion_id); + match (self.sample_index, self.sample_count) { + (Some(index), Some(count)) => format!("{base}:sample-{index}-of-{count}"), + _ => base, + } } /// A verdict is present once its response file exists and is non-empty — diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index 9433abf..a45f878 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -63,6 +63,7 @@ pub(super) fn write_dispatch( agent_model: opts.agent_model.map(str::to_owned), agent_env: opts.agent_env.clone(), judge_model: opts.judge_model.map(str::to_owned), + judge_samples: opts.judge_samples, responder_model: opts.responder_model.map(str::to_owned), label: opts.label.map(str::to_owned), codebases: r.codebases.iter().map(super::RunCodebase::usage).collect(), @@ -297,6 +298,12 @@ pub(super) fn write_dispatch( .expect("dispatch envelope is an object") .insert("agent_env".to_string(), json!(conditions.agent_env)); } + if let Some(samples) = conditions.judge_samples { + dispatch_json + .as_object_mut() + .expect("dispatch envelope is an object") + .insert("judge_samples".to_string(), json!(samples)); + } // Unconditional: `dispatch` drives every task from this envelope, so the // descriptor it freezes and the guard state it dispatches under are needed // whether or not any eval declares scripted turns. diff --git a/src/cli/run/orchestrate/mod.rs b/src/cli/run/orchestrate/mod.rs index 5acf13b..6cc0eae 100644 --- a/src/cli/run/orchestrate/mod.rs +++ b/src/cli/run/orchestrate/mod.rs @@ -19,8 +19,8 @@ use crate::adapters::{CliDispatchContext, adapter_for}; use crate::cli::command_target_args; use crate::core::fs::artifact_path; use crate::core::{ - CodebaseRecord, CodebaseSource, CodebaseUse, Eval, GuardPolicyConfig, Mode, RunContext, - SkillSource, SourceKind, SourceRecord, + Assertion, CodebaseRecord, CodebaseSource, CodebaseUse, Eval, GuardPolicyConfig, Mode, + RunContext, SkillSource, SourceKind, SourceRecord, }; use crate::source::ResolvedSource; @@ -61,6 +61,9 @@ pub struct RunOptions<'a> { /// Resolved descriptor defaults plus run-level agent environment overrides. pub agent_env: BTreeMap, pub judge_model: Option<&'a str>, + /// Non-default judge sample count. Absence means one and keeps legacy + /// manifests byte-compatible. + pub judge_samples: Option, pub responder_model: Option<&'a str>, pub label: Option<&'a str>, } @@ -360,12 +363,29 @@ fn print_run_plan(ctx: &RunContext, opts: &RunOptions, r: &Resolved) { ids.join(", ") ); } - let effective_run_counts: BTreeSet = r - .selected_evals - .iter() - .map(|eval| eval.runs.unwrap_or(opts.runs)) - .collect(); - for runs in effective_run_counts { + let mut binary_run_counts = BTreeSet::new(); + let mut sampled_endpoints: BTreeSet<(u32, Vec)> = BTreeSet::new(); + for eval in &r.selected_evals { + let runs = eval.runs.unwrap_or(opts.runs); + let sample_counts: BTreeSet = eval + .assertions + .as_deref() + .unwrap_or(&[]) + .iter() + .filter_map(|assertion| match assertion { + Assertion::LlmJudge(judge) => { + Some(judge.samples.or(opts.judge_samples).unwrap_or(1)) + } + _ => None, + }) + .collect(); + if sample_counts.iter().any(|count| *count > 1) { + sampled_endpoints.insert((runs, sample_counts.into_iter().collect())); + } else { + binary_run_counts.insert(runs); + } + } + for runs in binary_run_counts { let run_label = if runs == 1 { "run" } else { "runs" }; println!( " statistical floor: 2 conditions × {runs} {run_label}; minimum attainable \ @@ -373,6 +393,19 @@ fn print_run_plan(ctx: &RunContext, opts: &RunOptions, r: &Resolved) { format_minimum_attainable_fisher_p_value(runs) ); } + for (runs, sample_counts) in sampled_endpoints { + let run_label = if runs == 1 { "run" } else { "runs" }; + let counts = sample_counts + .iter() + .map(u32::to_string) + .collect::>() + .join(", "); + println!( + " statistical endpoint: 2 conditions × {runs} {run_label}; LLM judge sample counts \ + per assertion: {counts}; report vote proportion and pass^k; the binary Fisher exact \ + floor does not apply" + ); + } if opts.no_stage { println!( " staging: disabled (--no-stage) — skills will be inlined into dispatch_prompt for harnesses without project-local skill discovery" diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index b7ce0f9..f387c19 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -491,6 +491,7 @@ mod tests { id: "a2".into(), rubric: "r".into(), model: None, + samples: None, }); assert!(evals_use_transcript_check(&[eval_with(Some(vec![ diff --git a/src/core/grading.rs b/src/core/grading.rs new file mode 100644 index 0000000..f2938f5 --- /dev/null +++ b/src/core/grading.rs @@ -0,0 +1,178 @@ +//! Grading artifact types shared by finalization and aggregation. + +use serde::{Deserialize, Serialize}; + +/// The result of grading one binary assertion. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AssertionResult { + pub id: String, + pub passed: bool, + pub evidence: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub grader: Option, +} + +/// One verdict inside a multi-sample LLM assertion result. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct JudgeSampleResult { + pub sample_index: u32, + pub passed: bool, + pub evidence: String, + pub confidence: f64, +} + +/// Vote totals and derived consistency metrics for one sampled assertion. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct JudgeVotes { + pub passed: u32, + pub failed: u32, + pub total: u32, + pub proportion: f64, + pub pass_power_k: f64, +} + +/// A substantive LLM assertion represented by its independent judge verdicts, +/// without an ambiguous assertion-level boolean. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct SampledAssertionResult { + pub id: String, + pub grader: Grader, + pub votes: JudgeVotes, + pub judge_samples: Vec, +} + +/// A substantive assertion result. The untagged variants preserve the exact +/// legacy boolean shape while admitting sampled LLM results. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum GradedAssertionResult { + Sampled(SampledAssertionResult), + Binary(AssertionResult), +} + +impl From for GradedAssertionResult { + fn from(result: AssertionResult) -> Self { + Self::Binary(result) + } +} + +impl GradedAssertionResult { + pub fn id(&self) -> &str { + match self { + Self::Sampled(result) => &result.id, + Self::Binary(result) => &result.id, + } + } + + pub fn vote_proportion(&self) -> f64 { + match self { + Self::Sampled(result) => result.votes.proportion, + Self::Binary(result) => { + if result.passed { + 1.0 + } else { + 0.0 + } + } + } + } + + pub fn pass_power_k(&self) -> f64 { + match self { + Self::Sampled(result) => result.votes.pass_power_k, + Self::Binary(result) => { + if result.passed { + 1.0 + } else { + 0.0 + } + } + } + } +} + +/// Which grader produced an assertion result. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Grader { + TranscriptCheck, + LlmJudge, + CommandCheck, + DiffScope, +} + +/// The full grading output for one run. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct GradingResult { + pub assertion_results: Vec, + // Substantive results + summary first, then the optional meta block — + // grading.json reads as "the verdict, then the validity check on it". + pub summary: GradingSummary, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta_results: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub meta_summary: Option, +} + +/// Legacy pass/fail tallies for an entirely binary grading. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct BinaryGradingSummary { + pub passed: u32, + pub failed: u32, + pub total: u32, + pub pass_rate: f64, +} + +/// Equal-assertion-weight endpoints for a grading containing sampled judges. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct SampledGradingSummary { + pub total: u32, + /// Compatibility alias for `vote_proportion`. + pub pass_rate: f64, + pub vote_proportion: f64, + pub pass_power_k: f64, +} + +/// Per-run grading summary, preserving the exact legacy shape until an +/// assertion requests more than one judge verdict. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum GradingSummary { + Sampled(SampledGradingSummary), + Binary(BinaryGradingSummary), +} + +impl GradingSummary { + pub fn pass_rate(self) -> f64 { + match self { + Self::Sampled(summary) => summary.pass_rate, + Self::Binary(summary) => summary.pass_rate, + } + } + + pub fn vote_proportion(self) -> Option { + match self { + Self::Sampled(summary) => Some(summary.vote_proportion), + Self::Binary(_) => None, + } + } + + pub fn pass_power_k(self) -> Option { + match self { + Self::Sampled(summary) => Some(summary.pass_power_k), + Self::Binary(_) => None, + } + } +} + +/// Tallies for the meta-assertions, plus the skill-invocation determination. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +pub struct MetaSummary { + pub passed: u32, + pub failed: u32, + pub total: u32, + /// `None` (serialized `null`) when invocation could not be determined. + pub skill_invoked: Option, +} diff --git a/src/core/mod.rs b/src/core/mod.rs index 9f3a381..7446d42 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,6 +1,7 @@ //! Shared kernel used by nearly every other module. //! -//! - [`types`] — domain types (`Eval`, `RunRecord`, `Assertion`, `GradingResult`, …) +//! - [`types`] — domain types (`Eval`, `RunRecord`, `Assertion`, …) +//! - [`grading`] — binary and sampled grading artifact types //! - [`context`] — `RunContext` detection from parsed flags / environment //! - [`capabilities`] — per-harness run-option capabilities //! - [`git`] — git spawned with the operator's configuration held off @@ -13,6 +14,7 @@ pub mod capabilities; pub mod context; pub mod fs; pub mod git; +pub mod grading; pub mod runtime; pub mod types; diff --git a/src/core/types.rs b/src/core/types.rs index 8e2600d..eb62d42 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -9,6 +9,10 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; + +// Preserve the established `core::types::*` artifact API while the focused +// implementation lives in `core::grading`. +pub use super::grading::*; use serde_json::Value; use crate::core::context::Harness; @@ -44,6 +48,10 @@ pub struct AssertionLlmJudge { pub rubric: String, #[serde(skip_serializing_if = "Option::is_none")] pub model: Option, + /// Independent judge verdicts requested for this assertion. Absence resolves + /// through the run-level default and ultimately to one. + #[serde(skip_serializing_if = "Option::is_none")] + pub samples: Option, } /// A runner-owned command assertion evaluated against the final task environment. @@ -398,6 +406,10 @@ pub struct ConditionsRecord { /// Operator-declared judge model (provenance, like `agent_model`). #[serde(skip_serializing_if = "Option::is_none")] pub judge_model: Option, + /// Non-default judge verdict count selected for authored `llm_judge` + /// assertions. Absence means one for compatibility with older iterations. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub judge_samples: Option, /// Operator-declared responder model (provenance, like `agent_model`). A /// responder eval puts a third model in the attribution picture, so a /// report that names the agent and the judge has to name this one too. @@ -645,60 +657,6 @@ impl ResponderStopCause { } } -/// The result of grading one assertion. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct AssertionResult { - pub id: String, - pub passed: bool, - pub evidence: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub confidence: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub grader: Option, -} - -/// Which grader produced an assertion result. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum Grader { - TranscriptCheck, - LlmJudge, - CommandCheck, - DiffScope, -} - -/// The full grading output for one run. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct GradingResult { - pub assertion_results: Vec, - // Substantive results + summary first, then the optional meta block — - // grading.json reads as "the verdict, then the validity check on it". - pub summary: GradingSummary, - #[serde(skip_serializing_if = "Option::is_none")] - pub meta_results: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub meta_summary: Option, -} - -/// Pass/fail tallies for the main assertions. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct GradingSummary { - pub passed: u32, - pub failed: u32, - pub total: u32, - pub pass_rate: f64, -} - -/// Tallies for the meta-assertions, plus the skill-invocation determination. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -pub struct MetaSummary { - pub passed: u32, - pub failed: u32, - pub total: u32, - /// `None` (serialized `null`) when invocation could not be determined. - pub skill_invoked: Option, -} - /// Token/duration provenance for a run. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TimingRecord { @@ -724,7 +682,7 @@ pub enum TimingSource { #[cfg(test)] mod tests { use super::*; - use crate::core::context::Harness; + use crate::core::{MetaSummary, context::Harness}; use serde_json::{Value, json}; #[test] @@ -895,6 +853,7 @@ mod tests { agent_model: None, agent_env: BTreeMap::new(), judge_model: None, + judge_samples: None, responder_model: None, label: None, codebases: Vec::new(), diff --git a/src/core/types/artifact_tests.rs b/src/core/types/artifact_tests.rs index 040f2fc..da3ae6f 100644 --- a/src/core/types/artifact_tests.rs +++ b/src/core/types/artifact_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::core::Grader; use serde_json::{Value, json}; #[test] diff --git a/src/pipeline/aggregate.rs b/src/pipeline/aggregate.rs index 42a3678..561e94f 100644 --- a/src/pipeline/aggregate.rs +++ b/src/pipeline/aggregate.rs @@ -1,12 +1,13 @@ //! Stage 5 — `aggregate`. //! //! Compares exactly two conditions: collects -//! `pass_rate` (from `grading.json`), `total_tokens`/`duration_ms` (from -//! `timing.json`), per-assertion pass counts, raw per-run diff scope, and the -//! skill-invocation determination per condition; computes mean/stddev and the -//! `a - b` delta; accumulates validity warnings (mixed timing sources, sub-100% -//! invocation rate, stray-write violations + live-source reads, guard denials, -//! permission-denied tool calls, plugin shadows); and writes `benchmark.json`. +//! grading endpoints (binary pass rate or sampled vote proportion and pass^k), +//! `total_tokens`/`duration_ms` (from `timing.json`), per-assertion pass or vote +//! counts, raw per-run diff scope, and the skill-invocation determination per +//! condition; computes mean/stddev and the `a - b` delta; accumulates validity +//! warnings (mixed timing sources, sub-100% invocation rate, stray-write +//! violations + live-source reads, guard denials, permission-denied tool calls, +//! plugin shadows); and writes `benchmark.json`. mod assertions; @@ -80,6 +81,10 @@ pub struct Stats { #[derive(Debug, Clone, Serialize)] struct ConditionSummary { pass_rate: Stats, + #[serde(skip_serializing_if = "Option::is_none")] + vote_proportion: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pass_power_k: Option, duration_ms: Stats, total_tokens: Stats, #[serde(skip_serializing_if = "Option::is_none")] @@ -94,6 +99,10 @@ struct ConditionSummary { struct Delta { direction: String, pass_rate: f64, + #[serde(skip_serializing_if = "Option::is_none")] + vote_proportion: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pass_power_k: Option, duration_ms: f64, total_tokens: f64, } @@ -146,6 +155,8 @@ struct DiffScopeRun { #[derive(Default)] struct Bucket { pass_rates: Vec, + vote_proportions: Vec, + pass_power_k: Vec, durations: Vec, tokens: Vec, skill_invoked: Vec, @@ -216,6 +227,7 @@ pub fn aggregate( let mut warnings: Vec = Vec::new(); let mut timing_sources: HashSet = HashSet::new(); let mut assertion_counts = AssertionRollup::default(); + let mut has_sampled_gradings = false; let mut diff_scope_by_condition: HashMap> = condition_names .iter() .map(|condition| (condition.clone(), Vec::new())) @@ -277,9 +289,20 @@ pub fn aggregate( .strip_prefix("eval-") .unwrap_or(eval_dir) .to_string(); - assertion_counts.record(&eval_id, cond, &grading.assertion_results); + assertion_counts.record(&eval_id, cond, &grading.assertion_results)?; let bucket = by_condition.get_mut(cond).expect("condition bucket"); - bucket.pass_rates.push(grading.summary.pass_rate); + bucket.pass_rates.push(grading.summary.pass_rate()); + let vote_proportion = grading + .summary + .vote_proportion() + .unwrap_or_else(|| grading.summary.pass_rate()); + let pass_power_k = grading + .summary + .pass_power_k() + .unwrap_or_else(|| grading.summary.pass_rate()); + has_sampled_gradings |= grading.summary.vote_proportion().is_some(); + bucket.vote_proportions.push(vote_proportion); + bucket.pass_power_k.push(pass_power_k); if let Some(meta) = &grading.meta_summary && let Some(invoked) = meta.skill_invoked { @@ -324,6 +347,8 @@ pub fn aggregate( }; let summary = ConditionSummary { pass_rate: stats(&bucket.pass_rates, 3), + vote_proportion: has_sampled_gradings.then(|| stats(&bucket.vote_proportions, 3)), + pass_power_k: has_sampled_gradings.then(|| stats(&bucket.pass_power_k, 6)), duration_ms: stats(&bucket.durations, 0), total_tokens: stats(&bucket.tokens, 0), skill_invocation_n, @@ -340,6 +365,20 @@ pub fn aggregate( let delta = Delta { direction: format!("{a} - {b}"), pass_rate: round(sa.pass_rate.mean - sb.pass_rate.mean, 3), + vote_proportion: has_sampled_gradings.then(|| { + round( + sa.vote_proportion.expect("sampled vote stats").mean + - sb.vote_proportion.expect("sampled vote stats").mean, + 3, + ) + }), + pass_power_k: has_sampled_gradings.then(|| { + round( + sa.pass_power_k.expect("sampled pass^k stats").mean + - sb.pass_power_k.expect("sampled pass^k stats").mean, + 6, + ) + }), duration_ms: round(sa.duration_ms.mean - sb.duration_ms.mean, 0), total_tokens: round(sa.total_tokens.mean - sb.total_tokens.mean, 0), }; diff --git a/src/pipeline/aggregate/assertions.rs b/src/pipeline/aggregate/assertions.rs index 8beb855..0864021 100644 --- a/src/pipeline/aggregate/assertions.rs +++ b/src/pipeline/aggregate/assertions.rs @@ -2,15 +2,23 @@ use std::collections::HashMap; -use serde::Serialize; -use serde_json::{Map, Value}; +use serde_json::{Map, Value, json}; -use crate::core::AssertionResult; +use crate::core::{GradedAssertionResult, SampledAssertionResult}; +use crate::pipeline::error::PipelineError; -#[derive(Debug, Default, Serialize)] -struct AssertionCount { - passed: usize, - n: usize, +#[derive(Debug)] +enum AssertionCount { + Binary { + passed: u32, + n: u32, + }, + Sampled { + passed: u32, + total: u32, + samples_per_run: u32, + run_count: u32, + }, } type Counts = HashMap>>; @@ -21,19 +29,60 @@ pub(super) struct AssertionRollup { } impl AssertionRollup { - pub(super) fn record(&mut self, eval_id: &str, condition: &str, results: &[AssertionResult]) { + pub(super) fn record( + &mut self, + eval_id: &str, + condition: &str, + results: &[GradedAssertionResult], + ) -> Result<(), PipelineError> { let eval_counts = self.counts.entry(eval_id.to_string()).or_default(); for result in results { - let count = eval_counts - .entry(result.id.clone()) + let condition_counts = eval_counts + .entry(result.id().to_string()) .or_default() - .entry(condition.to_string()) - .or_default(); - count.n += 1; - if result.passed { - count.passed += 1; + .entry(condition.to_string()); + match result { + GradedAssertionResult::Binary(result) => { + let count = + condition_counts.or_insert(AssertionCount::Binary { passed: 0, n: 0 }); + let AssertionCount::Binary { passed, n } = count else { + return Err(inconsistent_shape(eval_id, result.id.as_str(), condition)); + }; + *n += 1; + if result.passed { + *passed += 1; + } + } + GradedAssertionResult::Sampled(SampledAssertionResult { votes, .. }) => { + let count = condition_counts.or_insert(AssertionCount::Sampled { + passed: 0, + total: 0, + samples_per_run: votes.total, + run_count: 0, + }); + let AssertionCount::Sampled { + passed, + total, + samples_per_run, + run_count, + } = count + else { + return Err(inconsistent_shape(eval_id, result.id(), condition)); + }; + if *samples_per_run != votes.total { + return Err(PipelineError::Message(format!( + "inconsistent judge sample counts for {eval_id}/{}/{condition}: expected {samples_per_run}, found {}", + result.id(), + votes.total + ))); + } + *passed += votes.passed; + *total += votes.total; + *run_count += 1; + } } } + Ok(()) } /// Render stable eval/assertion ordering while preserving the declared @@ -53,10 +102,7 @@ impl AssertionRollup { let mut by_condition = Map::new(); for condition in condition_names { if let Some(count) = eval_counts[assertion_id].get(condition) { - by_condition.insert( - condition.clone(), - serde_json::to_value(count).expect("assertion counts serialize"), - ); + by_condition.insert(condition.clone(), count.to_value()); } } by_assertion.insert(assertion_id.clone(), Value::Object(by_condition)); @@ -66,3 +112,36 @@ impl AssertionRollup { Value::Object(by_eval) } } + +impl AssertionCount { + fn to_value(&self) -> Value { + match self { + Self::Binary { passed, n } => json!({ "passed": passed, "n": n }), + Self::Sampled { + passed, + total, + samples_per_run, + run_count, + } => { + let proportion = f64::from(*passed) / f64::from(*total); + json!({ + "votes": { + "passed": passed, + "failed": total - passed, + "total": total, + "proportion": proportion + }, + "samples_per_run": samples_per_run, + "run_count": run_count, + "pass_power_k": proportion.powf(f64::from(*samples_per_run)) + }) + } + } + } +} + +fn inconsistent_shape(eval_id: &str, assertion_id: &str, condition: &str) -> PipelineError { + PipelineError::Message(format!( + "inconsistent grading shapes for {eval_id}/{assertion_id}/{condition}: cannot combine binary and sampled assertion results" + )) +} diff --git a/src/pipeline/grade/finalize.rs b/src/pipeline/grade/finalize.rs index f501929..0ac7f7f 100644 --- a/src/pipeline/grade/finalize.rs +++ b/src/pipeline/grade/finalize.rs @@ -3,9 +3,9 @@ //! For each //! `(eval, condition)` it grades `transcript_check` assertions directly, folds in //! persisted `command_check` results, deterministic `diff_scope` thresholds, -//! and the `llm_judge` responses written by the orchestrator (missing → FAIL), -//! assembles the skill-invocation meta result, and writes a schema-valid -//! `grading.json` with pass/fail summaries. +//! and the `llm_judge` responses written by the orchestrator (a missing response +//! fails only that verdict), assembles the skill-invocation meta result, and +//! writes a schema-valid `grading.json` with binary or sampled vote summaries. use std::fs; @@ -14,8 +14,9 @@ use serde::Deserialize; use crate::adapters::adapter_for; use crate::core::fs::write_json; use crate::core::{ - Assertion, AssertionResult, Grader, GradingResult, GradingSummary, MetaSummary, RunRecord, - SKILL_INVOKED_META_ID, ToolInvocation, + Assertion, AssertionResult, BinaryGradingSummary, GradedAssertionResult, Grader, GradingResult, + GradingSummary, JudgeSampleResult, JudgeVotes, MetaSummary, RunRecord, SKILL_INVOKED_META_ID, + SampledAssertionResult, SampledGradingSummary, ToolInvocation, }; use crate::pipeline::DiffScopeMetrics; use crate::pipeline::error::PipelineError; @@ -95,7 +96,7 @@ pub fn finalize(ctx: &GradeContext) -> Result { None }; - let mut assertion_results: Vec = Vec::new(); + let mut assertion_results: Vec = Vec::new(); if has_assertions { for assertion in assertions { match assertion { @@ -107,12 +108,15 @@ pub fn finalize(ctx: &GradeContext) -> Result { let conversation = run_record .as_ref() .and_then(|run| run.conversation.as_ref()); - assertion_results.push(grade_transcript_check_with_context( - tc, - invocations, - conversation, - &transcript_vocabulary, - )); + assertion_results.push( + grade_transcript_check_with_context( + tc, + invocations, + conversation, + &transcript_vocabulary, + ) + .into(), + ); let unverifiable = match tc.check.as_str() { "assistant_message_matches" => conversation.is_none(), _ => invocations.is_empty(), @@ -124,6 +128,62 @@ pub fn finalize(ctx: &GradeContext) -> Result { } } Assertion::LlmJudge(j) => { + let sample_count = + j.samples.or(ctx.conditions.judge_samples).unwrap_or(1); + if sample_count > 1 { + let mut judge_samples = + Vec::with_capacity(sample_count as usize); + for sample_index in 1..=sample_count { + let response_path = judge_responses_dir + .join(format!("{}__sample-{sample_index}.json", j.id)); + if !response_path.exists() { + summary.warnings.push(format!( + "missing judge response: {} (sample will be FAIL)", + response_path.display() + )); + judge_samples.push(JudgeSampleResult { + sample_index, + passed: false, + evidence: format!( + "judge response missing at {}", + response_path.display() + ), + confidence: 0.0, + }); + continue; + } + let response: JudgeResponse = serde_json::from_str( + &fs::read_to_string(&response_path)?, + )?; + judge_samples.push(JudgeSampleResult { + sample_index, + passed: response.passed, + evidence: response.evidence.unwrap_or_default(), + confidence: response.confidence.unwrap_or(0.0), + }); + } + let passed = + judge_samples.iter().filter(|sample| sample.passed).count() + as u32; + let proportion = f64::from(passed) / f64::from(sample_count); + assertion_results.push(GradedAssertionResult::Sampled( + SampledAssertionResult { + id: j.id.clone(), + grader: Grader::LlmJudge, + votes: JudgeVotes { + passed, + failed: sample_count - passed, + total: sample_count, + proportion, + pass_power_k: proportion + .powf(f64::from(sample_count)), + }, + judge_samples, + }, + )); + summary.total_graded += 1; + continue; + } let response_path = judge_responses_dir.join(format!("{}.json", j.id)); if !response_path.exists() { @@ -131,27 +191,33 @@ pub fn finalize(ctx: &GradeContext) -> Result { "missing judge response: {} (assertion will be FAIL)", response_path.display() )); - assertion_results.push(AssertionResult { - id: j.id.clone(), - passed: false, - evidence: format!( - "judge response missing at {}", - response_path.display() - ), - confidence: Some(0.0), - grader: Some(Grader::LlmJudge), - }); + assertion_results.push( + AssertionResult { + id: j.id.clone(), + passed: false, + evidence: format!( + "judge response missing at {}", + response_path.display() + ), + confidence: Some(0.0), + grader: Some(Grader::LlmJudge), + } + .into(), + ); continue; } let response: JudgeResponse = serde_json::from_str(&fs::read_to_string(&response_path)?)?; - assertion_results.push(AssertionResult { - id: j.id.clone(), - passed: response.passed, - evidence: response.evidence.unwrap_or_default(), - confidence: Some(response.confidence.unwrap_or(0.0)), - grader: Some(Grader::LlmJudge), - }); + assertion_results.push( + AssertionResult { + id: j.id.clone(), + passed: response.passed, + evidence: response.evidence.unwrap_or_default(), + confidence: Some(response.confidence.unwrap_or(0.0)), + grader: Some(Grader::LlmJudge), + } + .into(), + ); summary.total_graded += 1; } Assertion::CommandCheck(check) => { @@ -170,13 +236,16 @@ pub fn finalize(ctx: &GradeContext) -> Result { &serde_json::from_str(&fs::read_to_string(&result_path)?)?, &result_path.to_string_lossy(), )?; - assertion_results.push(AssertionResult { - id: check.id.clone(), - passed: result.passed, - evidence: result.evidence, - confidence: Some(1.0), - grader: Some(Grader::CommandCheck), - }); + assertion_results.push( + AssertionResult { + id: check.id.clone(), + passed: result.passed, + evidence: result.evidence, + confidence: Some(1.0), + grader: Some(Grader::CommandCheck), + } + .into(), + ); summary.total_graded += 1; } Assertion::DiffScope(check) => { @@ -192,7 +261,7 @@ pub fn finalize(ctx: &GradeContext) -> Result { &serde_json::from_str(&fs::read_to_string(&result_path)?)?, &result_path.to_string_lossy(), )?; - assertion_results.push(grade_diff_scope(check, metrics)); + assertion_results.push(grade_diff_scope(check, metrics).into()); summary.total_graded += 1; } } @@ -237,17 +306,47 @@ pub fn finalize(ctx: &GradeContext) -> Result { } } - let passed = assertion_results.iter().filter(|r| r.passed).count() as u32; let total = assertion_results.len() as u32; let meta_len = meta_results.len() as u32; let meta_passed = meta_results.iter().filter(|r| r.passed).count() as u32; let has_meta = !meta_results.is_empty(); let skill_invoked = has_meta.then(|| meta_results.iter().all(|r| r.passed)); - let grading = GradingResult { - assertion_results, - meta_results: has_meta.then_some(meta_results), - summary: GradingSummary { + let has_sampled = assertion_results + .iter() + .any(|result| matches!(result, GradedAssertionResult::Sampled(_))); + let grading_summary = if has_sampled { + let divisor = f64::from(total); + let vote_proportion = if total == 0 { + 0.0 + } else { + assertion_results + .iter() + .map(GradedAssertionResult::vote_proportion) + .sum::() + / divisor + }; + let pass_power_k = if total == 0 { + 0.0 + } else { + assertion_results + .iter() + .map(GradedAssertionResult::pass_power_k) + .sum::() + / divisor + }; + GradingSummary::Sampled(SampledGradingSummary { + total, + pass_rate: vote_proportion, + vote_proportion, + pass_power_k, + }) + } else { + let passed = assertion_results + .iter() + .filter(|result| result.vote_proportion() == 1.0) + .count() as u32; + GradingSummary::Binary(BinaryGradingSummary { passed, failed: total - passed, total, @@ -256,7 +355,13 @@ pub fn finalize(ctx: &GradeContext) -> Result { } else { f64::from(passed) / f64::from(total) }, - }, + }) + }; + + let grading = GradingResult { + assertion_results, + meta_results: has_meta.then_some(meta_results), + summary: grading_summary, meta_summary: has_meta.then_some(MetaSummary { passed: meta_passed, failed: meta_len - meta_passed, diff --git a/src/pipeline/grade/judge_tasks.rs b/src/pipeline/grade/judge_tasks.rs index b9d0720..c152cd4 100644 --- a/src/pipeline/grade/judge_tasks.rs +++ b/src/pipeline/grade/judge_tasks.rs @@ -7,6 +7,7 @@ //! per-assertion prompt files. `transcript_check` assertions are not dispatched //! here — they are graded directly in `finalize`. +use std::collections::HashMap; use std::fs; use std::path::Path; @@ -35,6 +36,13 @@ pub struct JudgeTask { #[serde(skip_serializing_if = "Option::is_none")] pub run_index: Option, pub assertion_id: String, + /// 1-based verdict index when this assertion requests more than one sample. + #[serde(skip_serializing_if = "Option::is_none")] + pub sample_index: Option, + /// Total verdicts requested for a sampled assertion. Paired with + /// `sample_index`; both stay absent for the legacy single-verdict shape. + #[serde(skip_serializing_if = "Option::is_none")] + pub sample_count: Option, pub rubric: String, pub model: Option, pub is_meta: bool, @@ -271,6 +279,7 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result = HashMap::new(); for assertion in assertions { let j = match assertion { Assertion::LlmJudge(j) => j, @@ -282,28 +291,50 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result continue, }; - let response_path = judge_responses_dir.join(format!("{}.json", j.id)); - let dispatch_prompt = - build_judge_prompt(&j.id, &j.rubric, &evidence.content, &response_path)?; - let prompt_path = judge_prompts_dir.join(format!("{}.txt", j.id)); - fs::write(&prompt_path, &dispatch_prompt)?; - tasks.push(JudgeTask { - eval_id: ev.id.clone(), - condition: cond.clone(), - run_index: slot.run_index, - assertion_id: j.id.clone(), - rubric: j.rubric.clone(), - model: j.model.clone().or_else(|| default_judge_model.clone()), - is_meta: false, - run_record_path: artifact_path(&run_record_path), - outputs_dir: artifact_path(&outputs_dir), - response_path: artifact_path(&response_path), - dispatch_prompt_path: artifact_path(&prompt_path), - evidence_bundle: evidence.reference.clone(), - dispatch_prompt_bytes: dispatch_prompt.len(), - dispatch_prompt_byte_limit: JUDGE_PROMPT_BYTE_LIMIT, - dispatch_prompt, - }); + let sample_count = j.samples.or(ctx.conditions.judge_samples).unwrap_or(1); + for index in 1..=sample_count { + let sampled_index = (sample_count > 1).then_some(index); + let stem = sampled_index.map_or_else( + || j.id.clone(), + |sample| format!("{}__sample-{sample}", j.id), + ); + if let Some(first_assertion) = + task_stem_owners.insert(stem.clone(), j.id.clone()) + { + return Err(PipelineError::Message(format!( + "judge task filename collision for {}/{cond}: assertions '{}' and '{}' both resolve to '{stem}'. Rename one assertion id.", + ev.id, first_assertion, j.id + ))); + } + let response_path = judge_responses_dir.join(format!("{stem}.json")); + let dispatch_prompt = build_judge_prompt( + &j.id, + &j.rubric, + &evidence.content, + &response_path, + )?; + let prompt_path = judge_prompts_dir.join(format!("{stem}.txt")); + fs::write(&prompt_path, &dispatch_prompt)?; + tasks.push(JudgeTask { + eval_id: ev.id.clone(), + condition: cond.clone(), + run_index: slot.run_index, + assertion_id: j.id.clone(), + sample_index: sampled_index, + sample_count: (sample_count > 1).then_some(sample_count), + rubric: j.rubric.clone(), + model: j.model.clone().or_else(|| default_judge_model.clone()), + is_meta: false, + run_record_path: artifact_path(&run_record_path), + outputs_dir: artifact_path(&outputs_dir), + response_path: artifact_path(&response_path), + dispatch_prompt_path: artifact_path(&prompt_path), + evidence_bundle: evidence.reference.clone(), + dispatch_prompt_bytes: dispatch_prompt.len(), + dispatch_prompt_byte_limit: JUDGE_PROMPT_BYTE_LIMIT, + dispatch_prompt, + }); + } } // Skill-invocation meta-check. Negative evals (skill_should_trigger: @@ -360,6 +391,8 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result bool { #[cfg(test)] mod tests { use super::validate_evals_config; - use crate::core::CodebaseSource; + use crate::core::{Assertion, CodebaseSource}; use serde_json::{Value, json}; /// The minimal valid config the cases below mutate. @@ -338,6 +338,39 @@ mod tests { assert_eq!(parsed.evals[0].skill_should_trigger, None); } + #[test] + fn llm_judge_accepts_a_positive_sample_count() { + let mut config = base(); + config["evals"][0]["assertions"] = json!([{ + "id": "quality", + "type": "llm_judge", + "rubric": "Is the implementation well designed?", + "samples": 10 + }]); + + let parsed = validate_evals_config(&config, "evals.json").unwrap(); + let Assertion::LlmJudge(judge) = &parsed.evals[0].assertions.as_ref().unwrap()[0] else { + panic!("expected llm_judge assertion"); + }; + assert_eq!(judge.samples, Some(10)); + } + + #[test] + fn llm_judge_rejects_zero_samples() { + let mut config = base(); + config["evals"][0]["assertions"] = json!([{ + "id": "quality", + "type": "llm_judge", + "rubric": "Is the implementation well designed?", + "samples": 0 + }]); + + let error = validate_evals_config(&config, "evals.json") + .unwrap_err() + .to_string(); + assert!(error.contains("samples"), "error was: {error}"); + } + #[test] fn rejects_an_empty_files_root() { let mut config = base(); diff --git a/src/workspace/promote.rs b/src/workspace/promote.rs index 9cc6c1a..405ed84 100644 --- a/src/workspace/promote.rs +++ b/src/workspace/promote.rs @@ -464,9 +464,9 @@ fn provenance(opts: &PromoteOptions, conditions: Option<&ConditionsRecord>, head format!("| Promoted from commit | {head} |"), String::new(), "Files:".to_string(), - "- `benchmark.json` — aggregate pass-rate / duration / token deltas plus per-assertion pass counts." + "- `benchmark.json` — aggregate grading / duration / token deltas plus per-assertion pass or sampled-vote counts." .to_string(), - "- `grading/__.json` (multi-run cells add an `__r` suffix per run) — assertion results and judge rationales." + "- `grading/__.json` (multi-run cells add an `__r` suffix per run) — assertion results, sampled verdicts, and judge rationales." .to_string(), "- `evidence/__.md` (multi-run cells add an `__r` suffix per run) — the exact bounded run evidence inlined for judge tasks." .to_string(), diff --git a/src/workspace/promote/tests.rs b/src/workspace/promote/tests.rs index 291cda3..18b4df5 100644 --- a/src/workspace/promote/tests.rs +++ b/src/workspace/promote/tests.rs @@ -95,7 +95,7 @@ fn copies_benchmark_and_per_run_gradings_into_baseline() { assert!(provenance.contains("Agent model | unspecified")); assert!(provenance.contains("Judge model | unspecified")); assert!(provenance.contains("Responder model | unspecified")); - assert!(provenance.contains("per-assertion pass counts")); + assert!(provenance.contains("per-assertion pass or sampled-vote counts")); } #[test] diff --git a/tests/cli/aggregate/assertions.rs b/tests/cli/aggregate/assertions.rs index bd4dcf6..553ca89 100644 --- a/tests/cli/aggregate/assertions.rs +++ b/tests/cli/aggregate/assertions.rs @@ -10,6 +10,42 @@ fn write_grading_json_in(run_dir: &std::path::Path, grading: serde_json::Value) .unwrap(); } +fn sampled_grading(passed: u32, total: u32) -> serde_json::Value { + let proportion = f64::from(passed) / f64::from(total); + let judge_samples: Vec = (1..=total) + .map(|sample_index| { + let sample_passed = sample_index <= passed; + json!({ + "sample_index": sample_index, + "passed": sample_passed, + "evidence": format!("sample {sample_index}"), + "confidence": 0.8 + }) + }) + .collect(); + let pass_power_k = proportion.powf(f64::from(total)); + json!({ + "assertion_results": [{ + "id": "quality", + "grader": "llm_judge", + "votes": { + "passed": passed, + "failed": total - passed, + "total": total, + "proportion": proportion, + "pass_power_k": pass_power_k + }, + "judge_samples": judge_samples + }], + "summary": { + "total": 1, + "pass_rate": proportion, + "vote_proportion": proportion, + "pass_power_k": pass_power_k + } + }) +} + /// `aggregate`: substantive assertion results are counted separately for each /// eval, assertion, and condition, while framework meta-results stay out of the /// effectiveness report. @@ -125,3 +161,56 @@ fn aggregate_rolls_up_substantive_assertions_by_eval_and_condition() { .contains("__skill_invoked") ); } + +#[test] +fn aggregate_surfaces_sampled_votes_and_pass_power_k_by_condition() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_md, iteration_dir, cwd) = setup_agg(&root); + new_skill_conditions(&iteration_dir, &skill_md); + + for (condition, run_index, passed) in [ + ("with_skill", 1, 3), + ("with_skill", 2, 4), + ("without_skill", 1, 2), + ("without_skill", 2, 2), + ] { + let run_dir = iteration_dir + .join("eval-e1") + .join(condition) + .join(format!("run-{run_index}")); + write_grading_json_in(&run_dir, sampled_grading(passed, 4)); + write_timing_in(&run_dir, json!({"total_tokens": 1000, "duration_ms": 100})); + } + + agg_cmd(&cwd, &skill_dir).assert().success(); + + let benchmark = read_benchmark(&iteration_dir); + assert_eq!( + benchmark["assertions"]["e1"]["quality"]["with_skill"], + json!({ + "votes": {"passed": 7, "failed": 1, "total": 8, "proportion": 0.875}, + "samples_per_run": 4, + "run_count": 2, + "pass_power_k": 0.586181640625 + }) + ); + assert_eq!( + benchmark["assertions"]["e1"]["quality"]["without_skill"], + json!({ + "votes": {"passed": 4, "failed": 4, "total": 8, "proportion": 0.5}, + "samples_per_run": 4, + "run_count": 2, + "pass_power_k": 0.0625 + }) + ); + assert_eq!( + benchmark["run_summary"]["with_skill"]["vote_proportion"], + json!({"mean": 0.875, "stddev": 0.125, "n": 2}) + ); + assert_eq!( + benchmark["run_summary"]["with_skill"]["pass_power_k"], + json!({"mean": 0.658203, "stddev": 0.341797, "n": 2}) + ); + assert_eq!(benchmark["delta"]["vote_proportion"], 0.375); + assert_eq!(benchmark["delta"]["pass_power_k"], 0.595703); +} diff --git a/tests/cli/docs.rs b/tests/cli/docs.rs index d738e9e..095ec98 100644 --- a/tests/cli/docs.rs +++ b/tests/cli/docs.rs @@ -220,7 +220,8 @@ fn docs_guard_keeps_configuration_defaults_and_boundary_contracts() { } /// Judge evidence is the primary grading input, so the shipped reference must -/// keep the bounds, trust boundary, source fallback, and retention contract. +/// keep the bounds, sampling semantics, trust boundary, source fallback, and +/// retention contract. #[test] fn docs_judging_keeps_bundle_bounds_truncation_and_retention_contract() { skill_eval() @@ -239,7 +240,21 @@ fn docs_judging_keeps_bundle_bounds_truncation_and_retention_contract() { .stdout(contains("truncated")) .stdout(contains("untrusted")) .stdout(contains("read-only")) + .stdout(contains("\"samples\": 10")) + .stdout(contains("--judge-samples")) + .stdout(contains("6 / 10")) + .stdout(contains("0.6^10")) + .stdout(contains("__sample-N")) + .stdout(contains("missing response")) + .stdout(contains("__skill_invoked")) .stdout(contains("evals/baseline/evidence")); + + skill_eval() + .args(["run", "--help"]) + .assert() + .success() + .stdout(contains("--judge-samples")) + .stdout(contains("pass^k")); } #[test] diff --git a/tests/cli/grade.rs b/tests/cli/grade.rs index bbf843c..8d28d61 100644 --- a/tests/cli/grade.rs +++ b/tests/cli/grade.rs @@ -5,6 +5,8 @@ use assert_cmd::Command; use predicates::str::contains; use std::fs; +mod sampling; + /// Write `/SKILL.md` and `/evals/evals.json`. fn write_skill(skill_sub: &std::path::Path, skill_md: &str, evals: &serde_json::Value) { fs::create_dir_all(skill_sub.join("evals")).unwrap(); diff --git a/tests/cli/grade/sampling.rs b/tests/cli/grade/sampling.rs new file mode 100644 index 0000000..111b574 --- /dev/null +++ b/tests/cli/grade/sampling.rs @@ -0,0 +1,298 @@ +//! Multi-sample judge-task emission and finalization. + +use super::*; + +#[test] +fn sampled_judge_paths_reject_colliding_authored_assertion_ids() { + use serde_json::json; + let (_tmp, root) = canonical_root(); + let skill_dir = root.join("skill-dir"); + let skill_sub = skill_dir.join("mr-review"); + write_skill( + &skill_sub, + "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n", + &json!({"skill_name": "mr-review", "evals": [{ + "id": "sampled", "prompt": "Review it.", "expected_output": "a review", + "skill_should_trigger": false, + "assertions": [ + {"id": "quality", "type": "llm_judge", "rubric": "Good?", "samples": 2}, + {"id": "quality__sample-1", "type": "llm_judge", "rubric": "Clear?"} + ] + }]}), + ); + + let cwd = root.join("work"); + let iteration_dir = cwd.join(".eval-magic/mr-review/iteration-1"); + let cell = iteration_dir.join("eval-sampled/with_skill"); + fs::create_dir_all(&cell).unwrap(); + fs::write( + iteration_dir.join("conditions.json"), + serde_json::to_string(&json!({ + "mode": "new-skill", + "conditions": [{"name": "with_skill", "skill_path": null}], + "timestamp": "2026-08-23T00:00:00Z", + "harness": "codex" + })) + .unwrap(), + ) + .unwrap(); + fs::write( + cell.join("run.json"), + serde_json::to_string(&json!({ + "eval_id": "sampled", "condition": "with_skill", "skill_path": null, + "prompt": "Review it.", "files": [], "final_message": "Done.", + "tool_invocations": [], "total_tokens": 10, "duration_ms": 20 + })) + .unwrap(), + ) + .unwrap(); + + grade_cmd(&cwd, &skill_dir, Some("codex")) + .assert() + .failure() + .stderr(contains("judge task filename collision")) + .stderr(contains("quality__sample-1")) + .stderr(contains("quality")); +} + +#[test] +fn grade_emits_resolved_judge_samples_with_unique_paths_and_shared_evidence() { + use serde_json::json; + let (_tmp, root) = canonical_root(); + let skill_dir = root.join("skill-dir"); + let skill_sub = skill_dir.join("mr-review"); + write_skill( + &skill_sub, + "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n", + &json!({"skill_name": "mr-review", "evals": [{ + "id": "sampled", "prompt": "Review it.", "expected_output": "a review", + "skill_should_trigger": false, + "assertions": [ + {"id": "run-default", "type": "llm_judge", "rubric": "Good?"}, + {"id": "explicit-single", "type": "llm_judge", "rubric": "Clear?", "samples": 1}, + {"id": "explicit-three", "type": "llm_judge", "rubric": "Safe?", "samples": 3} + ] + }]}), + ); + + let cwd = root.join("work"); + let iteration_dir = cwd.join(".eval-magic/mr-review/iteration-1"); + let cell = iteration_dir.join("eval-sampled/with_skill"); + fs::create_dir_all(&cell).unwrap(); + fs::write( + iteration_dir.join("conditions.json"), + serde_json::to_string(&json!({ + "mode": "new-skill", + "conditions": [{"name": "with_skill", "skill_path": null}], + "timestamp": "2026-08-23T00:00:00Z", + "harness": "codex", + "judge_samples": 2 + })) + .unwrap(), + ) + .unwrap(); + fs::write( + cell.join("run.json"), + serde_json::to_string(&json!({ + "eval_id": "sampled", "condition": "with_skill", "skill_path": null, + "prompt": "Review it.", "files": [], "final_message": "Done.", + "tool_invocations": [], "total_tokens": 10, "duration_ms": 20 + })) + .unwrap(), + ) + .unwrap(); + + grade_cmd(&cwd, &skill_dir, Some("codex")) + .assert() + .success() + .stdout(contains( + "Judge tasks: 6 (0 skill-invocation meta-judge(s))", + )); + + let artifact: serde_json::Value = + serde_json::from_str(&fs::read_to_string(iteration_dir.join("judge-tasks.json")).unwrap()) + .unwrap(); + let tasks = artifact["tasks"].as_array().unwrap(); + assert_eq!(tasks.len(), 6); + + let single = tasks + .iter() + .find(|task| task["assertion_id"] == "explicit-single") + .unwrap(); + assert!(single.get("sample_index").is_none()); + assert!(single.get("sample_count").is_none()); + assert!( + single["response_path"] + .as_str() + .unwrap() + .ends_with("explicit-single.json") + ); + assert!( + single["dispatch_prompt_path"] + .as_str() + .unwrap() + .ends_with("explicit-single.txt") + ); + + for (assertion, expected) in [("run-default", 2_u64), ("explicit-three", 3_u64)] { + let sampled: Vec<&serde_json::Value> = tasks + .iter() + .filter(|task| task["assertion_id"] == assertion) + .collect(); + assert_eq!(sampled.len() as u64, expected); + for (offset, task) in sampled.into_iter().enumerate() { + let index = offset as u64 + 1; + assert_eq!(task["sample_index"], json!(index)); + assert_eq!(task["sample_count"], json!(expected)); + assert!( + task["response_path"] + .as_str() + .unwrap() + .ends_with(&format!("{assertion}__sample-{index}.json")) + ); + assert!( + task["dispatch_prompt_path"] + .as_str() + .unwrap() + .ends_with(&format!("{assertion}__sample-{index}.txt")) + ); + } + } + + let evidence_paths: std::collections::HashSet<&str> = tasks + .iter() + .map(|task| task["evidence_bundle"]["path"].as_str().unwrap()) + .collect(); + assert_eq!( + evidence_paths.len(), + 1, + "every sample reuses one run bundle" + ); +} + +#[test] +fn finalize_keeps_each_sample_and_counts_a_missing_response_as_one_fail_vote() { + use serde_json::json; + let (_tmp, root) = canonical_root(); + let skill_dir = root.join("skill-dir"); + let skill_sub = skill_dir.join("mr-review"); + write_skill( + &skill_sub, + "---\nname: mr-review\ndescription: review MRs\n---\n\nbody\n", + &json!({"skill_name": "mr-review", "evals": [{ + "id": "sampled", "prompt": "Review it.", "expected_output": "a review", + "skill_should_trigger": false, + "assertions": [ + {"id": "quality", "type": "llm_judge", "rubric": "Is it good?", "samples": 4}, + {"id": "single", "type": "llm_judge", "rubric": "Is it clear?", "samples": 1} + ] + }]}), + ); + + let cwd = root.join("work"); + let iteration_dir = cwd.join(".eval-magic/mr-review/iteration-1"); + let cell = iteration_dir.join("eval-sampled/without_skill"); + fs::create_dir_all(&cell).unwrap(); + fs::write( + iteration_dir.join("conditions.json"), + serde_json::to_string(&json!({ + "mode": "new-skill", + "conditions": [{"name": "without_skill", "skill_path": null}], + "timestamp": "2026-08-23T00:00:00Z", + "harness": "codex" + })) + .unwrap(), + ) + .unwrap(); + fs::write( + cell.join("run.json"), + serde_json::to_string(&json!({ + "eval_id": "sampled", "condition": "without_skill", "skill_path": null, + "prompt": "Review it.", "files": [], "final_message": "Done.", + "tool_invocations": [], "total_tokens": 10, "duration_ms": 20 + })) + .unwrap(), + ) + .unwrap(); + + grade_cmd(&cwd, &skill_dir, Some("codex")) + .assert() + .success(); + let responses = cell.join("judge-responses"); + for (index, passed) in [(1, true), (2, false), (3, true)] { + fs::write( + responses.join(format!("quality__sample-{index}.json")), + serde_json::to_string(&json!({ + "passed": passed, + "evidence": format!("sample {index} evidence"), + "confidence": 0.8 + })) + .unwrap(), + ) + .unwrap(); + } + fs::write( + responses.join("single.json"), + serde_json::to_string(&json!({ + "passed": true, + "evidence": "the result is clear", + "confidence": 0.9 + })) + .unwrap(), + ) + .unwrap(); + + let finalized = grade_cmd(&cwd, &skill_dir, Some("codex")) + .arg("--finalize") + .assert() + .success(); + let stderr = String::from_utf8_lossy(&finalized.get_output().stderr); + assert!( + stderr.contains("quality__sample-4.json") && stderr.contains("sample will be FAIL"), + "missing sample warning was: {stderr}" + ); + + let grading: serde_json::Value = + serde_json::from_str(&fs::read_to_string(cell.join("grading.json")).unwrap()).unwrap(); + let result = &grading["assertion_results"][0]; + assert_eq!(result["id"], "quality"); + assert_eq!(result["grader"], "llm_judge"); + assert!(result.get("passed").is_none()); + assert!(result.get("evidence").is_none()); + assert!(result.get("confidence").is_none()); + assert_eq!( + result["votes"], + json!({ + "passed": 2, + "failed": 2, + "total": 4, + "proportion": 0.5, + "pass_power_k": 0.0625 + }) + ); + let samples = result["judge_samples"].as_array().unwrap(); + assert_eq!(samples.len(), 4); + assert_eq!(samples[0]["sample_index"], 1); + assert_eq!(samples[0]["evidence"], "sample 1 evidence"); + assert_eq!(samples[3]["sample_index"], 4); + assert_eq!(samples[3]["passed"], false); + assert_eq!(samples[3]["confidence"], 0.0); + assert!( + samples[3]["evidence"] + .as_str() + .unwrap() + .contains("quality__sample-4.json") + ); + assert_eq!(grading["assertion_results"][1]["id"], "single"); + assert_eq!(grading["assertion_results"][1]["passed"], true); + assert!(grading["assertion_results"][1].get("votes").is_none()); + assert_eq!( + grading["summary"], + json!({ + "total": 2, + "pass_rate": 0.75, + "vote_proportion": 0.75, + "pass_power_k": 0.53125 + }) + ); +} diff --git a/tests/cli/workspace.rs b/tests/cli/workspace.rs index 5ee5189..27bb77c 100644 --- a/tests/cli/workspace.rs +++ b/tests/cli/workspace.rs @@ -84,6 +84,67 @@ fn promote_baseline_copies_artifacts_and_reports() { assert!(iteration_dir.join(".promoted.json").exists()); } +/// Promotion keeps every independent judge vote in the grading artifact while +/// retaining the one bounded evidence bundle all of those votes inspected. +#[test] +fn promote_baseline_preserves_sampled_grading_and_shared_evidence() { + let (_tmp, root) = canonical_root(); + let (skill_dir, skill_sub) = write_skill_md(&root, "---\nname: mr-review\n---\nbody\n"); + + let cwd = root.join("work"); + let iteration_dir = cwd + .join(".eval-magic") + .join("mr-review") + .join("iteration-2"); + let cond_dir = iteration_dir.join("eval-e1").join("new_skill"); + fs::create_dir_all(&cond_dir).unwrap(); + fs::write( + iteration_dir.join("benchmark.json"), + r#"{"delta":{"pass_rate":0.5,"vote_proportion":0.5,"pass_power_k":0.0625}}"#, + ) + .unwrap(); + let grading = r#"{ + "summary":{"total":1,"pass_rate":0.5,"vote_proportion":0.5,"pass_power_k":0.0625}, + "assertion_results":[{ + "id":"clear","grader":"llm_judge", + "votes":{"passed":2,"failed":2,"total":4,"proportion":0.5,"pass_power_k":0.0625}, + "judge_samples":[ + {"sample_index":1,"passed":true,"evidence":"yes","confidence":0.8}, + {"sample_index":2,"passed":true,"evidence":"yes","confidence":0.8}, + {"sample_index":3,"passed":false,"evidence":"no","confidence":0.7}, + {"sample_index":4,"passed":false,"evidence":"no","confidence":0.7} + ] + }], + "meta_results":[] + }"#; + fs::write(cond_dir.join("grading.json"), grading).unwrap(); + fs::write( + cond_dir.join("judge-evidence.md"), + "# Judge evidence bundle\n\nshared by four samples\n", + ) + .unwrap(); + + skill_eval() + .current_dir(&cwd) + .args(["promote-baseline", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--iteration", "2"]) + .assert() + .success() + .stdout(contains("1 grading file ")) + .stdout(contains("1 evidence bundle")); + + let baseline = skill_sub.join("evals").join("baseline"); + assert_eq!( + fs::read_to_string(baseline.join("grading/e1__new_skill.json")).unwrap(), + grading + ); + assert_eq!( + fs::read_to_string(baseline.join("evidence/e1__new_skill.md")).unwrap(), + "# Judge evidence bundle\n\nshared by four samples\n" + ); +} + /// `promote-baseline`: a multi-run (`runs > 1`) cell stores each run's grading /// and exact bounded evidence under matching `__r` filenames. #[test] diff --git a/tests/run/codebase.rs b/tests/run/codebase.rs index e6f939e..401f4d7 100644 --- a/tests/run/codebase.rs +++ b/tests/run/codebase.rs @@ -440,11 +440,24 @@ fn revision_mode_provisions_both_arms_from_the_cached_codebase() { .current_dir(&cwd) .args(["run", "--skill-dir"]) .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "revision", "--dry-run"]) + .args([ + "--skill", + "mr-review", + "--mode", + "revision", + "--judge-samples", + "3", + "--dry-run", + ]) .assert() .success(); let iteration = iteration_dir(&cwd); + let conditions = read_json(&iteration.join("conditions.json")); + assert_eq!(conditions["mode"], "revision"); + assert_eq!(conditions["judge_samples"], 3); + assert_eq!(conditions["codebases"][0]["source"], wire_path(&origin)); + assert!(conditions["codebases"][0]["revision"].is_string()); let cached: Vec<_> = fs::read_dir(iteration.join(".codebase")).unwrap().collect(); assert_eq!( cached.len(), diff --git a/tests/run/judges.rs b/tests/run/judges.rs index 3c5b915..f173685 100644 --- a/tests/run/judges.rs +++ b/tests/run/judges.rs @@ -20,6 +20,18 @@ const JUDGED_EVALS: &str = r#"{ }] }"#; +const SAMPLED_JUDGED_EVALS: &str = r#"{ + "skill_name": "mr-review", + "evals": [{ + "id": "reviewed", + "prompt": "Review this MR.", + "expected_output": "a clear review", + "assertions": [ + {"id": "clear", "type": "llm_judge", "rubric": "Was the review clear?", "samples": 2} + ] + }] +}"#; + /// The runner dispatches judge tasks the same way it dispatches eval tasks: it /// skips verdicts that already exist, runs the ones that do not, and reports /// how many are present. Before this, an operator pasted a `jq`/`xargs` @@ -153,6 +165,49 @@ fn dispatch_judges_exits_nonzero_while_a_verdict_is_missing() { .stderr(contains("verdict")); } +/// Sample coordinates belong in dispatch failures so an operator can rerun +/// the exact missing judge without confusing it with another independent vote. +#[test] +fn sampled_judge_failures_name_the_sample() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), SAMPLED_JUDGED_EVALS); + prepare_and_dispatch(tmp.path(), &skill_dir, &cwd); + + let script = tmp.path().join("fail-second-sample.sh"); + fs::write( + &script, + r#"#!/bin/sh +outputs=$1 +case "$outputs" in + *__sample-2) exit 7 ;; + *) printf '%s\n' '{"passed":true,"evidence":"stub verdict","confidence":0.8}' > "${outputs}.json" ;; +esac +"#, + ) + .unwrap(); + stub_judge_template( + &cwd, + &format!("sh \"{}\" ", script.to_string_lossy()), + ); + + skill_eval() + .current_dir(&cwd) + .args(["dispatch", "--judges", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--iteration", + "1", + "--harness", + "codex", + ]) + .assert() + .failure() + .stderr(contains("reviewed:with_skill:clear:sample-2-of-2")) + .stderr(contains("reviewed:without_skill:clear:sample-2-of-2")); +} + /// Prepare an iteration, dispatch its eval tasks through a stub, and ingest, so /// `judge-tasks.json` exists to dispatch judges from. fn prepare_and_dispatch(tmp: &Path, skill_dir: &Path, cwd: &Path) { diff --git a/tests/run/lifecycle.rs b/tests/run/lifecycle.rs index 6f22dd0..6107923 100644 --- a/tests/run/lifecycle.rs +++ b/tests/run/lifecycle.rs @@ -347,6 +347,37 @@ fn omitted_models_and_label_are_absent_from_conditions() { assert!(conditions.get("judge_model").is_none()); assert!(conditions.get("label").is_none()); assert!(conditions.get("agent_env").is_none()); + assert!(conditions.get("judge_samples").is_none()); + + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + assert!(dispatch.get("judge_samples").is_none()); +} + +#[test] +fn records_a_non_default_judge_sample_count_in_manifests() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "new-skill", + "--judge-samples", + "10", + "--dry-run", + ]) + .assert() + .success(); + + let iteration = iteration_dir(&cwd); + let conditions = read_json(&iteration.join("conditions.json")); + let dispatch = read_json(&iteration.join("dispatch.json")); + assert_eq!(conditions["judge_samples"], serde_json::json!(10)); + assert_eq!(dispatch["judge_samples"], serde_json::json!(10)); } #[test] @@ -513,6 +544,28 @@ fn runs_zero_is_rejected() { .failure(); } +#[test] +fn judge_samples_zero_is_rejected() { + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "new-skill", + "--judge-samples", + "0", + "--dry-run", + ]) + .assert() + .failure() + .stderr(contains("invalid value '0' for '--judge-samples")); +} + #[test] fn per_eval_runs_overrides_the_flag() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/tests/run/statistical_floor.rs b/tests/run/statistical_floor.rs index 29507dd..bf8b0f4 100644 --- a/tests/run/statistical_floor.rs +++ b/tests/run/statistical_floor.rs @@ -84,3 +84,69 @@ fn excluded_evals_do_not_influence_the_statistical_floor() { "excluded evals must not influence the notice: {stdout}" ); } + +#[test] +fn sampled_judging_prints_the_non_binary_endpoint_instead_of_a_fisher_floor() { + let tmp = tempfile::TempDir::new().unwrap(); + let evals = r#"{ "skill_name": "mr-review", "evals": [{ + "id": "quality", "prompt": "review it", "expected_output": "a review", + "assertions": [{ + "id": "design", "type": "llm_judge", "rubric": "Is it well designed?", "samples": 3 + }] + }] }"#; + let (skill_dir, cwd) = setup(tmp.path(), evals); + let assert = skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + + assert!( + stdout.contains( + "statistical endpoint: 2 conditions × 1 run; LLM judge sample counts per assertion: 3" + ), + "{stdout}" + ); + assert!( + stdout.contains("vote proportion and pass^k; the binary Fisher exact floor does not apply"), + "{stdout}" + ); + assert!(!stdout.contains("statistical floor:"), "{stdout}"); +} + +#[test] +fn sampled_endpoint_resolves_run_default_and_assertion_overrides() { + let tmp = tempfile::TempDir::new().unwrap(); + let evals = r#"{ "skill_name": "mr-review", "evals": [{ + "id": "quality", "prompt": "review it", "expected_output": "a review", + "assertions": [ + {"id": "defaulted", "type": "llm_judge", "rubric": "Good?"}, + {"id": "overridden", "type": "llm_judge", "rubric": "Safe?", "samples": 3} + ] + }] }"#; + let (skill_dir, cwd) = setup(tmp.path(), evals); + let assert = skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--mode", + "new-skill", + "--judge-samples", + "5", + "--dry-run", + ]) + .assert() + .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + + assert!( + stdout.contains("LLM judge sample counts per assertion: 3, 5"), + "{stdout}" + ); +}