diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index 94ec77feb4..7f7029b5e6 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -189,3 +189,17 @@ running best from the first observed score, with a new step on improvement. Expand **All score observations** for the full range, including unsuccessful iterations. Each chart uses its own labeled score axis. Lines end at observed samples. The figure is descriptive and includes provisional runs. + +For fixed-duration comparisons, set `duration_policy.target_ms` and list the arm IDs +permitted to finish a verified Goal early in `duration_policy.allow_goal_completion`. +For each selected logical run, supply `active_intervals` with observed solver start +and end times in Unix milliseconds and an `evidence_ref` per interval. Include only +verified continuations of that run. Omit the field when coverage is unknown. An +empty array explicitly records zero observed runtime. Overlapping intervals count +once and downtime between intervals contributes zero. + +The JSON and Markdown reports cover every selected case and arm. A short terminal +run enters `needs_makeup`, even with a valid score or a successful process exit. +Only a matching completed run with an audited normal end and a satisfied Goal +can use the early-finish allowance. Incomplete and unverified durations exclude a +case from the duration-qualified comparison. Retain raw results for diagnosis. diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 972d03624a..1ac64a6e0d 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -36,6 +36,7 @@ type Selection = { arm_id: string; case_id: string; run_id: string; status: Status; score: Score | null; protocol_id: string; exclude_reason: string | null; rerun_reason: Rerun | null; + active_intervals?: { start_ms: number; end_ms: number; evidence_ref: string }[]; effort?: Effort; score_samples?: ScoreSample[]; configuration?: RunConfiguration; }; type Incident = { @@ -55,6 +56,7 @@ export type CampaignInput = { observed_at: string; title: string; case_ids: string[]; arms: { arm_id: string; label: string }[]; selection: Selection[]; incidents: Incident[]; notes: string[]; + duration_policy?: { target_ms: number; allow_goal_completion: string[] }; execution_audit?: ExecutionAudit[]; }; @@ -89,6 +91,12 @@ export function validateInput(raw: unknown): CampaignInput { const caseIds = new Set(d.case_ids); requireThat(Array.isArray(d.selection) && d.selection.length === d.arms.length * d.case_ids.length, 'selection must cover every arm/case exactly once'); requireThat(distinct(d.selection.map(key)) && distinct(d.selection.map(r => r.run_id)), 'duplicate selection/run id'); + if (d.duration_policy !== undefined) { + const policy = d.duration_policy; + requireThat(policy && Number.isSafeInteger(policy.target_ms) && policy.target_ms > 0, 'positive active duration target required'); + requireThat(Array.isArray(policy.allow_goal_completion) && distinct(policy.allow_goal_completion) && + policy.allow_goal_completion.every(id => armIds.has(id)), 'unknown or duplicate early completion arm'); + } for (const r of d.selection) { requireThat(armIds.has(r.arm_id) && caseIds.has(r.case_id) && token(r.run_id), 'unknown selection identity'); requireThat(statuses.includes(r.status) && token(r.protocol_id), 'invalid status/protocol'); @@ -101,6 +109,14 @@ export function validateInput(raw: unknown): CampaignInput { requireThat(s.origin === 'official' || (s.origin === 'authorized_failure_zero' && s.value === 0), 'score origin invalid'); requireThat(!s.countable || (r.status === 'completed' && s.origin === 'official' && r.exclude_reason === null), 'countable score contradicts status/origin/exclusion'); } + if (r.active_intervals !== undefined) { + requireThat(Array.isArray(r.active_intervals), 'active intervals must be an array'); + for (const interval of r.active_intervals) { + requireThat(interval && Number.isSafeInteger(interval.start_ms) && interval.start_ms >= 0 && + Number.isSafeInteger(interval.end_ms) && interval.end_ms >= interval.start_ms && + interval.end_ms <= Date.parse(d.observed_at) && text(interval.evidence_ref), 'invalid observed solver interval'); + } + } if (r.effort !== undefined) { const e = r.effort; requireThat(e && nonnegativeIntegerOrNull(e.duration_ms) && nonnegativeIntegerOrNull(e.token_count) && @@ -156,8 +172,35 @@ export function validateInput(raw: unknown): CampaignInput { return d; } +// Each row contains only the verified segments of one selected logical run. +function durationCoverage(d: CampaignInput, policy: NonNullable) { + return d.selection.map(row => { + const { arm_id, case_id, run_id } = row; + let end = 0; + let elapsed = 0; + for (const interval of [...row.active_intervals ?? []].sort((a,b) => a.start_ms - b.start_ms)) { + elapsed += Math.max(0, interval.end_ms - Math.max(end, interval.start_ms)); + end = Math.max(end, interval.end_ms); + } + const active_ms = row.active_intervals === undefined ? null : elapsed; + const remaining_ms = active_ms === null ? null : Math.max(0, policy.target_ms - active_ms); + const audit = d.execution_audit?.find(item => item.run_id === run_id); + const acceptedGoal = row.status === 'completed' && policy.allow_goal_completion.includes(arm_id) && + audit?.outcome === 'normal_end' && audit.goal_verdict === 'satisfied'; + let disposition: string; + if (remaining_ms === 0) disposition = 'covered'; + else if (acceptedGoal) disposition = 'goal_completed'; + else if (row.status === 'planned' || row.status === 'running') disposition = row.status; + else if (remaining_ms === null) disposition = 'verify_duration'; + else disposition = 'needs_makeup'; + return { case_id, arm_id, run_id, active_ms, remaining_ms, disposition }; + }); +} + export function buildReport(raw: unknown) { const d = validateInput(raw); + const coverage = d.duration_policy && durationCoverage(d, d.duration_policy); + const covered = new Set(coverage?.filter(row => ['covered', 'goal_completed'].includes(row.disposition)).map(key)); const selected = new Map(d.selection.map(r => [key(r), r])); const summary = d.arms.map(a => { const rows = d.selection.filter(r => r.arm_id === a.arm_id); @@ -212,7 +255,7 @@ export function buildReport(raw: unknown) { return !reasons.length; }); const scores = (a: string, cs: string[]) => cs.map(c => selected.get(key({arm_id:a,case_id:c}))!.score!.value); - const qualifiedCases = cases.filter(c => arms.every(a => selected.get(key({arm_id:a,case_id:c}))!.score!.countable)); + const qualifiedCases = cases.filter(c => arms.every(a => selected.get(key({arm_id:a,case_id:c}))!.score!.countable && (!coverage || covered.has(key({arm_id:a,case_id:c}))))); const means = Object.fromEntries(arms.map(a => [a, mean(scores(a, cases))])); const deltas = arms.length === 2 && cases.length ? cases.map(c => scores(arms[1], [c])[0] - scores(arms[0], [c])[0]) : []; return { arms, case_ids: cases, n: cases.length, case_set_sha256: digest(cases), means, @@ -252,6 +295,7 @@ export function buildReport(raw: unknown) { b.run_count - a.run_count || a.profile_id.localeCompare(b.profile_id)); return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d), observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection, + ...(coverage ? { duration_coverage: coverage } : {}), configuration_profiles, score_time_series, incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes }; } @@ -287,6 +331,8 @@ export function renderMarkdown(r: Report): string { table(['实验组','运行数','Source','Scorer','Model','预算(h)','主动提交冷却(s)','后台评测(s)','资源档位','证据覆盖'],r.configuration_profiles.map(p => [labels.get(p.arm_id),p.run_count,known(p.source_revision_id),known(p.scorer_revision_id),known(p.model_id),hours(p.budget_ms),seconds(p.active_submission_cooldown_ms),seconds(p.background_evaluation_interval_ms),known(p.resource_profile),`${p.configuration_supplied ? p.run_count : 0}/${p.run_count}`])) + '\n\n均分使用相同题目交集、相同归一化规则,尺度 0–100。未出分不填 0;授权的故障判零单独保留来源。以下为描述性比较;源版本、协议、资源差异和完整性复核仍需独立审查。\n\n' + table(['对比范围','共同题数',...r.summary.map(a => a.label),'第二组−第一组','共同题中双方/各方可计数'],r.comparisons.map(c => [c.arms.map(a => labels.get(a)).join(' / '),c.n,...r.summary.map(a => number(c.means[a.arm_id] ?? null)),number(c.delta_second_minus_first),c.qualified_common_n])) + + (r.duration_coverage ? '\n\n## 各臂有效运行时间\n\n' + table(['题目','实验臂','已运行(小时)','距目标(小时)','状态'], + r.duration_coverage.map(row => [row.case_id, labels.get(row.arm_id), hours(row.active_ms), hours(row.remaining_ms), row.disposition])) : '') + '\n\n' + r.notes.map(n => '- ' + cell(n)).join('\n') + '\n\n## 共同题目与排除记录\n\n' + r.comparisons.map(c => `- ${c.arms.map(a => labels.get(a)).join(' / ')}:${c.case_ids.join('、') || '无'}。`).join('\n') + `\n\n分数/时间序列:${r.score_time_series.length} 个带来源的采样点,覆盖 ${new Set(r.score_time_series.map(s => s.run_id)).size} 个运行。采样覆盖不同,不从缺失点补零,也不自动计算跨题均值曲线;同名 JSON 保留可绘图序列。\n\n` + diff --git a/skills/loopx-benchmark/SKILL.md b/skills/loopx-benchmark/SKILL.md index 3d146d4943..21b7844ff2 100644 --- a/skills/loopx-benchmark/SKILL.md +++ b/skills/loopx-benchmark/SKILL.md @@ -339,3 +339,12 @@ become a second source of score, integrity, or countability truth. only when a complete authorized post-run review observed no such transition; otherwise absence is `unknown`. Keep terminal settlement separate. - Do not send a repetitive user update when nothing material changed. + +For fixed-duration studies, use the campaign report's duration policy for **every +declared arm**, including Official. At each terminal transition reconcile its +active solver intervals and any maintained make-up obligation with the report. +Only waive a short run when its arm is expressly allowed and its own audited Goal +completed normally. Keep unknown duration pending evidence. Keep provider holds +visible with the outstanding duration and the condition for resuming. Preserve +session, scoring opportunities and cooldown when resuming; record fresh runs +separately. A duration shortfall alone never authorizes overriding a stop. diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 1d00d913c0..09a7269db2 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -196,3 +196,23 @@ test('opt-in HTML separates the record trajectory from all samples and splits di assert.match(charts[0], /H932\.00"/); // retain the best through the last observed regression } finally { rmSync(dir, {recursive: true, force: true}); } }); + +test('duration policy checks every selected arm and only accepts verified voluntary goal completion', () => { + const input = fixture(); + input.duration_policy = {target_ms: 100, allow_goal_completion: ['b']}; + input.selection[0].active_intervals = [[0,30],[10,40],[80,100]].map(([start_ms,end_ms]) => + ({start_ms,end_ms,evidence_ref:'solver:segment'})); + input.selection[1].active_intervals = [{start_ms:10,end_ms:110,evidence_ref:'solver:segment'}]; + input.selection[2].active_intervals = []; + input.execution_audit = [{case_id:'one', arm_id:'b', run_id:'b-one',initial_attempt:true, + outcome:'normal_end',goal_verdict:'satisfied',attribution:'not_applicable',evidence_refs:['receipt:goal']}]; + const report = buildReport(input); + const rows = report.duration_coverage!; + assert.equal(rows.length,6); + assert.deepEqual(rows.map(row => row.disposition), ['needs_makeup','covered','needs_makeup','goal_completed','verify_duration','verify_duration']); + assert.deepEqual(rows.map(row => row.remaining_ms),[40,0,100,null,null,null]); + assert.equal(report.comparisons[0].qualified_common_n,0); + assert.match(renderMarkdown(report),/needs_makeup/); + input.selection[0].active_intervals[0].end_ms = -1; + assert.throws(() => buildReport(input),/observed solver interval/); +});