diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index 7f7029b5e6..1eaebbf95b 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -203,3 +203,21 @@ run enters `needs_makeup`, even with a valid score or a successful process exit. Only a matching completed run with an audited normal end and a satisfied Goal can use the early-finish allowance. Incomplete and unverified durations exclude a case from the duration-qualified comparison. Retain raw results for diagnosis. + +An operator can explicitly set `duration_policy.makeup_tolerance_ms` when small +terminal shortfalls do not warrant another run. It defaults to zero and must be +a nonnegative safe integer smaller than `target_ms`. For example, a tolerance of +`900000` waives at most fifteen minutes. After unioning the verified active +intervals, a terminal run with a positive observed duration and a remaining gap +at or below that tolerance is marked `within_tolerance`, rather than `covered`. +The report retains the actual `active_ms`, `remaining_ms` and selected policy. +Unknown duration still needs verification; short planned/running rows retain +their status, and no runtime deadline or stopping rule changes. + +`within_tolerance` satisfies only the report's declared duration condition. It +does not change run status, scores, integrity/countability, or matched-comparison +eligibility. A cancelled result stays out of the descriptive comparison, and an +uncountable score stays out of `qualified_common_n` even when its duration qualifies. +The Markdown report labels the waiver and its threshold. Callers should use this +same projection when selecting makeup work rather than rounding the raw duration +or inventing a second threshold. diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 1ac64a6e0d..acc0d0aac8 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -56,7 +56,7 @@ export type CampaignInput = { observed_at: string; title: string; case_ids: string[]; arms: { arm_id: string; label: string }[]; selection: Selection[]; incidents: Incident[]; notes: string[]; - duration_policy?: { target_ms: number; allow_goal_completion: string[] }; + duration_policy?: { target_ms: number; allow_goal_completion: string[]; makeup_tolerance_ms?: number }; execution_audit?: ExecutionAudit[]; }; @@ -94,6 +94,9 @@ export function validateInput(raw: unknown): CampaignInput { if (d.duration_policy !== undefined) { const policy = d.duration_policy; requireThat(policy && Number.isSafeInteger(policy.target_ms) && policy.target_ms > 0, 'positive active duration target required'); + requireThat(policy.makeup_tolerance_ms === undefined || + (Number.isSafeInteger(policy.makeup_tolerance_ms) && policy.makeup_tolerance_ms >= 0 && + policy.makeup_tolerance_ms < policy.target_ms), 'makeup tolerance must be a nonnegative integer below target'); requireThat(Array.isArray(policy.allow_goal_completion) && distinct(policy.allow_goal_completion) && policy.allow_goal_completion.every(id => armIds.has(id)), 'unknown or duplicate early completion arm'); } @@ -192,6 +195,7 @@ function durationCoverage(d: CampaignInput, policy: NonNullable 0 && remaining_ms <= (policy.makeup_tolerance_ms ?? 0)) disposition = 'within_tolerance'; else disposition = 'needs_makeup'; return { case_id, arm_id, run_id, active_ms, remaining_ms, disposition }; }); @@ -200,7 +204,7 @@ function durationCoverage(d: CampaignInput, policy: NonNullable ['covered', 'goal_completed'].includes(row.disposition)).map(key)); + const covered = new Set(coverage?.filter(row => ['covered', 'goal_completed', 'within_tolerance'].includes(row.disposition)).map(key)); const selected = new Map(d.selection.map(r => [key(r), r])); const summary = d.arms.map(a => { const rows = d.selection.filter(r => r.arm_id === a.arm_id); @@ -296,6 +300,11 @@ export function buildReport(raw: unknown) { return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d), observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection, ...(coverage ? { duration_coverage: coverage } : {}), + ...(d.duration_policy?.makeup_tolerance_ms !== undefined ? { duration_policy: { + target_ms: d.duration_policy.target_ms, + allow_goal_completion: d.duration_policy.allow_goal_completion, + makeup_tolerance_ms: d.duration_policy.makeup_tolerance_ms, + } } : {}), configuration_profiles, score_time_series, incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes }; } @@ -331,7 +340,10 @@ export function renderMarkdown(r: Report): string { table(['实验组','运行数','Source','Scorer','Model','预算(h)','主动提交冷却(s)','后台评测(s)','资源档位','证据覆盖'],r.configuration_profiles.map(p => [labels.get(p.arm_id),p.run_count,known(p.source_revision_id),known(p.scorer_revision_id),known(p.model_id),hours(p.budget_ms),seconds(p.active_submission_cooldown_ms),seconds(p.background_evaluation_interval_ms),known(p.resource_profile),`${p.configuration_supplied ? p.run_count : 0}/${p.run_count}`])) + '\n\n均分使用相同题目交集、相同归一化规则，尺度 0–100。未出分不填 0；授权的故障判零单独保留来源。以下为描述性比较；源版本、协议、资源差异和完整性复核仍需独立审查。\n\n' + table(['对比范围','共同题数',...r.summary.map(a => a.label),'第二组−第一组','共同题中双方/各方可计数'],r.comparisons.map(c => [c.arms.map(a => labels.get(a)).join(' / '),c.n,...r.summary.map(a => number(c.means[a.arm_id] ?? null)),number(c.delta_second_minus_first),c.qualified_common_n])) + - (r.duration_coverage ? '\n\n## 各臂有效运行时间\n\n' + table(['题目','实验臂','已运行（小时）','距目标（小时）','状态'], + (r.duration_coverage ? '\n\n## 各臂有效运行时间\n\n' + + ((r.duration_policy?.makeup_tolerance_ms ?? 0) > 0 + ? `已终止运行的补时容差：${r.duration_policy!.makeup_tolerance_ms! / 60000} 分钟。within_tolerance 表示容差内免补；实际时长和缺口保持原值，不提前停止正在运行的任务。\n\n` : '') + + table(['题目','实验臂','已运行（小时）','距目标（小时）','状态'], r.duration_coverage.map(row => [row.case_id, labels.get(row.arm_id), hours(row.active_ms), hours(row.remaining_ms), row.disposition])) : '') + '\n\n' + r.notes.map(n => '- ' + cell(n)).join('\n') + '\n\n## 共同题目与排除记录\n\n' + r.comparisons.map(c => `- ${c.arms.map(a => labels.get(a)).join(' / ')}：${c.case_ids.join('、') || '无'}。`).join('\n') + diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 09a7269db2..46f195b0e4 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -216,3 +216,67 @@ test('duration policy checks every selected arm and only accepts verified volunt input.selection[0].active_intervals[0].end_ms = -1; assert.throws(() => buildReport(input),/observed solver interval/); }); + +test('makeup tolerance waives only known terminal gaps, retaining actual duration and qualification gates', () => { + const input = fixture(); + input.duration_policy = {target_ms: 100, allow_goal_completion: [], makeup_tolerance_ms: 15}; + for (const [i, elapsed] of [85, 84, 100, 86, 0, 85].entries()) { + input.selection[i].active_intervals = [{start_ms: 0, end_ms: elapsed, evidence_ref: 'solver:segment'}]; + } + // Overlap is not extra active time: 50 + 45 - 10 = 85. + input.selection[0].active_intervals = [[0,50],[40,85]].map(([start_ms,end_ms]) => + ({start_ms,end_ms,evidence_ref:'solver:segment'})); + let report = buildReport(input); + assert.deepEqual(report.duration_coverage!.map(row => row.disposition), + ['within_tolerance', 'needs_makeup', 'covered', 'within_tolerance', 'needs_makeup', 'within_tolerance']); + assert.equal(report.duration_coverage![0].active_ms, 85); + assert.equal(report.duration_coverage![0].remaining_ms, 15); + assert.equal(report.comparisons[0].qualified_common_n, 2); + assert.deepEqual(report.duration_policy, input.duration_policy); + assert.match(renderMarkdown(report), /容差内免补/); + input.selection[0].score!.countable = false; + report = buildReport(input); + assert.equal(report.comparisons[0].qualified_common_n, 1); + input.selection[0].status = 'cancelled'; + assert.equal(buildReport(input).comparisons[0].n, 2); + input.selection[0].status = 'running'; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'running'); + input.selection[0].status = 'planned'; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'planned'); + input.selection[0].status = 'completed'; + delete input.selection[0].active_intervals; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'verify_duration'); + input.selection[0].active_intervals = [{start_ms:0,end_ms:85,evidence_ref:'solver:segment'}]; + delete input.duration_policy.makeup_tolerance_ms; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'needs_makeup'); +}); + +test('duration tolerance rejects invalid policy and real CLI retains the declared waiver', () => { + const input = fixture(); + input.duration_policy = {target_ms: 100, allow_goal_completion: []}; + for (const bad of [-1, 100, 101, 1.5, NaN, Infinity, true, null]) { + input.duration_policy.makeup_tolerance_ms = bad as number; + assert.throws(() => buildReport(input), /makeup tolerance/); + } + input.duration_policy.makeup_tolerance_ms = 15; + (input.duration_policy as typeof input.duration_policy & {operator_annotation: string}).operator_annotation = + 'must not enter report output'; + input.selection[0].active_intervals = [{start_ms:0,end_ms:85,evidence_ref:'solver:segment'}]; + const dir = mkdtempSync(join(tmpdir(), 'duration-tolerance-')); + try { + const path = join(dir, 'input.json'), prefix = join(dir, 'report'); + writeFileSync(path, JSON.stringify(input)); + const run = spawnSync(process.execPath, ['--experimental-strip-types', resolve('scripts/benchmark-campaign-report.ts'), path, prefix], {encoding:'utf8'}); + assert.equal(run.status, 0, run.stderr); + const actual = JSON.parse(readFileSync(prefix + '.json', 'utf8')); + assert.equal(actual.duration_policy.makeup_tolerance_ms, 15); + assert.deepEqual(actual.duration_policy, { + target_ms: 100, + allow_goal_completion: [], + makeup_tolerance_ms: 15, + }); + assert.equal(actual.duration_coverage[0].disposition, 'within_tolerance'); + assert.equal(actual.duration_coverage[0].remaining_ms, 15); + assert.match(readFileSync(prefix + '.md', 'utf8'), /容差内免补/); + } finally { rmSync(dir, {recursive: true, force: true}); } +});