From fb43dd2f0b64285a309769ac256cac2613828e53 Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:40:42 +0800 Subject: [PATCH 1/3] feat(benchmark): support explicit terminal makeup tolerance Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/benchmark-campaign-report.md | 17 ++++++ scripts/benchmark-campaign-report.ts | 14 ++++- .../benchmark_campaign_report.test.ts | 57 +++++++++++++++++++ 3 files changed, 85 insertions(+), 3 deletions(-) diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index 7f7029b5e6..e2a0ba7c31 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -203,3 +203,20 @@ run enters `needs_makeup`, even with a valid score or a successful process exit. Only a matching completed run with an audited normal end and a satisfied Goal can use the early-finish allowance. Incomplete and unverified durations exclude a case from the duration-qualified comparison. Retain raw results for diagnosis. + +An operator can explicitly set `duration_policy.makeup_tolerance_ms` when small +terminal shortfalls do not warrant another run. It defaults to zero and must be +a nonnegative safe integer smaller than `target_ms`. For example, a tolerance of +`900000` waives at most fifteen minutes. After unioning the verified active +intervals, a terminal run with a positive observed duration and a remaining gap +at or below that tolerance is marked `within_tolerance`, rather than `covered`. +The report retains the actual `active_ms`, `remaining_ms` and selected policy. +Unknown duration still needs verification; planned/running rows retain their +status, and no runtime deadline or stopping rule changes. + +`within_tolerance` satisfies only the report's declared duration condition. It +does not change run status, scores, integrity/countability, or matched-comparison +eligibility. A cancelled or uncountable result remains excluded by those gates. +The Markdown report labels the waiver and its threshold. Callers should use this +same projection when selecting makeup work rather than rounding the raw duration +or inventing a second threshold. diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 1ac64a6e0d..136b67f6c8 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -56,7 +56,7 @@ export type CampaignInput = { observed_at: string; title: string; case_ids: string[]; arms: { arm_id: string; label: string }[]; selection: Selection[]; incidents: Incident[]; notes: string[]; - duration_policy?: { target_ms: number; allow_goal_completion: string[] }; + duration_policy?: { target_ms: number; allow_goal_completion: string[]; makeup_tolerance_ms?: number }; execution_audit?: ExecutionAudit[]; }; @@ -94,6 +94,9 @@ export function validateInput(raw: unknown): CampaignInput { if (d.duration_policy !== undefined) { const policy = d.duration_policy; requireThat(policy && Number.isSafeInteger(policy.target_ms) && policy.target_ms > 0, 'positive active duration target required'); + requireThat(policy.makeup_tolerance_ms === undefined || + (Number.isSafeInteger(policy.makeup_tolerance_ms) && policy.makeup_tolerance_ms >= 0 && + policy.makeup_tolerance_ms < policy.target_ms), 'makeup tolerance must be a nonnegative integer below target'); requireThat(Array.isArray(policy.allow_goal_completion) && distinct(policy.allow_goal_completion) && policy.allow_goal_completion.every(id => armIds.has(id)), 'unknown or duplicate early completion arm'); } @@ -192,6 +195,7 @@ function durationCoverage(d: CampaignInput, policy: NonNullable 0 && remaining_ms <= (policy.makeup_tolerance_ms ?? 0)) disposition = 'within_tolerance'; else disposition = 'needs_makeup'; return { case_id, arm_id, run_id, active_ms, remaining_ms, disposition }; }); @@ -200,7 +204,7 @@ function durationCoverage(d: CampaignInput, policy: NonNullable ['covered', 'goal_completed'].includes(row.disposition)).map(key)); + const covered = new Set(coverage?.filter(row => ['covered', 'goal_completed', 'within_tolerance'].includes(row.disposition)).map(key)); const selected = new Map(d.selection.map(r => [key(r), r])); const summary = d.arms.map(a => { const rows = d.selection.filter(r => r.arm_id === a.arm_id); @@ -296,6 +300,7 @@ export function buildReport(raw: unknown) { return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d), observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection, ...(coverage ? { duration_coverage: coverage } : {}), + ...(d.duration_policy?.makeup_tolerance_ms !== undefined ? { duration_policy: d.duration_policy } : {}), configuration_profiles, score_time_series, incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes }; } @@ -331,7 +336,10 @@ export function renderMarkdown(r: Report): string { table(['实验组','运行数','Source','Scorer','Model','预算(h)','主动提交冷却(s)','后台评测(s)','资源档位','证据覆盖'],r.configuration_profiles.map(p => [labels.get(p.arm_id),p.run_count,known(p.source_revision_id),known(p.scorer_revision_id),known(p.model_id),hours(p.budget_ms),seconds(p.active_submission_cooldown_ms),seconds(p.background_evaluation_interval_ms),known(p.resource_profile),`${p.configuration_supplied ? p.run_count : 0}/${p.run_count}`])) + '\n\n均分使用相同题目交集、相同归一化规则，尺度 0–100。未出分不填 0；授权的故障判零单独保留来源。以下为描述性比较；源版本、协议、资源差异和完整性复核仍需独立审查。\n\n' + table(['对比范围','共同题数',...r.summary.map(a => a.label),'第二组−第一组','共同题中双方/各方可计数'],r.comparisons.map(c => [c.arms.map(a => labels.get(a)).join(' / '),c.n,...r.summary.map(a => number(c.means[a.arm_id] ?? null)),number(c.delta_second_minus_first),c.qualified_common_n])) + - (r.duration_coverage ? '\n\n## 各臂有效运行时间\n\n' + table(['题目','实验臂','已运行（小时）','距目标（小时）','状态'], + (r.duration_coverage ? '\n\n## 各臂有效运行时间\n\n' + + ((r.duration_policy?.makeup_tolerance_ms ?? 0) > 0 + ? `已终止运行的补时容差：${r.duration_policy!.makeup_tolerance_ms! / 60000} 分钟。within_tolerance 表示容差内免补；实际时长和缺口保持原值，不提前停止正在运行的任务。\n\n` : '') + + table(['题目','实验臂','已运行（小时）','距目标（小时）','状态'], r.duration_coverage.map(row => [row.case_id, labels.get(row.arm_id), hours(row.active_ms), hours(row.remaining_ms), row.disposition])) : '') + '\n\n' + r.notes.map(n => '- ' + cell(n)).join('\n') + '\n\n## 共同题目与排除记录\n\n' + r.comparisons.map(c => `- ${c.arms.map(a => labels.get(a)).join(' / ')}：${c.case_ids.join('、') || '无'}。`).join('\n') + diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 09a7269db2..974db6beca 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -216,3 +216,60 @@ test('duration policy checks every selected arm and only accepts verified volunt input.selection[0].active_intervals[0].end_ms = -1; assert.throws(() => buildReport(input),/observed solver interval/); }); + +test('makeup tolerance waives only known terminal gaps, retaining actual duration and qualification gates', () => { + const input = fixture(); + input.duration_policy = {target_ms: 100, allow_goal_completion: [], makeup_tolerance_ms: 15}; + for (const [i, elapsed] of [85, 84, 100, 86, 0, 85].entries()) { + input.selection[i].active_intervals = [{start_ms: 0, end_ms: elapsed, evidence_ref: 'solver:segment'}]; + } + // Overlap is not extra active time: 50 + 45 - 10 = 85. + input.selection[0].active_intervals = [[0,50],[40,85]].map(([start_ms,end_ms]) => + ({start_ms,end_ms,evidence_ref:'solver:segment'})); + let report = buildReport(input); + assert.deepEqual(report.duration_coverage!.map(row => row.disposition), + ['within_tolerance', 'needs_makeup', 'covered', 'within_tolerance', 'needs_makeup', 'within_tolerance']); + assert.equal(report.duration_coverage![0].active_ms, 85); + assert.equal(report.duration_coverage![0].remaining_ms, 15); + assert.equal(report.comparisons[0].qualified_common_n, 2); + assert.deepEqual(report.duration_policy, input.duration_policy); + assert.match(renderMarkdown(report), /容差内免补/); + input.selection[0].score!.countable = false; + report = buildReport(input); + assert.equal(report.comparisons[0].qualified_common_n, 1); + input.selection[0].status = 'cancelled'; + assert.equal(buildReport(input).comparisons[0].n, 2); + input.selection[0].status = 'running'; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'running'); + input.selection[0].status = 'planned'; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'planned'); + input.selection[0].status = 'completed'; + delete input.selection[0].active_intervals; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'verify_duration'); + input.selection[0].active_intervals = [{start_ms:0,end_ms:85,evidence_ref:'solver:segment'}]; + delete input.duration_policy.makeup_tolerance_ms; + assert.equal(buildReport(input).duration_coverage![0].disposition, 'needs_makeup'); +}); + +test('duration tolerance rejects invalid policy and real CLI retains the declared waiver', () => { + const input = fixture(); + input.duration_policy = {target_ms: 100, allow_goal_completion: []}; + for (const bad of [-1, 100, 101, 1.5, NaN, Infinity, true, null]) { + input.duration_policy.makeup_tolerance_ms = bad as number; + assert.throws(() => buildReport(input), /makeup tolerance/); + } + input.duration_policy.makeup_tolerance_ms = 15; + input.selection[0].active_intervals = [{start_ms:0,end_ms:85,evidence_ref:'solver:segment'}]; + const dir = mkdtempSync(join(tmpdir(), 'duration-tolerance-')); + try { + const path = join(dir, 'input.json'), prefix = join(dir, 'report'); + writeFileSync(path, JSON.stringify(input)); + const run = spawnSync(process.execPath, ['--experimental-strip-types', resolve('scripts/benchmark-campaign-report.ts'), path, prefix], {encoding:'utf8'}); + assert.equal(run.status, 0, run.stderr); + const actual = JSON.parse(readFileSync(prefix + '.json', 'utf8')); + assert.equal(actual.duration_policy.makeup_tolerance_ms, 15); + assert.equal(actual.duration_coverage[0].disposition, 'within_tolerance'); + assert.equal(actual.duration_coverage[0].remaining_ms, 15); + assert.match(readFileSync(prefix + '.md', 'utf8'), /容差内免补/); + } finally { rmSync(dir, {recursive: true, force: true}); } +}); From d67599eff74e64f62468b9145b47dbf89de6dd7c Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:42:38 +0800 Subject: [PATCH 2/3] docs: clarify duration tolerance qualification boundaries Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- docs/reference/benchmark-campaign-report.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/reference/benchmark-campaign-report.md b/docs/reference/benchmark-campaign-report.md index e2a0ba7c31..1eaebbf95b 100644 --- a/docs/reference/benchmark-campaign-report.md +++ b/docs/reference/benchmark-campaign-report.md @@ -211,12 +211,13 @@ a nonnegative safe integer smaller than `target_ms`. For example, a tolerance of intervals, a terminal run with a positive observed duration and a remaining gap at or below that tolerance is marked `within_tolerance`, rather than `covered`. The report retains the actual `active_ms`, `remaining_ms` and selected policy. -Unknown duration still needs verification; planned/running rows retain their -status, and no runtime deadline or stopping rule changes. +Unknown duration still needs verification; short planned/running rows retain +their status, and no runtime deadline or stopping rule changes. `within_tolerance` satisfies only the report's declared duration condition. It does not change run status, scores, integrity/countability, or matched-comparison -eligibility. A cancelled or uncountable result remains excluded by those gates. +eligibility. A cancelled result stays out of the descriptive comparison, and an +uncountable score stays out of `qualified_common_n` even when its duration qualifies. The Markdown report labels the waiver and its threshold. Callers should use this same projection when selecting makeup work rather than rounding the raw duration or inventing a second threshold. From 6cb25b54ca0670b4e2c29a97715c80807c55bdef Mon Sep 17 00:00:00 2001 From: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:11:39 +0800 Subject: [PATCH 3/3] fix(benchmark): narrow duration policy report projection Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com> --- scripts/benchmark-campaign-report.ts | 6 +++++- tests/control_plane_ts/benchmark_campaign_report.test.ts | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/benchmark-campaign-report.ts b/scripts/benchmark-campaign-report.ts index 136b67f6c8..acc0d0aac8 100644 --- a/scripts/benchmark-campaign-report.ts +++ b/scripts/benchmark-campaign-report.ts @@ -300,7 +300,11 @@ export function buildReport(raw: unknown) { return { schema_version: 'benchmark_campaign_report_v1', input_sha256: digest(d), observed_at: d.observed_at, title: d.title, summary, comparisons, selection: d.selection, ...(coverage ? { duration_coverage: coverage } : {}), - ...(d.duration_policy?.makeup_tolerance_ms !== undefined ? { duration_policy: d.duration_policy } : {}), + ...(d.duration_policy?.makeup_tolerance_ms !== undefined ? { duration_policy: { + target_ms: d.duration_policy.target_ms, + allow_goal_completion: d.duration_policy.allow_goal_completion, + makeup_tolerance_ms: d.duration_policy.makeup_tolerance_ms, + } } : {}), configuration_profiles, score_time_series, incidents: d.incidents, execution_audit: d.execution_audit ?? [], notes: d.notes }; } diff --git a/tests/control_plane_ts/benchmark_campaign_report.test.ts b/tests/control_plane_ts/benchmark_campaign_report.test.ts index 974db6beca..46f195b0e4 100644 --- a/tests/control_plane_ts/benchmark_campaign_report.test.ts +++ b/tests/control_plane_ts/benchmark_campaign_report.test.ts @@ -259,6 +259,8 @@ test('duration tolerance rejects invalid policy and real CLI retains the declare assert.throws(() => buildReport(input), /makeup tolerance/); } input.duration_policy.makeup_tolerance_ms = 15; + (input.duration_policy as typeof input.duration_policy & {operator_annotation: string}).operator_annotation = + 'must not enter report output'; input.selection[0].active_intervals = [{start_ms:0,end_ms:85,evidence_ref:'solver:segment'}]; const dir = mkdtempSync(join(tmpdir(), 'duration-tolerance-')); try { @@ -268,6 +270,11 @@ test('duration tolerance rejects invalid policy and real CLI retains the declare assert.equal(run.status, 0, run.stderr); const actual = JSON.parse(readFileSync(prefix + '.json', 'utf8')); assert.equal(actual.duration_policy.makeup_tolerance_ms, 15); + assert.deepEqual(actual.duration_policy, { + target_ms: 100, + allow_goal_completion: [], + makeup_tolerance_ms: 15, + }); assert.equal(actual.duration_coverage[0].disposition, 'within_tolerance'); assert.equal(actual.duration_coverage[0].remaining_ms, 15); assert.match(readFileSync(prefix + '.md', 'utf8'), /容差内免补/);