diff --git a/docs/v2/BLIND-TEST-RUNBOOK.md b/docs/v2/BLIND-TEST-RUNBOOK.md index 40a9fb2e..d8389cec 100644 --- a/docs/v2/BLIND-TEST-RUNBOOK.md +++ b/docs/v2/BLIND-TEST-RUNBOOK.md @@ -15,6 +15,22 @@ 三种角色都必须是**模型或人**,不是本脚本。同一模型不得同时充当蒸馏器与裁判。 +蒸馏者、裁判、核对者必须是三个不同模型,且都不是 deepseek-flash。本程序固定: + +| 角色 | 模型 | 看到什么 | +|---|---|---| +| 蒸馏者 | 不得是下面两行,也不得是 deepseek-flash | A 半段(实验组读 `deriver-input.md`) | +| 裁判 | `claude-sonnet-5-thinking-medium` | 只看 `profile.html` 与 `judge-prompt.md` | +| 核对者 | `gpt-5.6-sol-medium` | 只读 `split/B.srt` 与裁判的 10 条 | + +效果数字不是代码。第一组数字由后续 swarm 按上表产出,写到: + +`/cursor/stores/bc-0e6a1836-4b6d-4ce2-8177-5061476fde52/media/distilly-atomic-pr/pr-8-blind/` + +本拍不跑三模型盲测 Lane 7 与 Lane 8。机械层(切分、相对路径 prepare、finalize、control、空表 score)由本 PR 自证。 + +`--out`、`--a`、`--view`、`--scores` 可以是相对路径。脚本对 `process.cwd()` 做一次 `path.resolve`,传给 `harvest` / `retrospect` / `view` 的 `--base-dir` 与文件路径都是绝对路径。HANDOFF §8 的 `prepare --a split/A.srt --out run` 必须退出 0。 + ## 1. 切分(检查者一个人做) ```bash @@ -110,4 +126,5 @@ node scripts/blind-test.mjs score --scores /tmp/blind-run/scores.json --out /tmp 报告里必须写明这一点。 - 身份盲:`prepare --person ` 决定页面上的 `slug`;要让裁判连人名也看不到, 用一个中性 slug(如 `blind-sample-01`)跑,目录名与页面都不含真名。 -- 裁判与蒸馏器若用同一模型,须在结论里注明;同模型自评会让命中率偏高。 +- 裁判、核对者、蒸馏者必须是三个不同模型,都不是 deepseek-flash。裁判固定 + `claude-sonnet-5-thinking-medium`,核对者固定 `gpt-5.6-sol-medium`。同模型自评不作数。 diff --git a/scripts/blind-test.mjs b/scripts/blind-test.mjs index 92dc11a6..b9c2f71a 100644 --- a/scripts/blind-test.mjs +++ b/scripts/blind-test.mjs @@ -56,6 +56,11 @@ function arg(name, fallback) { } const flag = (name) => process.argv.includes(`--${name}`); +/** Resolve a user path once so spawn cwd cannot reinterpret --out/--a/--base-dir. */ +function abs(value) { + return resolve(process.cwd(), value); +} + const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex"); const round = (value) => (Number.isInteger(value) ? String(value) : value.toFixed(2).replace(/0$/, "")); @@ -304,19 +309,21 @@ function writeSheet(out) { } function prepare() { - const a = arg("a"); + const aArg = arg("a"); const person = arg("person"); - const out = arg("out"); - if (!a || !person || !out) { + const outArg = arg("out"); + if (!aArg || !person || !outArg) { console.error("usage: node scripts/blind-test.mjs prepare --a --person --out [--skeleton-only]"); process.exit(2); } + const a = abs(aArg); + const out = abs(outArg); const work = join(out, "work"); rmSync(work, { recursive: true, force: true }); mkdirSync(join(work, "corpus"), { recursive: true }); cpSync(a, join(work, "corpus", basename(a))); distilly(["harvest", join(work, "corpus"), "--person", person, "--base-dir", work, "--json"], work); - const retro = distilly(["retrospect", "--person", person, "--json"], work); + const retro = distilly(["retrospect", "--person", person, "--base-dir", work, "--json"], work); const personDir = join(work, "skills", "colleague", person); const derivation = loadDerivation(personDir); @@ -413,10 +420,18 @@ function countRenderedAnchors(html) { } function finalizeView({ out, viewPath, slug, personDir, receipt, source }) { - const checked = distilly(["view", "check", "--file", viewPath, "--json"], personDir, { tolerate: true }); - const rendered = distilly(["view", "render", "--file", viewPath, "--out", join(out, "profile.html"), "--json"], personDir, { - tolerate: true, - }); + const work = join(out, "work"); + const htmlPath = join(out, "profile.html"); + const checked = distilly( + ["view", "check", "--file", viewPath, "--person", slug, "--base-dir", work, "--json"], + personDir, + { tolerate: true }, + ); + const rendered = distilly( + ["view", "render", "--file", viewPath, "--out", htmlPath, "--person", slug, "--base-dir", work, "--json"], + personDir, + { tolerate: true }, + ); const diagnostics = []; for (const entry of checked.receipt?.diagnostics ?? []) diagnostics.push(`${entry.code}: ${entry.message}`); const html = readFileSync(join(out, "profile.html"), "utf8"); @@ -438,17 +453,19 @@ function finalizeView({ out, viewPath, slug, personDir, receipt, source }) { } function finalize() { - const out = arg("out"); - const view = arg("view"); - if (!out || !view) { + const outArg = arg("out"); + const viewArg = arg("view"); + if (!outArg || !viewArg) { console.error("usage: node scripts/blind-test.mjs finalize --out --view [--strict]"); process.exit(2); } + const out = abs(outArg); + const view = abs(viewArg); const receiptPath = join(out, "receipt.json"); const receipt = JSON.parse(readFileSync(receiptPath, "utf8")); const person = receipt.view?.slug ?? JSON.parse(readFileSync(view, "utf8")).meta?.slug; const personDir = join(out, "work", "skills", "colleague", person); - const result = finalizeView({ out, viewPath: resolve(view), slug: person, personDir, receipt, source: "authored" }); + const result = finalizeView({ out, viewPath: view, slug: person, personDir, receipt, source: "authored" }); Object.assign(receipt, result, { view_source: "authored", next_steps: receipt.next_steps }); writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, "utf8"); console.log(`finalized ${out}/profile.html (${result.html.bytes} bytes, check ok: ${result.check.ok}, external links: ${result.external_links})`); @@ -469,12 +486,14 @@ const CONTROL_PROMPT = (source) => `# 对照任务(裸 prompt,无证据层 `; function control() { - const a = arg("a"); - const out = arg("out"); - if (!a || !out) { + const aArg = arg("a"); + const outArg = arg("out"); + if (!aArg || !outArg) { console.error("usage: node scripts/blind-test.mjs control --a --out "); process.exit(2); } + const a = abs(aArg); + const out = abs(outArg); mkdirSync(out, { recursive: true }); writeFileSync(join(out, "control-prompt.md"), CONTROL_PROMPT(basename(a)) + readFileSync(a, "utf8"), "utf8"); writeSheet(out); @@ -525,11 +544,12 @@ export function verdictFor(metrics) { } function score() { - const scoresPath = arg("scores"); - if (!scoresPath) { + const scoresArg = arg("scores"); + if (!scoresArg) { console.error("usage: node scripts/blind-test.mjs score --scores [--out report.md]"); process.exit(2); } + const scoresPath = abs(scoresArg); const scores = JSON.parse(readFileSync(scoresPath, "utf8")); const arms = ["evidence", "control"].filter((arm) => Array.isArray(scores[arm])); if (arms.length === 0) throw new Error("scores.json needs an `evidence` and/or `control` array"); @@ -560,8 +580,9 @@ function score() { } const report = lines.join("\n") + "\n"; - const out = arg("out"); - if (out) { + const outArg = arg("out"); + if (outArg) { + const out = abs(outArg); writeFileSync(out, report, "utf8"); console.log(`report → ${out}`); } diff --git a/tests/blind-test.test.mjs b/tests/blind-test.test.mjs index d82cfa5a..44b32696 100644 --- a/tests/blind-test.test.mjs +++ b/tests/blind-test.test.mjs @@ -10,7 +10,7 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { after, describe, it } from "node:test"; @@ -190,4 +190,31 @@ describe("blind-test prepare", () => { assert.equal(sheet.control.length, 10); assert.equal(sheet.evidence.length, 10, "both arms share one sheet so they cannot diverge"); }); + + it("accepts the relative --out and --a the handoff (§8) uses", () => { + const dir = scratch("blind-rel"); + const splitResult = run("split-corpus.mjs", ["--in", FIXTURE, "--out", "split", "--ratio", "0.7"], { cwd: dir }); + assert.equal(splitResult.status, 0, splitResult.stderr || splitResult.stdout); + + const prepare = run( + "blind-test.mjs", + ["prepare", "--a", "split/A.srt", "--person", "lin-gong", "--out", "run"], + { cwd: dir }, + ); + assert.equal(prepare.status, 0, prepare.stderr || prepare.stdout); + assert.equal(existsSync(join(dir, "run", "profile.html")), true); + + const finalize = run("blind-test.mjs", ["finalize", "--out", "run", "--view", "run/view.baseline.json"], { cwd: dir }); + assert.equal(finalize.status, 0, finalize.stderr || finalize.stdout); + + const control = run("blind-test.mjs", ["control", "--a", "split/A.srt", "--out", "control"], { cwd: dir }); + assert.equal(control.status, 0, control.stderr || control.stdout); + assert.equal(existsSync(join(dir, "control", "control-prompt.md")), true); + + const score = run("blind-test.mjs", ["score", "--scores", "run/scores.template.json", "--out", "report.md"], { + cwd: dir, + }); + assert.equal(score.status, 0, score.stderr || score.stdout); + assert.match(readFileSync(join(dir, "report.md"), "utf8"), /evidence → FAIL/); + }); }); diff --git a/tests/gates/baseline/audit-tests.txt b/tests/gates/baseline/audit-tests.txt index 128430a4..5226f85d 100644 --- a/tests/gates/baseline/audit-tests.txt +++ b/tests/gates/baseline/audit-tests.txt @@ -9,7 +9,6 @@ R23 `parse-doc` advertises pdf alongside docx and xlsx (CONTRACT §1 line 22) R08 `collect feishu --mode browser` (SKILL.md 迁移对照表) reaches the browser route and asks for consent R08 `collect feishu --mode mcp` (SKILL.md 迁移对照表: feishu_mcp_client.py → collect feishu) reaches the MCP route R08 a contract channel that is not shipped is reported as planned (PR \#185 正文: collect/planned-channel), never as an unknown channel -R56 `blind-test prepare` works with the relative --out the handoff (§8) and the blind-test README use R21 identity.json: a handle claimed twice exits 2 with the exact message and writes nothing R21 identity.json with a person that lists no handles is a warning, not a crash R28 a document's labels (根因:/ 时间线:) are not counted as participants in a multi-source person