Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion docs/v2/BLIND-TEST-RUNBOOK.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,22 @@

三种角色都必须是**模型或人**,不是本脚本。同一模型不得同时充当蒸馏器与裁判。

蒸馏者、裁判、核对者必须是三个不同模型,且都不是 deepseek-flash。本程序固定:

| 角色 | 模型 | 看到什么 |
|---|---|---|
| 蒸馏者 | 不得是下面两行,也不得是 deepseek-flash | A 半段(实验组读 `deriver-input.md`) |
| 裁判 | `claude-sonnet-5-thinking-medium` | 只看 `profile.html` 与 `judge-prompt.md` |
| 核对者 | `gpt-5.6-sol-medium` | 只读 `split/B.srt` 与裁判的 10 条 |

效果数字不是代码。第一组数字由后续 swarm 按上表产出,写到:

`/cursor/stores/bc-0e6a1836-4b6d-4ce2-8177-5061476fde52/media/distilly-atomic-pr/pr-8-blind/`

本拍不跑三模型盲测 Lane 7 与 Lane 8。机械层(切分、相对路径 prepare、finalize、control、空表 score)由本 PR 自证。

`--out`、`--a`、`--view`、`--scores` 可以是相对路径。脚本对 `process.cwd()` 做一次 `path.resolve`,传给 `harvest` / `retrospect` / `view` 的 `--base-dir` 与文件路径都是绝对路径。HANDOFF §8 的 `prepare --a split/A.srt --out run` 必须退出 0。

## 1. 切分(检查者一个人做)

```bash
Expand Down Expand Up @@ -110,4 +126,5 @@ node scripts/blind-test.mjs score --scores /tmp/blind-run/scores.json --out /tmp
报告里必须写明这一点。
- 身份盲:`prepare --person <slug>` 决定页面上的 `slug`;要让裁判连人名也看不到,
用一个中性 slug(如 `blind-sample-01`)跑,目录名与页面都不含真名。
- 裁判与蒸馏器若用同一模型,须在结论里注明;同模型自评会让命中率偏高。
- 裁判、核对者、蒸馏者必须是三个不同模型,都不是 deepseek-flash。裁判固定
`claude-sonnet-5-thinking-medium`,核对者固定 `gpt-5.6-sol-medium`。同模型自评不作数。
59 changes: 40 additions & 19 deletions scripts/blind-test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,11 @@ function arg(name, fallback) {
}
const flag = (name) => process.argv.includes(`--${name}`);

/** Resolve a user path once so spawn cwd cannot reinterpret --out/--a/--base-dir. */
function abs(value) {
return resolve(process.cwd(), value);
}

const sha256 = (text) => createHash("sha256").update(text, "utf8").digest("hex");
const round = (value) => (Number.isInteger(value) ? String(value) : value.toFixed(2).replace(/0$/, ""));

Expand Down Expand Up @@ -304,19 +309,21 @@ function writeSheet(out) {
}

function prepare() {
const a = arg("a");
const aArg = arg("a");
const person = arg("person");
const out = arg("out");
if (!a || !person || !out) {
const outArg = arg("out");
if (!aArg || !person || !outArg) {
console.error("usage: node scripts/blind-test.mjs prepare --a <A.srt> --person <slug> --out <dir> [--skeleton-only]");
process.exit(2);
}
const a = abs(aArg);
const out = abs(outArg);
const work = join(out, "work");
rmSync(work, { recursive: true, force: true });
mkdirSync(join(work, "corpus"), { recursive: true });
cpSync(a, join(work, "corpus", basename(a)));
distilly(["harvest", join(work, "corpus"), "--person", person, "--base-dir", work, "--json"], work);
const retro = distilly(["retrospect", "--person", person, "--json"], work);
const retro = distilly(["retrospect", "--person", person, "--base-dir", work, "--json"], work);

const personDir = join(work, "skills", "colleague", person);
const derivation = loadDerivation(personDir);
Expand Down Expand Up @@ -413,10 +420,18 @@ function countRenderedAnchors(html) {
}

function finalizeView({ out, viewPath, slug, personDir, receipt, source }) {
const checked = distilly(["view", "check", "--file", viewPath, "--json"], personDir, { tolerate: true });
const rendered = distilly(["view", "render", "--file", viewPath, "--out", join(out, "profile.html"), "--json"], personDir, {
tolerate: true,
});
const work = join(out, "work");
const htmlPath = join(out, "profile.html");
const checked = distilly(
["view", "check", "--file", viewPath, "--person", slug, "--base-dir", work, "--json"],
personDir,
{ tolerate: true },
);
const rendered = distilly(
["view", "render", "--file", viewPath, "--out", htmlPath, "--person", slug, "--base-dir", work, "--json"],
personDir,
{ tolerate: true },
);
const diagnostics = [];
for (const entry of checked.receipt?.diagnostics ?? []) diagnostics.push(`${entry.code}: ${entry.message}`);
const html = readFileSync(join(out, "profile.html"), "utf8");
Expand All @@ -438,17 +453,19 @@ function finalizeView({ out, viewPath, slug, personDir, receipt, source }) {
}

function finalize() {
const out = arg("out");
const view = arg("view");
if (!out || !view) {
const outArg = arg("out");
const viewArg = arg("view");
if (!outArg || !viewArg) {
console.error("usage: node scripts/blind-test.mjs finalize --out <dir> --view <view.json> [--strict]");
process.exit(2);
}
const out = abs(outArg);
const view = abs(viewArg);
const receiptPath = join(out, "receipt.json");
const receipt = JSON.parse(readFileSync(receiptPath, "utf8"));
const person = receipt.view?.slug ?? JSON.parse(readFileSync(view, "utf8")).meta?.slug;
const personDir = join(out, "work", "skills", "colleague", person);
const result = finalizeView({ out, viewPath: resolve(view), slug: person, personDir, receipt, source: "authored" });
const result = finalizeView({ out, viewPath: view, slug: person, personDir, receipt, source: "authored" });
Object.assign(receipt, result, { view_source: "authored", next_steps: receipt.next_steps });
writeFileSync(receiptPath, `${JSON.stringify(receipt, null, 2)}\n`, "utf8");
console.log(`finalized ${out}/profile.html (${result.html.bytes} bytes, check ok: ${result.check.ok}, external links: ${result.external_links})`);
Expand All @@ -469,12 +486,14 @@ const CONTROL_PROMPT = (source) => `# 对照任务(裸 prompt,无证据层
`;

function control() {
const a = arg("a");
const out = arg("out");
if (!a || !out) {
const aArg = arg("a");
const outArg = arg("out");
if (!aArg || !outArg) {
console.error("usage: node scripts/blind-test.mjs control --a <A.srt> --out <dir>");
process.exit(2);
}
const a = abs(aArg);
const out = abs(outArg);
mkdirSync(out, { recursive: true });
writeFileSync(join(out, "control-prompt.md"), CONTROL_PROMPT(basename(a)) + readFileSync(a, "utf8"), "utf8");
writeSheet(out);
Expand Down Expand Up @@ -525,11 +544,12 @@ export function verdictFor(metrics) {
}

function score() {
const scoresPath = arg("scores");
if (!scoresPath) {
const scoresArg = arg("scores");
if (!scoresArg) {
console.error("usage: node scripts/blind-test.mjs score --scores <scores.json> [--out report.md]");
process.exit(2);
}
const scoresPath = abs(scoresArg);
const scores = JSON.parse(readFileSync(scoresPath, "utf8"));
const arms = ["evidence", "control"].filter((arm) => Array.isArray(scores[arm]));
if (arms.length === 0) throw new Error("scores.json needs an `evidence` and/or `control` array");
Expand Down Expand Up @@ -560,8 +580,9 @@ function score() {
}

const report = lines.join("\n") + "\n";
const out = arg("out");
if (out) {
const outArg = arg("out");
if (outArg) {
const out = abs(outArg);
writeFileSync(out, report, "utf8");
console.log(`report → ${out}`);
}
Expand Down
29 changes: 28 additions & 1 deletion tests/blind-test.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { existsSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { after, describe, it } from "node:test";
Expand Down Expand Up @@ -190,4 +190,31 @@ describe("blind-test prepare", () => {
assert.equal(sheet.control.length, 10);
assert.equal(sheet.evidence.length, 10, "both arms share one sheet so they cannot diverge");
});

it("accepts the relative --out and --a the handoff (§8) uses", () => {
const dir = scratch("blind-rel");
const splitResult = run("split-corpus.mjs", ["--in", FIXTURE, "--out", "split", "--ratio", "0.7"], { cwd: dir });
assert.equal(splitResult.status, 0, splitResult.stderr || splitResult.stdout);

const prepare = run(
"blind-test.mjs",
["prepare", "--a", "split/A.srt", "--person", "lin-gong", "--out", "run"],
{ cwd: dir },
);
assert.equal(prepare.status, 0, prepare.stderr || prepare.stdout);
assert.equal(existsSync(join(dir, "run", "profile.html")), true);

const finalize = run("blind-test.mjs", ["finalize", "--out", "run", "--view", "run/view.baseline.json"], { cwd: dir });
assert.equal(finalize.status, 0, finalize.stderr || finalize.stdout);

const control = run("blind-test.mjs", ["control", "--a", "split/A.srt", "--out", "control"], { cwd: dir });
assert.equal(control.status, 0, control.stderr || control.stdout);
assert.equal(existsSync(join(dir, "control", "control-prompt.md")), true);

const score = run("blind-test.mjs", ["score", "--scores", "run/scores.template.json", "--out", "report.md"], {
cwd: dir,
});
assert.equal(score.status, 0, score.stderr || score.stdout);
assert.match(readFileSync(join(dir, "report.md"), "utf8"), /evidence → FAIL/);
});
});
1 change: 0 additions & 1 deletion tests/gates/baseline/audit-tests.txt
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ R23 `parse-doc` advertises pdf alongside docx and xlsx (CONTRACT §1 line 22)
R08 `collect feishu --mode browser` (SKILL.md 迁移对照表) reaches the browser route and asks for consent
R08 `collect feishu --mode mcp` (SKILL.md 迁移对照表: feishu_mcp_client.py → collect feishu) reaches the MCP route
R08 a contract channel that is not shipped is reported as planned (PR \#185 正文: collect/planned-channel), never as an unknown channel
R56 `blind-test prepare` works with the relative --out the handoff (§8) and the blind-test README use
R21 identity.json: a handle claimed twice exits 2 with the exact message and writes nothing
R21 identity.json with a person that lists no handles is a warning, not a crash
R28 a document's labels (根因:/ 时间线:) are not counted as participants in a multi-source person
Expand Down
Loading