From 62d2c0f7ddac0116b75a32cc638039f1e2bb4850 Mon Sep 17 00:00:00 2001 From: "Elias S.W." <310511041+eliasruntime@users.noreply.github.com> Date: Mon, 21 Sep 2026 03:43:14 +0800 Subject: [PATCH] Add a reproducible external importer trial with optional evidence recording --- CHANGELOG.md | 1 + README.md | 9 ++ .../driftproof-import-trial-2026-09-21.json | 125 ++++++++++++++++++ .../evidence/macos-replacement-candidate.json | 12 +- .../driftproof-import-counts/README.md | 82 ++++++++++++ .../driftproof-import-counts/prepare.mjs | 31 +++++ .../driftproof-import-counts/probe.mjs | 77 +++++++++++ test/external-probe.test.js | 76 +++++++++++ 8 files changed, 407 insertions(+), 6 deletions(-) create mode 100644 docs/evidence/driftproof-import-trial-2026-09-21.json create mode 100644 examples/external/driftproof-import-counts/README.md create mode 100644 examples/external/driftproof-import-counts/prepare.mjs create mode 100644 examples/external/driftproof-import-counts/probe.mjs create mode 100644 test/external-probe.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f25ace3..37c0941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased — after 0.2.0-rc.1 +- Add a pinned, offline external-source trial for Driftproof's discussed import-count semantics, with a standalone probe and optional non-overwriting Proofline sidecar. Keep maintainer execution separate from independent adoption and the upstream full gate. - Add explicit `init --command-only` scaffolding for CLI and CI maintenance. Start with one command criterion; retain input discovery, evidence states and overwrite refusal. The default scaffold continues to include a visual criterion. ## 0.2.0-rc.1 — prerelease candidate diff --git a/README.md b/README.md index e0a8191..d26eaa3 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,15 @@ The first screen answers three questions without reading a log: 2. Which exact proof line blocks readiness? 3. What environment and receipt produced the evidence? +## Try a reported problem in external code + +The [Driftproof import-count trial](examples/external/driftproof-import-counts/README.md) +uses a maintainer-discussed issue and a pinned public source revision. Run a small +offline probe first, then optionally record its passing control and failing count +checks with Proofline. No model API or dependency installation is needed for these +paths. The maintainer-run reproduction is complete; independent trial and repeat +use remain unobserved. The sidecar is available from this source checkout. + ## The complete loop Use `proofline doctor --json` to inspect configuration omissions and `proofline query --contract target.json --dependencies dependencies.json --gaps` to read compact evidence rows for a named target. Bind observations with the same two options on `run` or `capture`. See [machine query and configuration semantics](docs/MACHINE_QUERIES.md). diff --git a/docs/evidence/driftproof-import-trial-2026-09-21.json b/docs/evidence/driftproof-import-trial-2026-09-21.json new file mode 100644 index 0000000..be88321 --- /dev/null +++ b/docs/evidence/driftproof-import-trial-2026-09-21.json @@ -0,0 +1,125 @@ +{ + "evidenceClass": "maintainer-executed-public-source-reproduction", + "date": "2026-09-20T19:40:10.978Z", + "platform": "win32", + "node": "v24.19.0", + "upstreamRevision": "871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8", + "upstreamTrackedFilesUnchanged": true, + "prooflineVersion": "0.2.0-rc.1", + "direct": { + "schemaVersion": "lab-import-count-probe/1", + "check": "all", + "passed": false, + "source": { + "packageVersion": "0.11.1", + "importerSha256": "7b78b0d1787eab80ee7b111657109014b76c6e70b7aac5ef8283c0cbe28e1297" + }, + "observations": { + "sourceJudgeSampleCountSupplied": false, + "agentSkillsEvalJudgeSamples": 1, + "skillgradeJudgeSamples": 2, + "skillgradeTrialRewards": [ + 1, + 0 + ], + "verificationLevels": [ + "DECLARED", + "DECLARED" + ] + }, + "assertions": [ + { + "id": "imports-stay-declared", + "passed": true + }, + { + "id": "imports-stay-external", + "passed": true + }, + { + "id": "absent-agent-skills-judge-count-stays-unknown", + "passed": false + }, + { + "id": "trial-count-is-not-judge-samples", + "passed": false + }, + { + "id": "observed-trial-rewards-are-preserved", + "passed": true + } + ], + "boundary": "Authored inputs against a selected upstream importer; not a live eval, full upstream gate or agent-benefit trial." + }, + "finalCounts": { + "verified": 1, + "missing": 0, + "stale": 0, + "declared-only": 0, + "failed": 1 + }, + "receipts": [ + { + "receipt": "sha256:41cf4dca7ad8f11d66dad93884a6b068676b91893e3b31bb91f1a7f2579e9fcc", + "kind": "command", + "exitCode": 0 + }, + { + "receipt": "sha256:66ed91d3e5b5bdcee1d5459dc0d28b970027ba3ecf8886ea5e750142a59db441", + "kind": "command", + "exitCode": 1 + } + ], + "commands": [ + { + "label": "installed runtime version", + "expectedExit": 0, + "observedExit": 0, + "subprocessMs": 291.865 + }, + { + "label": "standalone probe", + "expectedExit": 1, + "observedExit": 1, + "subprocessMs": 95.964 + }, + { + "label": "prepare optional sidecar", + "expectedExit": 0, + "observedExit": 0, + "subprocessMs": 72.932 + }, + { + "label": "observe declared", + "expectedExit": 0, + "observedExit": 0, + "subprocessMs": 240.514 + }, + { + "label": "observe counts", + "expectedExit": 1, + "observedExit": 1, + "subprocessMs": 250.53 + }, + { + "label": "query gaps", + "expectedExit": 0, + "observedExit": 0, + "subprocessMs": 142.158 + } + ], + "limits": { + "externalUser": false, + "fullUpstreamGate": false, + "liveProviderRun": false, + "completeTaskCost": null, + "modelUsage": null, + "repeatUse": null, + "netBenefit": "unmeasured" + }, + "publicSource": "https://github.com/driftproofhq/driftproof/tree/871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8", + "discussion": "https://github.com/agentskills/agentskills/discussions/544#discussioncomment-18409986", + "probeSha256": "7ef56c721b9ff6bf45c6e075575fd34a23dd56275d22005b7dd7982153051a70", + "timingMeaning": "One maintainer-run demonstration on a shared Windows host. Subprocess milliseconds exclude research, cloning, probe authoring, setup review, human time and model usage; they are not a whole-task benefit comparison.", + "receiptBoundary": "Receipt hashes associate private local command records with these observations; they do not independently prove authenticity, adequacy or adoption." +} diff --git a/docs/evidence/macos-replacement-candidate.json b/docs/evidence/macos-replacement-candidate.json index cbf89a3..c61c137 100644 --- a/docs/evidence/macos-replacement-candidate.json +++ b/docs/evidence/macos-replacement-candidate.json @@ -3,13 +3,13 @@ "product": "codex-proofline", "gateVersion": 2, "scope": "runtime-gate-tests-v1", - "id": "codex-proofline-macos-rc-ec04ead371789c89", - "contentSha256": "ec04ead371789c89607f4e8ab063342fee7e2ea9fed3ac579479e44db66c573b", - "contentFiles": 53, - "installedVersion": "0.2.0-rc.1+codex.macos-ec04ead371789c89", + "id": "codex-proofline-macos-rc-b220e89283de1b7d", + "contentSha256": "b220e89283de1b7dfb842e4b49f32ac8ce30ad05728ed587762cefe306cbfea1", + "contentFiles": 54, + "installedVersion": "0.2.0-rc.1+codex.macos-b220e89283de1b7d", "status": "awaiting-full-real-mac-rerun", "previousMacEvidenceImported": false, - "generatedOn": "2026-09-12", + "generatedOn": "2026-09-21", "evidenceClass": "source-derived-candidate-identity-not-macos-execution", - "documentationUpdate": "Unreleased command-only init and regression/package coverage. Release v0.2.0-rc.1 stays frozen; physical Mac and fresh-task activation remain pending." + "documentationUpdate": "Unreleased external-source count probe and sidecar regression coverage. Frozen release artifacts are unchanged; current physical-Mac and fresh-task activation evidence remain pending." } diff --git a/examples/external/driftproof-import-counts/README.md b/examples/external/driftproof-import-counts/README.md new file mode 100644 index 0000000..e309b20 --- /dev/null +++ b/examples/external/driftproof-import-counts/README.md @@ -0,0 +1,82 @@ +# A real reported importer question, with an optional Proofline trial + +This small probe follows [the maintainer's public response](https://github.com/agentskills/agentskills/discussions/544#discussioncomment-18409986) +about keeping absent counts unknown and distinguishing generation from grading. +It checks two **count** paths in actual Driftproof source. It does not propose a +replacement receipt schema, implement the clock change, or audit the full project. + +On upstream commit `871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8` (package 0.11.1): + +- `agent-skills-eval` supplies no judge-sample count; the importer writes `1`. +- `skillgrade` supplies two trial rewards and no judge-sample count; the importer + writes `2` into that count. The two observed rewards should still be retained. +- Both results remain `DECLARED` and use the `external` surface. This control passes. + +These are authored minimal inputs run through the real, unmodified importers. +They are not provider runs or independently collected external-user data. The +count assertions implement the rule discussed with the maintainer; the shipped +schema and documentation may still require an older representation. + +## Run the independent probe first + +Use Node.js 22+ (the upstream requirement) and Git. Starting in a directory for +the two checkouts: + +```sh +git clone https://github.com/codex-improvement-lab/codex-proofline.git +git clone https://github.com/driftproofhq/driftproof.git +git -C driftproof checkout 871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8 +node codex-proofline/examples/external/driftproof-import-counts/probe.mjs driftproof +``` + +**Expected exit: 1**, with `agentSkillsEvalJudgeSamples: 1`, +`skillgradeJudgeSamples: 2`, and both count assertions false. The declaration and +reward-preservation checks pass. Exit 2 is a probe/setup error, not this result. +No dependency installation, API key or model invocation is needed for these +library paths. The script does not edit the upstream checkout. The subprocess and +HTTP guards catch accidental provider calls; they are not a security sandbox. + +The probe works independently of Proofline. If that is sufficient for the task, +there is no reason to add the optional record below. + +## Optional: keep a queryable record through the fix + +Continue from the same parent directory. This uses existing Proofline 0.2.0-rc.1 +CLI behavior; `--command-only` or an unreleased runtime feature is unnecessary. + +```sh +node codex-proofline/examples/external/driftproof-import-counts/prepare.mjs driftproof +node codex-proofline/bin/proofline.js run DECLARED/imports --manifest driftproof/proofline-import-trial.json -- node .proofline/lab-import-trial/probe.mjs . --check declared +node codex-proofline/bin/proofline.js run COUNTS/imports --manifest driftproof/proofline-import-trial.json -- node .proofline/lab-import-trial/probe.mjs . --check counts +node codex-proofline/bin/proofline.js query --manifest driftproof/proofline-import-trial.json --gaps +``` + +The second observation exits **1 intentionally**; run the final query afterward. +The summary should show one verified line and one failed line. `verified` here +means our local declaration-preservation **check** passed; it does not promote +the imported Driftproof result from `DECLARED` to `TESTED`. + +The preparer adds `proofline-import-trial.json` and the private +`.proofline/lab-import-trial/` directory in the selected checkout. It refuses +existing sidecars. It does not replace a pre-existing `proofline.json`, modify +tracked upstream files, install a plugin, or contact a service. Keep the generated +ledger private: command receipts may contain local paths. + +After a maintainer changes the import logic or reviewed schema, query again and +rerun relevant checks. Inputs deliberately cover the importer dependencies +conservatively: shared source changes may stale the previously passing control. +An observed failure remains failed until a new observation; it is never silently +upgraded. A passing exit code does not establish that our chosen assertions are a +complete business contract. + +## Feedback that would help + +Run the standalone probe before deciding whether the optional record earns its +cost. If trying it on a real fix, report the source revision, where setup failed +or needed help, which receipt was useful on the next session, and whether you +would choose this flow again. Additional steps and a preference for plain tests +are useful results. Do not upload private ledgers or production eval data. + +Our [maintainer-run observation](../../../docs/evidence/driftproof-import-trial-2026-09-21.json) +is separate from external execution. No independent trial, repeat use, time or +token saving is claimed. diff --git a/examples/external/driftproof-import-counts/prepare.mjs b/examples/external/driftproof-import-counts/prepare.mjs new file mode 100644 index 0000000..b33c9c7 --- /dev/null +++ b/examples/external/driftproof-import-counts/prepare.mjs @@ -0,0 +1,31 @@ +// Explicitly install a small optional sidecar into the caller-selected checkout. +import { access, lstat, mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +if (process.argv.length !== 3) throw new Error("Usage: node prepare.mjs "); +const root = path.resolve(process.argv[2]); +await access(path.join(root, "lib", "importers.js")); +const manifestPath = path.join(root, "proofline-import-trial.json"); +try { + await lstat(manifestPath); + throw new Error("Refusing to overwrite an existing proofline-import-trial.json."); +} catch (error) { if (error.code !== "ENOENT") throw error; } +const evidenceRoot = path.join(root, ".proofline"); +await mkdir(evidenceRoot, { recursive: true }); +if ((await lstat(evidenceRoot)).isSymbolicLink()) throw new Error("Refusing a linked .proofline directory."); +const sidecar = path.join(evidenceRoot, "lab-import-trial"); +await mkdir(sidecar); // Existing owned or user content is never replaced. +await writeFile(path.join(sidecar, "probe.mjs"), await readFile(new URL("probe.mjs", import.meta.url)), { flag: "wx" }); +const inputs = ["lib", "config", "config.js", "package.json", ".proofline/lab-import-trial/probe.mjs"]; +const manifest = { + version: 1, project: "Driftproof importer count review", completionPolicy: "warn", + ledger: ".proofline/lab-import-trial/evidence.jsonl", + criteria: [ + { id: "DECLARED", statement: "Imported results remain declarations from an external harness.", + proof: [{ id: "imports", kind: "command", label: "Imported evidence level", inputs, expect: { exitCode: 0 } }] }, + { id: "COUNTS", statement: "Absent judge-sample counts remain unknown; observed trial rewards are preserved separately.", + proof: [{ id: "imports", kind: "command", label: "Missing count and trial-count semantics", inputs, expect: { exitCode: 0 } }] } + ] +}; +await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" }); +process.stdout.write("Prepared proofline-import-trial.json and .proofline/lab-import-trial/probe.mjs. Review the declared criteria and input scopes before recording checks.\n"); diff --git a/examples/external/driftproof-import-counts/probe.mjs b/examples/external/driftproof-import-counts/probe.mjs new file mode 100644 index 0000000..0274093 --- /dev/null +++ b/examples/external/driftproof-import-counts/probe.mjs @@ -0,0 +1,77 @@ +// Original Lab probe for a publicly reported importer contract; no model calls. +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import path from "node:path"; + +const require = createRequire(import.meta.url); +const args = process.argv.slice(2); +if (args.length === 1 && args[0] === "--help") { + process.stdout.write("Usage: node probe.mjs [--check declared|counts|all]\nNo model, network, installation or target-file writes. Exit 0 pass, 1 unmet contract, 2 probe error.\n"); +} else { + try { + if (![1, 3].includes(args.length) || (args.length === 3 && args[1] !== "--check")) { + throw new Error("Supply a checkout and optional --check declared|counts|all."); + } + const check = args[2] ?? "all"; + if (!["declared", "counts", "all"].includes(check)) throw new Error("Unknown check."); + const root = path.resolve(args[0]); + const importerPath = path.join(root, "lib", "importers.js"); + const importerBytes = readFileSync(importerPath); + const packageInfo = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")); + + // These guards catch accidental provider calls in this reviewed code path. + // They are not a sandbox for arbitrary or hostile JavaScript. + const blocked = () => { throw new Error("Unexpected network or subprocess in the offline importer probe."); }; + const child = require("node:child_process"); + for (const key of ["spawn", "spawnSync", "exec", "execSync", "execFile", "execFileSync", "fork"]) child[key] = blocked; + globalThis.fetch = blocked; + for (const protocol of ["node:http", "node:https"]) { + const transport = require(protocol); + transport.request = transport.get = blocked; + } + const { importAgentSkillsEval, importSkillgrade } = require(importerPath); + const options = { importedAt: "2026-09-21T00:00:00.000Z" }; + const agent = importAgentSkillsEval({ + skill_name: "lab-public-count-probe", target: "gpt-4o-mini", judge: "gpt-4o-mini", + evals: [{ id: "one", with_skill: { pass: true } }] + }, options); + const trials = importSkillgrade({ + skill: "lab-public-count-probe", agent: "codex", + tasks: [{ name: "one", trials: [{ reward: 1 }, { reward: 0 }] }] + }, options); + const records = [agent, trials]; + const assertions = []; + function verify(id, task) { + try { task(); assertions.push({ id, passed: true }); } + catch { assertions.push({ id, passed: false }); } + } + if (["declared", "all"].includes(check)) { + verify("imports-stay-declared", () => assert.ok(records.every(record => record.verification_level === "DECLARED"))); + verify("imports-stay-external", () => assert.ok(records.every(record => record.run.surface === "external"))); + } + if (["counts", "all"].includes(check)) { + verify("absent-agent-skills-judge-count-stays-unknown", () => assert.equal(agent.run.judge.samples ?? null, null)); + verify("trial-count-is-not-judge-samples", () => assert.equal(trials.run.judge.samples ?? null, null)); + verify("observed-trial-rewards-are-preserved", () => assert.deepEqual(trials.results.cases[0].samples, [1, 0])); + } + const passed = assertions.every(item => item.passed); + process.stdout.write(`${JSON.stringify({ + schemaVersion: "lab-import-count-probe/1", check, passed, + source: { packageVersion: packageInfo.version, + importerSha256: createHash("sha256").update(importerBytes).digest("hex") }, + observations: { sourceJudgeSampleCountSupplied: false, + agentSkillsEvalJudgeSamples: agent.run.judge.samples ?? null, + skillgradeJudgeSamples: trials.run.judge.samples ?? null, + skillgradeTrialRewards: trials.results.cases[0].samples, + verificationLevels: records.map(record => record.verification_level) }, + assertions, + boundary: "Authored inputs against a selected upstream importer; not a live eval, full upstream gate or agent-benefit trial." + }, null, 2)}\n`); + process.exitCode = passed ? 0 : 1; + } catch (error) { + process.stdout.write(`${JSON.stringify({ schemaVersion: "lab-import-count-probe/1", error: "probe-error", message: error.message })}\n`); + process.exitCode = 2; + } +} diff --git a/test/external-probe.test.js b/test/external-probe.test.js new file mode 100644 index 0000000..2c11a89 --- /dev/null +++ b/test/external-probe.test.js @@ -0,0 +1,76 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; +import { createTestDirectory, removeTestDirectory, writeJson } from "../test-support/helpers.js"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const example = path.join(root, "examples", "external", "driftproof-import-counts"); +const probe = path.join(example, "probe.mjs"); +const prepare = path.join(example, "prepare.mjs"); +const bin = path.join(root, "bin", "proofline.js"); +function invoke(script, args, cwd) { + const result = spawnSync(process.execPath, [script, ...args], { cwd, encoding: "utf8", timeout: 30000 }); + assert.ifError(result.error); + return result; +} +async function fixture(directory, { counts = [1, 2], rewards = [1, 0] } = {}) { + await mkdir(path.join(directory, "lib"), { recursive: true }); + await mkdir(path.join(directory, "config"), { recursive: true }); + await writeJson(path.join(directory, "package.json"), { name: "synthetic-importer-control", version: "0.0.0" }); + await writeFile(path.join(directory, "config.js"), "module.exports = {};\n"); + await writeFile(path.join(directory, "lib", "importers.js"), ` +const receipt = samples => ({ verification_level: 'DECLARED', run: { surface: 'external', judge: { samples } }, results: { cases: [{ samples: ${JSON.stringify(rewards)} }] } }); +module.exports = { importAgentSkillsEval: () => receipt(${JSON.stringify(counts[0])}), importSkillgrade: () => receipt(${JSON.stringify(counts[1])}) }; +`); +} + +test("external probe distinguishes missing counts, declaration controls and preserved rewards", async (t) => { + const directory = await createTestDirectory("external-probe"); + t.after(() => removeTestDirectory(directory)); + await fixture(directory); + let result = invoke(probe, [directory], directory); + assert.equal(result.status, 1, result.stderr); + assert.deepEqual(JSON.parse(result.stdout).assertions.filter(row => !row.passed).map(row => row.id), + ["absent-agent-skills-judge-count-stays-unknown", "trial-count-is-not-judge-samples"]); + assert.equal(invoke(probe, [directory, "--check", "declared"], directory).status, 0); + await fixture(directory, { counts: [null, null] }); + assert.equal(invoke(probe, [directory], directory).status, 0); + await fixture(directory, { counts: [null, null], rewards: [1] }); + result = invoke(probe, [directory], directory); + assert.equal(result.status, 1); + assert.deepEqual(JSON.parse(result.stdout).assertions.filter(row => !row.passed).map(row => row.id), + ["observed-trial-rewards-are-preserved"]); + assert.equal(invoke(probe, [directory, "--check", "unknown"], directory).status, 2); +}); + +test("optional external sidecar preserves existing files and tracks actual source changes", async (t) => { + const directory = await createTestDirectory("external-sidecar with spaces"); + t.after(() => removeTestDirectory(directory)); + await fixture(directory); + await writeFile(path.join(directory, "proofline.json"), "existing user content\n"); + const prepared = invoke(prepare, [directory], directory); + assert.equal(prepared.status, 0, prepared.stderr); + const selected = path.join(directory, "proofline-import-trial.json"); + const before = await readFile(selected); + assert.notEqual(invoke(prepare, [directory], directory).status, 0); + assert.deepEqual(await readFile(selected), before); + assert.equal(await readFile(path.join(directory, "proofline.json"), "utf8"), "existing user content\n"); + const observe = check => invoke(bin, ["run", `${check === "declared" ? "DECLARED" : "COUNTS"}/imports`, + "--manifest", selected, "--", process.execPath, ".proofline/lab-import-trial/probe.mjs", ".", "--check", check], directory); + const query = () => { + const result = invoke(bin, ["query", "--manifest", selected], directory); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); + }; + assert.equal(observe("declared").status, 0); + assert.equal(observe("counts").status, 1); + assert.deepEqual(query().evidence.map(row => row.status), ["verified", "failed"]); + await fixture(directory, { counts: [null, null] }); + assert.deepEqual(query().evidence.map(row => row.status), ["stale", "failed"]); + assert.equal(observe("declared").status, 0); + assert.equal(observe("counts").status, 0); + assert.equal(query().summary.counts.verified, 2); +});