Add an offline external-source trial for importer count evidence

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased — after 0.2.0-rc.1
 
+- Add a pinned, offline external-source trial for Driftproof's discussed import-count semantics, with a standalone probe and optional non-overwriting Proofline sidecar. Keep maintainer execution separate from independent adoption and the upstream full gate.
 - Add explicit `init --command-only` scaffolding for CLI and CI maintenance. Start with one command criterion; retain input discovery, evidence states and overwrite refusal. The default scaffold continues to include a visual criterion.
 
 ## 0.2.0-rc.1 — prerelease candidate

README.md

@@ -55,6 +55,15 @@ The first screen answers three questions without reading a log:
 2. Which exact proof line blocks readiness?
 3. What environment and receipt produced the evidence?
 
+## Try a reported problem in external code
+
+The [Driftproof import-count trial](examples/external/driftproof-import-counts/README.md)
+uses a maintainer-discussed issue and a pinned public source revision. Run a small
+offline probe first, then optionally record its passing control and failing count
+checks with Proofline. No model API or dependency installation is needed for these
+paths. The maintainer-run reproduction is complete; independent trial and repeat
+use remain unobserved. The sidecar is available from this source checkout.
+
 ## The complete loop
 
 Use `proofline doctor --json` to inspect configuration omissions and `proofline query --contract target.json --dependencies dependencies.json --gaps` to read compact evidence rows for a named target. Bind observations with the same two options on `run` or `capture`. See [machine query and configuration semantics](docs/MACHINE_QUERIES.md).

docs/evidence/driftproof-import-trial-2026-09-21.json

@@ -0,0 +1,125 @@
+{
+  "evidenceClass": "maintainer-executed-public-source-reproduction",
+  "date": "2026-09-20T19:40:10.978Z",
+  "platform": "win32",
+  "node": "v24.19.0",
+  "upstreamRevision": "871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8",
+  "upstreamTrackedFilesUnchanged": true,
+  "prooflineVersion": "0.2.0-rc.1",
+  "direct": {
+    "schemaVersion": "lab-import-count-probe/1",
+    "check": "all",
+    "passed": false,
+    "source": {
+      "packageVersion": "0.11.1",
+      "importerSha256": "7b78b0d1787eab80ee7b111657109014b76c6e70b7aac5ef8283c0cbe28e1297"
+    },
+    "observations": {
+      "sourceJudgeSampleCountSupplied": false,
+      "agentSkillsEvalJudgeSamples": 1,
+      "skillgradeJudgeSamples": 2,
+      "skillgradeTrialRewards": [
+        1,
+        0
+      ],
+      "verificationLevels": [
+        "DECLARED",
+        "DECLARED"
+      ]
+    },
+    "assertions": [
+      {
+        "id": "imports-stay-declared",
+        "passed": true
+      },
+      {
+        "id": "imports-stay-external",
+        "passed": true
+      },
+      {
+        "id": "absent-agent-skills-judge-count-stays-unknown",
+        "passed": false
+      },
+      {
+        "id": "trial-count-is-not-judge-samples",
+        "passed": false
+      },
+      {
+        "id": "observed-trial-rewards-are-preserved",
+        "passed": true
+      }
+    ],
+    "boundary": "Authored inputs against a selected upstream importer; not a live eval, full upstream gate or agent-benefit trial."
+  },
+  "finalCounts": {
+    "verified": 1,
+    "missing": 0,
+    "stale": 0,
+    "declared-only": 0,
+    "failed": 1
+  },
+  "receipts": [
+    {
+      "receipt": "sha256:41cf4dca7ad8f11d66dad93884a6b068676b91893e3b31bb91f1a7f2579e9fcc",
+      "kind": "command",
+      "exitCode": 0
+    },
+    {
+      "receipt": "sha256:66ed91d3e5b5bdcee1d5459dc0d28b970027ba3ecf8886ea5e750142a59db441",
+      "kind": "command",
+      "exitCode": 1
+    }
+  ],
+  "commands": [
+    {
+      "label": "installed runtime version",
+      "expectedExit": 0,
+      "observedExit": 0,
+      "subprocessMs": 291.865
+    },
+    {
+      "label": "standalone probe",
+      "expectedExit": 1,
+      "observedExit": 1,
+      "subprocessMs": 95.964
+    },
+    {
+      "label": "prepare optional sidecar",
+      "expectedExit": 0,
+      "observedExit": 0,
+      "subprocessMs": 72.932
+    },
+    {
+      "label": "observe declared",
+      "expectedExit": 0,
+      "observedExit": 0,
+      "subprocessMs": 240.514
+    },
+    {
+      "label": "observe counts",
+      "expectedExit": 1,
+      "observedExit": 1,
+      "subprocessMs": 250.53
+    },
+    {
+      "label": "query gaps",
+      "expectedExit": 0,
+      "observedExit": 0,
+      "subprocessMs": 142.158
+    }
+  ],
+  "limits": {
+    "externalUser": false,
+    "fullUpstreamGate": false,
+    "liveProviderRun": false,
+    "completeTaskCost": null,
+    "modelUsage": null,
+    "repeatUse": null,
+    "netBenefit": "unmeasured"
+  },
+  "publicSource": "https://github.com/driftproofhq/driftproof/tree/871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8",
+  "discussion": "https://github.com/agentskills/agentskills/discussions/544#discussioncomment-18409986",
+  "probeSha256": "7ef56c721b9ff6bf45c6e075575fd34a23dd56275d22005b7dd7982153051a70",
+  "timingMeaning": "One maintainer-run demonstration on a shared Windows host. Subprocess milliseconds exclude research, cloning, probe authoring, setup review, human time and model usage; they are not a whole-task benefit comparison.",
+  "receiptBoundary": "Receipt hashes associate private local command records with these observations; they do not independently prove authenticity, adequacy or adoption."
+}

docs/evidence/macos-replacement-candidate.json

@@ -3,13 +3,13 @@
   "product": "codex-proofline",
   "gateVersion": 2,
   "scope": "runtime-gate-tests-v1",
-  "id": "codex-proofline-macos-rc-ec04ead371789c89",
-  "contentSha256": "ec04ead371789c89607f4e8ab063342fee7e2ea9fed3ac579479e44db66c573b",
-  "contentFiles": 53,
-  "installedVersion": "0.2.0-rc.1+codex.macos-ec04ead371789c89",
+  "id": "codex-proofline-macos-rc-b220e89283de1b7d",
+  "contentSha256": "b220e89283de1b7dfb842e4b49f32ac8ce30ad05728ed587762cefe306cbfea1",
+  "contentFiles": 54,
+  "installedVersion": "0.2.0-rc.1+codex.macos-b220e89283de1b7d",
   "status": "awaiting-full-real-mac-rerun",
   "previousMacEvidenceImported": false,
-  "generatedOn": "2026-09-12",
+  "generatedOn": "2026-09-21",
   "evidenceClass": "source-derived-candidate-identity-not-macos-execution",
-  "documentationUpdate": "Unreleased command-only init and regression/package coverage. Release v0.2.0-rc.1 stays frozen; physical Mac and fresh-task activation remain pending."
+  "documentationUpdate": "Unreleased external-source count probe and sidecar regression coverage. Frozen release artifacts are unchanged; current physical-Mac and fresh-task activation evidence remain pending."
 }

examples/external/driftproof-import-counts/README.md

@@ -0,0 +1,82 @@
+# A real reported importer question, with an optional Proofline trial
+
+This small probe follows [the maintainer's public response](https://github.com/agentskills/agentskills/discussions/544#discussioncomment-18409986)
+about keeping absent counts unknown and distinguishing generation from grading.
+It checks two **count** paths in actual Driftproof source. It does not propose a
+replacement receipt schema, implement the clock change, or audit the full project.
+
+On upstream commit `871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8` (package 0.11.1):
+
+- `agent-skills-eval` supplies no judge-sample count; the importer writes `1`.
+- `skillgrade` supplies two trial rewards and no judge-sample count; the importer
+  writes `2` into that count. The two observed rewards should still be retained.
+- Both results remain `DECLARED` and use the `external` surface. This control passes.
+
+These are authored minimal inputs run through the real, unmodified importers.
+They are not provider runs or independently collected external-user data. The
+count assertions implement the rule discussed with the maintainer; the shipped
+schema and documentation may still require an older representation.
+
+## Run the independent probe first
+
+Use Node.js 22+ (the upstream requirement) and Git. Starting in a directory for
+the two checkouts:
+
+```sh
+git clone https://github.com/codex-improvement-lab/codex-proofline.git
+git clone https://github.com/driftproofhq/driftproof.git
+git -C driftproof checkout 871c8c6a82b57934e42a8e4f4b1bcec5f49e08f8
+node codex-proofline/examples/external/driftproof-import-counts/probe.mjs driftproof
+```
+
+**Expected exit: 1**, with `agentSkillsEvalJudgeSamples: 1`,
+`skillgradeJudgeSamples: 2`, and both count assertions false. The declaration and
+reward-preservation checks pass. Exit 2 is a probe/setup error, not this result.
+No dependency installation, API key or model invocation is needed for these
+library paths. The script does not edit the upstream checkout. The subprocess and
+HTTP guards catch accidental provider calls; they are not a security sandbox.
+
+The probe works independently of Proofline. If that is sufficient for the task,
+there is no reason to add the optional record below.
+
+## Optional: keep a queryable record through the fix
+
+Continue from the same parent directory. This uses existing Proofline 0.2.0-rc.1
+CLI behavior; `--command-only` or an unreleased runtime feature is unnecessary.
+
+```sh
+node codex-proofline/examples/external/driftproof-import-counts/prepare.mjs driftproof
+node codex-proofline/bin/proofline.js run DECLARED/imports --manifest driftproof/proofline-import-trial.json -- node .proofline/lab-import-trial/probe.mjs . --check declared
+node codex-proofline/bin/proofline.js run COUNTS/imports --manifest driftproof/proofline-import-trial.json -- node .proofline/lab-import-trial/probe.mjs . --check counts
+node codex-proofline/bin/proofline.js query --manifest driftproof/proofline-import-trial.json --gaps
+```
+
+The second observation exits **1 intentionally**; run the final query afterward.
+The summary should show one verified line and one failed line. `verified` here
+means our local declaration-preservation **check** passed; it does not promote
+the imported Driftproof result from `DECLARED` to `TESTED`.
+
+The preparer adds `proofline-import-trial.json` and the private
+`.proofline/lab-import-trial/` directory in the selected checkout. It refuses
+existing sidecars. It does not replace a pre-existing `proofline.json`, modify
+tracked upstream files, install a plugin, or contact a service. Keep the generated
+ledger private: command receipts may contain local paths.
+
+After a maintainer changes the import logic or reviewed schema, query again and
+rerun relevant checks. Inputs deliberately cover the importer dependencies
+conservatively: shared source changes may stale the previously passing control.
+An observed failure remains failed until a new observation; it is never silently
+upgraded. A passing exit code does not establish that our chosen assertions are a
+complete business contract.
+
+## Feedback that would help
+
+Run the standalone probe before deciding whether the optional record earns its
+cost. If trying it on a real fix, report the source revision, where setup failed
+or needed help, which receipt was useful on the next session, and whether you
+would choose this flow again. Additional steps and a preference for plain tests
+are useful results. Do not upload private ledgers or production eval data.
+
+Our [maintainer-run observation](../../../docs/evidence/driftproof-import-trial-2026-09-21.json)
+is separate from external execution. No independent trial, repeat use, time or
+token saving is claimed.

examples/external/driftproof-import-counts/prepare.mjs

@@ -0,0 +1,31 @@
+// Explicitly install a small optional sidecar into the caller-selected checkout.
+import { access, lstat, mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+
+if (process.argv.length !== 3) throw new Error("Usage: node prepare.mjs <driftproof-checkout>");
+const root = path.resolve(process.argv[2]);
+await access(path.join(root, "lib", "importers.js"));
+const manifestPath = path.join(root, "proofline-import-trial.json");
+try {
+  await lstat(manifestPath);
+  throw new Error("Refusing to overwrite an existing proofline-import-trial.json.");
+} catch (error) { if (error.code !== "ENOENT") throw error; }
+const evidenceRoot = path.join(root, ".proofline");
+await mkdir(evidenceRoot, { recursive: true });
+if ((await lstat(evidenceRoot)).isSymbolicLink()) throw new Error("Refusing a linked .proofline directory.");
+const sidecar = path.join(evidenceRoot, "lab-import-trial");
+await mkdir(sidecar); // Existing owned or user content is never replaced.
+await writeFile(path.join(sidecar, "probe.mjs"), await readFile(new URL("probe.mjs", import.meta.url)), { flag: "wx" });
+const inputs = ["lib", "config", "config.js", "package.json", ".proofline/lab-import-trial/probe.mjs"];
+const manifest = {
+  version: 1, project: "Driftproof importer count review", completionPolicy: "warn",
+  ledger: ".proofline/lab-import-trial/evidence.jsonl",
+  criteria: [
+    { id: "DECLARED", statement: "Imported results remain declarations from an external harness.",
+      proof: [{ id: "imports", kind: "command", label: "Imported evidence level", inputs, expect: { exitCode: 0 } }] },
+    { id: "COUNTS", statement: "Absent judge-sample counts remain unknown; observed trial rewards are preserved separately.",
+      proof: [{ id: "imports", kind: "command", label: "Missing count and trial-count semantics", inputs, expect: { exitCode: 0 } }] }
+  ]
+};
+await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx" });
+process.stdout.write("Prepared proofline-import-trial.json and .proofline/lab-import-trial/probe.mjs. Review the declared criteria and input scopes before recording checks.\n");

examples/external/driftproof-import-counts/probe.mjs

@@ -0,0 +1,77 @@
+// Original Lab probe for a publicly reported importer contract; no model calls.
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import path from "node:path";
+
+const require = createRequire(import.meta.url);
+const args = process.argv.slice(2);
+if (args.length === 1 && args[0] === "--help") {
+  process.stdout.write("Usage: node probe.mjs <driftproof-checkout> [--check declared|counts|all]\nNo model, network, installation or target-file writes. Exit 0 pass, 1 unmet contract, 2 probe error.\n");
+} else {
+  try {
+    if (![1, 3].includes(args.length) || (args.length === 3 && args[1] !== "--check")) {
+      throw new Error("Supply a checkout and optional --check declared|counts|all.");
+    }
+    const check = args[2] ?? "all";
+    if (!["declared", "counts", "all"].includes(check)) throw new Error("Unknown check.");
+    const root = path.resolve(args[0]);
+    const importerPath = path.join(root, "lib", "importers.js");
+    const importerBytes = readFileSync(importerPath);
+    const packageInfo = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
+
+    // These guards catch accidental provider calls in this reviewed code path.
+    // They are not a sandbox for arbitrary or hostile JavaScript.
+    const blocked = () => { throw new Error("Unexpected network or subprocess in the offline importer probe."); };
+    const child = require("node:child_process");
+    for (const key of ["spawn", "spawnSync", "exec", "execSync", "execFile", "execFileSync", "fork"]) child[key] = blocked;
+    globalThis.fetch = blocked;
+    for (const protocol of ["node:http", "node:https"]) {
+      const transport = require(protocol);
+      transport.request = transport.get = blocked;
+    }
+    const { importAgentSkillsEval, importSkillgrade } = require(importerPath);
+    const options = { importedAt: "2026-09-21T00:00:00.000Z" };
+    const agent = importAgentSkillsEval({
+      skill_name: "lab-public-count-probe", target: "gpt-4o-mini", judge: "gpt-4o-mini",
+      evals: [{ id: "one", with_skill: { pass: true } }]
+    }, options);
+    const trials = importSkillgrade({
+      skill: "lab-public-count-probe", agent: "codex",
+      tasks: [{ name: "one", trials: [{ reward: 1 }, { reward: 0 }] }]
+    }, options);
+    const records = [agent, trials];
+    const assertions = [];
+    function verify(id, task) {
+      try { task(); assertions.push({ id, passed: true }); }
+      catch { assertions.push({ id, passed: false }); }
+    }
+    if (["declared", "all"].includes(check)) {
+      verify("imports-stay-declared", () => assert.ok(records.every(record => record.verification_level === "DECLARED")));
+      verify("imports-stay-external", () => assert.ok(records.every(record => record.run.surface === "external")));
+    }
+    if (["counts", "all"].includes(check)) {
+      verify("absent-agent-skills-judge-count-stays-unknown", () => assert.equal(agent.run.judge.samples ?? null, null));
+      verify("trial-count-is-not-judge-samples", () => assert.equal(trials.run.judge.samples ?? null, null));
+      verify("observed-trial-rewards-are-preserved", () => assert.deepEqual(trials.results.cases[0].samples, [1, 0]));
+    }
+    const passed = assertions.every(item => item.passed);
+    process.stdout.write(`${JSON.stringify({
+      schemaVersion: "lab-import-count-probe/1", check, passed,
+      source: { packageVersion: packageInfo.version,
+        importerSha256: createHash("sha256").update(importerBytes).digest("hex") },
+      observations: { sourceJudgeSampleCountSupplied: false,
+        agentSkillsEvalJudgeSamples: agent.run.judge.samples ?? null,
+        skillgradeJudgeSamples: trials.run.judge.samples ?? null,
+        skillgradeTrialRewards: trials.results.cases[0].samples,
+        verificationLevels: records.map(record => record.verification_level) },
+      assertions,
+      boundary: "Authored inputs against a selected upstream importer; not a live eval, full upstream gate or agent-benefit trial."
+    }, null, 2)}\n`);
+    process.exitCode = passed ? 0 : 1;
+  } catch (error) {
+    process.stdout.write(`${JSON.stringify({ schemaVersion: "lab-import-count-probe/1", error: "probe-error", message: error.message })}\n`);
+    process.exitCode = 2;
+  }
+}