feat(review): drop registry-refutable findings from review output

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## 0.5.1 - Unreleased
 
+- Added opt-in npm registry verification that drops only matching single-package, whole-title-and-reasoning public-npm publication claims when the exact version is confirmed published, thanks @coletebou.
 - Fixed revalidation to include linked patch attempts, validation results, feature context, and current relevant files so repaired findings can move out of `uncertain`.
 - Added `clawpatch review --feature-list <path>` for reviewing an explicit ordered, de-duplicated set of feature IDs, thanks @camwest.
 

docs/code-review.md

@@ -145,3 +145,43 @@ CUDA-specific category. Deslopify mode is unaffected.
 
 Review does not edit files. Use `clawpatch fix --finding <id>` for the explicit
 patch loop.
+
+## Registry verifier
+
+After per-finding evidence validation, review can run an opt-in npm-registry
+verifier. Findings whose entire title and reasoning both state the same bounded
+`pkg@semver` publication claim such as `mongodb@7.0.0 is unpublished on npm`
+get resolved against
+`https://registry.npmjs.org/{name}/{version}`. When the registry confirms
+the version is published, the finding is partitioned into
+`droppedFindings` with `layer: "registry-verifier"` instead of being
+surfaced as a real finding.
+
+This addresses a recurring failure mode where providers backed by an LLM
+with a fixed knowledge cutoff confidently flag post-cutoff package
+versions as nonexistent. (See _We Have a Package for You!_ — Spracklen et
+al., USENIX Security 2025, [arXiv:2406.10279][slop-paper] — for measured
+hallucination rates of the symmetric failure: invented package names.
+The registry-grounded mitigation is the same.)
+
+The verifier is intentionally biased toward keeping findings:
+
+| Registry response                      | Verdict              | Action       |
+| -------------------------------------- | -------------------- | ------------ |
+| 200 with matching `name` AND `version` | `verified-published` | drop finding |
+| 404                                    | `verified-missing`   | keep finding |
+| 5xx, transport error, timeout          | `unknown`            | keep finding |
+| 200 with non-JSON content-type         | `unknown`            | keep finding |
+| 200 with mismatched body name/version  | `unknown`            | keep finding |
+| 200 with body > 1 MiB                  | `unknown`            | keep finding |
+| Any redirect (`redirect: "error"`)     | `unknown`            | keep finding |
+
+Failure of the verifier never creates a false negative — only refutable
+single-package claims drop. Compound, multi-package, or context-disagreeing findings are always kept. The verifier is
+disabled by default because it sends package
+coordinates to the public npm registry. Enable it explicitly with
+`registryVerifier.enabled = true` in `.clawpatch/config.json`; the
+`--no-registry-verify` flag can still disable it for a single run. Within a single review run it
+deduplicates registry calls per `(name, version)`.
+
+[slop-paper]: https://arxiv.org/abs/2406.10279

docs/configuration.md

@@ -51,10 +51,20 @@ Default shape:
     "requireCleanWorktreeForFix": true,
     "commit": false,
     "openPr": false
+  },
+  "registryVerifier": {
+    "enabled": false
   }
 }
 ```
 
+`registryVerifier.enabled` controls the npm-registry post-validator that
+drops direct `pkg@semver` public-npm publication claims refuted by
+the public npm registry. It is disabled by default because lookups disclose
+package coordinates; set it to `true` only when that network access is acceptable. See
+[Code review > Registry verifier](code-review.md#registry-verifier) for
+the full verdict matrix.
+
 Environment overrides:
 
 - `CLAWPATCH_STATE_DIR`

docs/spec.md

@@ -440,6 +440,9 @@ Initial config:
     "requireCleanWorktreeForFix": true,
     "commit": false,
     "openPr": false
+  },
+  "registryVerifier": {
+    "enabled": false
   }
 }
 ```

src/app.test.ts

@@ -1,10 +1,12 @@
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { __testing as appTesting, AppContext } from "./app.js";
+import { defaultConfig } from "./config.js";
 import { ClawpatchError } from "./errors.js";
 import type { ReviewOutput } from "./types.js";
 
 // eslint-disable-next-line no-underscore-dangle
-const { isRetryableReviewError, reviewRetries, runProviderReviewWithRetry } = appTesting;
+const { isRetryableReviewError, reviewFlagSubset, reviewRetries, runProviderReviewWithRetry } =
+  appTesting;
 
 const QUIET_CONTEXT: AppContext = {
   root: "/tmp/test-root",
@@ -24,6 +26,14 @@ function emptyReview(): ReviewOutput {
   return { findings: [], inspected: { files: [], symbols: [], notes: ["ok"] } };
 }
 
+it("forwards the registry-verifier opt-out into CI review flags", () => {
+  expect(reviewFlagSubset({ noRegistryVerify: true })).toEqual({ noRegistryVerify: true });
+});
+
+it("keeps public registry verification opt-in", () => {
+  expect(defaultConfig().registryVerifier.enabled).toBe(false);
+});
+
 function withEnv(name: string, value: string | undefined, fn: () => void): void {
   const previous = process.env[name];
   if (value === undefined) {

src/app.ts

@@ -32,7 +32,12 @@ import {
   renderFindingDetail,
   renderReport,
 } from "./reporting.js";
-import { validateReviewOutputPartitioned } from "./review-validation.js";
+import {
+  buildRegistryVerifierValidator,
+  validateReviewOutputPartitioned,
+  type FindingPostValidator,
+  type ValidatePartitionedOptions,
+} from "./review-validation.js";
 import {
   filterFeaturesByChangedFiles,
   filterFeaturesByProject,
@@ -320,6 +325,9 @@ export async function reviewCommand(
   const limiter = createRpmLimiter(
     rpmFromFlag(stringFlag(flags, "rateLimitPerMinute"), process.env["CLAWPATCH_RPM"]),
   );
+  const registryPostValidator = config.registryVerifier.enabled
+    ? buildRegistryVerifierValidator()
+    : undefined;
   let cursor = 0;
   emitProgress(context, "review", "start", {
     run: currentRunId,
@@ -348,13 +356,19 @@ export async function reviewCommand(
             mode,
             customPrompt,
             limiter,
+            registryPostValidator,
             allowNonPendingFeatureReview:
               stringFlag(flags, "feature") !== undefined ||
               stringFlag(flags, "featureList") !== undefined,
           });
           findingIds.push(...reviewed.findingIds);
           for (const dropped of reviewed.droppedFindings) {
-            const code = dropped.layer === "validation" ? "validation-drop" : "schema-drop";
+            const code =
+              dropped.layer === "validation"
+                ? "validation-drop"
+                : dropped.layer === "registry-verifier"
+                  ? "registry-verifier-drop"
+                  : "schema-drop";
             errors.push({
               message:
                 `dropped 1 finding from feature ${feature.featureId} ` +
@@ -374,7 +388,10 @@ export async function reviewCommand(
     }),
   );
   const fatalErrors = errors.filter(
-    (entry) => entry.code !== "schema-drop" && entry.code !== "validation-drop",
+    (entry) =>
+      entry.code !== "schema-drop" &&
+      entry.code !== "validation-drop" &&
+      entry.code !== "registry-verifier-drop",
   );
   if (fatalErrors.length > 0) {
     await writeRun(loaded.paths, {
@@ -660,6 +677,7 @@ type ReviewFeatureOptions = {
   mode: ReviewMode;
   customPrompt: string | null;
   limiter: RpmLimiter;
+  registryPostValidator: FindingPostValidator | undefined;
   allowNonPendingFeatureReview: boolean;
 };
 
@@ -678,6 +696,7 @@ async function reviewFeature(
     mode,
     customPrompt,
     limiter,
+    registryPostValidator,
     allowNonPendingFeatureReview,
   } = options;
   const started = Date.now();
@@ -729,12 +748,19 @@ async function reviewFeature(
     // Layer 2 drops: per-finding evidence validation (line ranges, quotes,
     // included files). Partition so a single bad finding doesn't lose the
     // whole feature.
+    // Layer 3 drops (optional): registry verifier rejects findings whose
+    // "package X@Y is unpublished" claim is refuted by the npm registry.
+    const validatePartitionedOptions: ValidatePartitionedOptions = {};
+    if (registryPostValidator !== undefined) {
+      validatePartitionedOptions.postValidator = registryPostValidator;
+    }
     const validated = await validateReviewOutputPartitioned(
       loaded.root,
       lockedFeature,
       config,
       reviewPrompt.manifest,
       reviewOutput,
+      validatePartitionedOptions,
     );
     droppedFindings.push(...validated.droppedFindings);
     const records = validated.findings.map((finding) =>
@@ -1410,6 +1436,14 @@ function applyProviderFlags(
       reasoningEffort: reasoningEffort ?? config.provider.reasoningEffort,
       skipGitRepoCheck: flags["skipGitRepoCheck"] === true,
     },
+    registryVerifier: {
+      ...config.registryVerifier,
+      // CLI flag is one-way: --no-registry-verify forces off, but absence
+      // of the flag preserves whatever config.json says. This matches the
+      // negative-flag convention (`--no-color`, `--no-input`) used
+      // elsewhere in the CLI surface.
+      enabled: flags["noRegistryVerify"] === true ? false : config.registryVerifier.enabled,
+    },
   };
 }
 
@@ -1442,6 +1476,9 @@ function reviewFlagSubset(
   if (flags["includeDirty"] === true) {
     subset["includeDirty"] = true;
   }
+  if (flags["noRegistryVerify"] === true) {
+    subset["noRegistryVerify"] = true;
+  }
   return subset;
 }
 
@@ -2229,6 +2266,7 @@ function stringFlag(flags: Record<string, string | boolean>, name: string): stri
 // eslint-disable-next-line no-underscore-dangle
 export const __testing = {
   isRetryableReviewError,
+  reviewFlagSubset,
   reviewRetries,
   runProviderReviewWithRetry,
 };

src/cli.ts

@@ -171,6 +171,7 @@ const commandFlags = {
     "promptFile",
     "exportTribunalLedger",
     "includeDirty",
+    "noRegistryVerify",
   ]),
   ci: new Set([
     "limit",
@@ -183,6 +184,7 @@ const commandFlags = {
     "skipGitRepoCheck",
     "output",
     "includeDirty",
+    "noRegistryVerify",
   ]),
   report: new Set(["status", "severity", "feature", "project", "category", "triage", "output"]),
   show: new Set(["finding"]),
@@ -262,6 +264,7 @@ const booleanFlagNames = new Set([
   "all",
   "draft",
   "include-dirty",
+  "no-registry-verify",
 ]);
 
 const shortFlagNames = new Set(["-h", "-q", "-v", "-o"]);
@@ -454,6 +457,11 @@ Flags:
                           JSONL file with one line per finding shaped
                           for downstream Tribunal-style signed-ledger
                           ingest. Opt-in; no effect when omitted.
+  --no-registry-verify    disable a configured npm-registry post-validator that
+                          drops findings whose "package X@Y is
+                          unpublished" claim is refuted by the registry.
+                          Set registryVerifier.enabled=true in config.json
+                          to opt in; this flag disables it for one run.
   --json
   -q, --quiet
 `);
@@ -494,6 +502,7 @@ Flags:
   --reasoning-effort <none|minimal|low|medium|high|xhigh>
   --skip-git-repo-check
   --output <path>
+  --no-registry-verify    see clawpatch review --help for details
   --json
 `);
     return;

src/config.ts

@@ -60,6 +60,9 @@ export function defaultConfig(): ClawpatchConfig {
       commit: false,
       openPr: false,
     },
+    registryVerifier: {
+      enabled: false,
+    },
   };
 }
 

src/provider.ts

@@ -157,16 +157,20 @@ export type ProviderOptions = {
  * One review finding rejected by per-finding validation. `layer` records
  * which gate dropped it: `schema` is the per-finding `reviewFindingSchema`
  * Zod parse, `validation` is the evidence/quote/line-range check in
- * `validateReviewOutputPartitioned`. Operators can use `layer` to tell
- * "the model emitted nonsense for this finding" (schema) apart from
- * "the model cited a real-looking finding but pointed at the wrong file
- * or quoted text that isn't there" (validation).
+ * `validateReviewOutputPartitioned`, and `registry-verifier` is the
+ * post-validation pass that drops findings whose central claim about a
+ * package version's nonexistence is refuted by the npm registry.
+ * Operators can use `layer` to tell these apart: "the model emitted
+ * nonsense" (schema), "the model cited a real-looking finding but quoted
+ * text that isn't there" (validation), or "the model asserted a package
+ * version is unpublished but the registry says otherwise"
+ * (registry-verifier).
  */
 export type DroppedFinding = {
   path: (string | number)[];
   message: string;
   sample: string;
-  layer?: "schema" | "validation";
+  layer?: "schema" | "validation" | "registry-verifier";
 };
 
 export type PartitionedReviewOutput = {