diff --git a/CHANGELOG.md b/CHANGELOG.md index ffa001d..f89f2e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 0.5.1 - Unreleased +- Fixed revalidation to include linked patch attempts, validation results, feature context, and current relevant files so repaired findings can move out of `uncertain`. - Added `clawpatch review --feature-list ` for reviewing an explicit ordered, de-duplicated set of feature IDs, thanks @camwest. ## 0.5.0 - 2026-05-31 diff --git a/src/app.ts b/src/app.ts index 9ba4315..839e1a6 100644 --- a/src/app.ts +++ b/src/app.ts @@ -875,6 +875,10 @@ export async function revalidateCommand( const config = applyProviderFlags(loaded.config, flags); const provider = providerByName(config.provider.name); const findings = await selectRevalidationFindings(loaded, flags); + const [features, patchAttempts] = await Promise.all([ + readFeatures(loaded.paths), + readPatchAttempts(loaded.paths), + ]); const currentRunId = runId(); const currentGit = await discoverGit(loaded.root); const run = newRun(currentRunId, "revalidate", context, loaded.root, currentGit.headSha); @@ -898,7 +902,20 @@ export async function revalidateCommand( finding: finding.findingId, title: finding.title, }); - const prompt = await buildRevalidatePrompt(loaded.root, JSON.stringify(finding, null, 2)); + const feature = assertDefined( + features.find((candidate) => candidate.featureId === finding.featureId), + `feature not found: ${finding.featureId}`, + ); + const linkedPatchAttempts = patchAttempts.filter((patch) => + finding.linkedPatchAttemptIds.includes(patch.patchAttemptId), + ); + const prompt = await buildRevalidatePrompt( + loaded.root, + finding, + feature, + linkedPatchAttempts, + config, + ); const output = await provider.revalidate(loaded.root, prompt, providerOptions(config)); const updated = appendFindingHistory( { diff --git a/src/prompt.ts b/src/prompt.ts index c53ebda..32797df 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -1,10 +1,19 @@ import { readFile, realpath } from "node:fs/promises"; -import { isAbsolute, relative, resolve } from "node:path"; -import { ClawpatchConfig, FeatureRecord, FindingRecord, ProjectRecord } from "./types.js"; +import { createHash } from "node:crypto"; +import { basename, isAbsolute, relative, resolve } from "node:path"; +import { + ClawpatchConfig, + FeatureRecord, + FindingRecord, + PatchAttempt, + ProjectRecord, +} from "./types.js"; +import { validationCommandsForFeature } from "./validation.js"; export type ReviewMode = "default" | "deslopify"; export const REVIEW_PROMPT_FILE_CHAR_LIMIT = 24_000; +const REVALIDATE_FILE_CONTEXT_CHAR_LIMIT = 120_000; export type ReviewPromptFileRole = "owned" | "context" | "test"; @@ -384,18 +393,107 @@ function reviewModeInstructions(mode: ReviewMode): string { throw new Error(`Unsupported review mode: ${mode}`); } -export async function buildRevalidatePrompt(root: string, findingJson: string): Promise { +export async function buildRevalidatePrompt( + root: string, + finding: FindingRecord, + feature: FeatureRecord, + patchAttempts: PatchAttempt[], + config: ClawpatchConfig, +): Promise { + const fileBlocks: string[] = []; + const newestPatchAttempts = patchAttempts.toSorted((left, right) => + right.updatedAt.localeCompare(left.updatedAt), + ); + const promptPatchAttempts = newestPatchAttempts.slice(0, 3); + const paths = [ + ...fixPromptPaths(finding, feature, config), + ...promptPatchAttempts.flatMap((patch) => patch.filesChanged.slice(0, 50)), + ].filter((path, index, allPaths) => allPaths.indexOf(path) === index); + const expectedValidationCommands = validationCommandsForFeature(feature, config.commands); + let fileContextChars = 0; + for (const [index, path] of paths.entries()) { + const block = await rawFileBlock(root, path); + if (fileContextChars + block.length > REVALIDATE_FILE_CONTEXT_CHAR_LIMIT) { + fileBlocks.push(`[omitted ${paths.length - index} files due to revalidation context budget]`); + break; + } + fileBlocks.push(block); + fileContextChars += block.length; + } return `Revalidate this clawpatch finding against the current repository at ${root}. Check whether the original evidence paths/lines still exist. If evidence moved or changed, decide whether the issue is fixed, stale/false-positive, still open elsewhere, or uncertain. -Use tests and current code as evidence; do not assume a missing line means fixed. +Use the linked patch attempts, command results, and current files as evidence. Do not assume a +missing line means fixed. Do not return fixed when targeted validation failed or the current code +does not support the repair. Return strict JSON only: {"outcome":"fixed|open|false-positive|uncertain","reasoning":"string","commands":["string"]} Finding: -${findingJson}`; +${JSON.stringify(finding, null, 2)} + +Feature: +${JSON.stringify(feature, null, 2)} + +Linked patch attempts: +${JSON.stringify( + { + attempts: promptPatchAttempts.map((patch) => + revalidationPatchEvidence(patch, expectedValidationCommands), + ), + omittedAttempts: Math.max(0, newestPatchAttempts.length - promptPatchAttempts.length), + }, + null, + 2, +)} + +Relevant current files: +${fileBlocks.join("\n\n")}`; +} + +function revalidationPatchEvidence( + patch: PatchAttempt, + expectedValidationCommands: readonly string[], +): object { + return { + patchAttemptId: patch.patchAttemptId, + status: patch.status, + filesChanged: patch.filesChanged.slice(0, 50), + omittedFiles: Math.max(0, patch.filesChanged.length - 50), + commandsRun: patch.commandsRun + .slice(0, 20) + .map((result) => revalidationCommandEvidence(result, expectedValidationCommands)), + omittedCommands: Math.max(0, patch.commandsRun.length - 20), + testResults: patch.testResults + .slice(0, 20) + .map((result) => revalidationCommandEvidence(result, expectedValidationCommands)), + omittedTestResults: Math.max(0, patch.testResults.length - 20), + }; +} + +function revalidationCommandEvidence( + result: PatchAttempt["testResults"][number], + expectedValidationCommands: readonly string[], +): object { + const expectedValidationIndex = expectedValidationCommands.indexOf(result.command); + return { + command: safeCommandIdentifier(result.command), + matchedExpectedValidation: expectedValidationIndex >= 0, + expectedValidationIndex: expectedValidationIndex >= 0 ? expectedValidationIndex : null, + exitCode: result.exitCode, + durationMs: result.durationMs, + }; +} + +function safeCommandIdentifier(command: string): object { + const tokens = command.trim().split(/\s+/u); + const executable = tokens.find((token) => token !== "env" && !token.includes("=")); + return { + executable: executable === undefined ? "unknown" : basename(executable).slice(0, 80), + fingerprint: createHash("sha256").update(command).digest("hex").slice(0, 12), + }; } export async function buildFixPrompt( diff --git a/src/provider.ts b/src/provider.ts index 30de981..2dc60b5 100644 --- a/src/provider.ts +++ b/src/provider.ts @@ -1502,6 +1502,17 @@ const mockProvider: Provider = { }; }, async revalidate(_root: string, prompt: string): Promise { + if (prompt.includes("REVALIDATE_PATCH_EVIDENCE")) { + const hasValidatedPatch = prompt.includes('"status": "validated"'); + const hasMatchedValidation = prompt.includes('"matchedExpectedValidation": true'); + const hasSuccessfulValidation = prompt.includes('"exitCode": 0'); + const leakedOutput = + prompt.includes("SECRET_OUTPUT_MUST_NOT_REACH_REVALIDATION") || + prompt.includes("PRIVATE_ERROR_MUST_NOT_REACH_REVALIDATION"); + return hasValidatedPatch && hasMatchedValidation && hasSuccessfulValidation && !leakedOutput + ? { outcome: "fixed", reasoning: "mock patch evidence supports fix", commands: [] } + : { outcome: "uncertain", reasoning: "mock patch evidence incomplete", commands: [] }; + } if (prompt.includes("REVALIDATE_FIXED")) { return { outcome: "fixed", reasoning: "mock fixed outcome", commands: ["mock fixed"] }; } diff --git a/src/workflow.test.ts b/src/workflow.test.ts index 1185493..978b9cd 100644 --- a/src/workflow.test.ts +++ b/src/workflow.test.ts @@ -1188,6 +1188,76 @@ describe("workflow", () => { } }); + it("uses linked C# patch evidence when revalidating a fixed finding", async () => { + const root = await fixtureRoot("clawpatch-csharp-revalidate-"); + await writeFixture( + root, + "Sample.csproj", + 'Exenet8.0\n', + ); + await writeFixture( + root, + "Program.cs", + "var completedBatches = 0;\nConsole.WriteLine(42 / completedBatches); // TODO_BUG\n", + ); + process.env["CLAWPATCH_PROVIDER"] = "mock"; + const context = await makeContext(testOptions(root)); + + try { + await initCommand(context, {}); + await mapCommand(context); + await reviewCommand(context, { limit: "20" }); + const paths = statePaths(join(root, ".clawpatch")); + const finding = (await readFindings(paths)).find((candidate) => + candidate.evidence.some((evidence) => evidence.path === "Program.cs"), + ); + expect(finding).toBeDefined(); + + await writeFixture( + root, + "Program.cs", + "var completedBatches = 0;\nConsole.WriteLine(completedBatches == 0 ? 0 : 42 / completedBatches); // REVALIDATE_PATCH_EVIDENCE\n", + ); + const timestamp = new Date().toISOString(); + const patch: PatchAttempt = { + schemaVersion: 1, + patchAttemptId: "pat_csharp_fixed", + findingIds: [finding!.findingId], + featureIds: [finding!.featureId], + status: "validated", + plan: "SECRET_OUTPUT_MUST_NOT_REACH_REVALIDATION", + filesChanged: ["Program.cs"], + commandsRun: [], + testResults: [ + { + command: "dotnet build Sample.csproj", + cwd: root, + exitCode: 0, + durationMs: 10, + stdout: "SECRET_OUTPUT_MUST_NOT_REACH_REVALIDATION", + stderr: "PRIVATE_ERROR_MUST_NOT_REACH_REVALIDATION", + }, + ], + provider: null, + git: { baseSha: null, commitSha: null, branchName: null, prUrl: null }, + createdAt: timestamp, + updatedAt: timestamp, + }; + await writePatchAttempt(paths, patch); + await writeFinding(paths, { + ...finding!, + linkedPatchAttemptIds: [patch.patchAttemptId], + }); + + const result = await revalidateCommand(context, { finding: finding!.findingId }); + + expect(result).toMatchObject({ finding: finding!.findingId, outcome: "fixed" }); + expect((await readFinding(paths, finding!.findingId))?.status).toBe("fixed"); + } finally { + delete process.env["CLAWPATCH_PROVIDER"]; + } + }); + it("shows, prioritizes, and triages findings with history", async () => { const root = await fixtureRoot("clawpatch-finding-lifecycle-"); await writeFixture(