From fb1a038ff1f78a2b98d490841bc57ce7755273fd Mon Sep 17 00:00:00 2001 From: elaav Date: Tue, 28 Jul 2026 17:45:44 +0300 Subject: [PATCH 01/30] feat(patch): integrate Auto Patcher into OpenAnt - add patch CLI command - merge Auto Patcher engine - generate Trust Reports for findings - add integration tests and live smoke validation --- apps/openant-cli/cmd/patch.go | 110 + apps/openant-cli/cmd/patch_test.go | 73 + apps/openant-cli/cmd/root.go | 1 + apps/openant-cli/internal/output/formatter.go | 13 + libs/openant-core/core/patch.py | 228 ++ libs/openant-core/core/verdict_taxonomy.py | 14 + libs/openant-core/openant/cli.py | 45 + .../fixtures/examples/curl-cve-2022-27774.md | 69 + .../examples/node-semver-cve-2022-25883.md | 62 + .../examples/urllib3-cve-2023-43804.md | 63 + .../patch/fixtures/examples/vulnerability.md | 51 + .../patch/test_behavior_recommendation.py | 29 + .../patch/test_behavior_suggested_tests.py | 33 + .../tests/patch/test_behavior_summary.py | 74 + .../patch/test_behavior_validation_plan.py | 42 + .../tests/patch/test_diff_hunk_repair.py | 487 ++++ .../tests/patch/test_diff_parsing.py | 150 + .../tests/patch/test_finding_calibration.py | 106 + .../tests/patch/test_impact_surface.py | 490 ++++ .../patch/test_investigation_adapters.py | 52 + .../tests/patch/test_investigation_models.py | 80 + .../tests/patch/test_language_support.py | 70 + .../tests/patch/test_llm_client.py | 374 +++ .../tests/patch/test_patch_applicability.py | 418 +++ .../tests/patch/test_patch_challenger.py | 124 + .../tests/patch/test_patch_generator.py | 293 ++ .../tests/patch/test_patch_hygiene.py | 274 ++ .../patch/test_patch_wrapper_contract.py | 246 ++ .../openant-core/tests/patch/test_pipeline.py | 1324 +++++++++ .../tests/patch/test_pipeline_repair.py | 604 ++++ .../tests/patch/test_pipeline_retry.py | 505 ++++ .../tests/patch/test_repo_locator.py | 1800 ++++++++++++ .../tests/patch/test_run_metadata.py | 225 ++ .../tests/patch/test_sink_extractor.py | 372 +++ .../tests/patch/test_test_suggester.py | 77 + .../tests/patch/test_trust_package.py | 1545 ++++++++++ .../patch/test_vulnerability_patterns.py | 252 ++ .../tests/test_patch_eligible_verdicts.py | 33 + .../utilities/autopatcher/__init__.py | 7 + .../utilities/autopatcher/behavior_summary.py | 80 + .../autopatcher/confidence_scorer.py | 63 + .../utilities/autopatcher/diff_hunk_repair.py | 186 ++ .../utilities/autopatcher/diff_parsing.py | 68 + .../autopatcher/finding_calibration.py | 110 + .../utilities/autopatcher/impact_surface.py | 496 ++++ .../autopatcher/investigation_adapters.py | 52 + .../autopatcher/investigation_models.py | 114 + .../utilities/autopatcher/language_support.py | 78 + .../utilities/autopatcher/llm_client.py | 481 ++++ .../utilities/autopatcher/llm_config.py | 37 + .../autopatcher/patch_applicability.py | 126 + .../utilities/autopatcher/patch_challenger.py | 107 + .../utilities/autopatcher/patch_generator.py | 86 + .../utilities/autopatcher/patch_hygiene.py | 206 ++ .../utilities/autopatcher/patch_reviewer.py | 48 + .../utilities/autopatcher/pipeline.py | 2538 +++++++++++++++++ .../autopatcher/prompts/confidence_scorer.md | 52 + .../prompts/finding_calibration.md | 61 + .../autopatcher/prompts/patch_challenger.md | 39 + .../autopatcher/prompts/patch_generator.md | 119 + .../autopatcher/prompts/patch_reviewer.md | 29 + .../utilities/autopatcher/repo_locator.py | 1117 ++++++++ .../repository_grounding_models.py | 56 + .../utilities/autopatcher/run_metadata.py | 146 + .../utilities/autopatcher/test_suggester.py | 174 ++ .../utilities/autopatcher/testing_support.py | 167 ++ .../autopatcher/vulnerability_patterns.py | 406 +++ 67 files changed, 18057 insertions(+) create mode 100644 apps/openant-cli/cmd/patch.go create mode 100644 apps/openant-cli/cmd/patch_test.go create mode 100644 libs/openant-core/core/patch.py create mode 100644 libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md create mode 100644 libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md create mode 100644 libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md create mode 100644 libs/openant-core/tests/patch/fixtures/examples/vulnerability.md create mode 100644 libs/openant-core/tests/patch/test_behavior_recommendation.py create mode 100644 libs/openant-core/tests/patch/test_behavior_suggested_tests.py create mode 100644 libs/openant-core/tests/patch/test_behavior_summary.py create mode 100644 libs/openant-core/tests/patch/test_behavior_validation_plan.py create mode 100644 libs/openant-core/tests/patch/test_diff_hunk_repair.py create mode 100644 libs/openant-core/tests/patch/test_diff_parsing.py create mode 100644 libs/openant-core/tests/patch/test_finding_calibration.py create mode 100644 libs/openant-core/tests/patch/test_impact_surface.py create mode 100644 libs/openant-core/tests/patch/test_investigation_adapters.py create mode 100644 libs/openant-core/tests/patch/test_investigation_models.py create mode 100644 libs/openant-core/tests/patch/test_language_support.py create mode 100644 libs/openant-core/tests/patch/test_llm_client.py create mode 100644 libs/openant-core/tests/patch/test_patch_applicability.py create mode 100644 libs/openant-core/tests/patch/test_patch_challenger.py create mode 100644 libs/openant-core/tests/patch/test_patch_generator.py create mode 100644 libs/openant-core/tests/patch/test_patch_hygiene.py create mode 100644 libs/openant-core/tests/patch/test_patch_wrapper_contract.py create mode 100644 libs/openant-core/tests/patch/test_pipeline.py create mode 100644 libs/openant-core/tests/patch/test_pipeline_repair.py create mode 100644 libs/openant-core/tests/patch/test_pipeline_retry.py create mode 100644 libs/openant-core/tests/patch/test_repo_locator.py create mode 100644 libs/openant-core/tests/patch/test_run_metadata.py create mode 100644 libs/openant-core/tests/patch/test_sink_extractor.py create mode 100644 libs/openant-core/tests/patch/test_test_suggester.py create mode 100644 libs/openant-core/tests/patch/test_trust_package.py create mode 100644 libs/openant-core/tests/patch/test_vulnerability_patterns.py create mode 100644 libs/openant-core/tests/test_patch_eligible_verdicts.py create mode 100644 libs/openant-core/utilities/autopatcher/__init__.py create mode 100644 libs/openant-core/utilities/autopatcher/behavior_summary.py create mode 100644 libs/openant-core/utilities/autopatcher/confidence_scorer.py create mode 100644 libs/openant-core/utilities/autopatcher/diff_hunk_repair.py create mode 100644 libs/openant-core/utilities/autopatcher/diff_parsing.py create mode 100644 libs/openant-core/utilities/autopatcher/finding_calibration.py create mode 100644 libs/openant-core/utilities/autopatcher/impact_surface.py create mode 100644 libs/openant-core/utilities/autopatcher/investigation_adapters.py create mode 100644 libs/openant-core/utilities/autopatcher/investigation_models.py create mode 100644 libs/openant-core/utilities/autopatcher/language_support.py create mode 100644 libs/openant-core/utilities/autopatcher/llm_client.py create mode 100644 libs/openant-core/utilities/autopatcher/llm_config.py create mode 100644 libs/openant-core/utilities/autopatcher/patch_applicability.py create mode 100644 libs/openant-core/utilities/autopatcher/patch_challenger.py create mode 100644 libs/openant-core/utilities/autopatcher/patch_generator.py create mode 100644 libs/openant-core/utilities/autopatcher/patch_hygiene.py create mode 100644 libs/openant-core/utilities/autopatcher/patch_reviewer.py create mode 100644 libs/openant-core/utilities/autopatcher/pipeline.py create mode 100644 libs/openant-core/utilities/autopatcher/prompts/confidence_scorer.md create mode 100644 libs/openant-core/utilities/autopatcher/prompts/finding_calibration.md create mode 100644 libs/openant-core/utilities/autopatcher/prompts/patch_challenger.md create mode 100644 libs/openant-core/utilities/autopatcher/prompts/patch_generator.md create mode 100644 libs/openant-core/utilities/autopatcher/prompts/patch_reviewer.md create mode 100644 libs/openant-core/utilities/autopatcher/repo_locator.py create mode 100644 libs/openant-core/utilities/autopatcher/repository_grounding_models.py create mode 100644 libs/openant-core/utilities/autopatcher/run_metadata.py create mode 100644 libs/openant-core/utilities/autopatcher/test_suggester.py create mode 100644 libs/openant-core/utilities/autopatcher/testing_support.py create mode 100644 libs/openant-core/utilities/autopatcher/vulnerability_patterns.py diff --git a/apps/openant-cli/cmd/patch.go b/apps/openant-cli/cmd/patch.go new file mode 100644 index 00000000..3b55b6da --- /dev/null +++ b/apps/openant-cli/cmd/patch.go @@ -0,0 +1,110 @@ +package cmd + +import ( + "os" + + "github.com/knostic/open-ant-cli/internal/output" + "github.com/knostic/open-ant-cli/internal/python" + "github.com/spf13/cobra" +) + +var patchCmd = &cobra.Command{ + Use: "patch [pipeline-output-path]", + Short: "Generate and evaluate a candidate remediation for a finding", + Long: `Patch invokes the merged Auto Patcher engine to generate a candidate +remediation for a single Finding and produce a Trust Report judging whether +that candidate should be trusted. + +The Trust Report is treated as an opaque artifact: it is written under the +active scan directory but never parsed, scored, or reinterpreted. + +Requires LLM_PROVIDER to be set explicitly (e.g. LLM_PROVIDER=anthropic) so +a run never silently falls back to a mock LLM. Set LLM_PROVIDER=mock only +if you intentionally want a mock run. + +If no path is given, the active project's pipeline_output.json is used.`, + Args: cobra.MaximumNArgs(1), + Run: runPatch, +} + +var ( + patchFindingID string + patchRepoRoot string + patchOutput string +) + +func init() { + patchCmd.Flags().StringVar(&patchFindingID, "finding-id", "", "ID of the finding to remediate (required)") + patchCmd.Flags().StringVar(&patchRepoRoot, "repo-root", "", "Path to the target repository root (defaults to the active project's repo path)") + patchCmd.Flags().StringVarP(&patchOutput, "output", "o", "", "Output directory (default: active scan directory)") +} + +func buildPatchPyArgs(pipelineOutputPath, findingID, repoRoot, outputDir string) []string { + pyArgs := []string{"patch", pipelineOutputPath, "--finding-id", findingID} + if repoRoot != "" { + pyArgs = append(pyArgs, "--repo-root", repoRoot) + } + if outputDir != "" { + pyArgs = append(pyArgs, "--output", outputDir) + } + return pyArgs +} + +func runPatch(cmd *cobra.Command, args []string) { + if patchFindingID == "" { + output.PrintError("openant patch requires --finding-id ") + os.Exit(2) + } + + pipelineOutputPath, ctx, err := resolveFileArg(args, "pipeline_output.json") + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + if _, err := os.Stat(pipelineOutputPath); err != nil { + output.PrintError("pipeline_output.json not found. Run 'openant build-output' first.") + os.Exit(2) + } + + outputDir := patchOutput + if outputDir == "" && ctx != nil { + outputDir = ctx.ScanDir + } + + repoRoot := patchRepoRoot + if repoRoot == "" && ctx != nil { + repoRoot = ctx.RepoPath + } + + rt, err := ensurePython() + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + pyArgs := buildPatchPyArgs(pipelineOutputPath, patchFindingID, repoRoot, outputDir) + + // Auto Patcher's LLM calls are independently configured via LLM_PROVIDER / + // OPENAI_API_KEY / ANTHROPIC_API_KEY -- never OpenAnt's own --api-key or + // llm_providers config, so no API key is forwarded here. + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, "") + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + if result.Envelope.Status == "interrupted" { + os.Exit(130) + } else if jsonOutput { + output.PrintJSON(result.Envelope) + } else if result.Envelope.Status == "success" { + if data, ok := result.Envelope.Data.(map[string]any); ok { + output.PrintPatchSummary(data) + } + } else { + output.PrintErrors(result.Envelope.Errors) + } + + os.Exit(result.ExitCode) +} diff --git a/apps/openant-cli/cmd/patch_test.go b/apps/openant-cli/cmd/patch_test.go new file mode 100644 index 00000000..c45d3832 --- /dev/null +++ b/apps/openant-cli/cmd/patch_test.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "testing" + + "github.com/spf13/cobra" +) + +func TestBuildPatchPyArgsBaseline(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "", "") + want := []string{"patch", "/scan/pipeline_output.json", "--finding-id", "F-001"} + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchPyArgsWithRepoRootAndOutput(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "/repo", "/scan") + want := []string{ + "patch", "/scan/pipeline_output.json", "--finding-id", "F-001", + "--repo-root", "/repo", + "--output", "/scan", + } + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchPyArgsOmitsRepoRootAndOutputWhenEmpty(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "", "") + if found, _ := findFlag(args, "--repo-root"); found { + t.Errorf("did not expect --repo-root in pyArgs when unset, got %v", args) + } + if found, _ := findFlag(args, "--output"); found { + t.Errorf("did not expect --output in pyArgs when unset, got %v", args) + } +} + +func TestPatchCmdHasFindingIDFlag(t *testing.T) { + flag := patchCmd.Flags().Lookup("finding-id") + if flag == nil { + t.Fatal("patchCmd is missing the --finding-id flag") + } + if flag.DefValue != "" { + t.Errorf("--finding-id default should be empty, got %q", flag.DefValue) + } +} + +func TestPatchCmdIsRegisteredOnRoot(t *testing.T) { + var found *cobra.Command + for _, c := range rootCmd.Commands() { + if c.Name() == "patch" { + found = c + break + } + } + if found == nil { + t.Fatal("patch command not registered on rootCmd") + } + if found.Flags().Lookup("finding-id") == nil { + t.Error("patch subcommand resolved from root is missing --finding-id flag") + } +} diff --git a/apps/openant-cli/cmd/root.go b/apps/openant-cli/cmd/root.go index 015d3099..75d64bca 100644 --- a/apps/openant-cli/cmd/root.go +++ b/apps/openant-cli/cmd/root.go @@ -146,6 +146,7 @@ func init() { rootCmd.AddCommand(buildOutputCmd) rootCmd.AddCommand(dynamicTestCmd) rootCmd.AddCommand(reportCmd) + rootCmd.AddCommand(patchCmd) rootCmd.AddCommand(projectCmd) rootCmd.AddCommand(configCmd) rootCmd.AddCommand(setAPIKeyCmd) diff --git a/apps/openant-cli/internal/output/formatter.go b/apps/openant-cli/internal/output/formatter.go index cd1ed81a..fd814b51 100644 --- a/apps/openant-cli/internal/output/formatter.go +++ b/apps/openant-cli/internal/output/formatter.go @@ -335,6 +335,19 @@ func PrintDynamicTestSummary(data map[string]any) { fmt.Println() } +// PrintPatchSummary outputs a formatted summary of a patch-trust run. +func PrintPatchSummary(data map[string]any) { + PrintHeader("Patch Trust Report") + + if id, ok := data["finding_id"].(string); ok { + PrintKeyValue("Finding", id) + } + if path, ok := data["trust_report_path"].(string); ok { + PrintKeyValue("Report", path) + } + fmt.Println() +} + // PrintBuildOutputSummary outputs a formatted summary of pipeline output generation. func PrintBuildOutputSummary(data map[string]any) { PrintHeader("Pipeline Output") diff --git a/libs/openant-core/core/patch.py b/libs/openant-core/core/patch.py new file mode 100644 index 00000000..0a3d97fb --- /dev/null +++ b/libs/openant-core/core/patch.py @@ -0,0 +1,228 @@ +""" +Patch-trust wrapper. + +Loads a Finding from pipeline_output.json, checks it is eligible for +remediation, renders it into a vulnerability description, and runs it +through the merged Auto Patcher engine (``utilities.autopatcher``) to +produce a Trust Report. Mirrors core/dynamic_tester.py's shape: a thin +wrapper around a heavier ``utilities.*`` engine. + +The Trust Report is treated as an opaque artifact: this module never parses +its Recommendation or Trust Signals, only the path it was written to. +""" + +import os +from dataclasses import dataclass, asdict +from datetime import datetime, timezone +from pathlib import Path + +from core.verdict_taxonomy import PATCH_ELIGIBLE +from utilities.file_io import read_json, normalize_results + + +@dataclass +class PatchStepResult: + """Result of `openant patch`.""" + finding_id: str + vulnerability_path: str + trust_report_path: str + + def to_dict(self) -> dict: + return asdict(self) + + +def find_finding_by_id(findings: list, finding_id: str) -> dict: + """Return the finding dict with the given id, or raise ValueError.""" + for f in findings: + if isinstance(f, dict) and f.get("id") == finding_id: + return f + raise ValueError(f"no finding with id {finding_id!r} in pipeline_output.json") + + +def effective_verdict(finding: dict) -> str: + """The verdict that governs eligibility: stage2_verdict if set, else stage1_verdict.""" + return finding.get("stage2_verdict") or finding.get("stage1_verdict") or "" + + +def check_eligible(finding: dict) -> None: + """Raise ValueError if finding's effective verdict is not patch-eligible. + + Deliberately an explicit allowlist (PATCH_ELIGIBLE), not a denylist, so + an empty, unknown, or future verdict value fails closed. + """ + verdict = effective_verdict(finding) + if verdict not in PATCH_ELIGIBLE: + raise ValueError( + f"finding {finding.get('id')} has verdict {verdict!r}, which is not " + f"eligible for remediation (eligible: {', '.join(sorted(PATCH_ELIGIBLE))})" + ) + + +def render_vulnerability_markdown(finding: dict) -> str: + """Render a Finding into deterministic Markdown for the patch engine's input. + + suggested_fix and rejection_reason are never read here: feeding OpenAnt's + own suggested fix into the patch engine could bias its independently + generated candidate patch. + """ + location = finding.get("location") or {} + lines = [ + f"# {finding.get('name', '')}", + "", + "## Vulnerability description", + "", + f"- **Finding ID:** {finding.get('id', '')}", + f"- **CWE:** CWE-{finding.get('cwe_id', '')} ({finding.get('cwe_name', '')})", + f"- **Location:** {location.get('file', '')} ({location.get('function', '')})", + f"- **Verdict:** {effective_verdict(finding)}", + ] + + description = finding.get("description") + if description: + lines += ["", description] + + vulnerable_code = finding.get("vulnerable_code") + if vulnerable_code: + lines += ["", "## Vulnerable code", "", "```", vulnerable_code, "```"] + + impact = finding.get("impact") or [] + if impact: + lines += ["", "## Impact", ""] + lines += [f"- {item}" for item in impact] + + steps = finding.get("steps_to_reproduce") or [] + if steps: + lines += ["", "## Attack scenario", ""] + lines += [f"{i + 1}. {step}" for i, step in enumerate(steps)] + + return "\n".join(lines) + "\n" + + +def _find_openant_root() -> Path | None: + """Walk up from this file to the OpenAnt repo root (contains libs/openant-core), + for the Run Metadata report's commit row. Returns None if not found (e.g. + installed as a wheel outside a git checkout) -- collect_git_info already + degrades to 'unknown' in that case.""" + here = Path(__file__).resolve() + for parent in here.parents: + if (parent / "libs" / "openant-core").is_dir(): + return parent + return None + + +def run_patch( + pipeline_output_path: str, + finding_id: str, + output_dir: str, + repo_root: str | None = None, +) -> PatchStepResult: + """Generate and evaluate a candidate remediation for one finding. + + Requires LLM_PROVIDER to be set explicitly in the environment: Auto + Patcher's engine silently falls back to a deterministic mock LLM when + LLM_PROVIDER is unset and stdin is non-interactive (always true when + invoked from OpenAnt's Go CLI). An OpenAnt-triggered run must never + silently produce a mock Trust Report that looks real -- LLM_PROVIDER=mock + is allowed, it just must have been explicitly chosen. + + Writes two artifacts under ``{output_dir}/patch/``: + {finding_id}-vulnerability.md -- the rendered input (for transparency) + {finding_id}-trust-report.md -- the engine's opaque Trust Report + + Raises: + RuntimeError: if LLM_PROVIDER is unset. + FileNotFoundError: if pipeline_output_path doesn't exist. + ValueError: if finding_id is unknown or ineligible. + """ + if not os.environ.get("LLM_PROVIDER"): + raise RuntimeError( + "LLM_PROVIDER is not set. The patch engine requires an explicit " + "provider (e.g. LLM_PROVIDER=anthropic) so a run triggered by " + "OpenAnt never silently falls back to mock mode. Set " + "LLM_PROVIDER=mock only if you intentionally want a mock run." + ) + + if not os.path.exists(pipeline_output_path): + raise FileNotFoundError(f"pipeline_output.json not found: {pipeline_output_path}") + + pipeline_data = read_json(pipeline_output_path) + if "findings" in pipeline_data: + normalize_results(pipeline_data, "findings") + findings = pipeline_data.get("findings", []) + + finding = find_finding_by_id(findings, finding_id) + check_eligible(finding) + + patch_dir = os.path.join(output_dir, "patch") + os.makedirs(patch_dir, exist_ok=True) + + vulnerability_text = render_vulnerability_markdown(finding) + vulnerability_path = os.path.join(patch_dir, f"{finding_id}-vulnerability.md") + with open(vulnerability_path, "w", encoding="utf-8") as f: + f.write(vulnerability_text) + + from utilities.autopatcher import llm_client as _llm + from utilities.autopatcher import run_metadata as _rm + from utilities.autopatcher.pipeline import run as _run_pipeline + + # Reset per-run LLM call metadata -- module-level state that would + # otherwise leak a stale stage entry from a prior run in this process. + _llm.clear_call_metadata() + + timestamp = datetime.now(timezone.utc) + openant_root = _find_openant_root() + patcher_commit = _rm.collect_git_info(openant_root) if openant_root else "unknown" + repo_commit = _rm.collect_git_info(Path(repo_root)) if repo_root else "-" + + api_key = os.environ.get("OPENAI_API_KEY", "") + report_body = _run_pipeline( + vulnerability_text=vulnerability_text, + api_key=api_key, + repo_root=repo_root, + ) + + provider = _llm._cached_provider or os.environ.get("LLM_PROVIDER", "unknown") + if provider and provider != "unknown": + model = _llm._cached_model.get(provider, "unknown") + else: + model = "unknown" + if provider == "mock": + model = "mock" + + llm_mode = "MOCK" if _llm.LLMClient(api_key=api_key).is_mock else "LIVE" + + call_metadata = _llm.get_call_metadata() + stage_stop_reasons = {stage: info.get("stop_reason") for stage, info in call_metadata.items()} + max_tokens_configured = next( + ( + info.get("max_tokens_configured") + for info in call_metadata.values() + if info.get("max_tokens_configured") is not None + ), + None, + ) + + trust_report_path = os.path.join(patch_dir, f"{finding_id}-trust-report.md") + meta = _rm.RunMetadata( + timestamp=timestamp.strftime("%Y-%m-%d %H:%M:%S UTC"), + input_source=vulnerability_path, + repo_root=repo_root or "", + repo_commit=repo_commit, + llm_provider=provider, + llm_model=model, + llm_mode=llm_mode, + output_path=trust_report_path, + patcher_commit=patcher_commit, + max_tokens_configured=max_tokens_configured, + stage_stop_reasons=stage_stop_reasons, + ) + full_report = report_body + "\n---\n\n" + _rm.render_metadata_section(meta) + + with open(trust_report_path, "w", encoding="utf-8") as f: + f.write(full_report) + + return PatchStepResult( + finding_id=finding_id, + vulnerability_path=vulnerability_path, + trust_report_path=trust_report_path, + ) diff --git a/libs/openant-core/core/verdict_taxonomy.py b/libs/openant-core/core/verdict_taxonomy.py index 7d29e659..5801fa77 100644 --- a/libs/openant-core/core/verdict_taxonomy.py +++ b/libs/openant-core/core/verdict_taxonomy.py @@ -100,3 +100,17 @@ "agreed", "vulnerable", }) + +# --- Patch-eligibility filter ------------------------------------------------- +# Findings eligible to be sent to the patch-trust pipeline (``core/patch.py``). +# Deliberately its OWN set, not an alias of DISCLOSURE_ELIGIBLE or +# DYNAMIC_TESTABLE: sending a finding to an external-generation patch pipeline +# is a stricter bar than including it in a report, and "bypassable" (a real, +# exploitable-if-bypassed finding) is worth patching even though it isn't +# worth an active dynamic-test reproduction attempt. +PATCH_ELIGIBLE = frozenset({ + "confirmed", + "agreed", + "vulnerable", + "bypassable", +}) diff --git a/libs/openant-core/openant/cli.py b/libs/openant-core/openant/cli.py index 5b78f872..40ae1cf1 100644 --- a/libs/openant-core/openant/cli.py +++ b/libs/openant-core/openant/cli.py @@ -581,6 +581,39 @@ def cmd_dynamic_test(args): return 2 +def cmd_patch(args): + """Generate and evaluate a candidate remediation for one finding.""" + from core.patch import run_patch + from core.schemas import success, error + from core.step_report import step_context + + output_dir = args.output or tempfile.mkdtemp(prefix="openant_patch_") + + try: + with step_context("patch", output_dir, inputs={ + "pipeline_output_path": os.path.abspath(args.pipeline_output), + "finding_id": args.finding_id, + }) as ctx: + result = run_patch( + pipeline_output_path=args.pipeline_output, + finding_id=args.finding_id, + output_dir=output_dir, + repo_root=args.repo_root, + ) + + ctx.outputs = { + "vulnerability_path": result.vulnerability_path, + "trust_report_path": result.trust_report_path, + } + + _output_json(success(result.to_dict())) + return 0 + + except Exception as e: + _output_json(error(str(e))) + return 2 + + def _default_report_output(results_path: str, fmt: str) -> str: """Derive a sensible default output path based on format.""" reports_dir = os.path.join(os.path.dirname(os.path.abspath(results_path)), "final-reports") @@ -1543,6 +1576,18 @@ def build_parser() -> argparse.ArgumentParser: ) dt_p.set_defaults(func=cmd_dynamic_test) + # --------------------------------------------------------------- + # patch — generate and evaluate a candidate remediation for a finding + # --------------------------------------------------------------- + patch_p = subparsers.add_parser( + "patch", help="Generate and evaluate a candidate remediation for a finding" + ) + patch_p.add_argument("pipeline_output", help="Path to pipeline_output.json") + patch_p.add_argument("--finding-id", required=True, help="ID of the finding to remediate") + patch_p.add_argument("--repo-root", help="Path to the target repository root") + patch_p.add_argument("--output", "-o", help="Output directory (default: temp dir)") + patch_p.set_defaults(func=cmd_patch) + # --------------------------------------------------------------- # report — generate reports from results # --------------------------------------------------------------- diff --git a/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md b/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md new file mode 100644 index 00000000..5d4be5bb --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md @@ -0,0 +1,69 @@ +# curl — Credential Leak on Redirect (CVE-2022-27774) + +## Vulnerability description + +**Type:** Insufficiently Protected Credentials (CWE-522) +**Severity:** Medium (CVSS 5.0, NVD) +**Component:** libcurl — HTTP(S) redirect handling + +When libcurl performs a request that includes authentication credentials +(for example HTTP Basic, Digest, Bearer, or NTLM authentication supplied via +`-u` / `CURLOPT_USERPWD` or an equivalent option) and the server responds +with a redirect, libcurl may continue sending those same credentials to the +destination of the redirect — even when that destination is a different +host, a different port on the same host, or a different protocol/scheme +than the original request. + +Because libcurl automatically follows redirects when configured to do so +(`CURLOPT_FOLLOWLOCATION`, or `-L` on the command line), a malicious or +compromised server can respond to an authenticated request with a redirect +that points somewhere entirely different from the original destination. If +libcurl carries the original request's credentials along to that new +destination, an attacker who controls or can influence the redirect target +can obtain the user's authentication credentials, even though the user +never intended to send them there. + +This is a credential-leak issue, not a request-forgery issue: the redirect +target does not need to be part of the original service at all. Any server +able to issue a redirect response to an authenticated client can potentially +be used to exfiltrate that client's credentials to a destination of its +choosing. + +## Attack scenario + +An application uses libcurl (or the `curl` command-line tool) to make an +authenticated HTTP request to a trusted service, for example: + +``` +curl -L -u alice:s3cr3t https://trusted-service.example/api/resource +``` + +The trusted service is compromised, or a man-in-the-middle is able to +influence its responses. Instead of returning the requested resource, it +responds with a redirect (HTTP 301, 302, 303, 307, or 308) to a URL under +the attacker's control — one that may differ from the original request in +host, port, or scheme. + +Because `-L` / `CURLOPT_FOLLOWLOCATION` causes libcurl to follow the +redirect automatically, and credentials set on the original request are not +necessarily cleared when the destination changes, libcurl may re-send +`alice`'s username and password to the attacker's server while following +that redirect — despite the user never having configured curl to send +credentials to that destination. + +## Affected versions + +curl versions 4.9 up to and including 7.82.0 are affected. Versions prior +to 4.9 are not affected. The issue is fixed starting in curl 7.83.0. + +## Additional context + +- This affects any authentication scheme where libcurl retains credential + state across a request (Basic, Digest, Bearer, NTLM, etc.) when redirect + following is enabled. +- The concern is specifically about credentials libcurl itself carries + forward across a redirect — not about credentials a user has separately + and explicitly configured to be sent to a specific destination. +- No source code is provided with this advisory. The patch generator should + produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md b/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md new file mode 100644 index 00000000..4b73535a --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md @@ -0,0 +1,62 @@ +# node-semver — Regular Expression Denial of Service via `new Range` (CVE-2022-25883) + +## Vulnerability description + +**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333 +(Inefficient Regular Expression Complexity) +**Severity:** High (CVSS 7.5 — AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H) +**Component:** `semver` — version range parsing (`new Range()`) + +Versions of the package `semver` before 7.5.2 on the 7.x branch, before +6.3.1 on the 6.x branch, and all other versions before 5.7.2 are +vulnerable to Regular Expression Denial of Service (ReDoS) via the +function `new Range`, when untrusted user data is provided as a range. + +`semver`'s range-parsing logic relies on regular expressions to tokenize +and validate version range strings. When a range string originates from an +untrusted source — for example, a dependency specifier, a request +parameter, or any other externally influenced input passed to +`new Range()` — a specially crafted string can cause the underlying +regular expression engine to take an extreme amount of time to process, +due to catastrophic backtracking. Because Node.js executes JavaScript on a +single thread, a call that hangs while backtracking blocks the entire +process, denying service to any other work sharing that thread. + +## Attack scenario + +An application accepts a version-range string from an untrusted source and +passes it to `semver`, for example: + +``` +const semver = require('semver') +semver.satisfies(userSuppliedVersion, untrustedRangeString) +``` + +Common real-world sources of an untrusted range string include a request +parameter, a header, a configuration value sourced from an external +system, or a package specifier resolved from an untrusted registry or +lockfile. If an attacker can control or influence the value passed as the +range, they can supply a string engineered to trigger catastrophic +backtracking in `semver`'s internal range-parsing regular expressions. The +resulting call to `new Range()` — directly, or indirectly via functions +such as `satisfies()` or `validRange()` that construct a `Range` +internally — can take an extreme amount of time to return, consuming CPU +and blocking further request processing. This is a denial-of-service +condition that requires no authentication or special privilege from the +attacker. + +## Affected versions + +- `semver` `>= 7.0.0, < 7.5.2` (7.x branch) +- `semver` `>= 6.0.0, < 6.3.1` (6.x branch) +- `semver` `>= 2.0.0-alpha, < 5.7.2` (all other versions) + +Fixed starting in 7.5.2 (7.x branch), 6.3.1 (6.x branch), and 5.7.2 (all +other versions). + +## Additional context + +- CVE: CVE-2022-25883. GHSA: GHSA-c2qf-rxjj-qqgw. +- No source code is provided with this advisory. The patch generator + should produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md b/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md new file mode 100644 index 00000000..2db1649b --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md @@ -0,0 +1,63 @@ +# urllib3 — Cookie HTTP header isn't stripped on cross-origin redirects (CVE-2023-43804) + +## Vulnerability description + +**Type:** Exposure of Sensitive Information to an Unauthorized Actor — +CWE-200. +**Component:** `urllib3` — HTTP redirect handling. + +The `urllib3` library fails to strip the `Cookie` HTTP header during +cross-origin redirects. `urllib3` does not provide built-in cookie-jar +management, so it is up to the calling application to attach a `Cookie` +header to a request when needed. When such a request receives an HTTP +redirect (3xx) response pointing to a different origin than the one the +request was originally sent to, `urllib3` automatically follows the +redirect and, prior to the fix, forwarded the `Cookie` header unchanged to +the new destination. + +## Attack scenario + +An application manually attaches a `Cookie` header to an outgoing request, +for example to carry a session token or other authentication material: + +```python +resp = http.request("GET", "https://example.com/start", headers={"Cookie": "session=..."}) +``` + +If `https://example.com` is malicious, compromised, or simply +misconfigured, it can respond with a redirect to a different origin (e.g. +`https://attacker.example`). `urllib3` automatically follows the redirect +and forwards the `Cookie` header, unmodified, to that different origin — +an unintended disclosure of potentially sensitive authentication data to +a third party the application never intended to share it with. + +All of the following conditions must hold for this to be exploitable: + +- an affected `urllib3` version is in use; +- the calling application manually sets a `Cookie` header on the request + (`urllib3` itself does not manage cookies automatically); +- HTTP redirect-following is not disabled; +- the server issues a redirect to a different origin. + +## Affected versions + +- `urllib3` `>= 2.0.0, < 2.0.6` +- `urllib3` `< 1.26.17` + +Fixed starting in `2.0.6` (2.x line) and `1.26.17` (1.26.x line). + +## Remediation options (as published) + +1. Update to `urllib3` `v1.26.17` or `v2.0.6` or later. +2. Disable automatic redirect-following (e.g. pass `redirects=False`). +3. Avoid setting the `Cookie` header directly on requests that may be + redirected cross-origin. + +## Additional context + +- CVE: CVE-2023-43804. GHSA: + [GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f). + Published 2023-10-02. +- No source code is provided with this advisory. The patch generator + should produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md b/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md new file mode 100644 index 00000000..06d3d6ee --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md @@ -0,0 +1,51 @@ +# SQL Injection Vulnerability — Example Input + +## Vulnerability description + +**Type:** SQL Injection (CWE-89) +**Severity:** Critical +**Component:** `app/auth.py` — `authenticate()` function + +A classic SQL injection vulnerability exists in the authentication module. +User-supplied `username` and `password` values are interpolated directly into a +SQL query string without sanitisation or parameterisation. An attacker can +craft a malicious input (e.g. `' OR '1'='1`) to bypass authentication entirely +or extract arbitrary data from the database. + +## Vulnerable code + +```python +# app/auth.py (lines 38-46) + +import sqlite3 + +db = sqlite3.connect("users.db") + +def authenticate(username: str, password: str) -> bool: + query = f"SELECT * FROM users WHERE username='{username}' AND password='{password}'" + cursor = db.execute(query) + return cursor.fetchone() is not None +``` + +## Attack scenario + +An attacker sends the following credentials via the login form: + +- **username:** `admin' --` +- **password:** *(anything)* + +The resulting query becomes: + +```sql +SELECT * FROM users WHERE username='admin' --' AND password='anything' +``` + +The `--` comment character causes the password check to be ignored, granting +the attacker full access to the `admin` account. + +## Additional context + +- Database: SQLite 3.x (driver: `sqlite3` from the Python standard library) +- Python version: 3.11 +- Framework: Flask 3.0 (no ORM; raw SQL is used throughout `app/`) +- There are no existing parameterised query helpers in the codebase diff --git a/libs/openant-core/tests/patch/test_behavior_recommendation.py b/libs/openant-core/tests/patch/test_behavior_recommendation.py new file mode 100644 index 00000000..749381c5 --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_recommendation.py @@ -0,0 +1,29 @@ +""" +Pipeline-level test: ensure Recommendation reason includes behavior mention +when behavior summary exists (mock mode). +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_recommendation_includes_behavior_phrase(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Recommendation" in report + start = report.find("## Recommendation") + assert start != -1 + block = report[start:] + + # Should include the word 'affects' from the appended phrase + assert "affects" in block + + # And include either the function name or file name from behavior + assert ("authenticate" in block) or ("app/auth.py" in block) diff --git a/libs/openant-core/tests/patch/test_behavior_suggested_tests.py b/libs/openant-core/tests/patch/test_behavior_suggested_tests.py new file mode 100644 index 00000000..26b436bd --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_suggested_tests.py @@ -0,0 +1,33 @@ +""" +Integration test: ensure Suggested Tests block contains behavior-derived skeletons +when a behavior summary exists (mock LLM mode). +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_pipeline_suggested_tests_include_behavior(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Suggested Tests" in report + start = report.find("## Suggested Tests") + assert start != -1 + after = report.find("## Confidence score", start) + block = report[start:after if after != -1 else start + 1000] + + # Suggested Tests no longer inlines pytest skeleton code (presentation + # shortening) — a behavior-derived suggestion is now identified by its + # test name and reason instead of the removed "# Behavior-focused + # validation" code-body marker. + assert "Based on finding:" in block + + # At least one behavior-derived test name present + assert "test_" in block diff --git a/libs/openant-core/tests/patch/test_behavior_summary.py b/libs/openant-core/tests/patch/test_behavior_summary.py new file mode 100644 index 00000000..7ecdf7b3 --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_summary.py @@ -0,0 +1,74 @@ +from pathlib import Path +import sys + + +from utilities.autopatcher.behavior_summary import BehaviorAnalyzer +from utilities.autopatcher.pipeline import TargetRepoContext + + +def write_file(p: Path, text: str): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8") + + +def test_behavior_auth_case(tmp_path): + repo = tmp_path / "repo_auth" + repo.mkdir() + auth = repo / "app" / "auth.py" + write_file(auth, "def authenticate(username, password):\n return db.query(username)\n") + + diff = """+++ b/app/auth.py +@@ -1,1 +1,3 @@ ++def authenticate(username, password): ++ return db.query(username) +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "authenticate" + assert r["file"] == "app/auth.py" + assert "authentication" in r["summary"] + assert any("login" in b for b in r["primary_behaviors"]) + + +def test_behavior_db_case(tmp_path): + repo = tmp_path / "repo_db" + repo.mkdir() + q = repo / "db" / "queries.py" + write_file(q, "def update_user(u):\n cursor.execute(\"UPDATE...\")\n") + + diff = """+++ b/db/queries.py +@@ -1,1 +1,3 @@ ++def update_user(u): ++ cursor.execute("UPDATE users SET ...") +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "update_user" + assert r["file"] == "db/queries.py" + assert "database" in r["summary"] or "database" in " ".join(r["primary_behaviors"]) + + +def test_behavior_fallback(tmp_path): + repo = tmp_path / "repo_fallback" + repo.mkdir() + f = repo / "utils" / "misc.py" + write_file(f, "# no function here\npass\n") + + diff = """+++ b/utils/misc.py +@@ -1 +1 @@ ++some minor change +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "" + assert r["file"] == "utils/misc.py" + assert isinstance(r["primary_behaviors"], list) diff --git a/libs/openant-core/tests/patch/test_behavior_validation_plan.py b/libs/openant-core/tests/patch/test_behavior_validation_plan.py new file mode 100644 index 00000000..9efe298f --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_validation_plan.py @@ -0,0 +1,42 @@ +""" +Test that Validation Actions includes a behavior-derived validation action +when a behavior summary is present (mock LLM mode). + +Note: this used to check a separate "Validation Plan" section further down +the report. That section was removed as a terminology/navigation cleanup — +it repeated the same (already capped-at-3) items shown in "Validation +Actions" near the top, with only a "Reason:" line added; "Validation +Actions" now includes Reason too, so nothing unique was lost. +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_validation_plan_includes_behavior(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Validation Actions" in report + start = report.find("## Validation Actions") + assert start != -1 + # Patch Hygiene now precedes Validation Actions (promoted next to the + # diff) — Review Results is the next heading that follows it. + after = report.find("## Review Results", start) + block = report[start:after if after != -1 else start + 800] + + # Expect the behavior-driven validation action to be present + assert "Validate behavior" in block + + # Behavior summary sentence should be present (BehaviorAnalyzer uses + # "This patch likely affects ...") — now carried via the Reason field. + assert "This patch likely affects" in block + + # At least one primary behavior item should appear (non-brittle check) + assert any(x in block for x in ("valid login", "invalid login", "valid input acceptance", "query correctness", "happy-path response")) diff --git a/libs/openant-core/tests/patch/test_diff_hunk_repair.py b/libs/openant-core/tests/patch/test_diff_hunk_repair.py new file mode 100644 index 00000000..4953c526 --- /dev/null +++ b/libs/openant-core/tests/patch/test_diff_hunk_repair.py @@ -0,0 +1,487 @@ +"""Tests for diff_hunk_repair.repair_hunk_headers. + +Covers: + - Minimist-exact corrupt header regression (primary case) + - Already-correct patch is returned unchanged (no-op guarantee) + - Multi-hunk with net-positive first hunk: offset propagation + - Multi-hunk where both count AND offset are wrong (cascaded error) + - Hunk suffix text preserved verbatim + - No-newline marker excluded from counts + - Count-omitted single-line form normalized to explicit count + - Multi-file: file_delta resets between files + - Non-diff / empty input passthrough + - New-file hunk (--- /dev/null) handled + - RepairResult metadata reflects actual changes +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.diff_hunk_repair import repair_hunk_headers, RepairResult + + +# --------------------------------------------------------------------------- +# Fixtures / shared patch strings +# --------------------------------------------------------------------------- + +# Exact minimist patch from the evaluation run — corrupt hunk headers. +# Hunk 1: claims old=7/new=9 (net +2). Actual body: old=8/new=8 (net 0). +# Hunk 2: claims old_start=79/new_start=81. Correct: 79/79 (delta carried wrong). +_MINIMIST_CORRUPT = """\ +--- a/index.js ++++ b/index.js +@@ -69,7 +69,9 @@ + function setKey (obj, keys, value) { + var o = obj; + for (var i = 0; i < keys.length-1; i++) { + var key = keys[i]; +- if (key === '__proto__') return; ++ if (key === '__proto__' || key === 'constructor' || key === 'prototype') return; + if (o[key] === undefined) o[key] = {}; + if (o[key] === Object.prototype || o[key] === Number.prototype + || o[key] === String.prototype) o[key] = {}; +@@ -79,7 +81,7 @@ + + var key = keys[keys.length - 1]; +- if (key === '__proto__') return; ++ if (key === '__proto__' || key === 'constructor' || key === 'prototype') return; + if (o === Object.prototype || o === Number.prototype + || o === String.prototype) o = {}; + if (o === Array.prototype) o = []; +""" + +# A correctly formed patch — headers must survive unchanged. +# Hunk adds 1 line (net +1): old=4 context+1 removed=5, new=4 context+2 added=6. +_CORRECT_PATCH = """\ +--- a/example.py ++++ b/example.py +@@ -10,5 +10,6 @@ + def foo(): + a = 1 + b = 2 +- return a ++ c = a + b ++ return c + +""" + + +# --------------------------------------------------------------------------- +# Primary regression: minimist corrupt headers +# --------------------------------------------------------------------------- + +class TestMinimistCorruptHeaders: + def test_first_hunk_old_count_corrected(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -69,8 +69,8 @@"), ( + f"Expected @@ -69,8 +69,8 @@, got: {hunk_lines[0]}" + ) + + def test_second_hunk_new_start_corrected(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + # file_delta after hunk1 = 0 (net zero), so new_start = 79 + 0 = 79 + assert hunk_lines[1].startswith("@@ -79,6 +79,6 @@"), ( + f"Expected @@ -79,6 +79,6 @@, got: {hunk_lines[1]}" + ) + + def test_body_content_unchanged(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + orig_body = [l for l in _MINIMIST_CORRUPT.splitlines() if not l.startswith("@@")] + repr_body = [l for l in repaired.splitlines() if not l.startswith("@@")] + assert orig_body == repr_body + + def test_metadata_reports_two_hunks_rewritten(self): + _, meta = repair_hunk_headers(_MINIMIST_CORRUPT) + assert meta.normalization_applied is True + assert meta.hunks_rewritten == 2 + assert meta.files_rewritten == 1 + + +# --------------------------------------------------------------------------- +# No-op guarantee: already-correct patch must be returned byte-for-byte +# --------------------------------------------------------------------------- + +class TestNoOpGuarantee: + def test_correct_patch_is_unchanged(self): + repaired, meta = repair_hunk_headers(_CORRECT_PATCH) + assert repaired == _CORRECT_PATCH + + def test_correct_patch_metadata_zero(self): + _, meta = repair_hunk_headers(_CORRECT_PATCH) + assert meta.normalization_applied is False + assert meta.hunks_rewritten == 0 + assert meta.files_rewritten == 0 + + +# --------------------------------------------------------------------------- +# Multi-hunk: net-positive first hunk propagates offset to second hunk +# --------------------------------------------------------------------------- + +class TestOffsetPropagation: + def test_net_positive_first_hunk_shifts_second_hunk_new_start(self): + # Hunk 1 adds 2 lines (net +2). Hunk 2's new_start must shift by +2. + patch = ( + "--- a/foo.py\n" + "+++ b/foo.py\n" + "@@ -1,3 +1,5 @@\n" # correct: old=3, new=5 (net +2) + " line1\n" + "+added1\n" + "+added2\n" + " line2\n" + " line3\n" + "@@ -10,3 +12,3 @@\n" # correct: old_start=10, new_start=12 (+2 from prior) + " lineA\n" + " lineB\n" + " lineC\n" + ) + repaired, meta = repair_hunk_headers(patch) + assert meta.normalization_applied is False, ( + "Patch with correct headers must not be modified" + ) + + def test_wrong_second_hunk_offset_is_corrected(self): + # Hunk 1 is correct (net +2). Hunk 2 has wrong new_start (still +10 not +12). + patch = ( + "--- a/foo.py\n" + "+++ b/foo.py\n" + "@@ -1,3 +1,5 @@\n" + " line1\n" + "+added1\n" + "+added2\n" + " line2\n" + " line3\n" + "@@ -10,3 +10,3 @@\n" # wrong: new_start should be 12 (10+2) + " lineA\n" + " lineB\n" + " lineC\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[1].startswith("@@ -10,3 +12,3 @@"), ( + f"Expected @@ -10,3 +12,3 @@, got: {hunk_lines[1]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Hunk suffix text preserved verbatim +# --------------------------------------------------------------------------- + +class TestSuffixPreserved: + def test_function_name_suffix_preserved(self): + patch = ( + "--- a/index.js\n" + "+++ b/index.js\n" + "@@ -69,7 +69,9 @@ function setKey\n" + " function setKey (obj, keys, value) {\n" + " var o = obj;\n" + " for (var i = 0; i < keys.length-1; i++) {\n" + " var key = keys[i];\n" + "- if (key === '__proto__') return;\n" + "+ if (key === '__proto__' || key === 'constructor') return;\n" + " if (o[key] === undefined) o[key] = {};\n" + " if (o[key] === Object.prototype || o[key] === Number.prototype\n" + " || o[key] === String.prototype) o[key] = {};\n" + ) + repaired, _ = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].endswith("@@ function setKey"), ( + f"Suffix not preserved: {hunk_lines[0]!r}" + ) + + +# --------------------------------------------------------------------------- +# No-newline marker excluded from counts +# --------------------------------------------------------------------------- + +class TestNoNewlineMarker: + def test_no_newline_marker_not_counted(self): + # 1 context + 1 removed + 1 added + marker = old=2, new=2 + patch = ( + "--- a/file.py\n" + "+++ b/file.py\n" + "@@ -5,99 +5,99 @@\n" # intentionally wrong counts + " context\n" + "-old line\n" + "+new line\n" + "\\ No newline at end of file\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -5,2 +5,2 @@"), ( + f"Got: {hunk_lines[0]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Count-omitted single-line form: @@ -5 +5 @@ → @@ -5,1 +5,1 @@ +# --------------------------------------------------------------------------- + +class TestCountOmittedForm: + def test_omitted_count_normalized_to_explicit(self): + patch = ( + "--- a/x.py\n" + "+++ b/x.py\n" + "@@ -5 +5 @@\n" # count omitted — means 1 + "-old\n" + "+new\n" + ) + repaired, _ = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -5,1 +5,1 @@"), ( + f"Got: {hunk_lines[0]}" + ) + + +# --------------------------------------------------------------------------- +# Multi-file: file_delta resets between files +# --------------------------------------------------------------------------- + +class TestMultiFileDeltaReset: + def test_delta_does_not_leak_across_files(self): + # File 1: net +1. File 2's hunk new_start must NOT be shifted by +1. + patch = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,2 +1,3 @@\n" # correct: old=2, new=3, net+1 + " context\n" + "-removed\n" + "+added1\n" + "+added2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,3 +10,3 @@\n" # correct: new_start=10, not 11 + " a\n" + " b\n" + " c\n" + ) + repaired, meta = repair_hunk_headers(patch) + assert meta.normalization_applied is False, ( + "Correct headers across two files must not be modified" + ) + + def test_wrong_count_in_second_file_corrected_without_delta_from_first(self): + # File 1: net +1 (correct). File 2: wrong counts, but no delta from file 1. + patch = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,2 +1,3 @@\n" + " context\n" + "-removed\n" + "+added1\n" + "+added2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,99 +10,99 @@\n" # wrong counts; no delta from file1 + " a\n" + " b\n" + " c\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[1].startswith("@@ -10,3 +10,3 @@"), ( + f"Got: {hunk_lines[1]}" + ) + assert meta.hunks_rewritten == 1 + assert meta.files_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Non-diff / empty input passthrough +# --------------------------------------------------------------------------- + +class TestPassthrough: + def test_empty_string(self): + repaired, meta = repair_hunk_headers("") + assert repaired == "" + assert meta.normalization_applied is False + + def test_whitespace_only(self): + repaired, meta = repair_hunk_headers(" \n \n") + assert meta.normalization_applied is False + + def test_non_diff_content(self): + text = "# just a comment\nnot a diff at all\n" + repaired, meta = repair_hunk_headers(text) + assert repaired == text + assert meta.normalization_applied is False + + def test_never_raises_on_garbage(self): + for garbage in ["@@@@", "@@ broken @@\n+line", "---\n+++\n@@bad"]: + repaired, meta = repair_hunk_headers(garbage) + assert isinstance(repaired, str) + assert isinstance(meta, RepairResult) + + +# --------------------------------------------------------------------------- +# New-file hunk (--- /dev/null) +# --------------------------------------------------------------------------- + +class TestNewFileHunk: + def test_new_file_hunk_counts_only_added_lines(self): + patch = ( + "--- /dev/null\n" + "+++ b/newfile.py\n" + "@@ -0,0 +1,99 @@\n" # wrong new count + "+line1\n" + "+line2\n" + "+line3\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + # old=0 (no context, no removed), new=3 + assert hunk_lines[0].startswith("@@ -0,0 +1,3 @@"), ( + f"Got: {hunk_lines[0]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Fenced patches: ``` must not be absorbed into hunk body +# --------------------------------------------------------------------------- + +# Regression patch: LLM-generated fenced diff with inverted +/- order and +# wrong hunk header counts — the exact pattern seen in the minimist v2 eval. +# Before the fix, the closing ``` was absorbed into hunk-2's body, inflating +# its count by 1 so that the "repaired" header still said @@ -81,8 +81,8 @@ +# (matching the over-counted body), causing git apply to fail with +# "corrupt patch at line 21" even after repair reported "2 hunks fixed". +_FENCED_MINIMIST_V2 = """\ +```diff +--- a/index.js ++++ b/index.js +@@ -69,6 +69,8 @@ + function setKey (obj, keys, value) { + var o = obj; + for (var i = 0; i < keys.length-1; i++) { + var key = keys[i]; ++ if (key === 'constructor' || key === '__proto__') return; +- if (key === '__proto__') return; + if (o[key] === undefined) o[key] = {}; + if (o[key] === Object.prototype || o[key] === Number.prototype +@@ -81,6 +81,6 @@ + } + + var key = keys[keys.length - 1]; ++ if (key === 'constructor' || key === '__proto__') return; +- if (key === '__proto__') return; + if (o === Object.prototype || o === Number.prototype + || o === String.prototype) o = {}; + if (o === Array.prototype) o = []; +```""" + +def _make_minimist_fixture(tmp_path: Path) -> Path: + """Init a minimal git repo whose index.js matches, line-for-line, what + _FENCED_MINIMIST_V2's hunk headers target (lines 69-75 and 81-87) — so + the repaired patch can be checked against a real `git apply` without + depending on an externally-managed minimist checkout. + """ + lines = [f"// filler line {i}" for i in range(1, 69)] + lines += [ + " function setKey (obj, keys, value) {", + " var o = obj;", + " for (var i = 0; i < keys.length-1; i++) {", + " var key = keys[i];", + " if (key === '__proto__') return;", + " if (o[key] === undefined) o[key] = {};", + " if (o[key] === Object.prototype || o[key] === Number.prototype", + ] + lines += [f"// filler line {i}" for i in range(76, 81)] + lines += [ + " }", + "", + " var key = keys[keys.length - 1];", + " if (key === '__proto__') return;", + " if (o === Object.prototype || o === Number.prototype", + " || o === String.prototype) o = {};", + " if (o === Array.prototype) o = [];", + ] + index_js = "\n".join(lines) + "\n" + + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "index.js").write_text(index_js, encoding="utf-8") + subprocess.run(["git", "add", "index.js"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=tmp_path, capture_output=True, check=True, + ) + return tmp_path + + +class TestFencedPatch: + def test_both_hunk_headers_corrected(self): + repaired, meta = repair_hunk_headers(_FENCED_MINIMIST_V2) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -69,7 +69,7 @@"), hunk_lines[0] + assert hunk_lines[1].startswith("@@ -81,7 +81,7 @@"), hunk_lines[1] + + def test_metadata_two_hunks_rewritten(self): + _, meta = repair_hunk_headers(_FENCED_MINIMIST_V2) + assert meta.hunks_rewritten == 2 + assert meta.files_rewritten == 1 + + def test_closing_fence_not_counted_as_context(self): + """``` must not inflate hunk-2 counts — old bug produced @@ -81,8 +81,8 @@.""" + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert not hunk_lines[1].startswith("@@ -81,8"), ( + f"Closing fence was counted as context: {hunk_lines[1]}" + ) + + def test_fences_preserved_in_output(self): + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + lines = repaired.splitlines() + assert lines[0].startswith("```"), "Opening fence should be preserved" + assert lines[-1].strip() == "```", "Closing fence should be preserved" + + def test_correct_fenced_patch_is_noop(self): + """A fenced patch with already-correct headers must not be modified.""" + fenced_correct = ( + "```diff\n" + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -10,5 +10,6 @@\n" + " def foo():\n" + " a = 1\n" + " b = 2\n" + "- return a\n" + "+ c = a + b\n" + "+ return c\n" + "\n" + "```" + ) + repaired, meta = repair_hunk_headers(fenced_correct) + assert meta.normalization_applied is False + assert repaired == fenced_correct + + def test_repaired_patch_applies_cleanly(self, tmp_path): + """End-to-end: repaired fenced patch must pass git apply --check.""" + import re as _re + repo = _make_minimist_fixture(tmp_path) + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + # Strip fences the same way patch_applicability does + lines = repaired.splitlines() + if lines and _re.match(r"^```", lines[0]): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + stripped = "\n".join(lines).strip() + result = subprocess.run( + ["git", "apply", "--check", "--whitespace=nowarn", "-"], + input=stripped + "\n", + cwd=str(repo), + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"git apply failed: {result.stderr.strip()}" diff --git a/libs/openant-core/tests/patch/test_diff_parsing.py b/libs/openant-core/tests/patch/test_diff_parsing.py new file mode 100644 index 00000000..88439032 --- /dev/null +++ b/libs/openant-core/tests/patch/test_diff_parsing.py @@ -0,0 +1,150 @@ +from utilities.autopatcher.diff_parsing import DiffHunk, parse_diff + + +def test_empty_input_returns_no_files_or_hunks(): + changed_files, file_hunks = parse_diff("") + assert changed_files == [] + assert file_hunks == {} + + +def test_non_diff_input_returns_no_files_or_hunks(): + changed_files, file_hunks = parse_diff("hello\nworld\nthis is not a diff\n") + assert changed_files == [] + assert file_hunks == {} + + +def test_one_changed_file_one_hunk(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,3 @@\n" + "+def foo():\n" + "+ pass\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py"] + assert list(file_hunks.keys()) == ["a.py"] + hunks = file_hunks["a.py"] + assert len(hunks) == 1 + assert hunks[0] == DiffHunk( + new_start=1, new_count=3, lines=["+def foo():", "+ pass"] + ) + + +def test_multiple_changed_files(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+a\n" + "--- a/b.py\n" + "+++ b/b.py\n" + "@@ -1,1 +1,1 @@\n" + "+b\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py", "b.py"] + assert file_hunks["a.py"][0].lines == ["+a"] + assert file_hunks["b.py"][0].lines == ["+b"] + + +def test_multiple_hunks_in_one_file(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "-old\n" + "+new\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py"] + hunks = file_hunks["a.py"] + assert len(hunks) == 2 + assert hunks[0].new_start == 1 and hunks[0].new_count == 2 + assert hunks[1].new_start == 7 and hunks[1].new_count == 2 + + +def test_hunk_header_metadata_with_explicit_count(): + diff = "+++ b/a.py\n@@ -1,5 +10,20 @@\n context\n" + _, file_hunks = parse_diff(diff) + hunk = file_hunks["a.py"][0] + assert hunk.new_start == 10 + assert hunk.new_count == 20 + + +def test_hunk_header_metadata_defaults_count_to_one_when_omitted(): + # "@@ -1 +1 @@" form: no ",N" count suffix on the new-file side. + diff = "+++ b/a.py\n@@ -1 +1 @@\n+x\n" + _, file_hunks = parse_diff(diff) + hunk = file_hunks["a.py"][0] + assert hunk.new_start == 1 + assert hunk.new_count == 1 + + +def test_added_removed_and_context_lines_are_preserved_with_markers(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,3 +1,3 @@\n" + " context line\n" + "-removed line\n" + "+added line\n" + ) + _, file_hunks = parse_diff(diff) + assert file_hunks["a.py"][0].lines == [ + " context line", + "-removed line", + "+added line", + ] + + +def test_lines_without_a_recognized_marker_are_dropped(): + # A line inside a hunk that isn't prefixed with ' ', '+', or '-' (e.g. a + # "\ No newline at end of file" marker) is not appended to hunk.lines. + diff = ( + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+added\n" + "\\ No newline at end of file\n" + ) + _, file_hunks = parse_diff(diff) + assert file_hunks["a.py"][0].lines == ["+added"] + + +def test_minus_a_and_plus_b_paths_used_correctly(): + # Only "+++ b/..." sets the tracked filename; "--- a/..." is recognized + # only as a hunk-flush boundary, and its own path is never used. + diff = ( + "--- a/old_name.py\n" + "+++ b/new_name.py\n" + "@@ -1,1 +1,1 @@\n" + "+x\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["new_name.py"] + assert "old_name.py" not in file_hunks + + +def test_dash_a_line_flushes_pending_hunk_before_next_file(): + # A trailing hunk for the first file must be flushed when the second + # file's "--- a/" line appears, not silently merged into the next file. + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+from a\n" + "--- a/b.py\n" + "+++ b/b.py\n" + "@@ -1,1 +1,1 @@\n" + "+from b\n" + ) + _, file_hunks = parse_diff(diff) + assert len(file_hunks["a.py"]) == 1 + assert file_hunks["a.py"][0].lines == ["+from a"] + assert len(file_hunks["b.py"]) == 1 + assert file_hunks["b.py"][0].lines == ["+from b"] diff --git a/libs/openant-core/tests/patch/test_finding_calibration.py b/libs/openant-core/tests/patch/test_finding_calibration.py new file mode 100644 index 00000000..aabedfb2 --- /dev/null +++ b/libs/openant-core/tests/patch/test_finding_calibration.py @@ -0,0 +1,106 @@ +"""Unit tests for the finding calibration stage (evidence-quality pass). + +Covers the two things that must be robust against imperfect LLM output: +_parse_response's fallback behavior (never drop a finding, never invent an +invalid group), and calibrate_findings's LLM-call contract (empty input never +calls the LLM; mock mode returns something parseable). +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +from utilities.autopatcher.finding_calibration import _parse_response, calibrate_findings + + +class TestParseResponse: + def test_well_formed_response_parses_in_order(self): + resp = ( + "1. Group: Observed\n" + " Reworded: The constructor normalizes casing via h.lower().\n\n" + "2. Group: Hypothesis\n" + " Reworded: Same-origin redirects may also strip Cookie if stripping is not scoped.\n" + ) + findings = ["finding one", "finding two"] + result = _parse_response(resp, findings) + assert len(result) == 2 + assert result[0] == { + "original": "finding one", "group": "observed", + "reworded": "The constructor normalizes casing via h.lower().", + } + assert result[1]["group"] == "hypothesis" + + def test_missing_block_falls_back_to_original_as_hypothesis(self): + """Fewer blocks than findings must not drop the uncovered finding.""" + resp = "1. Group: Observed\n Reworded: Reworded first finding.\n" + findings = ["first finding", "second finding with no block"] + result = _parse_response(resp, findings) + assert len(result) == 2 + assert result[1] == { + "original": "second finding with no block", + "group": "hypothesis", + "reworded": "second finding with no block", + } + + def test_invalid_group_name_falls_back_to_hypothesis(self): + resp = "1. Group: Definitely\n Reworded: Some reworded text.\n" + result = _parse_response(resp, ["original text"]) + assert result[0]["group"] == "hypothesis" + assert result[0]["reworded"] == "original text" + + def test_empty_reworded_falls_back_to_original(self): + resp = "1. Group: Observed\n Reworded: \n" + result = _parse_response(resp, ["original text"]) + assert result[0]["reworded"] == "original text" + assert result[0]["group"] == "hypothesis" + + def test_completely_unparseable_response_falls_back_for_every_finding(self): + result = _parse_response("not a structured response at all", ["a", "b", "c"]) + assert len(result) == 3 + assert all(r["group"] == "hypothesis" for r in result) + assert [r["reworded"] for r in result] == ["a", "b", "c"] + + def test_empty_findings_list_returns_empty(self): + assert _parse_response("anything", []) == [] + + def test_reworded_text_collapses_internal_whitespace(self): + resp = "1. Group: Hardening\n Reworded: Line one\n continues on line two.\n" + result = _parse_response(resp, ["x"]) + assert "\n" not in result[0]["reworded"] + + def test_group_name_case_insensitive(self): + resp = "1. Group: HARDENING\n Reworded: Some text.\n" + result = _parse_response(resp, ["x"]) + assert result[0]["group"] == "hardening" + + +class TestCalibrateFindings: + def test_empty_findings_returns_empty_without_calling_llm(self): + llm = mock.MagicMock() + result = calibrate_findings("vuln text", "patch", [], llm, code_context="ctx") + assert result == [] + llm.complete.assert_not_called() + + def test_calls_llm_with_stage_label_and_parses_result(self): + llm = mock.MagicMock() + llm.complete.return_value = "1. Group: Observed\n Reworded: Reworded finding.\n" + result = calibrate_findings("vuln text", "patch", ["a finding"], llm, code_context="ctx") + _, kwargs = llm.complete.call_args + assert kwargs.get("stage") == "finding_calibration" + assert result == [{"original": "a finding", "group": "observed", "reworded": "Reworded finding."}] + + def test_user_message_includes_code_context_and_findings(self): + llm = mock.MagicMock() + llm.complete.return_value = "1. Group: Hardening\n Reworded: x\n" + calibrate_findings("VULN_MARKER", "PATCH_MARKER", ["FINDING_MARKER"], llm, code_context="CONTEXT_MARKER") + args, kwargs = llm.complete.call_args + user_message = args[1] if len(args) > 1 else kwargs.get("user_message") + assert "VULN_MARKER" in user_message + assert "PATCH_MARKER" in user_message + assert "FINDING_MARKER" in user_message + assert "CONTEXT_MARKER" in user_message diff --git a/libs/openant-core/tests/patch/test_impact_surface.py b/libs/openant-core/tests/patch/test_impact_surface.py new file mode 100644 index 00000000..671fd9a4 --- /dev/null +++ b/libs/openant-core/tests/patch/test_impact_surface.py @@ -0,0 +1,490 @@ +import os +from pathlib import Path +import json + +from utilities.autopatcher.impact_surface import LightweightImpactAnalyzer +from utilities.autopatcher.pipeline import enhance_findings_with_impact, TargetRepoContext + + +def write_file(p: Path, text: str): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8") + + +def make_diff_for(path: str) -> str: + # minimal unified diff header for file + return f"+++ b/{path}\n@@ -1,1 +1,3 @@\n+def placeholder():\n" + + +def test_low_impact_same_file(tmp_path): + # repo with a.py only; changed symbol used only in same file + repo = tmp_path / "repo" + repo.mkdir() + a = repo / "a.py" + write_file(a, "def foo(x):\n return x\n\nprint(foo(1))\n") + + diff = """+++ b/a.py +@@ -1,3 +1,5 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "low" + assert isinstance(report.impact_summary, str) + + +def test_medium_impact_two_files(tmp_path): + repo = tmp_path / "repo2" + repo.mkdir() + a = repo / "a.py" + b = repo / "b.py" + c = repo / "c.py" + write_file(a, "def foo(x):\n return x\n") + write_file(b, "from a import foo\nprint(foo(2))\n") + write_file(c, "x = foo(3)\n") + + diff = """+++ b/a.py +@@ -1,1 +1,3 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "medium" + assert len(report.affected_files) == 2 + + +def test_high_impact_three_files_or_entrypoint(tmp_path): + repo = tmp_path / "repo3" + repo.mkdir() + a = repo / "a.py" + b = repo / "b.py" + c = repo / "c.py" + d = repo / "api" / "routes.py" + write_file(a, "def foo(x):\n return x\n") + write_file(b, "print(foo(2))\n") + write_file(c, "print(foo(3))\n") + write_file(d, "from a import foo\n# route uses foo\n") + + diff = """+++ b/a.py +@@ -1,1 +1,3 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + # either >=3 external files or entrypoint hit should mark high + assert report.impact_level == "high" + + +def test_enhance_findings_with_impact(): + challenger = {} + # simulate impact report dicts + high = {"impact_level": "high"} + med = {"impact_level": "medium"} + low = {"impact_level": "low"} + + # high + c1 = dict(challenger) + enhance_findings_with_impact(c1, high) + assert "impact_annotations" in c1 and any("propagate" in s for s in c1["impact_annotations"]) + + # medium + c2 = dict(challenger) + enhance_findings_with_impact(c2, med) + assert "impact_annotations" in c2 and len(c2["impact_annotations"]) > 0 + + # low + c3 = dict(challenger) + enhance_findings_with_impact(c3, low) + assert "impact_annotations" not in c3 + + +def test_constant_hunk_does_not_extract_init(tmp_path): + """Regression: constant-only hunk must not fall back to __init__ and produce HIGH impact. + + Reproduces the urllib3 cookie-redirect case: a frozenset constant is + modified, no def appears in the hunk, and several other files define + __init__. Without the fix the upward scan grabs __init__, the search + matches every class file, and impact is incorrectly HIGH. + """ + repo = tmp_path / "repo_constant" + repo.mkdir() + + # The patched file: class with __init__ above a class-level constant. + write_file( + repo / "retry.py", + "\n".join([ + "class Retry:", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + ]), + ) + + # Five other files that each define __init__ — would inflate count if searched. + for i in range(5): + write_file( + repo / f"module{i}.py", + f"class Foo{i}:\n def __init__(self):\n pass\n", + ) + + # Diff touches only the constant; no def in the hunk. + diff = ( + "--- a/retry.py\n" + "+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert "__init__" not in report.changed_symbols, ( + "constant-only hunk must not fall back to __init__" + ) + assert report.impact_level == "low", ( + f"expected low impact for constant change, got {report.impact_level!r}; " + f"affected_files={report.affected_files}" + ) + + +def test_sensitive_auth_bumps_low_to_medium(tmp_path): + # changed auth file with no external usages should be bumped to medium + repo = tmp_path / "repo_sensitive" + (repo / "app").mkdir(parents=True) + auth = repo / "app" / "auth.py" + write_file(auth, "def authenticate(user, pwd):\n return True\n") + + diff = """+++ b/app/auth.py +@@ -1,1 +1,3 @@ ++def authenticate(user, pwd): ++ return True +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "medium" + assert "authenticate" in (report.changed_symbols or []) + + +def test_non_python_repo_reports_not_applicable_not_low(tmp_path): + # A C change with zero extractable Python symbols must not be scored + # "low impact" (which reads as a reassuring, verified-clean finding). + # It must be explicitly "not_applicable" instead. + repo = tmp_path / "repo_c" + repo.mkdir() + write_file(repo / "lib" / "http.c", "int Curl_follow(void) { return 0; }\n") + + diff = """+++ b/lib/http.c +@@ -1,1 +1,3 @@ ++int Curl_follow(void) { ++ return 0; ++} +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx, repo_language="c") + + assert report.impact_level == "not_applicable" + assert report.impact_level != "low" + assert report.changed_symbols == [] + assert report.usage_matches == [] + assert report.affected_files == [] + # Diff parsing itself is language-agnostic and must still work. + assert report.changed_files == ["lib/http.c"] + assert "Not Applicable" in report.impact_summary + + +def test_python_repo_impact_unchanged_by_default(tmp_path): + # Default repo_language="python" must reproduce prior behavior exactly. + repo = tmp_path / "repo_default" + repo.mkdir() + a = repo / "a.py" + write_file(a, "def foo(x):\n return x\n\nprint(foo(1))\n") + + diff = """+++ b/a.py +@@ -1,3 +1,5 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "low" + + +# --------------------------------------------------------------------------- +# Symbol-resolution robustness (the urllib3 hunk-drift incident and its +# resilience requirements: diff line offsets, nearby comments, whitespace-only +# edits, insertion/deletion before the target, equivalent patch formatting). +# --------------------------------------------------------------------------- + +_RETRY_FILE = "\n".join([ + "class Retry:", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + "", + " def new(self, **kw):", + " return Retry(**kw)", +]) + +# Other files that each define __init__ — if the analyzer ever falls back to +# that name, usage search inflates across all of them, exactly reproducing +# the false-HIGH-impact incident. +_OTHER_INIT_FILES = {f"module{i}.py": f"class Foo{i}:\n def __init__(self):\n pass\n" for i in range(5)} + + +def _write_retry_repo(tmp_path, retry_text=_RETRY_FILE, extra_files=None): + repo = tmp_path / "repo" + repo.mkdir() + write_file(repo / "retry.py", retry_text) + for name, content in {**_OTHER_INIT_FILES, **(extra_files or {})}.items(): + write_file(repo / name, content) + return repo + + +class TestShiftedHunkHeaders: + """The core incident: a hunk header claiming the wrong line number for + byte-identical content must not change which symbol is resolved.""" + + def test_correct_header_resolves_constant(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert report.impact_level == "low" + + def test_wildly_shifted_header_still_resolves_the_same_constant(self, tmp_path): + """Same file, same content change, header claims line 50 instead of + line 2 — the actual incident, reproduced deterministically.""" + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -50,2 +50,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert "__init__" not in report.changed_symbols + assert report.impact_level == "low" + + def test_shifted_header_on_a_function_change_still_resolves_correctly(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -999,2 +999,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + def test_two_runs_with_different_headers_same_content_agree(self, tmp_path): + """Direct stability check: semantically identical patches, differing + only in hunk header line number, must produce the identical report.""" + repo = _write_retry_repo(tmp_path) + body = ( + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + diff_a = f"--- a/retry.py\n+++ b/retry.py\n@@ -189,2 +189,2 @@\n{body}" + diff_b = f"--- a/retry.py\n+++ b/retry.py\n@@ -196,2 +196,2 @@\n{body}" + + report_a = LightweightImpactAnalyzer().analyze(diff_a, repo_context=TargetRepoContext(repo)) + report_b = LightweightImpactAnalyzer().analyze(diff_b, repo_context=TargetRepoContext(repo)) + + assert report_a.changed_symbols == report_b.changed_symbols + assert report_a.impact_level == report_b.impact_level + assert report_a.affected_files == report_b.affected_files + + +class TestWhitespaceOnlyChanges: + def test_reindentation_only_produces_no_symbol_and_low_impact(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == [] + assert report.impact_level == "low" + + def test_trailing_whitespace_only_produces_no_symbol(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,1 +7,1 @@\n" + "- def new(self, **kw):\n" + "+ def new(self, **kw): \n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == [] + + +class TestInsertionBeforeTarget: + def test_lines_inserted_above_target_do_not_break_resolution(self, tmp_path): + """The header's claimed line number is stale (as if computed before + 20 lines were inserted above the target) — content match must still + find the real, shifted location.""" + padded = "\n".join([f"# padding line {i}" for i in range(20)]) + "\n" + _RETRY_FILE + repo = _write_retry_repo(tmp_path, retry_text=padded) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" # stale: correct pre-padding location + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + + +class TestNearbyComments: + def test_misleading_def_in_comment_is_ignored(self, tmp_path): + """A comment that looks like a function definition must never be + mistaken for real code — ast parsing never sees comments at all.""" + text = "\n".join([ + "class Retry:", + " # def __init__(self): pass -- old implementation, removed", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + ]) + repo = _write_retry_repo(tmp_path, retry_text=text) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,3 +1,3 @@\n" + " class Retry:\n" + " # def __init__(self): pass -- old implementation, removed\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert "__init__" not in report.changed_symbols + + +class TestConstantAssignmentChanges: + def test_class_level_constant(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + + def test_module_level_constant(self, tmp_path): + text = "DEFAULT_TIMEOUT = 30\n\ndef connect():\n return DEFAULT_TIMEOUT\n" + repo = _write_retry_repo(tmp_path, retry_text=text) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,1 +1,1 @@\n" + "-DEFAULT_TIMEOUT = 30\n" + "+DEFAULT_TIMEOUT = 60\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_TIMEOUT"] + + +class TestFunctionChanges: + def test_change_inside_method_body_resolves_to_method_name(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + def test_change_to_signature_line_resolves_to_function_name(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,1 +7,1 @@\n" + "- def new(self, **kw):\n" + "+ def new(self, **kw, strict=False):\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + +class TestMultipleHunks: + def test_two_hunks_in_same_file_resolve_two_distinct_symbols(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert set(report.changed_symbols) == {"DEFAULT_REMOVE_HEADERS", "new"} + + def test_hunks_across_two_files_resolve_independently(self, tmp_path): + repo = _write_retry_repo(tmp_path, extra_files={ + "helpers.py": "def build_url():\n return ''\n", + }) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + "--- a/helpers.py\n+++ b/helpers.py\n" + "@@ -1,2 +1,2 @@\n" + "-def build_url():\n" + "- return ''\n" + "+def build_url():\n" + "+ return 'https://'\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert set(report.changed_symbols) == {"DEFAULT_REMOVE_HEADERS", "build_url"} + assert set(report.changed_files) == {"retry.py", "helpers.py"} diff --git a/libs/openant-core/tests/patch/test_investigation_adapters.py b/libs/openant-core/tests/patch/test_investigation_adapters.py new file mode 100644 index 00000000..6c4a3417 --- /dev/null +++ b/libs/openant-core/tests/patch/test_investigation_adapters.py @@ -0,0 +1,52 @@ +"""Characterization tests for the InvestigationCase adapter. + +Ported from the standalone Auto Patcher project's test_investigation_adapters.py, +trimmed to TestCaseFromVulnerabilityText -- the only class covering a +function this package still has (case_from_vulnerability_text). The +GHSA/CVE adapter classes and the evidence-collection classes tested +functionality that was excluded during the merge into OpenAnt (see +utilities/autopatcher/investigation_adapters.py's module docstring). + +These tests prove the one guarantee that matters here: an InvestigationCase +built from a rendered vulnerability_text projects back to byte-identical +text -- core/patch.py's render_vulnerability_markdown() output must reach +the patch engine unchanged. +""" + +from __future__ import annotations + +from pathlib import Path + +EXAMPLE_FILE = Path(__file__).parent / "fixtures" / "examples" / "vulnerability.md" + + +class TestCaseFromVulnerabilityText: + def test_round_trips_example_file_byte_identical(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + original = EXAMPLE_FILE.read_text(encoding="utf-8") + case = case_from_vulnerability_text(original, repo_root=Path("/tmp/repo")) + projection = case.to_context_projection() + + assert projection.vulnerability_text == original + assert projection.repo_root == Path("/tmp/repo") + + def test_repo_root_optional(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + case = case_from_vulnerability_text("# Some vuln\n\nDetails.") + assert case.to_context_projection().repo_root is None + + def test_framing_summary_extracted(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + case = case_from_vulnerability_text("# SQL Injection Vulnerability\n\nDetails.") + assert case.framing.summary == "SQL Injection Vulnerability" + + def test_raw_artifact_preserves_source_type_and_text(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + text = "# Vuln\n\nDetails." + case = case_from_vulnerability_text(text) + assert case.raw_artifact.source_type == "vulnerability_text" + assert case.raw_artifact.raw_text == text diff --git a/libs/openant-core/tests/patch/test_investigation_models.py b/libs/openant-core/tests/patch/test_investigation_models.py new file mode 100644 index 00000000..6db8d006 --- /dev/null +++ b/libs/openant-core/tests/patch/test_investigation_models.py @@ -0,0 +1,80 @@ +"""Unit tests for the canonical Investigation Engine data model (Slice 1).""" + +from __future__ import annotations + +import sys +from pathlib import Path + + + +class TestModelConstruction: + def test_raw_artifact_defaults(self): + from utilities.autopatcher.investigation_models import RawArtifact + a = RawArtifact(source_type="free_text", raw_text="hello") + assert a.source_url is None + assert a.structured_payload is None + + def test_problem_claims_defaults_are_independent(self): + from utilities.autopatcher.investigation_models import ProblemClaims + a = ProblemClaims() + b = ProblemClaims() + a.cwes.append("CWE-89") + assert b.cwes == [] # default_factory must not share state across instances + + def test_hypothesis_defaults(self): + from utilities.autopatcher.investigation_models import Hypothesis + h = Hypothesis(id="h1", statement="SQL injection in authenticate()") + assert h.status == "open" + assert h.rejection_reason is None + assert h.confirming_evidence_needed == [] + + def test_evidence_item_defaults(self): + from utilities.autopatcher.investigation_models import EvidenceItem + e = EvidenceItem(evidence_type="location", source_module="repo_locator", content="app/auth.py") + assert e.supports == [] + assert e.refutes == [] + + def test_context_projection_fields(self): + from utilities.autopatcher.investigation_models import ContextProjection + p = ContextProjection(vulnerability_text="# Vuln", repo_root=Path("/tmp/repo")) + assert p.vulnerability_text == "# Vuln" + assert p.repo_root == Path("/tmp/repo") + + +class TestInvestigationCase: + def _make_case(self): + from utilities.autopatcher.investigation_models import InvestigationCase, ProblemClaims, RawArtifact + return InvestigationCase( + raw_artifact=RawArtifact(source_type="vulnerability_text", raw_text="# Vuln\n"), + framing=ProblemClaims(summary="Vuln"), + rendered_text="# Vuln\n", + repo_root=Path("/tmp/repo"), + ) + + def test_defaults_are_empty(self): + case = self._make_case() + assert case.hypotheses == [] + assert case.evidence == [] + assert case.leading_hypothesis_id is None + assert case.open_questions == [] + + def test_to_context_projection_carries_text_and_repo_root(self): + case = self._make_case() + projection = case.to_context_projection() + assert projection.vulnerability_text == "# Vuln\n" + assert projection.repo_root == Path("/tmp/repo") + + def test_to_context_projection_repo_root_optional(self): + from utilities.autopatcher.investigation_models import InvestigationCase, ProblemClaims, RawArtifact + case = InvestigationCase( + raw_artifact=RawArtifact(source_type="vulnerability_text", raw_text="x"), + framing=ProblemClaims(), + rendered_text="x", + ) + assert case.to_context_projection().repo_root is None + + def test_default_lists_are_independent_across_cases(self): + case_a = self._make_case() + case_b = self._make_case() + case_a.open_questions.append("is this the right package?") + assert case_b.open_questions == [] diff --git a/libs/openant-core/tests/patch/test_language_support.py b/libs/openant-core/tests/patch/test_language_support.py new file mode 100644 index 00000000..9a2e3a7d --- /dev/null +++ b/libs/openant-core/tests/patch/test_language_support.py @@ -0,0 +1,70 @@ +"""Tests for src/language_support.py — dominant-language detection used to +gate Python-only deterministic signals on non-Python repositories.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +from utilities.autopatcher.language_support import detect_language, is_python_repo + + +def write(path: Path, content: str = "") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def test_detects_python_repo(tmp_path: Path): + write(tmp_path / "src" / "foo.py", "def foo():\n pass\n") + write(tmp_path / "tests" / "test_foo.py", "def test_foo():\n pass\n") + + assert detect_language(tmp_path) == "python" + assert is_python_repo(tmp_path) is True + + +def test_detects_c_repo(tmp_path: Path): + write(tmp_path / "lib" / "http.c", "int Curl_follow(void) { return 0; }\n") + write(tmp_path / "lib" / "http.h", "int Curl_follow(void);\n") + + assert detect_language(tmp_path) == "c" + assert is_python_repo(tmp_path) is False + + +def test_detects_javascript_repo(tmp_path: Path): + write(tmp_path / "index.js", "module.exports = function() {};\n") + write(tmp_path / "lib" / "parse.js", "function parse() {}\n") + + assert detect_language(tmp_path) == "javascript" + assert is_python_repo(tmp_path) is False + + +def test_mixed_repo_uses_dominant_extension(tmp_path: Path): + # A handful of stray .py maintenance scripts must not flip a C repo to "python". + write(tmp_path / "lib" / "a.c", "int a(void) { return 0; }\n") + write(tmp_path / "lib" / "b.c", "int b(void) { return 0; }\n") + write(tmp_path / "lib" / "c.c", "int c(void) { return 0; }\n") + write(tmp_path / "scripts" / "release.py", "print('release')\n") + + assert detect_language(tmp_path) == "c" + + +def test_empty_repo_is_unknown(tmp_path: Path): + assert detect_language(tmp_path) == "unknown" + assert is_python_repo(tmp_path) is False + + +def test_missing_repo_root_is_unknown(): + assert detect_language(None) == "unknown" + assert detect_language(Path("/definitely/does/not/exist/xyz")) == "unknown" + + +def test_ignored_dirs_excluded_from_detection(tmp_path: Path): + write(tmp_path / "lib" / "a.c", "int a(void) { return 0; }\n") + write( + tmp_path / ".venv" / "lib" / "python3.14" / "site-packages" / "pkg" / "mod.py", + "x = 1\n", + ) + + assert detect_language(tmp_path) == "c" diff --git a/libs/openant-core/tests/patch/test_llm_client.py b/libs/openant-core/tests/patch/test_llm_client.py new file mode 100644 index 00000000..4875be69 --- /dev/null +++ b/libs/openant-core/tests/patch/test_llm_client.py @@ -0,0 +1,374 @@ +import os +import sys +import types + +import pytest + +# Ensure `src` is on path so imports like `from llm_client import ...` +# work when running tests directly. + +import utilities.autopatcher.llm_client as llm_client +from utilities.autopatcher.llm_client import LLMClient, call_llm, _mock_response +from utilities.autopatcher.llm_config import DEFAULT_MAX_TOKENS + + +# --------------------------------------------------------------------------- +# LLMClient.is_mock — must reflect the active LLM_PROVIDER, not just OPENAI_API_KEY +# --------------------------------------------------------------------------- + +def test_is_mock_false_when_anthropic_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert not client.is_mock + + +def test_is_mock_false_when_openai_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setattr(llm_client, "_cached_provider", None) + client = LLMClient(api_key="sk-fake") + assert not client.is_mock + + +def test_is_mock_true_when_mock_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + client = LLMClient() + assert client.is_mock + + +def test_is_mock_true_when_no_provider_no_key(monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert client.is_mock + + +def test_is_mock_false_when_cached_provider_anthropic(monkeypatch): + # Simulates a session where the user already selected anthropic interactively. + monkeypatch.delenv("LLM_PROVIDER", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", "anthropic") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert not client.is_mock + + +# --------------------------------------------------------------------------- + + +def test_mock_provider_returns_string(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + res = call_llm("test prompt for mock") + assert isinstance(res, str) + assert res + + +def test_anthropic_explicit_provider_raises_on_api_error(monkeypatch): + """LLM_PROVIDER=anthropic + API failure must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + class FakeAnthropic: + def __init__(self, api_key=None): + pass + + class messages: + @staticmethod + def create(model, max_tokens, messages): + raise RuntimeError("simulated anthropic failure") + + fake_mod = types.SimpleNamespace(Anthropic=FakeAnthropic) + monkeypatch.setitem(sys.modules, "anthropic", fake_mod) + + with pytest.raises(RuntimeError, match="Anthropic API call failed"): + call_llm("prompt that triggers anthropic") + + +def test_openai_explicit_provider_raises_on_api_error(monkeypatch): + """LLM_PROVIDER=openai + API failure must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + class FakeOpenAI: + def __init__(self, api_key=None): + pass + + class chat: + class completions: + @staticmethod + def create(model, messages, temperature, max_tokens): + raise RuntimeError("simulated openai failure") + + fake_mod = types.SimpleNamespace(OpenAI=FakeOpenAI) + monkeypatch.setitem(sys.modules, "openai", fake_mod) + + with pytest.raises(RuntimeError, match="OpenAI API call failed"): + call_llm("prompt that triggers openai") + + +def test_anthropic_explicit_provider_raises_on_missing_key(monkeypatch): + """LLM_PROVIDER=anthropic with no key must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("builtins.input", lambda *a, **k: "") + + with pytest.raises(RuntimeError, match="ANTHROPIC_API_KEY is not set"): + call_llm("prompt with no anthropic key") + + +def test_openai_explicit_provider_raises_on_missing_key(monkeypatch): + """LLM_PROVIDER=openai with no key must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("builtins.input", lambda *a, **k: "") + + with pytest.raises(RuntimeError, match="OPENAI_API_KEY is not set"): + call_llm("prompt with no openai key") + + +def test_non_interactive_defaults_to_mock(monkeypatch): + # Ensure no provider env var + monkeypatch.delenv("LLM_PROVIDER", raising=False) + + # Ensure input() would raise if called + monkeypatch.setattr("builtins.input", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("input called"))) + + # Simulate non-interactive stdin + class FakeStdin: + def isatty(self): + return False + + monkeypatch.setattr("sys.stdin", FakeStdin()) + + res = call_llm("no interactive") + assert res == _mock_response("no interactive") + + +# --------------------------------------------------------------------------- +# Configurable max_tokens (LLM_MAX_TOKENS override + DEFAULT_MAX_TOKENS) +# --------------------------------------------------------------------------- + +def _fake_anthropic_module(captured, stop_reason="end_turn", text="ok"): + class FakeContentBlock: + def __init__(self, text): + self.text = text + + class FakeResponse: + def __init__(self): + self.content = [FakeContentBlock(text)] + self.stop_reason = stop_reason + + class FakeAnthropic: + def __init__(self, api_key=None): + pass + + class messages: + @staticmethod + def create(model, max_tokens, messages): + captured["max_tokens"] = max_tokens + return FakeResponse() + + return types.SimpleNamespace(Anthropic=FakeAnthropic) + + +def _fake_openai_module(captured, finish_reason="stop", text="ok"): + class FakeMessage: + def __init__(self, content): + self.content = content + + class FakeChoice: + def __init__(self, content, finish_reason): + self.message = FakeMessage(content) + self.finish_reason = finish_reason + + class FakeResponse: + def __init__(self): + self.choices = [FakeChoice(text, finish_reason)] + + class FakeOpenAI: + def __init__(self, api_key=None): + pass + + class chat: + class completions: + @staticmethod + def create(model, messages, temperature, max_tokens): + captured["max_tokens"] = max_tokens + return FakeResponse() + + return types.SimpleNamespace(OpenAI=FakeOpenAI) + + +def test_anthropic_uses_default_max_tokens_when_env_unset(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.delenv("LLM_MAX_TOKENS", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == DEFAULT_MAX_TOKENS + assert DEFAULT_MAX_TOKENS == 4096 + + +def test_anthropic_honors_llm_max_tokens_override(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "8000") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == 8000 + + +def test_invalid_llm_max_tokens_falls_back_to_default(monkeypatch, capsys): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "not-a-number") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == DEFAULT_MAX_TOKENS + # stderr, not stdout: progress/warning prints were redirected to stderr + # during the merge into OpenAnt (stdout carries only the JSON envelope). + assert "Invalid LLM_MAX_TOKENS" in capsys.readouterr().err + + +def test_openai_receives_explicit_max_tokens(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "2048") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "openai", _fake_openai_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == 2048 + + +# --------------------------------------------------------------------------- +# Per-stage call metadata (stop_reason / finish_reason capture) +# --------------------------------------------------------------------------- + +def test_anthropic_call_metadata_captures_max_tokens_stop_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + captured: dict = {} + monkeypatch.setitem( + sys.modules, "anthropic", _fake_anthropic_module(captured, stop_reason="max_tokens") + ) + + call_llm("prompt", stage="patch_generation") + meta = llm_client.get_call_metadata() + assert meta["patch_generation"]["stop_reason"] == "max_tokens" + assert meta["patch_generation"]["provider"] == "anthropic" + + +def test_openai_call_metadata_captures_finish_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + captured: dict = {} + monkeypatch.setitem( + sys.modules, "openai", _fake_openai_module(captured, finish_reason="length") + ) + + call_llm("prompt", stage="patch_review") + meta = llm_client.get_call_metadata() + assert meta["patch_review"]["stop_reason"] == "length" + + +def test_mock_call_populates_metadata_with_mock_stop_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + call_llm("prompt", stage="confidence_scorer") + meta = llm_client.get_call_metadata() + assert meta["confidence_scorer"]["stop_reason"] == "mock" + assert meta["confidence_scorer"]["max_tokens_configured"] is None + + +def test_complete_passes_stage_through_to_call_metadata(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + client = LLMClient(api_key="") + client.complete("system prompt", "user message", stage="challenger") + meta = llm_client.get_call_metadata() + assert "challenger" in meta + + +def test_clear_call_metadata_empties_the_store(monkeypatch): + monkeypatch.setattr(llm_client, "_call_metadata", { + "patch_generation": {"provider": "anthropic", "model": "x", "max_tokens_configured": 1000, "stop_reason": "max_tokens"}, + }) + llm_client.clear_call_metadata() + assert llm_client.get_call_metadata() == {} + + +def test_stale_metadata_does_not_leak_into_a_new_run(monkeypatch): + # Simulates two sequential runs in the same process (e.g. a batch + # runner) without the reset: a stage from run 1 must not survive into + # run 2's metadata once clear_call_metadata() is called between them. + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + # "Run 1": patch_generation stage recorded. + call_llm("prompt for run 1", stage="patch_generation") + assert "patch_generation" in llm_client.get_call_metadata() + + # Reset, as main.py now does before each run. + llm_client.clear_call_metadata() + assert llm_client.get_call_metadata() == {} + + # "Run 2": only a different stage is called. + call_llm("prompt for run 2", stage="challenger") + meta = llm_client.get_call_metadata() + assert "challenger" in meta + assert "patch_generation" not in meta, "stale stage from a previous run leaked into the new run's metadata" + + +def test_complete_default_stage_is_unknown_and_does_not_break(monkeypatch): + # Existing callers that don't pass `stage` must keep working unchanged. + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + client = LLMClient(api_key="") + result = client.complete("system prompt", "user message") + assert isinstance(result, str) + meta = llm_client.get_call_metadata() + assert "unknown" in meta diff --git a/libs/openant-core/tests/patch/test_patch_applicability.py b/libs/openant-core/tests/patch/test_patch_applicability.py new file mode 100644 index 00000000..14041ee2 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_applicability.py @@ -0,0 +1,418 @@ +"""Tests for patch_applicability.check_applicability.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path +from unittest import mock + +import pytest + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + +_FENCED_DIFF = """\ +```diff +--- a/auth.py ++++ b/auth.py +@@ -1,2 +1,2 @@ + def authenticate(u, p): +- return True ++ return check_credentials(u, p) +```""" + +_INNER_DIFF = """\ +--- a/auth.py ++++ b/auth.py +@@ -1,2 +1,2 @@ + def authenticate(u, p): +- return True ++ return check_credentials(u, p) +""" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_git_repo(tmp_path: Path) -> Path: + """Init a minimal git repo with one committed file.""" + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "auth.py").write_text( + "def authenticate(u, p):\n return True\n", encoding="utf-8" + ) + subprocess.run(["git", "add", "auth.py"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=tmp_path, capture_output=True, check=True, + ) + return tmp_path + + +def _mock_git_run(returncode: int, stderr: str = ""): + cm = mock.MagicMock() + cm.returncode = returncode + cm.stderr = stderr + return cm + + +# --------------------------------------------------------------------------- +# Skip conditions +# --------------------------------------------------------------------------- + +class TestSkipConditions: + def test_no_repo_root_skipped(self): + from utilities.autopatcher.patch_applicability import check_applicability + r = check_applicability(_FENCED_DIFF, None) + assert r["skipped"] is True + assert r["applicable"] is None + assert "no repo_root" in r["skipped_reason"] + + def test_not_git_repo_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is True + assert "git" in r["skipped_reason"].lower() + + def test_empty_patch_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + r = check_applicability("", tmp_path) + assert r["skipped"] is True + assert "empty" in r["skipped_reason"].lower() + + def test_fences_only_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + r = check_applicability("```diff\n```", tmp_path) + assert r["skipped"] is True + + def test_git_not_found_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + side_effect=FileNotFoundError): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is True + assert "git" in r["skipped_reason"].lower() + assert r["error"] is None + + +# --------------------------------------------------------------------------- +# Error state (not skipped) +# --------------------------------------------------------------------------- + +class TestErrorState: + def test_timeout_is_error_not_skip(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=subprocess.TimeoutExpired("git", 10), + ): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is None + assert r["error"] is not None + assert "timed out" in r["error"].lower() + + def test_unexpected_exception_is_error_not_skip(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=RuntimeError("something broke"), + ): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is False + assert r["error"] is not None + assert r["applicable"] is None + + +# --------------------------------------------------------------------------- +# Success / failure from git (mocked) +# --------------------------------------------------------------------------- + +class TestApplicabilityResult: + def test_applicable_true_on_returncode_0(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["applicable"] is True + assert r["skipped"] is False + assert r["error"] is None + assert r["exit_code"] == 0 + + def test_applicable_false_on_nonzero_returncode(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, "error: auth.py: does not exist in index")): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["applicable"] is False + assert r["exit_code"] == 1 + assert "does not exist" in r["stderr"] + + def test_fences_stripped_before_passing_to_git(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + captured = [] + def _capture(cmd, **kwargs): + captured.append(kwargs.get("input", "")) + return _mock_git_run(0) + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", side_effect=_capture): + check_applicability(_FENCED_DIFF, tmp_path) + assert captured + assert "```diff" not in captured[0] + assert "```" not in captured[0].strip().splitlines()[-1] + + def test_plain_diff_no_fences_also_accepted(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)) as m: + check_applicability(_INNER_DIFF, tmp_path) + assert m.called + + +# --------------------------------------------------------------------------- +# stderr truncation +# --------------------------------------------------------------------------- + +class TestStderrTruncation: + def test_truncated_at_20_lines(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + long_stderr = "\n".join(f"error line {i}" for i in range(50)) + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, long_stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + lines = r["stderr"].splitlines() + assert len(lines) <= 21 # 20 data + 1 truncation marker + assert "truncated" in r["stderr"].lower() + + def test_truncated_at_2000_chars(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + long_stderr = "x" * 3000 + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, long_stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert len(r["stderr"]) <= 2_100 # 2000 + marker + + def test_short_stderr_not_truncated(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + stderr = "error: auth.py: patch does not apply" + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["stderr"] == stderr + + +# --------------------------------------------------------------------------- +# Integration: real git (skipped if git unavailable) +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not shutil.which("git"), reason="git not available") +class TestRealGitApply: + def test_correct_patch_applies(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + " def authenticate(u, p):\n" + "- return True\n" + "+ return check_credentials(u, p)\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is True + assert r["error"] is None + + def test_nonexistent_file_fails(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + patch = ( + "```diff\n" + "--- a/nonexistent.py\n" + "+++ b/nonexistent.py\n" + "@@ -1,1 +1,1 @@\n" + "-old_line()\n" + "+new_line()\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is False + assert r["stderr"] # git names the problem + + def test_wrong_context_fails(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + # The file has "def authenticate" but the patch removes "def wrong_name" + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + "-def wrong_name(u, p):\n" + "+def authenticate(u, p):\n" + " return True\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is False + + +# --------------------------------------------------------------------------- +# Pipeline-level: section renders correctly +# --------------------------------------------------------------------------- + +class TestFenceStrippingPreservesTrailingBlankContext: + """Regression coverage for the byte-preserving _strip_fences fix. + + Historical bug: splitlines() + "\\n".join(...) followed by .strip() + silently deleted a trailing single-space context line, turning a valid + hunk into one whose body no longer matched its @@ header's declared + count — surfacing as `git apply`'s generic "corrupt patch at line N" + instead of the real applicability result. + """ + + def _repo_with_blank_line_before_brace(self, tmp_path: Path) -> Path: + """A hermetic git repo whose file has a real blank line before `}`.""" + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "redirect.c").write_text( + "int follow(int x) {\n" + " do_thing(x);\n" + "\n" + " return 0;\n" + "}\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "redirect.c"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=tmp_path, capture_output=True, check=True) + return tmp_path + + def _fenced_patch_with_trailing_blank_context(self) -> str: + """Fenced diff whose final hunk's last line is a single-space context + line (representing the real blank line before `}` in redirect.c).""" + body = ( + "--- a/redirect.c\n" + "+++ b/redirect.c\n" + "@@ -1,5 +1,8 @@\n" + " int follow(int x) {\n" + " do_thing(x);\n" + "+ if(x) {\n" + "+ clear_creds(x);\n" + "+ }\n" + " \n" + " return 0;\n" + " }\n" + ) + # No trailing newline after the closing fence, matching how + # patch_generator._extract_diff_block assembles the string. + return "```diff\n" + body + "```" + + def test_strip_fences_supports_tilde_closing_fence(self): + """~~~ is accepted as a closing fence, mirroring + diff_hunk_repair._strip_md_fences's same ("```", "~~~") check.""" + from utilities.autopatcher.patch_applicability import _strip_fences + patch = "```diff\n--- a/x\n+++ b/x\n@@ -1,1 +1,1 @@\n-a\n+b\n~~~" + stripped = _strip_fences(patch) + assert "~~~" not in stripped + assert "```" not in stripped + assert stripped == "--- a/x\n+++ b/x\n@@ -1,1 +1,1 @@\n-a\n+b\n" + + def test_strip_fences_preserves_single_space_last_line(self): + from utilities.autopatcher.patch_applicability import _strip_fences + patch = self._fenced_patch_with_trailing_blank_context() + stripped = _strip_fences(patch) + lines = stripped.splitlines(keepends=True) + # The blank context line (single space) two lines before the final + # ' }\n' context line must survive untouched, not be deleted. + assert lines[-3] == " \n" + assert lines[-2] == " return 0;\n" + assert lines[-1] == " }\n" + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_no_longer_reported_as_corrupt(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + patch = self._fenced_patch_with_trailing_blank_context() + r = check_applicability(patch, tmp_path) + assert "corrupt patch" not in (r["stderr"] or "").lower() + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_well_formed_patch_with_trailing_blank_context_applies(self, tmp_path): + """The patch above is genuinely well-formed against redirect.c — with + the fence-stripping bug fixed it must apply cleanly, not merely + avoid the word 'corrupt'.""" + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + patch = self._fenced_patch_with_trailing_blank_context() + r = check_applicability(patch, tmp_path) + assert r["applicable"] is True, r["stderr"] + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_genuine_context_mismatch_still_fails_normally(self, tmp_path): + """A real context mismatch (not a fence-stripping artifact) must + still be reported as a normal, non-corrupt applicability failure.""" + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + bad_patch = ( + "```diff\n" + "--- a/redirect.c\n" + "+++ b/redirect.c\n" + "@@ -1,5 +1,8 @@\n" + " int follow(int x) {\n" + " do_thing_that_does_not_exist(x);\n" + "+ if(x) {\n" + "+ clear_creds(x);\n" + "+ }\n" + " \n" + " return 0;\n" + " }\n" + "```" + ) + r = check_applicability(bad_patch, tmp_path) + assert r["applicable"] is False + assert "corrupt patch" not in (r["stderr"] or "").lower() + assert r["stderr"] # git still names the problem + + +class TestPipelineApplicabilitySection: + def test_section_present_in_report(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + assert "## Patch Applicability" in report + + def test_skipped_when_no_repo_root(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="", repo_root=None) + start = report.find("## Patch Applicability") + end = report.find("---", start + 1) + section = report[start:end] + assert "Skipped" in section + + def test_hygiene_before_applicability(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + idx_hygiene = report.find("## Patch Hygiene") + idx_applicability = report.find("## Patch Applicability") + assert idx_hygiene < idx_applicability diff --git a/libs/openant-core/tests/patch/test_patch_challenger.py b/libs/openant-core/tests/patch/test_patch_challenger.py new file mode 100644 index 00000000..5d559832 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_challenger.py @@ -0,0 +1,124 @@ +"""Unit tests for patch_challenger.challenge_patch, focused on the +code_context parameter (mirrors generate_patch/score_confidence grounding).""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + + +_CHALLENGER_RESPONSE = """\ +Still vulnerable: No + +Edge cases: +- Some edge case + +Potential issues: +- Some potential issue + +Summary: +- A concise paragraph summarising the adversarial findings. +""" + + +class TestCodeContextParameter: + """code_context is optional and, when provided, must reach the LLM call + the same way generate_patch/score_confidence already do.""" + + def test_default_omits_repository_evidence_section(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm) + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" not in user_message + + def test_code_context_included_when_provided(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" in user_message + assert "def foo(): pass" in user_message + + def test_empty_code_context_omits_section(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="") + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" not in user_message + + def test_code_context_precedes_vulnerability_report(self): + """Same ordering as score_confidence: repository evidence first, so + the model reads the real code before the advisory framing.""" + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert user_message.index("## Repository evidence") < user_message.index("## Vulnerability report") + + def test_backward_compatible_without_code_context_kwarg(self): + """Existing positional-only call sites must keep working unchanged.""" + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + result = challenge_patch("some vuln", "some diff", llm) + + assert result["still_vulnerable"] is False + assert result["edge_cases"] == ["Some edge case"] + assert result["potential_issues"] == ["Some potential issue"] + + +class TestChallengePatchBasicBehavior: + """Baseline behavior, unrelated to code_context, that the new parameter + must not disturb.""" + + def test_returns_expected_keys(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + result = challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert set(result.keys()) == {"still_vulnerable", "edge_cases", "potential_issues", "summary"} + + def test_still_vulnerable_yes_parsed_true(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE.replace( + "Still vulnerable: No", "Still vulnerable: Yes" + ) + + result = challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert result["still_vulnerable"] is True + + def test_stage_argument_is_challenger(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert llm.complete.call_args.kwargs.get("stage") == "challenger" diff --git a/libs/openant-core/tests/patch/test_patch_generator.py b/libs/openant-core/tests/patch/test_patch_generator.py new file mode 100644 index 00000000..4c0631f8 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_generator.py @@ -0,0 +1,293 @@ +"""Unit tests for patch_generator._extract_diff_block and generate_patch.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +```""" + +_PROSE_PREAMBLE = """\ +The vulnerability is in retry.py. Here is the minimal fix: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +``` + +Let me know if you need a more defensive approach.""" + +_TWO_ALTERNATIVES = """\ +Option A — minimal fix: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +``` + +Option B — more defensive: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization", "X-Csrf-Token"]) +```""" + +_MULTI_FILE_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/connectionpool.py ++++ b/src/urllib3/connectionpool.py +@@ -1,3 +1,4 @@ ++# Cookie stripping is handled via DEFAULT_REMOVE_HEADERS_ON_REDIRECT. +```""" + +_PATCH_TAG = """\ +```patch +--- a/src/utils.py ++++ b/src/utils.py +@@ -1,1 +1,1 @@ +-old_line() ++new_line() +```""" + +_UDIFF_TAG = """\ +```udiff +--- a/src/utils.py ++++ b/src/utils.py +@@ -1,1 +1,1 @@ +-old_line() ++new_line() +```""" + +_NO_DIFF_BLOCK = "The vulnerability requires manual intervention. No automated patch is possible." + +_WINDOWS_ENDINGS = "```diff\r\n--- a/foo.py\r\n+++ b/foo.py\r\n@@ -1 +1 @@\r\n-old\r\n+new\r\n```" + + +# --------------------------------------------------------------------------- +# Tests for _extract_diff_block +# --------------------------------------------------------------------------- + +class TestExtractDiffBlock: + def test_clean_diff_preserved(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_CLEAN_DIFF) + assert result.startswith("```diff\n") + assert result.strip().endswith("```") + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result + + def test_prose_preamble_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PROSE_PREAMBLE) + assert result.startswith("```diff\n") + assert "The vulnerability" not in result + assert "Let me know" not in result + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result + + def test_prose_postamble_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PROSE_PREAMBLE) + assert "Let me know" not in result + + def test_first_of_two_alternatives_returned(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_TWO_ALTERNATIVES) + assert result.count("```diff") == 1 + assert "Option A" not in result + assert "Option B" not in result + assert "X-Csrf-Token" not in result + assert '"Cookie", "Authorization"' in result + + def test_multi_file_diff_preserved(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_MULTI_FILE_DIFF) + assert "retry.py" in result + assert "connectionpool.py" in result + + def test_patch_tag_normalised_to_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PATCH_TAG) + assert result.startswith("```diff\n") + assert "+new_line()" in result + + def test_udiff_tag_normalised_to_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_UDIFF_TAG) + assert result.startswith("```diff\n") + assert "+new_line()" in result + + def test_no_fenced_block_returns_raw_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_NO_DIFF_BLOCK) + assert result == _NO_DIFF_BLOCK.strip() + + def test_empty_string_returns_empty(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + assert _extract_diff_block("") == "" + + def test_windows_line_endings_accepted(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_WINDOWS_ENDINGS) + assert result.startswith("```diff\n") + assert "+new" in result + + def test_output_always_starts_with_diff_fence(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + for raw in (_CLEAN_DIFF, _PROSE_PREAMBLE, _TWO_ALTERNATIVES, _PATCH_TAG): + result = _extract_diff_block(raw) + assert result.startswith("```diff\n"), f"Failed for input starting: {raw[:40]!r}" + + +# --------------------------------------------------------------------------- +# Integration: generate_patch extracts clean output from messy LLM response +# --------------------------------------------------------------------------- + +class TestGeneratePatchExtraction: + def test_extracts_diff_from_messy_response(self): + from utilities.autopatcher.patch_generator import generate_patch + messy = ( + "I'll fix this by adding Cookie to the frozenset:\n\n" + "```diff\n" + "--- a/src/urllib3/util/retry.py\n" + "+++ b/src/urllib3/util/retry.py\n" + "@@ -187,7 +187,7 @@\n" + '- DEFAULT_REMOVE = frozenset(["Authorization"])\n' + '+ DEFAULT_REMOVE = frozenset(["Cookie", "Authorization"])\n' + "```\n\n" + "Alternatively you could also strip X-Csrf-Token." + ) + llm = mock.MagicMock() + llm.complete.return_value = messy + result = generate_patch("some vuln", llm) + assert result.startswith("```diff\n") + assert "Alternatively" not in result + assert result.count("```diff") == 1 + + def test_fallback_when_no_fenced_block(self): + from utilities.autopatcher.patch_generator import generate_patch + plain = "No automated patch available for this vulnerability." + llm = mock.MagicMock() + llm.complete.return_value = plain + result = generate_patch("some vuln", llm) + assert result == plain.strip() + + +# --------------------------------------------------------------------------- +# AUTOPATCHER_DEBUG prompt dump +# --------------------------------------------------------------------------- + +class TestDebugDump: + def test_debug_file_written_when_env_set(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + # Redirect debug output to tmp_path + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("SQL injection in auth.py", llm, code_context="def auth(): pass") + + debug_dir = tmp_path / "reports" / "debug" + assert debug_dir.exists(), "reports/debug/ must be created" + files = list(debug_dir.glob("prompt_*.txt")) + assert len(files) == 1, f"Expected one debug file, found {files}" + content = files[0].read_text() + assert "SQL injection" in content + assert "def auth(): pass" in content + + def test_no_debug_file_when_env_unset(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.delenv("AUTOPATCHER_DEBUG", raising=False) + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("some vuln", llm) + + debug_dir = tmp_path / "reports" / "debug" + assert not debug_dir.exists(), "No debug directory should be created without the env var" + + def test_debug_file_does_not_contain_api_key(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("vuln text", llm, code_context="some code") + + debug_dir = tmp_path / "reports" / "debug" + files = list(debug_dir.glob("prompt_*.txt")) + assert files + content = files[0].read_text() + # user_message contains only vuln text + code context, never the API key + assert "sk-" not in content + assert "OPENAI_API_KEY" not in content + + +# --------------------------------------------------------------------------- +# retry_hint parameter +# --------------------------------------------------------------------------- + +class TestRetryHint: + def test_retry_hint_appended_to_user_message(self): + from utilities.autopatcher.patch_generator import generate_patch + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch( + "some vuln", + llm, + code_context="def foo(): pass", + retry_hint="Please fix the context lines.", + ) + + _system, user_message = llm.complete.call_args[0] + assert "## Retry instruction" in user_message + assert "Please fix the context lines." in user_message + + def test_no_retry_section_when_hint_empty(self): + from utilities.autopatcher.patch_generator import generate_patch + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("some vuln", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert "## Retry instruction" not in user_message diff --git a/libs/openant-core/tests/patch/test_patch_hygiene.py b/libs/openant-core/tests/patch/test_patch_hygiene.py new file mode 100644 index 00000000..0209e828 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_hygiene.py @@ -0,0 +1,274 @@ +"""Unit tests for patch_hygiene.check_patch and its three sub-checks.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures — raw diff strings (without fences, as _parse_file_patches sees them) +# --------------------------------------------------------------------------- + +_CLEAN_PATCH = """\ +--- a/app/auth.py ++++ b/app/auth.py +@@ -42,7 +42,7 @@ + def authenticate(username: str, password: str) -> bool: +- query = f"SELECT * FROM users WHERE username='{username}'" ++ query = "SELECT * FROM users WHERE username=?" + cursor = db.execute(query) + return cursor.fetchone() is not None +""" + +_EMPTY_HUNK = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/_collections.py ++++ b/src/urllib3/_collections.py +@@ -1,3 +1,3 @@ + # no actual changes here +""" + +_DUPLICATE_CONST = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,8 +187,9 @@ + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_CORRECT_CONST_REPLACEMENT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_UNUSED_IMPORT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,5 +1,6 @@ ++import re + class Retry: +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_USED_IMPORT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,5 +1,7 @@ ++import re ++PATTERN = re.compile(r'https?://') + class Retry: +- OLD_VAL = "x" ++ NEW_VAL = "y" +""" + +_NEW_FILE_WITH_CONST = """\ +--- /dev/null ++++ b/src/urllib3/util/headers.py +@@ -0,0 +1,3 @@ ++SENSITIVE_HEADERS = frozenset(["Cookie", "Authorization"]) ++ ++def strip_sensitive(headers): pass +""" + +# Simulates the dirty urllib3 output we observed in the live run +_URLLIB3_DIRTY = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,8 +187,9 @@ + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/_collections.py ++++ b/src/urllib3/_collections.py +@@ -1,3 +1,3 @@ + # unchanged +--- a/src/urllib3/connectionpool.py ++++ b/src/urllib3/connectionpool.py +@@ -1,4 +1,5 @@ ++import re + class HTTPConnectionPool: + pass +""" + +# The mock patch used throughout existing tests — should be clean +_MOCK_PATCH_INNER = """\ +--- a/app/auth.py ++++ b/app/auth.py +@@ -42,7 +42,7 @@ + def authenticate(username: str, password: str) -> bool: +- query = f"SELECT * FROM users WHERE username='{username}' AND password='{password}'" ++ query = "SELECT * FROM users WHERE username=? AND password=?" + cursor = db.execute(query) ++ # Use parameterized queries to prevent SQL injection + return cursor.fetchone() is not None +""" + + +# --------------------------------------------------------------------------- +# check_patch — public API +# --------------------------------------------------------------------------- + +class TestCheckPatchSafety: + def test_empty_string_returns_empty(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch("") == [] + + def test_none_like_string_returns_empty(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(" ") == [] + + def test_fenced_block_accepted(self): + from utilities.autopatcher.patch_hygiene import check_patch + fenced = "```diff\n" + _CLEAN_PATCH + "\n```" + result = check_patch(fenced) + assert isinstance(result, list) + + def test_clean_patch_no_findings(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(_CLEAN_PATCH) == [] + + def test_mock_patch_no_findings(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(_MOCK_PATCH_INNER) == [] + + +# --------------------------------------------------------------------------- +# Check A — empty / no-op hunks +# --------------------------------------------------------------------------- + +class TestEmptyHunkCheck: + def test_empty_hunk_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_EMPTY_HUNK) + empty = [f for f in findings if f["check"] == "empty_hunk"] + assert len(empty) == 1 + assert "_collections.py" in empty[0]["detail"] + assert empty[0]["severity"] == "HIGH" + + def test_file_with_changes_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_EMPTY_HUNK) + empty = [f for f in findings if f["check"] == "empty_hunk"] + details = " ".join(f["detail"] for f in empty) + assert "retry.py" not in details + + def test_multiple_empty_hunks_all_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + empty = [f for f in findings if f["check"] == "empty_hunk"] + filenames = " ".join(f["detail"] for f in empty) + assert "_collections.py" in filenames + + +# --------------------------------------------------------------------------- +# Check B — duplicate assignment +# --------------------------------------------------------------------------- + +class TestDuplicateAssignmentCheck: + def test_duplicate_constant_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_DUPLICATE_CONST) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert len(dups) == 1 + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in dups[0]["detail"] + assert dups[0]["severity"] == "HIGH" + + def test_correct_replacement_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_CORRECT_CONST_REPLACEMENT) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert dups == [] + + def test_new_file_constant_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_NEW_FILE_WITH_CONST) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert dups == [], "new-file constants should not be flagged as duplicates" + + def test_dirty_urllib3_flags_duplicate(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert any("DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in f["detail"] for f in dups) + + +# --------------------------------------------------------------------------- +# Check C — unused imports +# --------------------------------------------------------------------------- + +class TestUnusedImportCheck: + def test_unused_import_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_UNUSED_IMPORT) + imps = [f for f in findings if f["check"] == "unused_import"] + assert len(imps) == 1 + assert "re" in imps[0]["detail"] + assert imps[0]["severity"] == "MEDIUM" + + def test_used_import_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_USED_IMPORT) + imps = [f for f in findings if f["check"] == "unused_import"] + assert imps == [] + + def test_dirty_urllib3_flags_unused_re_import(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + imps = [f for f in findings if f["check"] == "unused_import"] + assert any("re" in f["detail"] for f in imps) + + +# --------------------------------------------------------------------------- +# Composite: dirty patch triggers all three checks +# --------------------------------------------------------------------------- + +class TestDirtyPatch: + def test_all_three_checks_fire_on_dirty_urllib3_patch(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + checks_found = {f["check"] for f in findings} + assert "empty_hunk" in checks_found + assert "duplicate_assignment" in checks_found + assert "unused_import" in checks_found + + def test_finding_structure_valid(self): + from utilities.autopatcher.patch_hygiene import check_patch + for finding in check_patch(_URLLIB3_DIRTY): + assert "severity" in finding + assert finding["severity"] in ("HIGH", "MEDIUM") + assert "check" in finding + assert "detail" in finding + assert isinstance(finding["detail"], str) and finding["detail"] + + +# --------------------------------------------------------------------------- +# Pipeline-level: hygiene section appears in report +# --------------------------------------------------------------------------- + +class TestPipelineHygieneSection: + def test_hygiene_section_present_in_report(self): + from utilities.autopatcher.pipeline import run + EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + assert "## Patch Hygiene" in report + + def test_clean_mock_patch_shows_no_issues(self): + from utilities.autopatcher.pipeline import run + EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + start = report.find("## Patch Hygiene") + end = report.find("---", start) + section = report[start:end] + assert "No obvious hygiene issues detected." in section diff --git a/libs/openant-core/tests/patch/test_patch_wrapper_contract.py b/libs/openant-core/tests/patch/test_patch_wrapper_contract.py new file mode 100644 index 00000000..e1168cd3 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_wrapper_contract.py @@ -0,0 +1,246 @@ +"""Contract tests for the openant-patch integration (core/patch.py + cmd_patch). + +These are the tests unique to THIS integration -- not ported from the +standalone Auto Patcher project, which had no notion of "a Go CLI triggered +this and must never get a report indistinguishable from a real one." + +Covers the specific safety/scope properties called out in the migration +plan: + - a run never silently falls back to Auto Patcher's mock mode + - suggested_fix/rejection_reason never reach the patch engine's input + - the eligibility gate is an explicit allowlist (fails closed) + - the envelope contract matches every other openant subcommand + - the Trust Report is written as an artifact whose content this + integration never inspects beyond existence + +All hermetic: LLM_PROVIDER=mock, no network, no real repo, no Docker. +""" + +import json +import os + +import pytest + +from core.patch import ( + PatchStepResult, + check_eligible, + effective_verdict, + find_finding_by_id, + render_vulnerability_markdown, + run_patch, +) + + +FIXTURE_FINDING_ELIGIBLE = { + "id": "F-001", + "name": "SQL Injection", + "location": {"file": "app.py", "function": "handle_request"}, + "cwe_id": 89, + "cwe_name": "SQL Injection", + "stage1_verdict": "vulnerable", + "stage2_verdict": "confirmed", + "description": "User input reaches a raw SQL query unsanitized.", + "vulnerable_code": "cursor.execute(query)", + "impact": ["Full database read access"], + "steps_to_reproduce": ["Send a crafted id"], + "suggested_fix": "DO-NOT-LEAK-THIS-SUGGESTED-FIX", + "rejection_reason": "DO-NOT-LEAK-THIS-REJECTION-REASON", +} + +FIXTURE_FINDING_REJECTED = { + **FIXTURE_FINDING_ELIGIBLE, + "id": "F-002", + "stage2_verdict": "rejected", +} + + +def _write_pipeline_output(tmp_path, findings): + path = tmp_path / "pipeline_output.json" + path.write_text(json.dumps({"findings": findings})) + return str(path) + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + +def test_find_finding_by_id_hit(): + f = find_finding_by_id([FIXTURE_FINDING_ELIGIBLE], "F-001") + assert f["id"] == "F-001" + + +def test_find_finding_by_id_miss(): + with pytest.raises(ValueError, match="F-999"): + find_finding_by_id([FIXTURE_FINDING_ELIGIBLE], "F-999") + + +@pytest.mark.parametrize( + "stage1,stage2,eligible", + [ + ("vulnerable", "confirmed", True), + ("vulnerable", "agreed", True), + ("vulnerable", "vulnerable", True), + ("bypassable", "bypassable", True), + ("vulnerable", "unverified", False), + ("vulnerable", "error", False), + ("vulnerable", "rejected", False), + ("safe", "safe", False), + ("vulnerable", "protected", False), + ("vulnerable", "inconclusive", False), + ("", "", False), + ("vulnerable", "Confirmed", False), # case-sensitive: fails closed, not normalized + ("vulnerable", "quarantined", False), # unknown future value: fails closed + ("vulnerable", "", True), # stage2 empty falls back to eligible stage1 + ("safe", "", False), # stage2 empty falls back to ineligible stage1 + ], +) +def test_check_eligible_allowlist(stage1, stage2, eligible): + finding = {"id": "X", "stage1_verdict": stage1, "stage2_verdict": stage2} + if eligible: + check_eligible(finding) # must not raise + else: + with pytest.raises(ValueError): + check_eligible(finding) + + +def test_effective_verdict_prefers_stage2(): + assert effective_verdict({"stage1_verdict": "vulnerable", "stage2_verdict": "confirmed"}) == "confirmed" + + +def test_effective_verdict_falls_back_to_stage1_when_stage2_empty(): + assert effective_verdict({"stage1_verdict": "vulnerable", "stage2_verdict": ""}) == "vulnerable" + + +def test_render_vulnerability_markdown_includes_key_fields(): + rendered = render_vulnerability_markdown(FIXTURE_FINDING_ELIGIBLE) + for expected in ("F-001", "SQL Injection", "app.py", "handle_request", + "CWE-89", "confirmed", "cursor.execute(query)", + "Full database read access", "Send a crafted id"): + assert expected in rendered + + +def test_render_vulnerability_markdown_never_includes_suggested_fix_or_rejection_reason(): + """Regression guard: OpenAnt's own suggested_fix/rejection_reason must + never reach the patch engine's input -- they could bias the + independently-generated candidate patch.""" + rendered = render_vulnerability_markdown(FIXTURE_FINDING_ELIGIBLE) + assert "DO-NOT-LEAK-THIS-SUGGESTED-FIX" not in rendered + assert "DO-NOT-LEAK-THIS-REJECTION-REASON" not in rendered + + +# --------------------------------------------------------------------------- +# run_patch end-to-end (mock mode) +# --------------------------------------------------------------------------- + +def test_run_patch_happy_path(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE, FIXTURE_FINDING_REJECTED]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + assert isinstance(result, PatchStepResult) + assert result.finding_id == "F-001" + assert os.path.exists(result.vulnerability_path) + assert os.path.exists(result.trust_report_path) + assert result.vulnerability_path == str(tmp_path / "patch" / "F-001-vulnerability.md") + assert result.trust_report_path == str(tmp_path / "patch" / "F-001-trust-report.md") + + +def test_run_patch_never_leaks_suggested_fix_into_artifact(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + vuln_text = open(result.vulnerability_path, encoding="utf-8").read() + assert "DO-NOT-LEAK-THIS-SUGGESTED-FIX" not in vuln_text + assert "DO-NOT-LEAK-THIS-REJECTION-REASON" not in vuln_text + + +def test_run_patch_mock_mode_is_self_disclosing(tmp_path, monkeypatch): + """The core safety property: a Go-triggered run must never produce a + mock report indistinguishable from a real one. LLM_PROVIDER=mock is + allowed, but the resulting Trust Report must say so, loudly.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + report_text = open(result.trust_report_path, encoding="utf-8").read() + assert "MOCK MODE" in report_text + assert "LLM mode | MOCK" in report_text + + +def test_run_patch_requires_llm_provider(tmp_path, monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + with pytest.raises(RuntimeError, match="LLM_PROVIDER"): + run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + +def test_run_patch_rejects_ineligible_finding(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_REJECTED]) + + with pytest.raises(ValueError, match="not eligible"): + run_patch(po_path, "F-002", str(tmp_path), repo_root=None) + + +def test_run_patch_rejects_unknown_finding(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + with pytest.raises(ValueError, match="no finding"): + run_patch(po_path, "does-not-exist", str(tmp_path), repo_root=None) + + +def test_run_patch_missing_pipeline_output(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + + with pytest.raises(FileNotFoundError): + run_patch(str(tmp_path / "nope.json"), "F-001", str(tmp_path), repo_root=None) + + +# --------------------------------------------------------------------------- +# Full CLI dispatch contract (openant/cli.py's cmd_patch) -- the exact +# envelope shape internal/python.Invoke() parses on the Go side. +# --------------------------------------------------------------------------- + +class _Args: + def __init__(self, **kw): + self.__dict__.update(kw) + + +def test_cmd_patch_success_envelope_shape(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + args = _Args(pipeline_output=po_path, finding_id="F-001", repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 0 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "success" + assert envelope["errors"] == [] + assert envelope["data"]["finding_id"] == "F-001" + assert os.path.exists(envelope["data"]["trust_report_path"]) + assert os.path.exists(envelope["data"]["vulnerability_path"]) + + +def test_cmd_patch_error_envelope_shape_for_unknown_finding(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + args = _Args(pipeline_output=po_path, finding_id="nope", repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert envelope["data"] == {} + assert len(envelope["errors"]) == 1 diff --git a/libs/openant-core/tests/patch/test_pipeline.py b/libs/openant-core/tests/patch/test_pipeline.py new file mode 100644 index 00000000..3b5a9639 --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline.py @@ -0,0 +1,1324 @@ +""" +Unit tests for the Auto Patcher MVP. + +All tests run in mock mode (no OPENAI_API_KEY required). +""" + +from __future__ import annotations + +import os +import sys +import textwrap +from pathlib import Path +from unittest import mock + +import pytest + + +import utilities.autopatcher as _ap_pkg +PROMPTS_DIR = Path(_ap_pkg.__file__).parent / "prompts" +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +# --------------------------------------------------------------------------- +# llm_client +# --------------------------------------------------------------------------- + +class TestLLMClientMock: + def test_mock_mode_when_no_key(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + assert client.is_mock is True + + def test_mock_patch_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("patch generator system prompt", "fix this") + assert "diff" in response.lower() or "---" in response or "+++" in response + + def test_mock_review_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("review and explain impact of patch", "review this") + # Should contain review-like content + assert len(response) > 20 + + def test_mock_score_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("confidence score assessment", "score this") + assert "confidence" in response.lower() or "score" in response.lower() + + def test_live_mode_when_key_provided(self): + import utilities.autopatcher.llm_client as llm_client + from utilities.autopatcher.llm_client import LLMClient + # Clear cached provider and LLM_PROVIDER so is_mock falls back to key check. + with mock.patch.object(llm_client, "_cached_provider", None), \ + mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("LLM_PROVIDER", None) + client = LLMClient(api_key="sk-fake-key-for-testing") + assert client.is_mock is False + + def test_env_var_key_used(self): + import utilities.autopatcher.llm_client as llm_client + from utilities.autopatcher.llm_client import LLMClient + with mock.patch.object(llm_client, "_cached_provider", None), \ + mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-env-key"}): + os.environ.pop("LLM_PROVIDER", None) + client = LLMClient() + assert client.is_mock is False + + +# --------------------------------------------------------------------------- +# Pipeline LLM mode log line +# --------------------------------------------------------------------------- + +class TestPipelineLLMModeLog: + """The early [pipeline] LLM mode: ... log must never include a model name + that hasn't been resolved yet. MOCK stays MOCK; LIVE carries no model. + + Progress logs go to stderr, not stdout: OpenAnt's Go CLI parses stdout + as a single final JSON envelope (internal/python.Invoke()), so any + engine progress print()s ported from the standalone Auto Patcher + project were redirected to stderr during the merge -- see + utilities/autopatcher/pipeline.py and llm_client.py.""" + + def test_mock_mode_log(self, monkeypatch, capsys): + import utilities.autopatcher.llm_client as llm_client + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + from utilities.autopatcher.pipeline import run + run("XSS in login form") + captured = capsys.readouterr() + assert "[pipeline] LLM mode: MOCK" in captured.err + + def test_live_mode_log_has_no_model_name(self, monkeypatch, capsys): + # With Anthropic configured, the early log should say LIVE with no model. + import utilities.autopatcher.llm_client as llm_client + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + + # Stub out the actual Anthropic call so the test stays offline. + import types, sys + class FakeAnthropic: + def __init__(self, api_key=None): pass + class messages: + @staticmethod + def create(model, max_tokens, messages): + import types + msg = types.SimpleNamespace(content=[types.SimpleNamespace(text="```diff\n--- a/f\n+++ b/f\n@@ -1,1 +1,1 @@\n-old\n+new\n```")]) + return types.SimpleNamespace(content=[msg.content[0]]) + monkeypatch.setitem(sys.modules, "anthropic", types.SimpleNamespace(Anthropic=FakeAnthropic)) + + from utilities.autopatcher.pipeline import run + run("path traversal in upload handler") + captured = capsys.readouterr() + assert "[pipeline] LLM mode: LIVE" in captured.err + # Must not include a specific model name in the early log. + assert "gpt-4o" not in captured.err.split("[pipeline] LLM mode:")[1].split("\n")[0] + assert "claude" not in captured.err.split("[pipeline] LLM mode:")[1].split("\n")[0].lower() + + +# --------------------------------------------------------------------------- +# patch_generator +# --------------------------------------------------------------------------- + +class TestPatchGenerator: + def test_returns_string(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_generator import generate_patch + llm = LLMClient(api_key="") + result = generate_patch("SQL injection in auth.py", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_patch_contains_diff_markers(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_generator import generate_patch + llm = LLMClient(api_key="") + result = generate_patch("SQL injection vulnerability", llm) + assert "---" in result or "+++" in result or "diff" in result.lower() + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "patch_generator.md" + assert prompt.exists(), "prompts/patch_generator.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# patch_reviewer +# --------------------------------------------------------------------------- + +class TestPatchReviewer: + def test_returns_string(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_reviewer import review_patch + llm = LLMClient(api_key="") + result = review_patch("vuln description", "some patch diff", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "patch_reviewer.md" + assert prompt.exists(), "prompts/patch_reviewer.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# confidence_scorer +# --------------------------------------------------------------------------- + +class TestConfidenceScorer: + def test_returns_string(self): + from utilities.autopatcher.confidence_scorer import score_confidence + from utilities.autopatcher.llm_client import LLMClient + llm = LLMClient(api_key="") + result = score_confidence("vuln", "patch", "review", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_response_mentions_score(self): + from utilities.autopatcher.confidence_scorer import score_confidence + from utilities.autopatcher.llm_client import LLMClient + llm = LLMClient(api_key="") + result = score_confidence("vuln", "patch", "review", llm) + assert "score" in result.lower() or "confidence" in result.lower() + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "confidence_scorer.md" + assert prompt.exists(), "prompts/confidence_scorer.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# pipeline +# --------------------------------------------------------------------------- + +class TestPipeline: + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + def test_run_produces_report(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert isinstance(report, str) + assert len(report) > 100 + + def test_report_contains_required_sections(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + required = [ + "Vulnerability summary", + "Vulnerability Sources", # renamed from "Primary Vulnerability References"; GHSA/CVE/Advisory URL only + "Proposed patch", + "Explanation", + "Affected areas", + "Reviewer Notes", # renamed from "Validation notes" (terminology cleanup) + "Recommendation", + "Trust Signals", # new Trust Package section + "Validation Actions", # new Trust Package section + "Review Results", # renamed from "Known Findings" (reviewer-experience terminology pass) + "Appendices", # new: consolidates diagnostics + legacy sections + ] + for section in required: + assert section in report, f"Report is missing section: '{section}'" + # "Validation Plan" was removed as a duplicate of Validation Actions + # (same underlying ≤3 items; Validation Actions now includes Reason + # too, so nothing unique was lost). + assert "## Validation Plan" not in report + # Superseded by the epistemic-category Review Results section. + assert "## Known Limitations" not in report + assert "## Known Security Gain" not in report + # Renamed to "Review Results" (reviewer-experience terminology pass). + assert "## Known Findings" not in report + # Reviewer-experience redesign: removed entirely as duplicated + # storytelling (restated Explanation/Known Findings/Reviewer Notes). + assert "## Patch Impact Summary" not in report + assert "## Impact Summary" not in report + assert "## Testing Notes" not in report + assert "### Testing Notes" not in report + # Upstream-reference policy: no fix-commit/remediation links in the + # reviewer report — those belong only in benchmark/evaluation artifacts. + assert "## Primary Vulnerability References" not in report + assert "Referenced Upstream Commit" not in report + # Presentation cleanup: legacy scoring sections, superseded by + # Trust Signals, are no longer rendered. + assert "Confidence Score" not in report + assert "Confidence Reasoning" not in report + assert "Confidence delta preview" not in report + # Presentation cleanup: the raw JSON dump duplicating the "Matching + # tests" list is no longer embedded in the human-facing report. + assert "```json" not in report + + def test_report_includes_validation_actions_format(self): + """Validation Actions is now the single, canonical checklist section + — it must include Reason as well as Next step (previously only the + now-removed "Validation Plan" section showed Reason).""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Validation Actions" in report + start = report.find("## Validation Actions") + assert start != -1 + # Patch Hygiene now precedes Validation Actions (promoted next to + # the diff) — Review Results is the next heading that follows it. + after = report.find("## Review Results", start) + block = report[start:after if after != -1 else start + 500] + assert "Reason:" in block and "Next step:" in block + actions = [ln for ln in block.splitlines() if re.match(r"^\d+\.\s+\*\*\[", ln.strip())] + assert len(actions) <= 3 + + def test_report_contains_recommendation_format(self): + """Trust Package V1: Recommendation uses the new decision vocabulary. + Trust Signals now precedes Recommendation (not after it), so this + just takes a fixed-width slice following the Recommendation heading.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Recommendation" in report + start = report.find("## Recommendation") + block = report[start:start + 400] + # New V1 format: bold decision on its own line, followed by reason text + bold_lines = [ln for ln in block.splitlines() if ln.strip().startswith("**") and ln.strip().endswith("**")] + assert bold_lines, "Recommendation must have a bold decision line" + decision = bold_lines[0].strip().strip("*").strip() + valid_decisions = ( + "Deploy After Validation", "Deploy With Caution", + "Manual Review Required", "Do Not Apply", + ) + assert decision in valid_decisions, f"Unexpected decision value: {decision!r}" + # Reason text must be a non-empty sentence after the decision line + non_empty = [ln.strip() for ln in block.splitlines() if ln.strip() and not ln.strip().startswith("#") and not ln.strip().startswith("**")] + assert non_empty, "Recommendation must contain reason text" + assert len(non_empty[0]) > 10 + + def test_recommendation_ordering(self): + """Reviewer-experience redesign: a reviewer first wants to know what + is broken and what patch is proposed — Vulnerability summary, + Vulnerability Sources, and Proposed patch now precede + Trust Signals and Recommendation entirely (reversing the previous + redesign's Trust-Signals-first placement). The legacy Confidence + Score/Reasoning sections that used to sit at the bottom, inside + Appendices, have been removed outright (superseded by Trust + Signals) rather than merely reordered. + + Patch Hygiene and Patch Applicability — the report's only two fully + deterministic checks — were promoted from Appendices to sit directly + after Proposed patch, before Trust Signals (a second reviewer- + experience pass): a reviewer who trusts deterministic evidence over + LLM narrative should not have to scroll past Explanation/Validation + Actions/Review Results to reach the actual git-apply result.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_vuln = report.find("## Vulnerability summary") + idx_refs = report.find("## Vulnerability Sources") + idx_patch = report.find("## Proposed patch") + idx_hygiene = report.find("## Patch Hygiene") + idx_applicability = report.find("## Patch Applicability") + idx_trust = report.find("## Trust Signals") + idx_rec = report.find("## Recommendation") + idx_val_actions = report.find("## Validation Actions") + idx_review_results = report.find("## Review Results") + for label, idx in [ + ("Vulnerability summary", idx_vuln), ("Vulnerability Sources", idx_refs), + ("Proposed patch", idx_patch), ("Patch Hygiene", idx_hygiene), + ("Patch Applicability", idx_applicability), ("Trust Signals", idx_trust), + ("Recommendation", idx_rec), ("Validation Actions", idx_val_actions), + ("Review Results", idx_review_results), + ]: + assert idx != -1, f"'{label}' section missing" + assert ( + idx_vuln < idx_refs < idx_patch < idx_hygiene < idx_applicability + < idx_trust < idx_rec < idx_val_actions < idx_review_results + ), ( + "Report order must be: Vulnerability summary, Primary Vulnerability " + "References, Proposed patch, Patch Hygiene, Patch Applicability, " + "Trust Signals, Recommendation, Validation Actions, then Review Results" + ) + + def test_report_omits_legacy_numeric_score(self): + """The legacy 'X / 1.0' confidence score was removed as a + presentation cleanup — it duplicated/contradicted the qualitative + Trust Signals verdicts. Trust Signals/Recommendation/decision logic + are unaffected; only this rendered artifact is gone.""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert not re.search(r"[0-9]+(?:\.[0-9]+)?\s*/\s*1\.0", report), ( + "Report should no longer contain the legacy numeric confidence score" + ) + + +# --------------------------------------------------------------------------- +# Report Structure v2, Phase 1 — Decision Card +# --------------------------------------------------------------------------- + +class TestDecisionCard: + """Unit tests for _render_decision_card as a Hero Banner: the decision + line itself is the heading (first visible line after the title), and + every field is a direct restatement of an already-computed value — + never a new derivation, never a separate "confidence" judgment.""" + + _EMOJI = { + "Deploy After Validation": "🟢", + "Deploy With Caution": "🟡", + "Manual Review Required": "🟠", + "Do Not Apply": "🔴", + } + + def _signals(self, patch_integrity="Clean"): + return {"patch_integrity": {"value": patch_integrity, "label": patch_integrity, "notes": ""}} + + def test_decision_line_is_first_line_and_uses_existing_emoji_mapping(self): + from utilities.autopatcher.pipeline import _render_decision_card + for decision, emoji in self._EMOJI.items(): + rec = {"decision": decision, "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + first_line = card.splitlines()[0] + assert first_line == f"## {emoji} {decision.upper()}" + + def test_patch_line_uses_existing_applicability_label(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Manual Review Required", "reason": "x"} + card = _render_decision_card(rec, self._signals("Critical Issues"), [], ["a.py"]) + assert "Patch has critical issues." in card + + def test_patch_line_clean(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals("Clean"), [], ["a.py"]) + assert "Patch applies cleanly." in card + + def test_no_separate_trust_or_confidence_field(self): + """Rule: never say 'High confidence' (or any confidence label) as a + separate Trust field — the banner has no independent confidence line.""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + assert "confidence" not in card.lower() + assert "Trust" not in card + + def test_validation_zero_actions(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + assert "No additional validation actions identified." in card + + def test_validation_one_action_is_singular(self): + """Hero wording must not name a section — it must stay valid even if + section names or positions change (they already have, twice).""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [{}], []) + assert "Complete the recommended validation check before deployment." in card + assert "section" not in card.lower() + assert "1 checks" not in card + + def test_validation_multiple_actions_is_plural(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [{}, {}, {}], []) + assert "Complete the recommended validation checks before deployment." in card + assert "section" not in card.lower() + + def test_files_changed_line(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], ["a.py", "b.py", "c.py"]) + assert "Files changed: 3" in card + + def test_do_not_apply_wording_does_not_imply_deployment(self): + """Reviewer-experience fix: 'before deployment' is contradictory when + the decision is Do Not Apply — there is no deployment to validate + toward. The same validation_actions still render; only the sentence + describing them changes.""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Do Not Apply", "reason": "x"} + card = _render_decision_card(rec, self._signals("Critical Issues"), [{}, {}], []) + assert "should not be deployed" in card + assert "before deployment" not in card + + def test_do_not_apply_wording_with_zero_actions(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Do Not Apply", "reason": "x"} + card = _render_decision_card(rec, self._signals("Does Not Apply"), [], []) + assert "should not be deployed" in card + assert "before deployment" not in card + + +class TestValidationActionsDecisionAware: + """Unit tests for _render_validation_actions_section's decision-aware + note (reviewer-experience fix): the actions themselves, their order, + priority, and count are untouched — only a leading note changes.""" + + _ACTION = {"priority": "MEDIUM", "title": "x", "reason": "y", "next_step": "z"} + + def test_do_not_apply_adds_note(self): + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION], "Do Not Apply") + assert "not recommended for deployment" in block + assert "1. **[MEDIUM]** x" in block + + def test_other_decisions_add_no_note(self): + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION], "Deploy After Validation") + assert "not recommended for deployment" not in block + assert "1. **[MEDIUM]** x" in block + + def test_default_decision_argument_adds_no_note(self): + """Backward-compatible default: omitting `decision` entirely must not + change existing callers' output.""" + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION]) + assert "not recommended for deployment" not in block + + +class TestDecisionCardInReport: + """End-to-end wiring: the Hero Banner sits first, Recommendation and + Trust Signals immediately follow it, and every other section keeps its + prior relative position.""" + + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + @staticmethod + def _find_hero_banner(report: str) -> int: + """Locate the Hero Banner heading (## ) — there is + no longer a literal '## Decision Card' label to search for.""" + import re + m = re.search(r"^## [🟢🟡🟠🔴] [A-Z ]+$", report, re.MULTILINE) + return m.start() if m else -1 + + def test_decision_card_appears_before_recommendation(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_card = self._find_hero_banner(report) + idx_rec = report.find("## Recommendation") + assert idx_card != -1, "Hero Banner heading not found" + assert idx_rec != -1 + assert idx_card < idx_rec + + def test_decision_card_is_first_section_after_title(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + title_idx = report.find("# Auto Patcher MVP — Security Patch Report") + first_heading_after_title = report.find("##", title_idx) + assert first_heading_after_title == self._find_hero_banner(report) + + def test_vulnerability_summary_immediately_follows_hero_banner(self): + """Reviewer-experience redesign: a reviewer first wants to know what + is broken — Vulnerability summary now directly follows the Hero + Banner (reversing the previous redesign, which put Trust Signals + there instead).""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_card = self._find_hero_banner(report) + hero_line_end = report.find("\n", idx_card) + idx_vuln = report.find("## Vulnerability summary") + between = report[hero_line_end:idx_vuln] + assert "## " not in between + + def test_trust_signals_follows_proposed_patch(self): + """Trust Signals only becomes meaningful once a reviewer already + knows what is broken and what patch is proposed — it now follows + Vulnerability summary, Vulnerability Sources, and + Proposed patch, and precedes Recommendation.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_vuln = report.find("## Vulnerability summary") + idx_patch = report.find("## Proposed patch") + idx_trust = report.find("## Trust Signals") + idx_rec = report.find("## Recommendation") + assert idx_vuln < idx_patch < idx_trust < idx_rec + + def test_security_gain_appears_within_explanation(self): + """Known Security Gain was merged into Explanation rather than kept + as its own heading — it was always an extracted sentence from (or, + on the fallback path, a truncated paragraph of) the same explanation + text, so it's now a labeled lead-in inside that section instead of a + separate one repeating the same information. "Impact Summary" no + longer exists at all (removed as duplicated storytelling).""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_patch = report.find("## Proposed patch") + idx_rec = report.find("## Recommendation") + idx_expl = report.find("## Explanation") + idx_val_actions = report.find("## Validation Actions") + assert idx_patch < idx_rec < idx_expl < idx_val_actions + assert "## Impact Summary" not in report + assert "## Known Security Gain" not in report + explanation_block = report[idx_expl:idx_val_actions] + assert "**Security gain:**" in explanation_block + + +class TestReportTerminologyCleanup: + """Terminology/navigation cleanup: one name per concept, and every + forward reference names a real, existing section heading.""" + + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + def test_validation_plan_removed(self): + """Validation Plan repeated the same <=3 items already shown, with + Reason added, in Validation Actions — verified lossless to fold in + and remove rather than keep as a second name for the same checklist.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Validation Plan" not in report + assert "## Validation Actions" in report + + def test_reviewer_notes_replaces_validation_notes_heading(self): + """Renamed to stop sharing the word "Validation" with the actions + checklist, since this section is unrelated LLM reviewer prose, not + part of that concept.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Reviewer Notes" in report + assert "## Validation notes" not in report + + def test_testing_notes_removed_as_duplicated_storytelling(self): + """Testing Notes was a third restatement of validation_notes/challenger + findings, fully duplicating Reviewer Notes and Known Findings — + removed entirely rather than promoted.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Testing Notes" not in report + assert "### Testing Notes" not in report + assert "### What should be tested" not in report + + def test_hero_banner_does_not_name_a_section(self): + """Hero wording must remain valid even if section names or positions + change elsewhere — it must not reference "Validation Actions" or any + other section name by name.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx = report.find("# Auto Patcher MVP") + banner = report[idx:report.find("## Vulnerability summary")] + assert "section" not in banner.lower() + assert "Complete the recommended validation check" in banner + + def test_no_dangling_see_analysis_below(self): + """Catches regression of the specific dangling phrase this cleanup + fixes in Trust Signals. Note: the Recommendation reason string + "Run the listed validation actions before deployment" (inside + _build_recommendation_v1, the Deploy After Validation branch) was + intentionally left untouched pending explicit confirmation that + editing that function's wording is in scope — not asserted here.""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert not re.search(r"see analysis below", report, re.IGNORECASE) +# --------------------------------------------------------------------------- +# Explanation rendering fix — dangling list-marker regression +# --------------------------------------------------------------------------- + +class TestExplanationDanglingListMarkerFix: + """Reviewer-experience fix: when the extracted security_gain sentence is + the entire body of a numbered/bulleted list item in the reviewer LLM's + explanation text, stripping it used to leave a bare marker behind (e.g. + a dangling "1." with nothing after it). This only touches how the + already-generated explanation text is rendered — _extract_security_gain + and the reviewer LLM's own output are untouched.""" + + @staticmethod + def _build(tmp_path, review): + from utilities.autopatcher.pipeline import _build_report, PipelineResult + result = PipelineResult( + vulnerability_text="# Test vulnerability\n\nSome description.", + patch=( + "--- a/mod.py\n+++ b/mod.py\n@@ -1,3 +1,3 @@\n" + " def foo():\n- return 1\n+ return 2\n" + ), + review=review, + score_text="**Confidence score:** 0.80\n\n**Reasons:**\n- ok", + challenger={"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""}, + impact={ + "impact_level": "low", "changed_files": [], "affected_files": [], + "impact_summary": "", "recommendations": [], "usage_matches": [], + }, + hygiene=[], + applicability={"applicable": True, "skipped": False, "skipped_reason": None, "error": None, "stderr": ""}, + repo_root=tmp_path, + detected_language="python", + ) + return _build_report(result) + + def test_no_dangling_list_marker_left_behind(self, tmp_path): + import re + review = ( + "**Explanation:**\n" + "1. This patch does not fix the vulnerability at all.\n\n" + "The underlying issue remains because the check was never added.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ) + report = self._build(tmp_path, review) + # The extracted sentence still appears once, as the Security gain callout. + assert "**Security gain:** This patch does not fix the vulnerability at all." in report + # No line consisting solely of a bare numbered/bulleted marker. + assert not re.search(r"^[ \t]*(?:\d+\.|[-*])[ \t]*$", report, re.MULTILINE) + # The rest of the explanation body is preserved. + assert "The underlying issue remains because the check was never added." in report + + def test_ordinary_explanation_unaffected(self, tmp_path): + """No list markers involved — behavior is unchanged from before this fix.""" + review = ( + "**Explanation:**\n" + "This patch fixes the vulnerability by validating input.\n\n" + "Additional context about the fix follows here.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ) + report = self._build(tmp_path, review) + assert "**Security gain:** This patch fixes the vulnerability by validating input." in report + assert "Additional context about the fix follows here." in report + + +# --------------------------------------------------------------------------- +# Slice 1 — Decision Consistency (report-level regression) +# --------------------------------------------------------------------------- + +class TestRecommendationConsistencyReport: + """Report-level regression coverage for Slice 1. + + Builds a PipelineResult directly and calls _build_report() — no LLM + calls, fully deterministic. Assertions check stable invariants (decision + label present, caveat text present/absent) rather than full-report + string equality, since equality would be brittle to unrelated formatting + and is not needed to prove this slice's behavior. + """ + + def _base_kwargs(self, tmp_path, challenger, applicability=None): + return dict( + vulnerability_text="# Test vulnerability\n\nSome description.", + patch=( + "--- a/mod.py\n" + "+++ b/mod.py\n" + "@@ -1,3 +1,3 @@\n" + " def foo():\n" + "- return 1\n" + "+ return 2\n" + ), + review=( + "**Explanation:**\n" + "The code was vulnerable because of X.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ), + score_text="**Confidence score:** 0.80\n\n**Reasons:**\n- ok", + challenger=challenger, + impact={ + "impact_level": "low", "changed_files": [], "affected_files": [], + "impact_summary": "", "recommendations": [], "usage_matches": [], + }, + hygiene=[], + applicability=applicability or { + "applicable": True, "skipped": False, "skipped_reason": None, + "error": None, "stderr": "", + }, + repo_root=tmp_path, + detected_language="python", + ) + + def test_no_tests_found_gets_test_caveat_only(self, tmp_path): + """minimist-representative: no matching tests, no challenger findings. + Top-tier recommendation is unchanged; the test-coverage caveat is new.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Deploy After Validation**" in report + assert "test coverage" in report + assert "adversarial coverage" not in report + # Correction: never say "0 confirmed" / imply the patch is broken. + assert "0 confirmed" not in report.lower() + assert "issue(s) flagged" not in report + + def test_low_coverage_confidence_gets_coverage_caveat_only(self, tmp_path): + """pip-representative: matching tests exist, one unresolved plausible + finding. Top-tier recommendation is unchanged; the coverage-confidence + caveat is new.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_mod.py").write_text("def test_foo(): pass\n", encoding="utf-8") + challenger = { + "still_vulnerable": False, + "edge_cases": ["custom configurations may not benefit from this change"], + "potential_issues": [], + "summary": "", + } + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Deploy After Validation**" in report + assert "adversarial coverage" in report + assert "test coverage" not in report + # Correction: never say "0 confirmed" / imply the patch is broken. + assert "0 confirmed" not in report.lower() + assert "issue(s) flagged" not in report + + def test_manual_review_required_gets_no_caveat(self, tmp_path): + """A non-top-tier decision must not gain caveats even when both + underlying signals are weak — it already reads as cautious.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": True, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Manual Review Required**" in report + assert "This recommendation currently has no automated test coverage" not in report + assert "adversarial coverage is heuristic" not in report + + def test_do_not_apply_gets_no_caveat(self, tmp_path): + """pygeoapi/curl-representative: applicability hard-block. Must remain + untouched by this slice.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs( + tmp_path, challenger, + applicability={ + "applicable": False, "skipped": False, "skipped_reason": None, + "error": None, "stderr": "error: patch does not apply", + }, + )) + report = _build_report(result) + + assert "**Do Not Apply**" in report + assert "This recommendation currently has no automated test coverage" not in report + assert "adversarial coverage is heuristic" not in report + + +# --------------------------------------------------------------------------- +# pipeline helpers +# --------------------------------------------------------------------------- + +class TestPipelineHelpers: + def test_extract_score(self): + from utilities.autopatcher.pipeline import _extract_score + text = "**Confidence score:** 0.85\n**Reasons:**\n- reason 1" + assert _extract_score(text) == "0.85" + + def test_extract_score_missing(self): + from utilities.autopatcher.pipeline import _extract_score + assert _extract_score("no score here") == "N/A" + + def test_extract_summary(self): + from utilities.autopatcher.pipeline import _extract_summary + text = "# SQL Injection\n\nSome details" + assert _extract_summary(text) == "SQL Injection" + + def test_split_review_sections(self): + from utilities.autopatcher.pipeline import _split_review + review = textwrap.dedent("""\ + **Explanation:** + The code was vulnerable because of X. + + **Affected areas:** + - app/auth.py + + **Validation notes:** + - Test with payload Y. + """) + sections = _split_review(review) + assert "vulnerable" in sections["explanation"] + assert "auth.py" in sections["affected_areas"] + assert "payload" in sections["validation_notes"] + + def test_build_recommendation_helper(self): + from utilities.autopatcher.pipeline import build_recommendation + # still_vulnerable -> Do not deploy yet + rec = build_recommendation({"still_vulnerable": True}, 0.9, "Good", {"impact_level": "low"}) + assert rec["decision"] == "Do not deploy yet" + + # high impact and no tests -> Do not deploy yet (override) + rec = build_recommendation({}, 0.95, "None", {"impact_level": "high"}) + assert rec["decision"] == "Do not deploy yet" + + # safe case + rec = build_recommendation({}, 0.80, "Good", {"impact_level": "low"}) + assert rec["decision"] == "Safe to deploy" + + # reason constraints + rec = build_recommendation({"edge_cases": ["x"]}, 0.80, "Some", {"impact_level": "medium"}) + reason = rec.get("reason", "") + assert len(reason) > 0 and len(reason) <= 120 + assert "-" not in reason and "*" not in reason + + +# --------------------------------------------------------------------------- +# Vulnerability Sources (GHSA/CVE/Advisory URL — no upstream remediation links) +# --------------------------------------------------------------------------- + +class TestPrimaryReferences: + """Direct unit coverage for _extract_primary_references/_render_primary_references + — previously only exercised indirectly through full report generation.""" + + def test_extracts_ghsa_and_cve_from_advisory_line(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "**Advisory:** GHSA-v845-jxx5-vc9f / CVE-2023-43804\n\nSome description.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] == "GHSA-v845-jxx5-vc9f" + assert refs["cve_id"] == "CVE-2023-43804" + assert refs["advisory_url"] == "https://github.com/advisories/GHSA-v845-jxx5-vc9f" + + def test_cve_only_advisory_line_uses_nvd_url(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "**Advisory:** CVE-2022-25883\n\nSome description.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] is None + assert refs["cve_id"] == "CVE-2022-25883" + assert refs["advisory_url"] == "https://nvd.nist.gov/vuln/detail/CVE-2022-25883" + + def test_file_mode_input_has_no_identifiers(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "# Some vulnerability\n\nA hand-written description with no advisory line.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] is None + assert refs["cve_id"] is None + assert refs["advisory_url"] is None + + def test_extraction_never_returns_a_commit_reference(self): + """The extractor has no notion of an upstream commit at all anymore — + not computed, not hidden, removed entirely.""" + from utilities.autopatcher.pipeline import _extract_primary_references + text = ( + "**Advisory:** GHSA-v845-jxx5-vc9f\n\n" + "## References\n\n- https://github.com/urllib3/urllib3/commit/abc123\n" + ) + refs = _extract_primary_references(text) + assert "referenced_commit" not in refs + assert set(refs.keys()) == {"ghsa_id", "cve_id", "advisory_url"} + + +class TestRenderPrimaryReferences: + def test_heading_is_vulnerability_sources(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({"ghsa_id": "GHSA-xxxx", "cve_id": None, "advisory_url": "https://github.com/advisories/GHSA-xxxx"}) + assert "## Vulnerability Sources" in block + assert "## Primary Vulnerability References" not in block + + def test_ghsa_and_advisory_url_are_clickable_links(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-v845-jxx5-vc9f", "cve_id": "CVE-2023-43804", + "advisory_url": "https://github.com/advisories/GHSA-v845-jxx5-vc9f", + }) + assert "[GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f)" in block + assert "[https://github.com/advisories/GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f)" in block + + def test_cve_only_mode_links_to_nvd(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": None, "cve_id": "CVE-2022-25883", + "advisory_url": "https://nvd.nist.gov/vuln/detail/CVE-2022-25883", + }) + assert "[CVE-2022-25883](https://nvd.nist.gov/vuln/detail/CVE-2022-25883)" in block + + def test_file_mode_degrades_to_single_line(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({"ghsa_id": None, "cve_id": None, "advisory_url": None}) + assert "## Vulnerability Sources" in block + assert "User-provided vulnerability description" in block + assert "|" not in block # no table at all, not a table of placeholders + + def test_no_upstream_commit_or_remediation_link_ever_rendered(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-xxxx", "cve_id": "CVE-2023-1", "advisory_url": "https://github.com/advisories/GHSA-xxxx", + }) + assert "Referenced Upstream Commit" not in block + assert "commit" not in block.lower() + + def test_only_three_rows_in_table(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-xxxx", "cve_id": "CVE-2023-1", "advisory_url": "https://github.com/advisories/GHSA-xxxx", + }) + row_lines = [l for l in block.splitlines() if l.startswith("| ") and "---" not in l and "Type" not in l] + assert len(row_lines) == 3 + + +# --------------------------------------------------------------------------- +# CLI (main.py) +# --------------------------------------------------------------------------- + +_FIXTURE_ADVISORY = { + "ghsa_id": "GHSA-1234-5678-9012", + "cve_id": "CVE-2021-12345", + "summary": "SQL injection in example library", + "description": "A SQL injection vulnerability exists in the authenticate() function.", + "severity": "critical", + "cvss": {"score": 9.8}, + "cwes": [{"cwe_id": "CWE-89", "name": "Improper Neutralization"}], + "vulnerabilities": [ + { + "package": {"name": "example-lib", "ecosystem": "pip"}, + "vulnerable_version_range": "< 1.2.3", + "first_patched_version": "1.2.3", + } + ], + "references": [{"url": "https://github.com/example/security/advisories/GHSA-1234"}], +} + + +# TestCLI (upstream): exercised main.py's argparse CLI, including GHSA mode. +# main.py is not ported -- its role is replaced by core/patch.py + cmd_patch +# in openant/cli.py, which have their own dedicated tests (see +# tests/patch/test_patch_wrapper_contract.py and openant-core's own CLI +# dispatch tests). GHSA mode (advisory_fetcher) is excluded entirely per the +# migration plan -- OpenAnt always supplies a rendered Finding, never a bare +# advisory ID. + + +# --------------------------------------------------------------------------- +# Pipeline code context integration +# --------------------------------------------------------------------------- + +class TestPipelineCodeContext: + def test_pipeline_passes_nonempty_code_context_when_repo_has_match(self, tmp_path): + """When repo_root contains a file matching the vulnerability, pipeline + should pass a non-empty code_context into generate_patch().""" + from utilities.autopatcher.pipeline import run + from unittest import mock as _mock + import utilities.autopatcher.patch_generator as _pg + + # Create a file that the example vulnerability.md references explicitly + auth_file = tmp_path / "app" / "auth.py" + auth_file.parent.mkdir(parents=True) + auth_file.write_text( + "def authenticate(username, password):\n" + " query = f\"SELECT * FROM users WHERE username='{username}'\"\n" + " return db.execute(query).fetchone()\n", + encoding="utf-8", + ) + + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + captured: list[str] = [] + _original = _pg.generate_patch + + def _capturing(vtext, llm, code_context=""): + captured.append(code_context) + return _original(vtext, llm) # use real (mock) implementation + + with _mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=_capturing): + run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + assert captured, "generate_patch was never called" + assert captured[0] != "", ( + "Expected non-empty code_context when repo contains app/auth.py" + ) + + def test_test_support_uses_target_repo_not_auto_patcher(self, tmp_path): + """Test Support must scan repo_root, not the Auto-Patcher's own test directory.""" + from utilities.autopatcher.pipeline import run + + # Target repo: a minimal Python project with one test file + target_test = tmp_path / "tests" / "test_auth.py" + target_test.parent.mkdir(parents=True) + target_test.write_text("def test_login(): pass\n", encoding="utf-8") + + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + # The report must reference the target repo's test, not Auto-Patcher's tests + assert "test_auth.py" in report, ( + "Test Support should list the target repo's test_auth.py" + ) + # Auto-Patcher's own tests must not appear + assert "test_pipeline.py" not in report, ( + "Test Support must not list Auto-Patcher's own test files" + ) + assert "test_advisory_fetcher.py" not in report, ( + "Test Support must not list Auto-Patcher's own test files" + ) + + +# --------------------------------------------------------------------------- +# Repository Context section (Repository Grounding surfaced in the report) +# --------------------------------------------------------------------------- + +class TestRepositoryContextSection: + """Coverage for _render_repository_context_section() and its + _selected_reason_kind() adapter — the report-facing surface of + Repository Grounding (ground_repository()/RepositoryGroundingResult). + + Builds synthetic RepositoryGroundingResult objects directly rather than + running a real repo scan — these tests exercise the renderer's mapping + and presentation rules in isolation from repo_locator's discovery + algorithm, which is already covered by tests/test_repo_locator.py. + """ + + @staticmethod + def _evidence(pass_name, tier): + from utilities.autopatcher.repository_grounding_models import DiscoveryEvidence + return DiscoveryEvidence( + pass_name=pass_name, tier=tier, matched_tokens=None, + total_occurrences=None, hit_line=0, resolution_strategy=None, + ) + + @staticmethod + def _candidate(path, evidence, best_tier): + from utilities.autopatcher.repository_grounding_models import RepositoryCandidate + return RepositoryCandidate(path=path, evidence=evidence, best_tier=best_tier) + + @staticmethod + def _decision(path, outcome): + from utilities.autopatcher.repository_grounding_models import GroundingDecision + return GroundingDecision( + path=path, outcome=outcome, snippet_ranges=None, + bytes_contributed=0, truncated=False, + ) + + @staticmethod + def _grounding(candidates, decisions): + from utilities.autopatcher.repository_grounding_models import RepositoryGroundingResult + return RepositoryGroundingResult( + rendered_context="irrelevant to these tests", + candidates=candidates, decisions=decisions, + extraction_signals={}, budget=None, + ) + + # --- renderer behavior --------------------------------------------------- + + def test_renderer_basic_layout(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("retry.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decision = self._decision("retry.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert section.startswith("---\n\n## Repository Context\n\n") + assert ( + "The following repository locations were selected to provide " + "context for patch generation and review." in section + ) + assert "**`retry.py`**" in section + assert "Selected because\n- Explicitly referenced in the security advisory" in section + assert "Used for\n- Primary reference (full file)" in section + + def test_renderer_multi_location_separated_by_rule(self): + """§4.3 of the approved plan: a horizontal rule separates entries + when more than one location is selected.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + cand_a = self._candidate("first.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + cand_b = self._candidate("second.py", [self._evidence("symbol_search", tier=2)], best_tier=2) + decisions = [ + self._decision("first.py", "primary_full_file"), + self._decision("second.py", "secondary_snippet"), + ] + grounding = self._grounding([cand_a, cand_b], decisions) + + section = _render_repository_context_section(grounding) + + assert "**`first.py`**" in section + assert "**`second.py`**" in section + between = section[section.index("first.py"):section.index("second.py")] + assert "\n\n---\n\n" in between + + # --- semantic reason mapping ---------------------------------------------- + + @pytest.mark.parametrize("pass_name,expected_phrase", [ + ("explicit_path", "Explicitly referenced in the security advisory"), + ("symbol_search", "References a symbol named in the advisory"), + ("cwe_keywords", "Contains terminology associated with this vulnerability type"), + ("class_definition_supplement", "Defines a class named in the advisory"), + ]) + def test_semantic_reason_mapping(self, pass_name, expected_phrase): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence(pass_name, tier=1)], best_tier=1) + decision = self._decision("a_file.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert expected_phrase in section + + def test_selected_reason_kind_picks_evidence_matching_best_tier(self): + """A candidate discovered by more than one pass must report the + reason for whichever pass produced its best_tier — not just the + first evidence entry appended. This is the adapter that keeps the + renderer itself ignorant of DiscoveryEvidence/best_tier/tier + matching.""" + from utilities.autopatcher.pipeline import _selected_reason_kind + candidate = self._candidate( + "a_file.py", + [self._evidence("cwe_keywords", tier=1), self._evidence("symbol_search", tier=2)], + best_tier=2, + ) + assert _selected_reason_kind(candidate) == "symbol_search" + + def test_selected_reason_kind_falls_back_to_first_evidence_when_best_tier_none(self): + """Class-definition-supplement-only candidates have best_tier=None + by construction (see repo_locator.py) — must not crash, must use + the one evidence entry present.""" + from utilities.autopatcher.pipeline import _selected_reason_kind + candidate = self._candidate( + "a_file.py", [self._evidence("class_definition_supplement", tier=None)], best_tier=None, + ) + assert _selected_reason_kind(candidate) == "class_definition_supplement" + + def test_selected_reason_kind_none_for_missing_or_empty_candidate(self): + from utilities.autopatcher.pipeline import _selected_reason_kind + assert _selected_reason_kind(None) is None + assert _selected_reason_kind(self._candidate("a_file.py", [], best_tier=None)) is None + + def test_unknown_reason_kind_falls_back_to_generic_phrase(self): + """An unrecognized pass_name (a future evidence kind not yet in the + lookup table) must degrade to a generic sentence, not KeyError.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("some_future_pass", tier=1)], best_tier=1) + decision = self._decision("a_file.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert "Identified during repository grounding" in section + + # --- usage-role mapping ----------------------------------------------- + + @pytest.mark.parametrize("outcome,expected_phrase", [ + ("primary_full_file", "Primary reference (full file)"), + ("primary_snippet", "Primary reference (excerpt)"), + ("secondary_snippet", "Supporting reference (excerpt)"), + ]) + def test_usage_role_mapping(self, outcome, expected_phrase): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decision = self._decision("a_file.py", outcome) + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert expected_phrase in section + + # --- rejected locations omitted ----------------------------------------- + + def test_rejected_locations_omitted(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + kept = self._candidate("kept_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + dropped = self._candidate("dropped_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + decisions = [ + self._decision("kept_file.py", "primary_full_file"), + self._decision("dropped_file.py", "rejected"), + ] + grounding = self._grounding([kept, dropped], decisions) + + section = _render_repository_context_section(grounding) + + assert "kept_file.py" in section + assert "dropped_file.py" not in section + + def test_all_rejected_renders_zero_selection_sentence(self): + """Every discovered candidate rejected must read the same as no + candidates at all — not a silently empty location list.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + decision = self._decision("a_file.py", "rejected") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + # --- None / empty grounding handling ------------------------------------- + + def test_none_grounding_renders_zero_selection_sentence(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + section = _render_repository_context_section(None) + assert "## Repository Context" in section + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + def test_empty_grounding_renders_zero_selection_sentence(self): + """A grounding result with no candidates/decisions at all (the real + empty/no-match exit path in ground_repository()) must render the + same zero-selection sentence as grounding=None.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + section = _render_repository_context_section(self._grounding([], [])) + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + # --- section ordering ---------------------------------------------------- + + def test_decision_order_preserved_not_resorted(self): + """The approved plan does not call for re-ordering by outcome — a + secondary location that comes before the primary in + grounding.decisions must still render in that order. No sort is + applied by the renderer.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + secondary = self._candidate("secondary_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + primary = self._candidate("primary_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decisions = [ + self._decision("secondary_file.py", "secondary_snippet"), + self._decision("primary_file.py", "primary_full_file"), + ] + grounding = self._grounding([secondary, primary], decisions) + + section = _render_repository_context_section(grounding) + + assert section.index("secondary_file.py") < section.index("primary_file.py") + + def test_repository_context_section_placed_before_impact_surface(self, tmp_path): + """§4.2 of the approved plan: Repository Context sits immediately + before Impact Surface, after Review Results — integration-level + check against a real ground_repository() result via run().""" + from utilities.autopatcher.pipeline import run + (tmp_path / "auth.py").write_text( + "def authenticate(u, p):\n return db.query(u, p)\n", encoding="utf-8" + ) + vuln_text = "SQL injection in authenticate() — see auth.py" + report = run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + idx_review = report.find("## Review Results") + idx_repo_context = report.find("## Repository Context") + idx_impact = report.find("## Impact Surface") + + assert idx_review != -1, "Review Results section missing" + assert idx_repo_context != -1, "Repository Context section missing" + assert idx_impact != -1, "Impact Surface section missing" + assert idx_review < idx_repo_context < idx_impact + + # --- path/reason/role pairing ----------------------------------------- + + def test_path_reason_role_pairing_not_mixed_up(self): + """Two locations with different reasons and different roles: each + path's own reason/role must appear paired with it, not swapped with + the other location's.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + cand_a = self._candidate("alpha_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + cand_b = self._candidate("beta_file.py", [self._evidence("symbol_search", tier=2)], best_tier=2) + decisions = [ + self._decision("alpha_file.py", "primary_full_file"), + self._decision("beta_file.py", "secondary_snippet"), + ] + grounding = self._grounding([cand_a, cand_b], decisions) + + section = _render_repository_context_section(grounding) + + block_a = section[section.index("alpha_file.py"):section.index("beta_file.py")] + block_b = section[section.index("beta_file.py"):] + + assert "Explicitly referenced in the security advisory" in block_a + assert "Primary reference (full file)" in block_a + assert "References a symbol named in the advisory" not in block_a + + assert "References a symbol named in the advisory" in block_b + assert "Supporting reference (excerpt)" in block_b diff --git a/libs/openant-core/tests/patch/test_pipeline_repair.py b/libs/openant-core/tests/patch/test_pipeline_repair.py new file mode 100644 index 00000000..99e054ab --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline_repair.py @@ -0,0 +1,604 @@ +"""Tests for the Phase C challenger-driven repair loop in pipeline.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Shared test fixtures +# --------------------------------------------------------------------------- + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ pass + pass +```""" + +_REPAIR_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ # repaired + pass +```""" + +# A challenger result that contains a confirmed defect (bypasses _EXPLICIT_DEFECT_RE) +_CHALLENGER_WITH_DEFECT = { + "still_vulnerable": False, + "edge_cases": ["An attacker can bypass this check via path traversal"], + "potential_issues": [], + "summary": "The patch has a confirmed bypass.", +} + +# A challenger result with BOTH a confirmed defect and a plausible-risk finding +_CHALLENGER_MIXED = { + "still_vulnerable": False, + "edge_cases": ["An attacker can bypass this check via crafted input"], + "potential_issues": ["Performance may be impacted under high load"], + "summary": "Mixed findings.", +} + +# A clean challenger result (no defects) +_CHALLENGER_CLEAN = { + "still_vulnerable": False, + "edge_cases": [], + "potential_issues": [], + "summary": "No issues found.", +} + +# A challenger result whose finding is a plausible_risk only +_CHALLENGER_RISK_ONLY = { + "still_vulnerable": False, + "edge_cases": ["This might behave unexpectedly under concurrent access"], + "potential_issues": [], + "summary": "Unverified risk only.", +} + +_APPLICABILITY_CLEAN = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, +} +_APPLICABILITY_FAIL = { + "applicable": False, "skipped": False, "stderr": "error: patch failed", + "exit_code": 1, "skipped_reason": None, "error": None, +} +_APPLICABILITY_SKIPPED = { + "applicable": None, "skipped": True, "stderr": "", + "exit_code": None, "skipped_reason": "no repo_root", "error": None, +} + + +def _capture_result(tmp_path, *, patches_gen, patches_app, patches_chall, repo_root=None): + """Run pipeline.run() with controlled mocks and capture the PipelineResult.""" + captured = {} + import utilities.autopatcher.pipeline as _pipeline_mod + orig_build = _pipeline_mod._build_report + + def _capture(r): + captured["result"] = r + return orig_build(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=patches_gen) as _mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=patches_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok review"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", side_effect=patches_chall) as _mock_chall, + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=_capture), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=repo_root or str(tmp_path)) + + return captured["result"], _mock_gen, _mock_chall + + +# --------------------------------------------------------------------------- +# _build_repair_hint +# --------------------------------------------------------------------------- + +class TestBuildRepairHint: + def test_contains_defect_text(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["attacker can bypass the check via path traversal"]) + assert "bypass" in hint + + def test_operation_level_instruction_present(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["some defect"]) + assert "dangerous operation" in hint.lower() or "perimeter" in hint.lower() + + def test_multiple_defects_produce_multiple_bullets(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["defect one", "defect two"]) + assert "defect one" in hint + assert "defect two" in hint + assert hint.count("- defect") == 2 + + def test_helper_wiring_instruction_present(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["helper not called"]) + assert "helper" in hint.lower() and "every" in hint.lower() + + def test_empty_list_returns_string(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint([]) + assert isinstance(hint, str) + assert len(hint) > 0 + + +# --------------------------------------------------------------------------- +# _render_repair_notice +# --------------------------------------------------------------------------- + +class TestRenderRepairNotice: + def _make_result(self, **kwargs): + from utilities.autopatcher.pipeline import PipelineResult + defaults = dict( + vulnerability_text="v", patch="p", review="r", + score_text="Confidence score: 0.8", challenger={}, + ) + defaults.update(kwargs) + return PipelineResult(**defaults) + + def test_no_notice_when_not_attempted(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result(repair_attempted=False) + assert _render_repair_notice(r) == "" + + def test_success_notice_contains_auto_repaired(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=True, + original_challenger_defect_count=2, + ) + notice = _render_repair_notice(r) + assert "auto-repaired" in notice.lower() + assert "2" in notice + + def test_failure_notice_contains_repair_attempted(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=True, + original_challenger_defect_count=1, repair_defect_count=1, + ) + notice = _render_repair_notice(r) + assert "repair attempted" in notice.lower() + assert "1" in notice + + def test_failure_notice_states_original_recommendation_stands(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=True, + original_challenger_defect_count=2, repair_defect_count=1, + ) + notice = _render_repair_notice(r) + assert "original recommendation stands" in notice.lower() + + def test_never_rechallenged_notice_does_not_claim_a_defect_count(self): + """When the repair patch never reached re-challenge (applicability + failure or an internal error), the notice must say so explicitly and + must not present the untouched repair_defect_count default as if it + were an observed finding.""" + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=False, + original_challenger_defect_count=2, repair_defect_count=0, + ) + notice = _render_repair_notice(r) + assert "repair attempted" in notice.lower() + assert "did not reach re-challenge" in notice.lower() + assert "no repair defect count is available" in notice.lower() + assert "original recommendation stands" in notice.lower() + # Must not claim the repair patch "had 0 confirmed defect(s)" — that + # number was never observed. + assert "repair patch still had" not in notice.lower() + assert "0 confirmed defect(s)" not in notice.lower() + + def test_never_rechallenged_notice_still_reports_original_count(self): + """The original (pre-repair) defect count IS a real, observed value + and should still be reported.""" + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=False, + original_challenger_defect_count=3, repair_defect_count=0, + ) + notice = _render_repair_notice(r) + assert "3" in notice + + +# --------------------------------------------------------------------------- +# Repair NOT triggered +# --------------------------------------------------------------------------- + +class TestRepairNotTriggered: + """Repair must NOT fire when there are no confirmed defects or patch does not apply.""" + + def test_no_repair_when_no_confirmed_defects(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + assert mock_chall.call_count == 1 + + def test_no_repair_when_plausible_risk_only(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_RISK_ONLY], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_no_repair_when_applicable_false(self, tmp_path): + # Patch doesn't apply — repair must not fire (applicability retry handles this) + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_no_repair_when_applicable_none(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_SKIPPED], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_urllib3_analog_passes_through_unchanged(self, tmp_path): + # urllib3 pattern: applicable, no confirmed defects → single generate, single challenge + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert result.repair_succeeded is False + assert result.repair_patch is None + assert mock_gen.call_count == 1 + assert mock_chall.call_count == 1 + + +# --------------------------------------------------------------------------- +# Repair triggered +# --------------------------------------------------------------------------- + +class TestRepairTriggered: + """Repair fires when applicable=True and confirmed_defect_count > 0.""" + + def test_repair_calls_generate_patch_twice(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_attempted is True + assert mock_gen.call_count == 2 + + def test_repair_calls_challenge_patch_twice(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert mock_chall.call_count == 2 + + def test_repair_hint_passed_to_second_generate_call(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + _args, kwargs = mock_gen.call_args_list[1] + hint = kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) or "" + assert "bypass" in hint.lower() + + def test_repair_hint_contains_confirmed_defect_not_plausible_risk(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_MIXED, _CHALLENGER_CLEAN], + ) + _args, kwargs = mock_gen.call_args_list[1] + hint = kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) or "" + # confirmed defect text is in the hint + assert "bypass" in hint.lower() + # plausible risk (performance) is NOT in the repair hint + assert "Performance" not in hint + + def test_repair_attempted_flag_set(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_attempted is True + + +# --------------------------------------------------------------------------- +# Repair outcomes +# --------------------------------------------------------------------------- + +class TestRepairOutcomes: + def test_repair_accepted_when_no_defects_remain(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_succeeded is True + # Final patch must be the repair diff content + assert "repaired" in result.patch + + def test_repair_rejected_when_defects_remain_after_repair(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + # repair challenger still has a confirmed defect + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + # patch must be the original, not the repair + assert "repaired" not in result.patch + + def test_repair_rejected_when_repair_patch_does_not_apply(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + # repair patch does not apply + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + assert result.repair_attempted is True + # Repair patch never reached re-challenge (applicability failed first), + # so repair_defect_count must not be presented as an observed value. + assert result.repair_rechallenged is False + + def test_review_runs_on_accepted_repair_patch(self, tmp_path): + """When repair is accepted, review_patch must be called with the repair diff.""" + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=[_CLEAN_DIFF, _REPAIR_DIFF]), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok") as mock_review, + mock.patch("utilities.autopatcher.pipeline.challenge_patch", + side_effect=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN]), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=str(tmp_path)) + # review_patch first arg is vuln text, second is the patch + _args, _ = mock_review.call_args + reviewed_patch = _args[1] + assert "repaired" in reviewed_patch + + def test_original_patch_kept_on_rejection(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + assert "repaired" not in result.patch + + +# --------------------------------------------------------------------------- +# Repair metadata +# --------------------------------------------------------------------------- + +class TestRepairMetadata: + def test_repair_patch_stored_even_when_not_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_patch is not None + assert "repaired" in result.repair_patch + + def test_repair_patch_stored_even_when_rejected_on_defects(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_patch is not None + + def test_original_challenger_defect_count_stored(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.original_challenger_defect_count == 1 + + def test_repair_challenger_stored_when_repair_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_challenger is not None + + def test_repair_challenger_none_when_repair_not_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_challenger is None + + def test_repair_defect_count_zero_on_success(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_defect_count == 0 + + def test_repair_defect_count_nonzero_on_rejection(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_defect_count > 0 + + def test_repair_rechallenged_true_when_repair_patch_applies(self, tmp_path): + """repair_defect_count is a real, observed value here — repair_rechallenged + must be True whether the re-challenge finds 0 or nonzero defects.""" + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_rechallenged is True + + def test_repair_rechallenged_false_when_repair_patch_does_not_apply(self, tmp_path): + """repair_defect_count stays at its untouched default (0) here — this + must be distinguishable from an actual re-challenge finding 0 defects.""" + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_defect_count == 0 + assert result.repair_rechallenged is False + + def test_defaults_when_repair_not_triggered(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert result.repair_succeeded is False + assert result.repair_patch is None + assert result.repair_challenger is None + assert result.repair_defect_count == 0 + assert result.repair_rechallenged is False + assert result.original_challenger_defect_count == 0 + + +# --------------------------------------------------------------------------- +# Report content +# --------------------------------------------------------------------------- + +class TestRepairReport: + def _run_and_get_report(self, tmp_path, patches_gen, patches_app, patches_chall): + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=patches_gen), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=patches_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="**Explanation**\nok\n" + "**Affected areas**\nok\n**Validation notes**\nok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", side_effect=patches_chall), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + return run("test vuln", api_key="", repo_root=str(tmp_path)) + + def test_report_contains_auto_repaired_on_success(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert "auto-repaired" in report.lower() + + def test_report_contains_repair_attempted_on_rejection(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert "repair attempted" in report.lower() + + def test_report_no_repair_notice_when_not_triggered(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert "auto-repaired" not in report.lower() + assert "repair attempted" not in report.lower() + + def test_report_states_no_rechallenge_when_repair_patch_does_not_apply(self, tmp_path): + """End-to-end: when the repair patch fails applicability, the rendered + report must not claim a repair defect count that was never observed.""" + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert "repair attempted" in report.lower() + assert "did not reach re-challenge" in report.lower() + assert "no repair defect count is available" in report.lower() + assert "repair patch still had" not in report.lower() + assert "0 confirmed defect(s)" not in report.lower() + + def test_report_states_defect_count_before_repair(self, tmp_path): + # original had 1 confirmed defect; notice should say 1 + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert "1 confirmed defect" in report diff --git a/libs/openant-core/tests/patch/test_pipeline_retry.py b/libs/openant-core/tests/patch/test_pipeline_retry.py new file mode 100644 index 00000000..ab5fd839 --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline_retry.py @@ -0,0 +1,505 @@ +"""Tests for applicability-aware retry logic in pipeline.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +_PIP_STDERR = ( + "error: patch failed: src/pip/_internal/download.py:1\n" + "error: src/pip/_internal/download.py: patch does not apply\n" +) + +_CORRUPT_STDERR = "error: corrupt patch at line 7\n" + +_PIP_DIFF_ORIG = """\ +```diff +--- a/src/pip/_internal/download.py ++++ b/src/pip/_internal/download.py +@@ -1,3 +1,4 @@ + from __future__ import absolute_import ++# security fix + import os +```""" + +_PIP_DIFF_RETRY = """\ +```diff +--- a/src/pip/_internal/download.py ++++ b/src/pip/_internal/download.py +@@ -1,3 +1,4 @@ + import os ++# security fix + import sys +```""" + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ # security fix + pass +```""" + + +# --------------------------------------------------------------------------- +# _extract_failed_file +# --------------------------------------------------------------------------- + +class TestExtractFailedFile: + def test_patch_failed_format(self): + from utilities.autopatcher.pipeline import _extract_failed_file + result = _extract_failed_file(_PIP_STDERR) + assert result == "src/pip/_internal/download.py" + + def test_does_not_apply_format(self): + from utilities.autopatcher.pipeline import _extract_failed_file + stderr = "error: some/path/file.py: patch does not apply\n" + assert _extract_failed_file(stderr) == "some/path/file.py" + + def test_corrupt_patch_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file(_CORRUPT_STDERR) is None + + def test_empty_stderr_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file("") is None + + def test_none_stderr_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file(None) is None + + def test_patch_failed_takes_priority_over_does_not_apply(self): + from utilities.autopatcher.pipeline import _extract_failed_file + # Both formats present — patch failed is matched first + result = _extract_failed_file(_PIP_STDERR) + assert result == "src/pip/_internal/download.py" + + +# --------------------------------------------------------------------------- +# _extract_patch_target +# --------------------------------------------------------------------------- + +class TestExtractPatchTarget: + def test_fenced_diff_returns_target(self): + from utilities.autopatcher.pipeline import _extract_patch_target + result = _extract_patch_target(_PIP_DIFF_ORIG) + assert result == "src/pip/_internal/download.py" + + def test_unfenced_diff_returns_target(self): + from utilities.autopatcher.pipeline import _extract_patch_target + unfenced = ( + "--- a/src/foo.py\n" + "+++ b/src/foo.py\n" + "@@ -1,1 +1,1 @@\n" + "-old\n" + "+new\n" + ) + assert _extract_patch_target(unfenced) == "src/foo.py" + + def test_no_plus_plus_line_returns_none(self): + from utilities.autopatcher.pipeline import _extract_patch_target + assert _extract_patch_target("no diff here") is None + + def test_empty_patch_returns_none(self): + from utilities.autopatcher.pipeline import _extract_patch_target + assert _extract_patch_target("") is None + + +# --------------------------------------------------------------------------- +# _build_retry_hint +# --------------------------------------------------------------------------- + +class TestBuildRetryHint: + def test_contains_failed_file(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "src/pip/_internal/download.py") + assert "src/pip/_internal/download.py" in hint + + def test_contains_stderr_excerpt(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "some/file.py") + assert "error: patch failed" in hint + + def test_instructs_not_to_use_training_memory(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "some/file.py") + assert "training" in hint.lower() or "ground truth" in hint.lower() + + def test_stderr_truncated_to_limit(self): + from utilities.autopatcher.pipeline import _build_retry_hint, _RETRY_STDERR_LINES + many_lines = "\n".join(f"line {i}" for i in range(50)) + hint = _build_retry_hint(many_lines, "f.py") + # Only first _RETRY_STDERR_LINES lines should appear + assert f"line {_RETRY_STDERR_LINES}" not in hint + assert "line 0" in hint + + +# --------------------------------------------------------------------------- +# Retry NOT triggered +# --------------------------------------------------------------------------- + +class TestRetryNotTriggered: + """Retry must NOT run when applicable=True, applicable=None, or no repo_root.""" + + def _run_with_mock(self, applicability_return, repo_root=None, vuln="test vuln"): + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", return_value=_CLEAN_DIFF) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", return_value=applicability_return), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok review"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run(vuln, api_key="", repo_root=repo_root) + return mock_gen.call_count + + def test_no_retry_when_applicable_true(self, tmp_path): + call_count = self._run_with_mock( + {"applicable": True, "skipped": False, "stderr": "", "exit_code": 0, + "skipped_reason": None, "error": None}, + repo_root=str(tmp_path), + ) + assert call_count == 1 + + def test_no_retry_when_applicable_none(self, tmp_path): + call_count = self._run_with_mock( + {"applicable": None, "skipped": True, "stderr": "", "exit_code": None, + "skipped_reason": "no repo_root", "error": None}, + repo_root=str(tmp_path), + ) + assert call_count == 1 + + def test_no_retry_when_no_repo_root(self): + call_count = self._run_with_mock( + {"applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None}, + repo_root=None, + ) + assert call_count == 1 + + +# --------------------------------------------------------------------------- +# Retry triggered — success and failure outcomes +# --------------------------------------------------------------------------- + +class TestRetryTriggered: + """applicable=False + repo_root → retry must be attempted.""" + + def _setup_mocks( + self, + tmp_path: Path, + retry_applicable: bool, + failed_file: str = "src/pip/_internal/download.py", + ): + """Return a context-manager tuple for the common retry scenario.""" + # Write the file the retry will read + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + return first_app, retry_app + + def test_retry_calls_generate_patch_twice(self, tmp_path): + first_app, retry_app = self._setup_mocks(tmp_path, retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 2 + + def test_retry_call_includes_retry_hint(self, tmp_path): + first_app, retry_app = self._setup_mocks(tmp_path, retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + _args, kwargs = mock_gen.call_args_list[1] + assert kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) + + +# --------------------------------------------------------------------------- +# Retry outcomes — patch and metadata +# --------------------------------------------------------------------------- + +class TestRetryOutcomes: + def _run_retry_scenario(self, tmp_path, retry_applicable): + failed_file = "src/pip/_internal/download.py" + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + + captured_result = {} + + original_build_report = None + import utilities.autopatcher.pipeline as _pipeline_mod + original_build_report = _pipeline_mod._build_report + + def capture_result(r): + captured_result["result"] = r + return original_build_report(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=capture_result), + ): + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + + return captured_result["result"] + + def test_retry_succeeded_uses_retry_patch(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=True) + assert result.retry_succeeded is True + # diff_hunk_repair rewrites @@ counts, so compare content not exact string + assert "import sys" in result.patch + assert result.patch != result.original_patch + + def test_retry_failed_keeps_original_patch(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert result.retry_succeeded is False + # patch must equal original_patch (both go through hunk repair, so compare relative identity) + assert result.patch == result.original_patch + + def test_retry_failed_keeps_original_applicability(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert result.applicability["applicable"] is False + + def test_retry_patch_stored_in_metadata_on_failure(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + # retry_patch is stored even on failure (for inspection) + assert result.retry_patch is not None + assert "import sys" in result.retry_patch + + def test_original_patch_preserved_in_metadata_on_success(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=True) + # original_patch must contain the original content (from __future__ not in retry) + assert "from __future__" in result.original_patch + + def test_original_patch_preserved_in_metadata_on_failure(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert "from __future__" in result.original_patch + + +# --------------------------------------------------------------------------- +# Retry metadata fields +# --------------------------------------------------------------------------- + +class TestRetryMetadata: + def _run_no_retry(self, tmp_path): + app = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, + } + captured_result = {} + import utilities.autopatcher.pipeline as _pipeline_mod + original_build_report = _pipeline_mod._build_report + + def capture(r): + captured_result["result"] = r + return original_build_report(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", return_value=_CLEAN_DIFF), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", return_value=app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=capture), + ): + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=str(tmp_path)) + return captured_result["result"] + + def test_no_retry_metadata_defaults(self, tmp_path): + result = self._run_no_retry(tmp_path) + assert result.retry_attempted is False + assert result.retry_succeeded is False + assert result.retry_patch is None + assert result.retry_failed_file is None + assert result.retry_error_before is None + + def test_original_patch_matches_patch_when_no_retry(self, tmp_path): + result = self._run_no_retry(tmp_path) + assert result.original_patch == result.patch + + +# --------------------------------------------------------------------------- +# _render_retry_notice — isolated unit tests (mirrors +# tests/test_pipeline_repair.py::TestRenderRepairNotice) +# --------------------------------------------------------------------------- + +class TestRenderRetryNotice: + def _make_result(self, **kwargs): + from utilities.autopatcher.pipeline import PipelineResult + defaults = dict( + vulnerability_text="v", patch="p", review="r", + score_text="Confidence score: 0.8", challenger={}, + ) + defaults.update(kwargs) + return PipelineResult(**defaults) + + def test_no_notice_when_not_attempted(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=False) + assert _render_retry_notice(r) == "" + + def test_success_notice_content(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=True, retry_succeeded=True) + notice = _render_retry_notice(r) + assert "Applicability-aware retry" in notice + assert "Initial patch did not apply." in notice + assert "Applicability-aware retry was attempted." in notice + assert "Retry succeeded" in notice + assert "Retry failed" not in notice + + def test_failure_notice_content(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=True, retry_succeeded=False) + notice = _render_retry_notice(r) + assert "Applicability-aware retry" in notice + assert "Initial patch did not apply." in notice + assert "Applicability-aware retry was attempted." in notice + assert "Retry failed to produce an applicable patch." in notice + assert "Retry succeeded" not in notice + + +# --------------------------------------------------------------------------- +# Retry notice — end-to-end wiring into the rendered report +# --------------------------------------------------------------------------- + +class TestRetryNoticeInReport: + """Proves the notice is actually wired into _build_report's output, in + the right place, and has zero footprint when no retry occurred.""" + + def _run_and_capture_report(self, tmp_path, *, trigger_retry: bool, retry_applicable=None): + if trigger_retry: + failed_file = "src/pip/_internal/download.py" + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + gen_side_effect = [_PIP_DIFF_ORIG, _PIP_DIFF_RETRY] + app_side_effect = [first_app, retry_app] + else: + clean_app = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, + } + gen_side_effect = [_CLEAN_DIFF] + app_side_effect = [clean_app] + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=gen_side_effect), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=app_side_effect), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + report = run("pip vuln", api_key="", repo_root=str(tmp_path)) + return report + + def test_no_notice_in_report_when_no_retry_occurred(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=False) + assert "Applicability-aware retry" not in report + + def test_notice_appears_in_report_on_retry_success(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=True) + assert "Applicability-aware retry" in report + assert "Retry succeeded" in report + + def test_notice_appears_in_report_on_retry_failure(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=False) + assert "Applicability-aware retry" in report + assert "Retry failed to produce an applicable patch." in report + + def test_notice_positioned_after_patch_applicability(self, tmp_path): + """Patch Applicability and the retry notice were promoted out of + Appendices (second reviewer-experience pass) to sit directly after + Proposed patch, before Trust Signals — Appendices now comes well + after both.""" + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=False) + applicability_idx = report.index("## Patch Applicability") + notice_idx = report.index("Applicability-aware retry") + trust_idx = report.index("## Trust Signals") + appendices_idx = report.index("## Appendices") + assert applicability_idx < notice_idx < trust_idx < appendices_idx diff --git a/libs/openant-core/tests/patch/test_repo_locator.py b/libs/openant-core/tests/patch/test_repo_locator.py new file mode 100644 index 00000000..cf7b4433 --- /dev/null +++ b/libs/openant-core/tests/patch/test_repo_locator.py @@ -0,0 +1,1800 @@ +"""Unit tests for repo_locator. All tests use tmp_path repos — no real repos needed.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +class TestExplicitFilePath: + def test_finds_file_mentioned_by_path(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "app" / "auth.py", "def authenticate(u, p):\n pass\n") + vuln = "Vulnerability in app/auth.py — SQL injection in authenticate()" + result = find_code_context(vuln, tmp_path) + assert "authenticate" in result + + def test_explicit_path_preferred_over_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context, _extract_file_paths + write(tmp_path / "app" / "auth.py", "def authenticate(u, p):\n pass\n") + write(tmp_path / "other" / "stuff.py", "def authenticate(x):\n pass\n") + vuln = "Vulnerability in app/auth.py — authenticate() is exploitable" + paths = _extract_file_paths(vuln) + assert "app/auth.py" in paths + result = find_code_context(vuln, tmp_path) + assert result != "" + # auth.py should appear (highest score) + assert "auth" in result + + +class TestRepositoryPathResolver: + """Unit tests for RepositoryPathResolver: exact match + unique suffix + match only. Basename-only matching is explicitly out of scope for this + slice — see test_bare_filename_does_not_fall_back_to_basename_match.""" + + def test_exact_match_resolves(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("app/auth.py") + assert result.strategy == "exact" + assert result.path == (tmp_path / "app" / "auth.py").resolve() + + def test_suffix_match_resolves_src_layout(self, tmp_path): + """Mirrors pip's shape: advisory names `_internal/download.py`, + the real file lives under a src-layout prefix the advisory text + never mentions.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "suffix" + assert result.path == ( + tmp_path / "src" / "pip" / "_internal" / "download.py" + ).resolve() + + def test_exact_match_preferred_over_suffix(self, tmp_path): + """When both an exact-join file and a deeper suffix-matching file + exist, the exact match wins — the fallback must never override a + working resolution.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "_internal" / "download.py", "# shallow\n") + write(tmp_path / "src" / "pip" / "_internal" / "download.py", "# deep\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "exact" + assert result.path == (tmp_path / "_internal" / "download.py").resolve() + + def test_ambiguous_suffix_match_is_not_resolved(self, tmp_path): + """Two files sharing the same suffix under different top-level + directories must not be silently guessed — this is the safety + property the whole fallback depends on.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "ambiguous" + assert result.path is None + + def test_bare_filename_does_not_fall_back_to_basename_match(self, tmp_path): + """A single-segment path (no directory component) must not trigger + suffix matching — that would be basename matching, which is + explicitly out of scope for this slice.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "deeply" / "nested" / "auth.py", "def authenticate(): pass\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("auth.py") + assert result.strategy == "unresolved" + assert result.path is None + + def test_no_match_returns_unresolved(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "other.py", "print('hello')\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("app/auth.py") + assert result.strategy == "unresolved" + assert result.path is None + + def test_suffix_match_respects_path_segment_boundaries(self, tmp_path): + """A directory named `my_internal` must not satisfy a suffix + search for `_internal/download.py` — matching is segment-aware, + not a raw string-suffix comparison.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "my_internal" / "download.py", "# decoy\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "unresolved" + assert result.path is None + + +class TestExplicitPathSrcLayoutIntegration: + """find_code_context() integration: the src-layout fallback must surface + the real file end-to-end, through Pass 1, not just at the resolver + unit-test level.""" + + def test_src_layout_file_included_via_suffix_fallback(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(link, location):\n pass\n", + ) + vuln = "Vulnerability in _internal/download.py — unpack_url() follows unsafe redirects" + result = find_code_context(vuln, tmp_path) + assert "unpack_url" in result + assert "src/pip/_internal/download.py" in result + + def test_ambiguous_src_layout_path_does_not_crash_or_leak_wrong_file(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "a" / "_internal" / "download.py", "def a_impl(): pass\n") + write(tmp_path / "b" / "_internal" / "download.py", "def b_impl(): pass\n") + vuln = "Vulnerability in _internal/download.py" + result = find_code_context(vuln, tmp_path) + assert isinstance(result, str) # no crash + # Neither ambiguous candidate should be silently chosen via Pass 1 + assert "a_impl" not in result + assert "b_impl" not in result + + +class TestSymbolMatch: + def test_finds_function_by_backtick_name(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "src" / "db.py", "def execute_query(sql):\n cursor.execute(sql)\n") + vuln = "The `execute_query` function does not use parameterized queries" + result = find_code_context(vuln, tmp_path) + assert "execute_query" in result + + def test_finds_snake_case_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "lib" / "auth.py", "def validate_token(tok):\n return True\n") + vuln = "validate_token does not check expiry — security bypass possible" + result = find_code_context(vuln, tmp_path) + assert "validate_token" in result + + def test_finds_pascal_case_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "models" / "user.py", "class UserManager:\n pass\n") + vuln = "UserManager lacks access control checks" + result = find_code_context(vuln, tmp_path) + assert "UserManager" in result + + +class TestCweKeywordFallback: + def test_sql_injection_cwe_finds_cursor(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "db" / "queries.py", + "def run(sql):\n cursor.execute(sql)\n return cursor.fetchone()\n", + ) + vuln = "SQL injection (CWE-89) in authentication module" + result = find_code_context(vuln, tmp_path) + assert result != "" + assert "execute" in result or "cursor" in result + + +# --------------------------------------------------------------------------- +# Stage 2: CWE-name token expansion (cwe_name_tokens) +# +# Per reports/implementation_plan_stage2_token_expansion_2026-07-14.md: +# union with _CWE_KEYWORDS (not replacement), lowercase tokens, minimal +# grammatical stopword filter only (no _GENERIC_TOKENS reuse — §0.2), +# parenthetical abbreviations kept (§0.3), no ranking/scoring/policy change. +# --------------------------------------------------------------------------- + +class TestCweNameTokens: + """Unit tests for cwe_name_tokens() — pure text function, no repo needed.""" + + def test_all_three_type_line_shapes_parse_without_bare_cwe_token(self): + """The three real **Type:** line shapes observed in this project's + corpus (CWE-id-first with parens, name-first with parens, name-first + with an em-dash and a nested parenthetical abbreviation) must all + parse without a bare CWE-NNN token surviving into the output.""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + curl_text = "**Type:** Insufficiently Protected Credentials (CWE-522)" + node_semver_text = ( + "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + ) + ghsa_text = "**Type:** CWE-798 (Use of Hard-coded Credentials)" + for text in (curl_text, node_semver_text, ghsa_text): + tokens = cwe_name_tokens(text) + assert not any("cwe" in t.lower() for t in tokens), tokens + + def test_curl_type_line_yields_expected_tokens(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Insufficiently Protected Credentials (CWE-522)" + tokens = cwe_name_tokens(text) + assert "credentials" in tokens + assert "protected" in tokens + assert "insufficiently" in tokens + + def test_node_semver_type_line_keeps_nested_abbreviation(self): + """The nested (ReDoS) abbreviation must survive as `redos` — approved + Stage 2 design decision (§0.3): kept, not discarded. `service` must + also survive (§0.2: _GENERIC_TOKENS reuse was rejected).""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + tokens = cwe_name_tokens(text) + assert "redos" in tokens + assert "regular" in tokens + assert "expression" in tokens + assert "service" in tokens + + def test_ghsa_style_cwe_id_first_shape_parses(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** CWE-798 (Use of Hard-coded Credentials)" + tokens = cwe_name_tokens(text) + assert "credentials" in tokens + assert "hard-coded" in tokens + + def test_generic_cwe_name_does_not_crash_or_special_case(self): + """CWE-200's real name is broad; must not crash, and no special-casing + for 'generic' CWE ids — the same minimal filter as every other case.""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = ( + "**Type:** CWE-200 " + "(Exposure of Sensitive Information to an Unauthorized Actor)" + ) + tokens = cwe_name_tokens(text) + assert isinstance(tokens, list) + assert all(isinstance(t, str) for t in tokens) + assert not any("cwe" in t.lower() for t in tokens) + + def test_no_type_line_returns_empty_list(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + assert cwe_name_tokens("No Type line present in this text at all.") == [] + + def test_tokens_are_lowercase(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + tokens = cwe_name_tokens(text) + assert tokens == [t.lower() for t in tokens] + assert "Regular" not in tokens + assert "Service" not in tokens + + def test_proven_valuable_words_survive_stopword_filter(self): + """expression/regular/prototype/credentials must never be filtered — + encodes the prior investigation's own false-negative lesson.""" + from utilities.autopatcher.repo_locator import _CWE_NAME_STOPWORDS + for word in ("expression", "regular", "prototype", "credentials"): + assert word not in _CWE_NAME_STOPWORDS + + def test_deterministic_ordering(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + assert cwe_name_tokens(text) == cwe_name_tokens(text) + + def test_bare_cwe_id_never_survives_prototype_pollution_name(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = ( + "**Type:** CWE-1321 (Improperly Controlled Modification of " + "Object Prototype Attributes ('Prototype Pollution'))" + ) + tokens = cwe_name_tokens(text) + assert not any("1321" in t for t in tokens) + assert "prototype" in tokens + assert "pollution" in tokens + + +class TestCweKeywordDictUnionUnaffected: + """Union, not replacement: CWE-22's existing _CWE_KEYWORDS coverage must + be unaffected by the addition of cwe_name_tokens().""" + + def test_cwe22_dict_token_still_reachable_via_find_code_context(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "fs.py", + "def resolve(p):\n return os.path.abspath(p)\n", + ) + vuln = "**Type:** CWE-22 (Path Traversal)\n\nA path traversal vulnerability." + ctx = find_code_context(vuln, tmp_path) + assert "abspath" in ctx + + +class TestCweNameTokensPass3Integration: + """find_code_context() integration: a CWE with no _CWE_KEYWORDS dict + entry (CWE-522) must still surface a real file via cwe_name_tokens, + with no Pass 1/2 signal available to find it any other way.""" + + def test_undicted_cwe_finds_file_via_derived_token(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "lib" / "vauth" / "digest.c", + "/* stores user credentials for digest authentication */\n" + "struct auth_state { char *credentials; };\n", + ) + vuln = ( + "**Type:** Insufficiently Protected Credentials (CWE-522)\n\n" + "curl leaks credentials to a different host on redirect." + ) + ctx = find_code_context(vuln, tmp_path) + assert "credentials" in ctx + assert "digest.c" in ctx + + +class TestNoMatch: + def test_returns_empty_string_when_nothing_found(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "unrelated.py", "print('hello')\n") + vuln = "Vulnerability in authenticate_user() — SQL injection (CWE-89)" + # authenticate_user is specific; unrelated.py has no such content + result = find_code_context(vuln, tmp_path) + # May or may not be empty depending on CWE fallback hits — key thing: no crash + assert isinstance(result, str) + + def test_empty_repo_returns_empty_string(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + result = find_code_context("SQL injection in authenticate()", tmp_path) + assert result == "" + + +class TestCharCap: + def test_snippet_mode_respects_char_cap(self, tmp_path): + """Files > 20 000 chars use snippet mode; result must stay near the cap.""" + from utilities.autopatcher.repo_locator import find_code_context + # File must exceed _FULL_FILE_THRESHOLD_CHARS (20 000) to trigger snippet mode + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000, "file must be large enough to bypass full-file mode" + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in app/auth.py — authenticate()" + result = find_code_context(vuln, tmp_path) + assert len(result) <= 4_500 # snippet budget + header overhead + + def test_small_file_fully_included(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + content = "def authenticate(u, p):\n pass\n" + write(tmp_path / "app" / "auth.py", content) + vuln = "SQL injection in app/auth.py" + result = find_code_context(vuln, tmp_path) + assert "def authenticate" in result + + +class TestGenericTokensIgnored: + def test_generic_tokens_produce_no_symbol_matches(self, tmp_path): + from utilities.autopatcher.repo_locator import _extract_symbols + vuln = "Error in handler: request response data output config settings" + symbols = _extract_symbols(vuln) + assert symbols == [] + + def test_specific_tokens_extracted(self, tmp_path): + from utilities.autopatcher.repo_locator import _extract_symbols + vuln = "SQL injection in `authenticate_user` and `validate_session`" + symbols = _extract_symbols(vuln) + assert "authenticate_user" in symbols + assert "validate_session" in symbols + + +# --------------------------------------------------------------------------- +# Change 1: ranking by total occurrence count +# --------------------------------------------------------------------------- + +class TestRankingByTotalOccurrences: + def test_higher_occurrence_count_ranks_first(self, tmp_path): + """File with more token hits should rank before file with fewer hits.""" + from utilities.autopatcher.repo_locator import _grep_repo + + # a.py: 4 occurrences of 'authenticate' + write(tmp_path / "a.py", + "authenticate()\nauthenticate()\nauthenticate()\nauthenticate()\n") + # b.py: 1 occurrence — sorts before a.py alphabetically but should rank lower + write(tmp_path / "b.py", "def authenticate(): pass\n") + + results = _grep_repo(tmp_path, ["authenticate"]) + assert results, "expected at least one result" + assert results[0][0].name == "a.py", ( + f"expected a.py (4 hits) first, got {results[0][0].name}" + ) + + def test_tie_broken_consistently(self, tmp_path): + """Two files with equal hit counts should both be returned.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "x.py", "frozenset(['Authorization'])\n") + write(tmp_path / "y.py", "frozenset(['Authorization'])\n") + results = _grep_repo(tmp_path, ["Authorization"]) + names = {r[0].name for r in results} + assert "x.py" in names and "y.py" in names + + +# --------------------------------------------------------------------------- +# Change 2: test-file exclusion +# --------------------------------------------------------------------------- + +class TestTestFileExclusion: + def test_test_prefix_file_excluded(self, tmp_path): + """test_*.py files must not appear in _grep_repo results.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "tests" / "test_auth.py", + "authenticate() " * 50) # many hits + write(tmp_path / "src" / "auth.py", + "def authenticate(): pass\n") # 1 hit + + results = _grep_repo(tmp_path, ["authenticate"]) + names = [r[0].name for r in results] + assert "test_auth.py" not in names, "test files must be excluded" + assert "auth.py" in names + + def test_tests_directory_excluded(self, tmp_path): + """Files inside a tests/ directory must not appear.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "tests" / "integration.py", + "Authorization " * 20) + write(tmp_path / "lib" / "http.py", + "Authorization = 'Bearer'\n") + + results = _grep_repo(tmp_path, ["Authorization"]) + names = [r[0].name for r in results] + assert "integration.py" not in names + assert "http.py" in names + + def test_spec_directory_excluded(self, tmp_path): + """Files inside a spec/ directory must not appear.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "spec" / "retry_spec.py", + "Authorization " * 20) + write(tmp_path / "retry.py", + "DEFAULT_REMOVE = frozenset(['Authorization'])\n") + + results = _grep_repo(tmp_path, ["Authorization"]) + names = [r[0].name for r in results] + assert "retry_spec.py" not in names + assert "retry.py" in names + + +# --------------------------------------------------------------------------- +# Change 3: backtick term extraction +# --------------------------------------------------------------------------- + +class TestBacktickTermExtraction: + def test_cookie_extracted_despite_generic_filter(self): + """'cookie' is in _GENERIC_TOKENS but `Cookie` in backticks must be extracted.""" + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "urllib3 doesn't strip the `Cookie` header on redirects." + terms = _extract_backtick_terms(text) + assert "Cookie" in terms + + def test_authorization_extracted(self): + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "The `Authorization` header is leaked to the redirect target." + terms = _extract_backtick_terms(text) + assert "Authorization" in terms + + def test_multiple_backtick_terms_extracted(self): + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "Use `Cookie` and `Authorization` with `remove_headers_on_redirect`." + terms = _extract_backtick_terms(text) + assert "Cookie" in terms + assert "Authorization" in terms + assert "remove_headers_on_redirect" in terms + + def test_generic_tokens_not_filtered_from_backtick_terms(self): + """Backtick extraction bypasses _GENERIC_TOKENS.""" + from utilities.autopatcher.repo_locator import _extract_backtick_terms, _GENERIC_TOKENS + # Build text where every token is in GENERIC_TOKENS + generic = " ".join(f"`{t}`" for t in list(_GENERIC_TOKENS)[:5]) + terms = _extract_backtick_terms(generic) + # All of them should be present (no filtering) + for t in list(_GENERIC_TOKENS)[:5]: + if len(t) >= 2: + assert t in terms, f"'{t}' should not be filtered from backtick terms" + + def test_backtick_term_used_as_search_signal(self, tmp_path): + """Advisory with backtick term finds file containing that token.""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "retry.py", + "DEFAULT_REMOVE = frozenset(['Authorization'])\n") + # Advisory mentions Authorization in backticks — should find retry.py + vuln = "The `Authorization` header is not stripped on cross-origin redirects." + ctx = find_code_context(vuln, tmp_path) + assert "Authorization" in ctx + assert "retry.py" in ctx + + +# --------------------------------------------------------------------------- +# Context grounding: _is_docstring_line, _find_code_block_after, _extract_snippet +# --------------------------------------------------------------------------- + +def _make_large_file( + tmp_path: "Path", name: str, docstring_lines: int = 60, _force_large: bool = False +) -> "Path": + """Create a >150-line Python file with a docstring section then constants. + + When _force_large=True, pad with trailing comments until the file exceeds + 20 000 chars (full-file threshold), keeping the constant close enough for + the code-anchor scan to find it. + """ + lines = ["class Retry:"] + lines.append(' """') + for i in range(docstring_lines): + lines.append(f" :param int p{i}: Param {i} description text here.") + lines.append(' """') + lines.append("") + lines.append(" DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])") + lines.append(" DEFAULT_BACKOFF_MAX = 120") + lines.append("") + lines.append(" def __init__(self, total=10, redirect=None):") + lines.append(" self.total = total") + lines.append(" self.redirect = redirect") + # Pad to exceed _SMALL_FILE_THRESHOLD (150 lines) + while len(lines) < 160: + lines.append("") + # Optionally pad with trailing comments to exceed the full-file threshold + # (20 000 chars), keeping the constant reachable within the anchor scan. + content = "\n".join(lines) + while len(content) < 22_000 if _force_large else False: + lines.append("# padding " + "x" * 70) + content = "\n".join(lines) + p = tmp_path / name + p.write_text(content, encoding="utf-8") + return p + + +class TestIsDocstringLine: + def test_rst_directive_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(":param int redirect: How many redirects") is True + + def test_indented_rst_directive_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" :param int redirect: How many redirects") is True + + def test_prose_without_operators_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" How many redirects to perform.") is True + + def test_triple_quote_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(' """') is True + + def test_constant_assignment_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line( + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])" + ) is False + + def test_def_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" def __init__(self, total=10):") is False + + def test_class_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("class Retry:") is False + + def test_empty_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("") is False + + def test_import_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("import re") is False + + +class TestFindCodeBlockAfter: + def test_finds_constant_after_docstring(self, tmp_path): + from utilities.autopatcher.repo_locator import _find_code_block_after + _make_large_file(tmp_path, "retry.py") + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + dq_close = next( + i for i, l in enumerate(lines) if l.strip() == '"""' and i > 0 + ) + result_tuple = _find_code_block_after(lines, dq_close + 1) + assert result_tuple is not None + text, start_0 = result_tuple + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert isinstance(start_0, int) + + def test_returns_none_when_no_code_found(self): + from utilities.autopatcher.repo_locator import _find_code_block_after + lines = [" prose line only" for _ in range(210)] + assert _find_code_block_after(lines, 0) is None + + def test_respects_max_lines_when_anchor_is_def(self): + """max_lines fallback applies when the first anchor is a def/class line.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # Class with only methods, no constants — anchor is a def. + lines = [ + " def method_a(self):", + " return 1", + " def method_b(self):", + " return 2", + ] * 20 # many lines; max_lines cap must kick in + result_tuple = _find_code_block_after(lines, 0, max_lines=3) + assert result_tuple is not None + text, _ = result_tuple + assert len(text.splitlines()) <= 3 + + def test_structural_collection_ignores_max_lines_for_constants(self): + """When anchor is a constant, all constants are collected even if > max_lines.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # 20 constants followed by a def — no max_lines truncation for constants. + lines = [f" CONST_{i} = {i}" for i in range(20)] + [ + " def __init__(self):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0, max_lines=5) + assert result_tuple is not None + text, _ = result_tuple + # All 20 constants must be present despite max_lines=5 + for i in range(20): + assert f"CONST_{i}" in text + # Method must not be included + assert "def __init__" not in text + + def test_stops_at_first_method_definition(self): + """Structural extraction stops exactly at the first def line.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + lines = [ + " DEFAULT_HEADERS = frozenset(['Authorization'])", + " DEFAULT_TIMEOUT = 30", + "", + " def __init__(self):", + " self.x = 1", + " def increment(self):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + assert "DEFAULT_HEADERS" in text + assert "DEFAULT_TIMEOUT" in text + assert "def __init__" not in text + assert "def increment" not in text + + def test_critical_constant_visible_when_not_first_in_block(self): + """A constant that is not the first anchor must still appear in the result.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # Mirrors urllib3 retry.py: DEFAULT_ALLOWED_METHODS is the first anchor, + # DEFAULT_REMOVE_HEADERS_ON_REDIRECT comes 9 lines later. + lines = [ + " #: Default methods", + " DEFAULT_ALLOWED_METHODS = frozenset(['GET', 'POST', 'HEAD'])", + "", + " #: Status codes", + " RETRY_AFTER_STATUS_CODES = frozenset([429, 503])", + "", + " #: Headers to strip on redirect", + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " #: Backoff", + " DEFAULT_BACKOFF_MAX = 120", + "", + " def __init__(self, total=10):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + # The critical constant must be visible even though it is not the first anchor + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert "frozenset(['Authorization'])" in text + # Method must not bleed into the result + assert "def __init__" not in text + + +class TestExtractSnippetGrounding: + def test_docstring_hit_appends_code_anchor(self, tmp_path): + """When hit_line is inside a docstring, snippet must include the constant.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + text, ranges = _extract_snippet(content, hit_line, 4000) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Code anchor must appear in snippet when hit is inside a docstring" + ) + assert len(ranges) == 2, "Non-contiguous snippet must return two ranges" + + def test_code_hit_does_not_add_anchor(self, tmp_path): + """When hit_line is already a code line, only the window range is returned.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next( + i for i, l in enumerate(lines) + if "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in l + ) + text, ranges = _extract_snippet(content, hit_line, 4000) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert len(ranges) == 1, "Code hit should produce a single range" + + def test_anchor_preserved_when_budget_tight(self, tmp_path): + """When budget is tight, code anchor survives even if window is cut.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + text, ranges = _extract_snippet(content, hit_line, 500) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Code anchor must be preserved over docstring window when budget is tight" + ) + + def test_small_file_returns_one_range(self, tmp_path): + """Files at or below _SMALL_FILE_THRESHOLD return a single range.""" + from utilities.autopatcher.repo_locator import _extract_snippet + small = "def foo():\n pass\n" * 5 # << 150 lines + text, ranges = _extract_snippet(small, 0, 4000) + assert "def foo" in text + assert len(ranges) == 1 + + +# --------------------------------------------------------------------------- +# Integration: urllib3 context now includes the vulnerable constant +# --------------------------------------------------------------------------- + +_URLLIB3_EVAL = Path("/tmp/urllib3-eval") +_URLLIB3_RETRY_PY = _URLLIB3_EVAL / "src" / "urllib3" / "util" / "retry.py" + +_run_live = ( + os.environ.get("RUN_LIVE_REPO_TESTS") == "1" + and _URLLIB3_RETRY_PY.exists() +) + + +@pytest.mark.skipif( + not _run_live, + reason=( + "Live repo tests opt-in only — " + "set RUN_LIVE_REPO_TESTS=1 and populate /tmp/urllib3-eval " + "with a urllib3 checkout containing src/urllib3/util/retry.py" + ), +) +class TestUrllib3ContextGrounding: + def test_default_remove_headers_in_context(self): + """After grounding improvement, the urllib3 context must include + DEFAULT_REMOVE_HEADERS_ON_REDIRECT so the LLM can produce the real fix.""" + import sys + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + from advisory_fetcher import fetch_ghsa + from advisory_converter import ghsa_to_vuln_text + from utilities.autopatcher.repo_locator import find_code_context + + adv = fetch_ghsa("GHSA-v845-jxx5-vc9f") + vuln_text = ghsa_to_vuln_text(adv) + ctx = find_code_context(vuln_text, _URLLIB3_EVAL) + + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in ctx, ( + "Context must include the vulnerable constant so the LLM can generate the fix" + ) + + def test_context_header_uses_full_relative_path(self): + """Context header must show src/urllib3/util/retry.py, not just retry.py.""" + import sys + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + from advisory_fetcher import fetch_ghsa + from advisory_converter import ghsa_to_vuln_text + from utilities.autopatcher.repo_locator import find_code_context + + adv = fetch_ghsa("GHSA-v845-jxx5-vc9f") + vuln_text = ghsa_to_vuln_text(adv) + ctx = find_code_context(vuln_text, _URLLIB3_EVAL) + + assert "# src/urllib3/util/retry.py" in ctx, ( + "Context header must contain the full repo-relative path" + ) + assert "# retry.py\n" not in ctx, ( + "Basename-only header must not appear" + ) + + +# --------------------------------------------------------------------------- +# Context header path grounding (no live repo required) +# --------------------------------------------------------------------------- + +class TestContextHeaderLineRanges: + def test_header_includes_line_range(self, tmp_path): + """Files > 20 000 chars use snippet mode and must show a (lines N-M) annotation.""" + from utilities.autopatcher.repo_locator import find_code_context + nested = tmp_path / "app" / "auth.py" + nested.parent.mkdir(parents=True) + # Build a large file (> 20 000 chars) so snippet mode is used + content = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(content) > 20_000 + nested.write_text(content, encoding="utf-8") + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + assert "(lines " in ctx, f"Expected line range in header, got:\n{ctx[:200]}" + + def test_non_contiguous_snippet_shows_two_ranges(self, tmp_path): + """When window + code anchor are non-contiguous, header shows both ranges. + Uses _force_large=True so the file exceeds 20 000 chars (snippet mode) + while keeping the constant close enough for the anchor scan. + """ + from utilities.autopatcher.repo_locator import find_code_context + _make_large_file(tmp_path, "retry.py", docstring_lines=60, _force_large=True) + vuln = "The `p0` parameter is not bounded correctly." + ctx = find_code_context(vuln, tmp_path) + import re as _re + header_match = _re.search(r'\(lines ([0-9]+-[0-9]+), ([0-9]+-[0-9]+)\)', ctx) + assert header_match is not None, ( + f"Expected two ranges in header for non-contiguous snippet, got:\n{ctx[:300]}" + ) + + def test_range_numbers_are_sensible(self, tmp_path): + """Line range start must be >= 1 and end >= start.""" + from utilities.autopatcher.repo_locator import find_code_context + import re as _re + _make_large_file(tmp_path, "retry.py", docstring_lines=60, _force_large=True) + vuln = "The `p0` parameter is not bounded correctly." + ctx = find_code_context(vuln, tmp_path) + for m in _re.finditer(r'(\d+)-(\d+)', ctx.split("\n")[0]): + start, end = int(m.group(1)), int(m.group(2)) + assert start >= 1 + assert end >= start + + +class TestContextHeaderPaths: + def test_nested_file_uses_full_relative_path(self, tmp_path): + """Header must be app/auth.py, not auth.py.""" + from utilities.autopatcher.repo_locator import find_code_context + + nested = tmp_path / "app" / "auth.py" + nested.parent.mkdir(parents=True) + nested.write_text("def authenticate(u, p):\n pass\n", encoding="utf-8") + + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + + assert "# app/auth.py" in ctx, f"Expected '# app/auth.py' in context, got:\n{ctx[:300]}" + assert "# auth.py\n" not in ctx + + def test_deeply_nested_file_uses_full_relative_path(self, tmp_path): + """Header must be src/urllib3/util/retry.py for deeply nested files.""" + from utilities.autopatcher.repo_locator import find_code_context + + deep = tmp_path / "src" / "urllib3" / "util" / "retry.py" + deep.parent.mkdir(parents=True) + deep.write_text( + "class Retry:\n" + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])\n", + encoding="utf-8", + ) + + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + + assert "# src/urllib3/util/retry.py" in ctx, ( + f"Expected full relative path in header, got:\n{ctx[:300]}" + ) + + +# --------------------------------------------------------------------------- +# Full-file mode +# --------------------------------------------------------------------------- + +class TestFullFileMode: + def test_small_file_uses_full_file_context(self, tmp_path): + """Files <= 20 000 chars must be sent in full, not as a snippet.""" + from utilities.autopatcher.repo_locator import find_code_context + content = ( + "class Retry:\n" + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])\n" + " def __init__(self): pass\n" + ) + assert len(content) < 20_000 + write(tmp_path / "retry.py", content) + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + # Full file content must be present + assert "class Retry:" in ctx + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in ctx + assert "def __init__" in ctx + + def test_full_file_header_format(self, tmp_path): + """Header must say '(full file, N lines)' in full-file mode.""" + from utilities.autopatcher.repo_locator import find_code_context + import re as _re + content = "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n" + write(tmp_path / "src" / "retry.py", content) + vuln = "The `Authorization` header is not stripped." + ctx = find_code_context(vuln, tmp_path) + assert "(full file," in ctx, f"Header must say full file, got:\n{ctx[:200]}" + n_lines = len(content.splitlines()) + assert f"{n_lines} lines)" in ctx + + def test_full_file_mode_includes_only_one_file(self, tmp_path): + """When only one candidate matches, full-file mode returns exactly that file.""" + from utilities.autopatcher.repo_locator import find_code_context + # Only retry.py contains Authorization — no backtick on redirects so + # connectionpool.py produces zero signal hits and is not a candidate. + write(tmp_path / "retry.py", + "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n") + write(tmp_path / "connectionpool.py", + "class ConnectionPool:\n redirects = True\n") + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, ( + f"With one matching candidate, output must have exactly one header, got: {headers}" + ) + + def test_large_file_falls_back_to_snippet_range_mode(self, tmp_path): + """Files > 20 000 chars must use snippet + range mode, not full-file.""" + from utilities.autopatcher.repo_locator import find_code_context + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000 + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + assert "(full file," not in ctx, "Large file must not use full-file mode" + assert "(lines " in ctx, "Large file must show line ranges" + + +# --------------------------------------------------------------------------- +# Structural constant extraction regression +# --------------------------------------------------------------------------- + +class TestStructuralConstantExtraction: + """Regression: structural extraction captures the complete constant block. + + Previously _find_code_block_after used a fixed 40-line window from the + first anchor. When many constants precede the target constant, it fell + outside the window. The structural approach (scan until first def/class) + is invariant to constant count and ordering. + """ + + def _build_many_constants_content(self, num_before: int = 50) -> str: + """Return file content with num_before constants before the critical one. + + The docstring is intentionally long (60 param lines) so that the ±30-line + window around the docstring hit contains only docstring — never the + constants section or method definition below it. This makes the anchor + block the sole source of constants in the snippet. + """ + lines = ["class Retry:", ' """'] + # 60 docstring param lines; hit_line will be somewhere in here (> 30 + # lines from the constants section, so the window never reaches them). + for i in range(60): + lines.append(f" :param int p{i}: Param {i} description.") + lines.append(' """') + lines.append("") + for i in range(num_before): + lines.append(f" PRECEDING_CONST_{i:02d} = {i}") + lines += [ + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " def __init__(self, redirect=None):", + " self.redirect = redirect", + ] + content = "\n".join(lines) + # Pad well past both thresholds (>150 lines, >20 000 chars) to force + # snippet mode and trigger the docstring-anchor path. + content += "\n" + (" # padding " + "x" * 70 + "\n") * 320 + return content + + def test_critical_constant_visible_with_many_preceding_constants(self): + """Structural extraction exposes the target even when > 40 constants precede it.""" + from utilities.autopatcher.repo_locator import _extract_snippet + + content = self._build_many_constants_content(num_before=50) + lines = content.splitlines() + + # Sanity: file is large enough to be in snippet mode + assert len(content) > 20_000 + + # Sanity: the constant is more than 40 lines from the start of the + # constants section (which is what old fixed-window would have given). + first_const_line = next( + i for i, l in enumerate(lines) if "PRECEDING_CONST_00" in l + ) + target_line = next( + i for i, l in enumerate(lines) + if "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in l + ) + assert (target_line - first_const_line) > 40, ( + "Test precondition: target must be > 40 lines from first constant " + "to prove the fixed-window approach would have missed it" + ) + + # Hit line is the docstring — triggers anchor mode + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + + text, ranges = _extract_snippet(content, hit_line, 4000) + + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Critical constant must appear in snippet even when preceded by " + "more than 40 constants" + ) + + def test_method_body_does_not_bleed_into_constant_block(self): + """Structural extraction stops before the first def — no method body lines.""" + from utilities.autopatcher.repo_locator import _extract_snippet + + content = self._build_many_constants_content(num_before=5) + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + + text, _ = _extract_snippet(content, hit_line, 4000) + + assert "def __init__" not in text, ( + "def __init__ must not appear in the constant block snippet" + ) + assert "self.redirect = redirect" not in text, ( + "Method body must not bleed into the constant block" + ) + + def test_snippet_size_bounded_by_constant_block_not_fixed_window(self): + """With few constants the result is smaller than the old 40-line window.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + + # 4 constants, then __init__ — same shape as urllib3 retry.py + lines = [ + " #: Default allowed methods", + " DEFAULT_ALLOWED_METHODS = frozenset(['GET', 'POST'])", + "", + " #: Status codes", + " RETRY_AFTER_STATUS_CODES = frozenset([429, 503])", + "", + " #: Headers to strip", + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " #: Max backoff", + " DEFAULT_BACKOFF_MAX = 120", + "", + " def __init__(self, total=10, redirect=None):", + " self.total = total", + ] + [" # body line\n"] * 50 # would inflate a fixed-40 result + + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + + # Structural result is tighter than the old 40-line block + n_lines = len(text.splitlines()) + assert n_lines < 40, ( + f"Structural extraction ({n_lines} lines) should be smaller than " + "old fixed 40-line window for a typical small constant block" + ) + # Critical constant is present + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + + +# --------------------------------------------------------------------------- +# Hybrid context budget: secondary snippets appended after full-file primary +# --------------------------------------------------------------------------- + +class TestHybridContextBudget: + """Full-file mode now injects snippets from ranked[1] and ranked[2] within + a bounded secondary budget so implementation files below the primary are + visible to the model.""" + + def _setup_two_candidates(self, tmp_path, primary_hits: int = 5, secondary_hits: int = 2): + """Create two files that both match on FileSystemProvider. + + primary.py has more hits and is always below _FULL_FILE_THRESHOLD_CHARS, + so it wins rank #1 and triggers full-file mode. + secondary.py has fewer hits and becomes the secondary candidate. + """ + primary_content = "class FileSystemProvider:\n pass\n" * primary_hits + " # pad\n" * 20 + secondary_content = ( + "class FileSystemProvider:\n def items(self, dirpath):\n" + " data_path = self.data + dirpath\n" + ) * secondary_hits + assert len(primary_content) < 20_000 + write(tmp_path / "api.py", primary_content) + write(tmp_path / "provider.py", secondary_content) + return "FileSystemProvider path traversal" + + def test_full_file_mode_includes_secondary_candidates(self, tmp_path): + """Primary is returned in full; secondary candidate snippet is appended.""" + from utilities.autopatcher.repo_locator import find_code_context + vuln = self._setup_two_candidates(tmp_path) + ctx = find_code_context(vuln, tmp_path) + + assert "api.py" in ctx, "primary file must appear in context" + assert "(full file," in ctx, "primary must use full-file header" + assert "provider.py" in ctx, "secondary candidate must be appended" + + def test_single_candidate_behavior_unchanged(self, tmp_path): + """When only one file matches, output is identical to pre-hybrid behavior.""" + from utilities.autopatcher.repo_locator import find_code_context + content = "class FileSystemProvider:\n pass\n" * 5 + assert len(content) < 20_000 + write(tmp_path / "only.py", content) + # unrelated file — no matching signal + write(tmp_path / "other.py", "def unrelated(): pass\n") + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + assert "only.py" in ctx + assert "(full file," in ctx + assert "other.py" not in ctx, "non-matching file must not appear" + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, f"single candidate must produce one header, got: {headers}" + + def test_secondary_budget_respected(self, tmp_path): + """Total secondary snippet size stays within _SECONDARY_CONTEXT_BUDGET.""" + from utilities.autopatcher.repo_locator import find_code_context, _SECONDARY_CONTEXT_BUDGET + # Primary: small, triggers full-file mode + primary_content = "class FileSystemProvider:\n pass\n" * 5 + assert len(primary_content) < 20_000 + write(tmp_path / "api.py", primary_content) + # Secondary: much larger than the secondary budget + big_secondary = "class FileSystemProvider:\n" + " # line\n" * 2000 + assert len(big_secondary) > _SECONDARY_CONTEXT_BUDGET + write(tmp_path / "provider.py", big_secondary) + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + # Locate the secondary section and measure its size + sep = "\n\n# provider.py" + assert sep in ctx, "secondary section must be present" + secondary_portion = ctx[ctx.index(sep) + 2:] # from "# provider.py" onward + assert len(secondary_portion) <= _SECONDARY_CONTEXT_BUDGET + 200, ( + f"secondary portion ({len(secondary_portion)} chars) exceeds budget " + f"{_SECONDARY_CONTEXT_BUDGET} + 200 header overhead" + ) + + def test_secondary_snippet_has_header(self, tmp_path): + """Secondary snippet header shows the file path and a line range.""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "api.py", + "class FileSystemProvider:\n pass\n" * 5) + write(tmp_path / "provider.py", + "class FileSystemProvider:\n" + " def items(self, dirpath):\n" + " data_path = self.data + dirpath\n") + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + # Secondary header must include path and line range annotation + assert "# provider.py (lines " in ctx, ( + f"secondary header must show path + line range; got context start:\n{ctx[:400]}" + ) + + +# --------------------------------------------------------------------------- +# _find_class_definitions unit tests +# --------------------------------------------------------------------------- + +class TestFindClassDefinitions: + """Unit tests for the class-definition supplement helper.""" + + def test_finds_defining_file(self, tmp_path): + """Returns the file that contains 'class FileSystemProvider'.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n def get_data_path(self): pass\n") + results = _find_class_definitions("FileSystemProvider path traversal", tmp_path) + assert len(results) == 1 + p, _, hit_line = results[0] + assert p.name == "filesystem.py" + assert hit_line == 0 + + def test_hit_line_points_to_class_not_file_top(self, tmp_path): + """hit_line is the 0-indexed line of the class definition, not always 0.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + preamble = "import os\n" * 20 # 20 lines before the class + write(tmp_path / "fs.py", preamble + "class FileSystemProvider:\n pass\n") + results = _find_class_definitions("FileSystemProvider path traversal", tmp_path) + assert results, "should find fs.py" + _, _, hit_line = results[0] + assert hit_line == 20, f"expected hit_line=20, got {hit_line}" + + def test_ignores_names_below_min_length(self, tmp_path): + """Names shorter than _MIN_CLASS_NAME_LENGTH (5) must not trigger a scan.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "foo.py", "class Foo:\n pass\n") # len 3 + write(tmp_path / "abcd.py", "class Abcd:\n pass\n") # len 4 + results = _find_class_definitions("Foo and Abcd have vulnerabilities", tmp_path) + assert results == [], f"short names must be ignored, got {results}" + + def test_excludes_test_files(self, tmp_path): + """Files inside tests/ or named test_*.py must not appear.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "tests" / "test_provider.py", + "class FileSystemProvider:\n pass\n") + write(tmp_path / "provider.py", + "class FileSystemProvider:\n pass\n") + results = _find_class_definitions("FileSystemProvider path traversal", tmp_path) + assert len(results) == 1 + assert results[0][0].name == "provider.py", "test file must be excluded" + + def test_returns_empty_when_no_pascal_case_in_advisory(self, tmp_path): + """Advisory with only snake_case and backtick terms produces no class-def scan.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "auth.py", "def authenticate_user(): pass\n") + results = _find_class_definitions( + "SQL injection in `authenticate_user` (CWE-89)", tmp_path + ) + assert results == [] + + def test_multiple_class_names_matched(self, tmp_path): + """Multiple PascalCase class names in the advisory each trigger discovery.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "stac_handler.py", + "class StacHandler:\n pass\n") + write(tmp_path / "filesystem.py", + "class FileSystemProvider:\n pass\n") + results = _find_class_definitions( + "FileSystemProvider and StacHandler both have path traversal", tmp_path + ) + names = {r[0].name for r in results} + assert "stac_handler.py" in names + assert "filesystem.py" in names + + def test_sorted_by_definition_count_descending(self, tmp_path): + """File with more class definitions appears first.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + # two.py defines FileSystemProvider twice (e.g. re-export + real class) + write(tmp_path / "two.py", + "class FileSystemProvider:\n pass\nclass FileSystemProvider:\n pass\n") + write(tmp_path / "one.py", + "class FileSystemProvider:\n pass\n") + results = _find_class_definitions("FileSystemProvider path traversal", tmp_path) + assert results[0][0].name == "two.py", "higher definition count must rank first" + + +# --------------------------------------------------------------------------- +# Class-definition supplement integration tests (find_code_context) +# --------------------------------------------------------------------------- + +class TestClassDefinitionGrounding: + """Class-definition supplement: advisory-named PascalCase classes appear + in secondary context even when occurrence-count ranking pushes them below + the ranked[1:3] window.""" + + def _write_high_hit_primary(self, tmp_path: Path, signal: str, n: int = 25) -> Path: + """Write api.py with many occurrences of signal but no class definition.""" + content = ( + f"# {signal} routing module\n" + + f"def handle_{signal}(req): pass # {signal}\n" * n + ) + assert len(content) < 20_000 + return write(tmp_path / "api.py", content) + + def test_class_def_file_injected_despite_low_occurrence_rank(self, tmp_path): + """filesystem.py defines the advisory-named class but is excluded by _grep_repo[:3]. + + Hit counts verified against \\bstac\\b (underscore is a word char so + "handle_stac" does NOT contribute a standalone "stac" hit; only the + trailing "# stac" comment in _write_high_hit_primary does): + api.py 26 hits (1 header + 25 comments) -> ranked[0], primary + urls.py 10 hits (1 header + 9 /stac/ paths) -> ranked[1] + flask_app 8 hits (1 header + 7 /stac/ paths) -> ranked[2] + filesystem 5 hits (4 standalone + 1 FileSystemProvider) -> 4th, outside [:3] + Without the supplement filesystem.py is never in candidates. + With it, _find_class_definitions fires and injects it as secondary slot 1. + """ + from utilities.autopatcher.repo_locator import find_code_context + # api.py: 26 stac hits (1 header + 25 trailing comments) -> primary + self._write_high_hit_primary(tmp_path, "stac", 25) + # urls.py: 10 stac hits (1 header + 9 "/stac/" route paths) + write(tmp_path / "urls.py", + "# stac route\n" + "path('/stac/')\n" * 9) + # flask_app.py: 8 stac hits (1 header + 7 "/stac/" route paths) + write(tmp_path / "flask_app.py", + "# stac routes\n" + "@app.route('/stac/')\n" * 7) + # filesystem.py: 5 hits (4 standalone stac + 1 FileSystemProvider) -> 4th + # Excluded by _grep_repo[:3]; only the class-def supplement can inject it. + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n") + ctx = find_code_context( + "FileSystemProvider path traversal in `stac` collection", tmp_path + ) + assert "filesystem.py" in ctx, ( + "class-def file must be injected via supplement despite being excluded by _grep_repo[:3]" + ) + + def test_primary_file_not_displaced(self, tmp_path): + """api.py (highest `stac` occurrence count) must remain the full-file primary + even when the class-def supplement fires for filesystem.py. + + Uses _write_high_hit_primary so api.py gets 26 standalone \\bstac\\b hits + (vs filesystem.py's 5), ensuring api.py wins ranked[0]. + """ + from utilities.autopatcher.repo_locator import find_code_context + # api.py: 26 stac hits -> primary (ranked[0]) + self._write_high_hit_primary(tmp_path, "stac", 25) + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n") + ctx = find_code_context( + "FileSystemProvider path traversal in `stac` collection", tmp_path + ) + assert "# api.py (full file," in ctx, "api.py must remain the full-file primary" + + def test_class_def_file_not_duplicated_when_already_primary(self, tmp_path): + """If the class-defining file IS the primary, it must not appear twice.""" + from utilities.autopatcher.repo_locator import find_code_context + content = ( + "class FileSystemProvider:\n" + " def get_data_path(self, p): return p\n" + ) * 5 + assert len(content) < 20_000 + write(tmp_path / "filesystem.py", content) + ctx = find_code_context("FileSystemProvider path traversal", tmp_path) + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, ( + f"class-def file as primary must not be duplicated; headers: {headers}" + ) + + def test_no_class_name_in_advisory_behavior_unchanged(self, tmp_path): + """Advisory without PascalCase class names: supplement is empty. + Output must be consistent with pre-supplement logic (no crash, non-empty).""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "auth.py", + "def authenticate_user(u, p):\n return u == 'admin'\n") + ctx = find_code_context("SQL injection in `authenticate_user` (CWE-89)", tmp_path) + assert "authenticate_user" in ctx + + def test_short_class_names_do_not_trigger_supplement(self, tmp_path): + """PascalCase names < _MIN_CLASS_NAME_LENGTH must not trigger supplement scan.""" + from utilities.autopatcher.repo_locator import _find_class_definitions + write(tmp_path / "foo.py", "class Foo:\n pass\n") + write(tmp_path / "base.py", "class Base:\n pass\n") + assert _find_class_definitions("Foo and Base vulnerability", tmp_path) == [] + + def test_class_def_snippet_includes_class_body(self, tmp_path): + """Snippet for class-def supplement must include the class definition itself, + not just the top-of-file preamble (hit_line points to the class line).""" + from utilities.autopatcher.repo_locator import find_code_context + self._write_high_hit_primary(tmp_path, "stac", 25) + preamble = "import os\nimport sys\n\n" # 3 lines before class + class_body = ( + "class FileSystemProvider:\n" + " def get_data_path(self, path):\n" + " return self.root + path\n" + ) + write(tmp_path / "provider.py", preamble + class_body) + ctx = find_code_context("FileSystemProvider path traversal in stac", tmp_path) + assert "class FileSystemProvider" in ctx, ( + "class definition must appear in the secondary snippet" + ) + + def test_secondary_budget_still_respected(self, tmp_path): + """Adding a supplement does not blow the secondary budget.""" + from utilities.autopatcher.repo_locator import find_code_context, _SECONDARY_CONTEXT_BUDGET + self._write_high_hit_primary(tmp_path, "stac", 25) + # Large class-def file that far exceeds the secondary budget + big_provider = ( + "class FileSystemProvider:\n" + + " # implementation line\n" * 2000 + ) + assert len(big_provider) > _SECONDARY_CONTEXT_BUDGET + write(tmp_path / "provider.py", big_provider) + ctx = find_code_context("FileSystemProvider path traversal in stac", tmp_path) + # Locate the secondary portion (everything after the primary header) + primary_end = ctx.index("\n\n") if "\n\n" in ctx else len(ctx) + secondary = ctx[primary_end:] + assert len(secondary) <= _SECONDARY_CONTEXT_BUDGET + 300, ( + f"secondary portion ({len(secondary)} chars) exceeds budget allowance" + ) + + +# --------------------------------------------------------------------------- +# Live pygeoapi integration (opt-in) +# --------------------------------------------------------------------------- + +_PYGEOAPI_EVAL = Path("/private/tmp/pygeoapi-eval") +_PYGEOAPI_STAC_PY = _PYGEOAPI_EVAL / "pygeoapi" / "api" / "stac.py" +_PYGEOAPI_FS_PY = _PYGEOAPI_EVAL / "pygeoapi" / "provider" / "filesystem.py" + +_run_pygeoapi_live = ( + os.environ.get("RUN_LIVE_REPO_TESTS") == "1" + and _PYGEOAPI_STAC_PY.exists() + and _PYGEOAPI_FS_PY.exists() +) + + +@pytest.mark.skipif( + not _run_pygeoapi_live, + reason=( + "Live pygeoapi tests opt-in only — " + "set RUN_LIVE_REPO_TESTS=1 and ensure " + "/private/tmp/pygeoapi-eval contains both " + "pygeoapi/api/stac.py and pygeoapi/provider/filesystem.py" + ), +) +class TestPygeoAPIClassDefinitionGrounding: + _VULN_TEXT = ( + "# pygeoapi 0.23.x: Path Traversal in STAC FileSystemProvider\n\n" + "**Type:** CWE-22\n\n" + "A raw string path concatenation vulnerability in pygeoapi's STAC " + "FileSystemProvider plugin allows path traversal via `stac-collection` " + "resources." + ) + + def test_filesystem_py_appears_in_context(self): + """After class-def supplement, filesystem.py must appear in injected context.""" + from utilities.autopatcher.repo_locator import find_code_context + ctx = find_code_context(self._VULN_TEXT, _PYGEOAPI_EVAL) + assert "filesystem.py" in ctx, ( + "filesystem.py must be injected via class-def supplement" + ) + + def test_stac_py_remains_primary(self): + """stac.py (highest occurrence count) must remain the full-file primary.""" + from utilities.autopatcher.repo_locator import find_code_context + ctx = find_code_context(self._VULN_TEXT, _PYGEOAPI_EVAL) + assert "# pygeoapi/api/stac.py (full file," in ctx, ( + "stac.py must remain the full-file primary" + ) + + +# --------------------------------------------------------------------------- +# Stage 6A: runtime observability — resolution_strategy propagation, +# explicit_path_resolutions, selected_pass, selected. Additive only: the +# pre-existing explicit_paths/explicit_paths_unresolved/explicit_paths_ambiguous +# fields must be unaffected by any of this. +# --------------------------------------------------------------------------- + +def _latest_debug_artifact(tmp_path: Path) -> dict: + debug_dir = tmp_path / "reports" / "debug" + files = sorted(debug_dir.glob("context_selection_*.json")) + assert files, f"expected a debug artifact under {debug_dir}" + return json.loads(files[-1].read_text(encoding="utf-8")) + + +class TestExplicitPathResolutionsAdditive: + """explicit_path_resolutions is added alongside the three existing + fields, which must remain exactly as they were.""" + + def test_existing_three_fields_unchanged_on_exact_match(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + assert signals["explicit_paths"] == ["app/auth.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == [] + + def test_existing_three_fields_unchanged_on_ambiguous_match(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + assert signals["explicit_paths"] == ["_internal/download.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == ["_internal/download.py"] + + def test_explicit_path_resolutions_present_alongside_old_fields(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + # Old fields still present and correct + assert signals["explicit_paths"] == ["_internal/download.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == [] + # New field, additive + assert signals["explicit_path_resolutions"] == [ + { + "raw_path": "_internal/download.py", + "strategy": "suffix", + "resolved_file": "src/pip/_internal/download.py", + } + ] + + def test_one_entry_per_extracted_path_including_unresolved_and_ambiguous( + self, tmp_path, monkeypatch + ): + """A single call whose advisory names three paths — one exact, one + ambiguous, one unresolved — must produce exactly three + explicit_path_resolutions entries, one per path, each with the + correct strategy.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + + vuln = ( + "Vulnerability in app/auth.py and _internal/download.py " + "and does/not/exist.py" + ) + find_code_context(vuln, tmp_path) + record = _latest_debug_artifact(tmp_path) + resolutions = record["extraction_signals"]["explicit_path_resolutions"] + + by_path = {r["raw_path"]: r for r in resolutions} + assert len(resolutions) == 3 + assert by_path["app/auth.py"]["strategy"] == "exact" + assert by_path["app/auth.py"]["resolved_file"] == "app/auth.py" + assert by_path["_internal/download.py"]["strategy"] == "ambiguous" + assert by_path["_internal/download.py"]["resolved_file"] is None + assert by_path["does/not/exist.py"]["strategy"] == "unresolved" + assert by_path["does/not/exist.py"]["resolved_file"] is None + + +class TestResolutionStrategyPropagation: + """The Stage 6A bugfix: resolution_strategy must survive into the + per-candidate `passes` entry written to disk.""" + + def test_resolution_strategy_exact_in_written_artifact(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + explicit_pass = next(p for p in candidate["passes"] if p["pass"] == "explicit_path") + + assert explicit_pass["resolution_strategy"] == "exact" + + def test_resolution_strategy_suffix_in_written_artifact(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next( + c for c in record["candidates"] + if c["file"] == "src/pip/_internal/download.py" + ) + explicit_pass = next(p for p in candidate["passes"] if p["pass"] == "explicit_path") + + assert explicit_pass["resolution_strategy"] == "suffix" + + +class TestSelectedPassAndSelected: + """selected_pass and selected are derived, single-source-of-truth + fields — never independently maintained state.""" + + def test_selected_pass_matches_final_score_owning_pass(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + + assert candidate["selected_pass"] == "explicit_path" + + def test_selected_true_for_primary_file(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + + assert candidate["selected"] is True + assert candidate["selection_outcome"] != "rejected" + + # Fixture shared by the next three tests: mirrors + # TestClassDefinitionGrounding's api.py/urls.py/flask_app.py/filesystem.py + # setup elsewhere in this file — api.py(26 `stac` hits) wins primary; + # urls.py(10) and flask_app.py(8) are _grep_repo's ranked[1]/ranked[2]; + # filesystem.py(5) falls outside _grep_repo's top-3 cut entirely and is + # only reachable via the class-definition supplement. Because the + # supplement is prepended to the secondary queue ahead of ranked[1:], + # and the secondary queue is capped at 2 slots, flask_app.py (ranked[2]) + # is pushed out and ends up "rejected" — giving a real, non-null + # final_score paired with a rejected outcome. + def _write_rejection_fixture(self, tmp_path: Path) -> None: + write( + tmp_path / "api.py", + "# stac routing module\n" + "def handle_stac(req): pass # stac\n" * 25, + ) + write(tmp_path / "urls.py", "# stac route\n" + "path('/stac/')\n" * 9) + write(tmp_path / "flask_app.py", "# stac routes\n" + "@app.route('/stac/')\n" * 7) + write( + tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n", + ) + + def test_selected_false_for_rejected_candidate(self, tmp_path, monkeypatch): + """A candidate outside the final secondary-slot cut must have + selected=False, matching its selection_outcome of 'rejected'.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + rejected = [c for c in record["candidates"] if c["selection_outcome"] == "rejected"] + + assert rejected, "expected at least one rejected candidate in this setup" + for c in rejected: + assert c["selected"] is False + + def test_selected_matches_selection_outcome_for_every_candidate(self, tmp_path, monkeypatch): + """selected is always exactly (selection_outcome != 'rejected') — + single source of truth, never independently wrong.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + + for c in record["candidates"]: + assert c["selected"] == (c["selection_outcome"] != "rejected") + + def test_selected_pass_never_null_for_any_candidate(self, tmp_path, monkeypatch): + """Every entry that made it into the candidates list arrived via at + least one pass, so selected_pass must always resolve to a name — + including a final_score=None, supplement-only entry.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + + for c in record["candidates"]: + assert c["selected_pass"] is not None, f"selected_pass unexpectedly null for {c['file']}" + + def test_selected_pass_class_definition_supplement_when_final_score_none( + self, tmp_path, monkeypatch + ): + """filesystem.py is injected solely via the class-definition + supplement (no Pass 1/2/3 hit of its own within the top-3 cut), so + final_score=None; selected_pass must still resolve to + 'class_definition_supplement', not None.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next( + c for c in record["candidates"] if c["file"] == "provider/filesystem.py" + ) + + assert candidate["final_score"] is None + assert candidate["selected_pass"] == "class_definition_supplement" + + +class TestGroundRepository: + """ground_repository() must return the exact same rendered_context as + find_code_context(), plus one RepositoryCandidate/GroundingDecision per + discovered file with today's exact outcome values, across the four + exit paths find_code_context() has: empty/no-match, primary full-file, + primary snippet, and multi-candidate secondary-context.""" + + def test_empty_no_match(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + vuln = "SQL injection in authenticate()" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected == "" + assert result.candidates == [] + assert result.decisions == [] + assert result.budget is None + + def test_primary_full_file(self, tmp_path): + """Single-candidate case: only retry.py matches (connectionpool.py + produces zero signal hits), so it must be the sole candidate and + be selected via the full-file path.""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + write(tmp_path / "retry.py", + "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n") + write(tmp_path / "connectionpool.py", + "class ConnectionPool:\n redirects = True\n") + vuln = "The `Authorization` header is not stripped on redirects." + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + assert [c.path for c in result.candidates] == ["retry.py"] + assert [d.path for d in result.decisions] == ["retry.py"] + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + assert result.decisions[0].outcome == "primary_full_file" + + def test_primary_snippet(self, tmp_path): + """Single-candidate case forced into snippet mode: the file exceeds + _FULL_FILE_THRESHOLD_CHARS, and only Pass 1 (explicit path) fires — + 'authenticate' has no underscore/PascalCase so Pass 2 stays empty.""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000 + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in app/auth.py — authenticate()" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + assert [c.path for c in result.candidates] == ["app/auth.py"] + assert [d.path for d in result.decisions] == ["app/auth.py"] + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + assert result.decisions[0].outcome == "primary_snippet" + + def test_secondary_context_selected_and_rejected(self, tmp_path): + """Reuses TestSelectedPassAndSelected's rejection fixture: 4 + candidates, 3 selected (1 primary full-file, 2 secondary snippets — + one of them class-definition-supplement-only, final_score=None), + 1 rejected by the 2-slot secondary cap despite having real, + non-null evidence (final_score=2).""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + write(tmp_path / "api.py", + "# stac routing module\n" + "def handle_stac(req): pass # stac\n" * 25) + write(tmp_path / "urls.py", "# stac route\n" + "path('/stac/')\n" * 9) + write(tmp_path / "flask_app.py", "# stac routes\n" + "@app.route('/stac/')\n" * 7) + write( + tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n", + ) + vuln = "FileSystemProvider path traversal in `stac` collection" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + + expected_paths = {"api.py", "urls.py", "flask_app.py", "provider/filesystem.py"} + assert {c.path for c in result.candidates} == expected_paths + assert len(result.decisions) == len(result.candidates) + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + + outcomes = {d.path: d.outcome for d in result.decisions} + assert outcomes["api.py"] == "primary_full_file" + assert outcomes["urls.py"] == "secondary_snippet" + assert outcomes["provider/filesystem.py"] == "secondary_snippet" + assert outcomes["flask_app.py"] == "rejected" + + selected = {p for p, o in outcomes.items() if o != "rejected"} + rejected = {p for p, o in outcomes.items() if o == "rejected"} + assert selected == {"api.py", "urls.py", "provider/filesystem.py"} + assert rejected == {"flask_app.py"} diff --git a/libs/openant-core/tests/patch/test_run_metadata.py b/libs/openant-core/tests/patch/test_run_metadata.py new file mode 100644 index 00000000..87e79751 --- /dev/null +++ b/libs/openant-core/tests/patch/test_run_metadata.py @@ -0,0 +1,225 @@ +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + + +from utilities.autopatcher.run_metadata import RunMetadata, auto_output_path, collect_git_info, render_metadata_section + +_TS = datetime(2026, 6, 16, 18, 35, 42, tzinfo=timezone.utc) + + +def _meta(**overrides) -> RunMetadata: + defaults = dict( + timestamp="2026-06-16 18:35:42 UTC", + input_source="GHSA-v845-jxx5-vc9f", + repo_root="/tmp/urllib3-eval", + repo_commit="d9f85a74", + llm_provider="anthropic", + llm_model="claude-sonnet-4-6", + llm_mode="LIVE", + output_path="reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md", + patcher_commit="e0f8737", + ) + defaults.update(overrides) + return RunMetadata(**defaults) + + +# --------------------------------------------------------------------------- +# auto_output_path +# --------------------------------------------------------------------------- + +class TestAutoOutputPath: + def test_ghsa_mode_full_format(self): + path = auto_output_path(_TS, "GHSA-v845-jxx5-vc9f", None, "/tmp/urllib3-eval") + assert path == "reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" + + def test_file_mode_no_repo(self): + path = auto_output_path(_TS, None, "examples/vulnerability.md", None) + assert path == "reports/20260616-183542-vulnerability.md" + + def test_file_mode_with_repo(self): + path = auto_output_path(_TS, None, "examples/vulnerability.md", "/tmp/myrepo") + assert path == "reports/20260616-183542-myrepo-vulnerability.md" + + def test_output_always_under_reports_dir(self): + path = auto_output_path(_TS, "GHSA-x-y-z", None, "/tmp/proj") + assert path.startswith("reports/") + assert path.endswith(".md") + + def test_ghsa_id_is_lowercased(self): + path = auto_output_path(_TS, "GHSA-V845-JXX5-VC9F", None, "/tmp/urllib3-eval") + assert "ghsa-v845-jxx5-vc9f" in path + assert "GHSA" not in path + + def test_repo_slug_is_lowercased(self): + path = auto_output_path(_TS, "GHSA-x-y-z", None, "/tmp/MyProject") + assert "myproject" in path + assert "MyProject" not in path + + def test_seconds_included_in_timestamp(self): + ts = datetime(2026, 6, 16, 9, 5, 3, tzinfo=timezone.utc) + path = auto_output_path(ts, "GHSA-a-b-c", None, "/tmp/repo") + assert "20260616-090503" in path + + def test_no_ghsa_no_file_fallback(self): + path = auto_output_path(_TS, None, None, None) + assert path == "reports/20260616-183542-report.md" + + +# --------------------------------------------------------------------------- +# collect_git_info +# --------------------------------------------------------------------------- + +class TestCollectGitInfo: + def test_non_repo_returns_unknown(self, tmp_path): + result = collect_git_info(tmp_path) + assert result == "unknown" + + def test_nonexistent_path_returns_unknown(self): + result = collect_git_info(Path("/nonexistent/path/xyz")) + assert result == "unknown" + + def test_never_raises(self, tmp_path): + result = collect_git_info(tmp_path / "does_not_exist") + assert isinstance(result, str) + + def test_real_repo_returns_short_sha(self): + project_root = Path(__file__).parent.parent + result = collect_git_info(project_root) + assert result != "unknown" + assert len(result) >= 7 + assert all(c in "0123456789abcdef" for c in result), f"Not a hex SHA: {result!r}" + + def test_real_repo_sha_matches_git_log(self): + project_root = Path(__file__).parent.parent + expected = subprocess.run( + ["git", "log", "-1", "--format=%h"], + cwd=str(project_root), + capture_output=True, + text=True, + ).stdout.strip() + assert collect_git_info(project_root) == expected + + +# --------------------------------------------------------------------------- +# render_metadata_section +# --------------------------------------------------------------------------- + +class TestRenderMetadataSection: + def test_all_required_fields_present(self): + md = render_metadata_section(_meta()) + for field in [ + "Generated", + "Input", + "Repository", + "Repo commit", + "LLM provider", + "LLM model", + "LLM mode", + "Output", + "Auto-patcher", + ]: + assert field in md, f"Required field missing from metadata: {field!r}" + + def test_values_rendered_in_table(self): + md = render_metadata_section(_meta()) + assert "GHSA-v845-jxx5-vc9f" in md + assert "d9f85a74" in md + assert "anthropic" in md + assert "claude-sonnet-4-6" in md + assert "LIVE" in md + assert "e0f8737" in md + + def test_mock_warning_present_in_mock_mode(self): + md = render_metadata_section(_meta(llm_mode="MOCK")) + assert "MOCK MODE" in md + assert "must not be used as benchmark evidence" in md + + def test_mock_warning_absent_in_live_mode(self): + md = render_metadata_section(_meta(llm_mode="LIVE")) + assert "MOCK MODE" not in md + + def test_empty_repo_root_renders_dash(self): + md = render_metadata_section(_meta(repo_root="")) + assert "| Repository | — |" in md + + def test_output_path_in_table(self): + md = render_metadata_section(_meta( + output_path="reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" + )) + assert "reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" in md + + +# --------------------------------------------------------------------------- +# render_metadata_section — max_tokens_configured / stage_stop_reasons +# --------------------------------------------------------------------------- + +class TestTokenBudgetAndStopReasons: + def test_max_tokens_row_rendered_when_present(self): + md = render_metadata_section(_meta(max_tokens_configured=4096)) + assert "| Max output tokens | 4096 |" in md + + def test_max_tokens_row_renders_dash_when_absent(self): + md = render_metadata_section(_meta()) + assert "| Max output tokens | — |" in md + + def test_stage_stop_reason_table_rendered_when_present(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "end_turn", + "patch_review": "end_turn", + "challenger": "end_turn", + "confidence_scorer": "end_turn", + })) + assert "### Stage Stop Reasons" in md + assert "| Patch generation | end_turn |" in md + assert "| Patch review | end_turn |" in md + assert "| Challenger | end_turn |" in md + assert "| Confidence scorer | end_turn |" in md + + def test_stage_stop_reason_table_absent_when_no_stages_recorded(self): + md = render_metadata_section(_meta()) + assert "Stage Stop Reasons" not in md + + def test_truncated_stage_gets_warning_icon_in_table(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "max_tokens", + "patch_review": "end_turn", + })) + assert "| Patch generation | ⚠️ max_tokens |" in md + assert "| Patch review | end_turn |" in md + + def test_openai_length_stop_reason_also_flagged(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "length", + })) + assert "| Patch generation | ⚠️ length |" in md + + def test_truncation_banner_present_when_any_stage_truncated(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "max_tokens", + "patch_review": "end_turn", + })) + assert "TRUNCATED OUTPUT DETECTED" in md + + def test_truncation_banner_absent_when_no_stage_truncated(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "end_turn", + "patch_review": "stop", + })) + assert "TRUNCATED OUTPUT DETECTED" not in md + + def test_truncation_banner_absent_when_no_stages_recorded(self): + md = render_metadata_section(_meta()) + assert "TRUNCATED OUTPUT DETECTED" not in md + + def test_truncation_banner_coexists_with_mock_warning(self): + md = render_metadata_section(_meta( + llm_mode="MOCK", + stage_stop_reasons={"patch_generation": "max_tokens"}, + )) + assert "MOCK MODE" in md + assert "TRUNCATED OUTPUT DETECTED" in md diff --git a/libs/openant-core/tests/patch/test_sink_extractor.py b/libs/openant-core/tests/patch/test_sink_extractor.py new file mode 100644 index 00000000..ee0beb86 --- /dev/null +++ b/libs/openant-core/tests/patch/test_sink_extractor.py @@ -0,0 +1,372 @@ +"""Tests for Phase D repo-wide sink extraction. + +Tests: extract_repo_sinks, _find_enclosing_def, _build_sink_table, and +the integration path through build_vulnerability_pattern_context. + +All file-system tests use pytest's tmp_path fixture — no real repos needed. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.vulnerability_patterns import ( + _build_sink_table, + _find_enclosing_def, + build_vulnerability_pattern_context, + extract_repo_sinks, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# _find_enclosing_def +# --------------------------------------------------------------------------- + + +class TestFindEnclosingDef: + def test_finds_simple_def(self): + lines = [ + "def my_function(self, x):", + " os.system(x)", + ] + assert _find_enclosing_def(lines, 1) == "my_function" + + def test_finds_async_def(self): + lines = [ + "async def async_func(self):", + " await do_thing()", + " os.system('x')", + ] + assert _find_enclosing_def(lines, 2) == "async_func" + + def test_returns_none_when_no_def_above(self): + lines = [ + "import os", + "os.system('cmd')", + ] + assert _find_enclosing_def(lines, 1) is None + + def test_finds_nearest_not_outer(self): + # Should return inner, not outer + lines = [ + "def outer(self):", + " x = 1", + " def inner(y):", + " os.system(y)", + ] + assert _find_enclosing_def(lines, 3) == "inner" + + def test_sink_on_def_line_itself(self): + lines = [ + "def risky(self, shell=True):", + ] + assert _find_enclosing_def(lines, 0) == "risky" + + def test_indented_method_inside_class(self): + lines = [ + "class Foo:", + " def method(self):", + " os.system('x')", + ] + assert _find_enclosing_def(lines, 2) == "method" + + def test_empty_lines_list(self): + assert _find_enclosing_def([], 0) is None + + def test_walks_past_blank_lines(self): + lines = [ + "def handler(self):", + "", + " x = 1", + " os.system(x)", + ] + assert _find_enclosing_def(lines, 3) == "handler" + + +# --------------------------------------------------------------------------- +# extract_repo_sinks +# --------------------------------------------------------------------------- + + +class TestExtractRepoSinks: + def test_multi_file_coverage(self, tmp_path): + write(tmp_path / "a.py", "def func_a(cmd):\n os.system(cmd)\n") + write(tmp_path / "b.py", "def func_b(cmd):\n x = cmd; os.system(x)\n") + write(tmp_path / "c.py", "def func_c(cmd):\n subprocess.call(cmd, shell=True)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + methods = {s["method"] for s in sinks} + assert "func_a" in methods + assert "func_b" in methods + assert "func_c" in methods + + def test_enclosing_method_detected(self, tmp_path): + write(tmp_path / "app.py", ( + "def safe_func(x):\n" + " return x\n" + "\n" + "def danger_func(path):\n" + " full = os.path.normpath(path)\n" + " return full\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "danger_func" + + def test_deduplication_keeps_first_occurrence(self, tmp_path): + write(tmp_path / "app.py", ( + "def multi_sink(a, b):\n" + " x = os.path.normpath(a)\n" + " y = os.path.normpath(b)\n" + " return x, y\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "multi_sink" + assert sinks[0]["line"] == 2 # first occurrence, 1-indexed + + def test_no_sinks_returns_empty(self, tmp_path): + write(tmp_path / "clean.py", ( + "from pathlib import Path\n" + "\n" + "def safe(path, base):\n" + " return Path(path).resolve().is_relative_to(Path(base).resolve())\n" + )) + assert extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") == [] + + def test_comment_lines_skipped(self, tmp_path): + write(tmp_path / "commented.py", ( + "def func(x):\n" + " # old: os.path.normpath(x)\n" + " return x\n" + )) + assert extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") == [] + + def test_test_files_by_name_skipped(self, tmp_path): + write(tmp_path / "test_app.py", "def test_f(cmd):\n os.system(cmd)\n") + write(tmp_path / "app.py", "def safe(x):\n return x\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert all(s["file"] != "test_app.py" for s in sinks) + + def test_tests_subdir_skipped(self, tmp_path): + write(tmp_path / "tests" / "test_foo.py", "def test_x():\n os.system('x')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks == [] + + def test_module_level_sink_has_none_method(self, tmp_path): + write(tmp_path / "script.py", "os.system('cmd')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks) == 1 + assert sinks[0]["method"] is None + + def test_results_sorted_by_file_then_line(self, tmp_path): + write(tmp_path / "z_file.py", "def zzz(x):\n os.system(x)\n") + write(tmp_path / "a_file.py", "def aaa(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + files = [s["file"] for s in sinks] + assert files == sorted(files) + + def test_unknown_vuln_class_returns_empty(self, tmp_path): + write(tmp_path / "app.py", "def f(x):\n os.system(x)\n") + assert extract_repo_sinks(tmp_path, "SQL_INJECTION") == [] + + def test_none_repo_root_returns_empty(self): + assert extract_repo_sinks(None, "COMMAND_INJECTION") == [] + + def test_capped_at_max_rows(self, tmp_path): + for i in range(25): + write(tmp_path / f"mod{i:02d}.py", f"def func_{i}(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks) <= 20 + + def test_line_number_is_1_indexed(self, tmp_path): + write(tmp_path / "app.py", ( + "def func(x):\n" + " safe()\n" + " os.system(x)\n" + )) + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks[0]["line"] == 3 + + def test_snippet_stripped(self, tmp_path): + write(tmp_path / "app.py", "def func(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks[0]["snippet"] == "os.system(x)" + + def test_path_traversal_normpath_detected(self, tmp_path): + write(tmp_path / "server.py", ( + "def serve(self, path):\n" + " full = os.path.normpath(self.root + path)\n" + " return open(full).read()\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "serve" + + def test_path_traversal_string_concat_detected(self, tmp_path): + write(tmp_path / "fs.py", ( + "def get_file(self, name):\n" + " path = self.root + name\n" + " return open(path).read()\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert any(s["method"] == "get_file" for s in sinks) + + def test_path_traversal_os_path_join_not_detected(self, tmp_path): + # os.path.join is excluded from repo_sink_patterns to avoid noise + write(tmp_path / "utils.py", ( + "def build_path(base, name):\n" + " return os.path.join(base, name)\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert sinks == [] + + def test_ignored_dirs_skipped(self, tmp_path): + write(tmp_path / ".venv" / "lib" / "bad.py", "os.system('x')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert all(".venv" not in s["file"] for s in sinks) + + def test_snippet_truncated_at_80_chars(self, tmp_path): + long_line = " os.system(" + "x" * 100 + ")" + write(tmp_path / "app.py", f"def f(x):\n{long_line}\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks[0]["snippet"]) <= 80 + + def test_multiple_methods_in_one_file(self, tmp_path): + write(tmp_path / "multi.py", ( + "def method_a(self, cmd):\n" + " os.system(cmd)\n" + "\n" + "def method_b(self, cmd):\n" + " os.system(cmd + ' extra')\n" + )) + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + methods = {s["method"] for s in sinks} + assert "method_a" in methods + assert "method_b" in methods + + +# --------------------------------------------------------------------------- +# _build_sink_table +# --------------------------------------------------------------------------- + + +class TestBuildSinkTable: + def test_empty_input_has_headers_only(self): + table = _build_sink_table([]) + assert "| File |" in table + assert "| Line |" in table + assert "| Method |" in table + assert "| Dangerous call |" in table + assert len(table.splitlines()) == 2 # header + separator + + def test_row_content_present(self): + sinks = [{"file": "app/cmd.py", "line": 42, "method": "execute", "snippet": "os.system(cmd)"}] + table = _build_sink_table(sinks) + assert "app/cmd.py" in table + assert "42" in table + assert "execute" in table + assert "os.system(cmd)" in table + + def test_none_method_shown_as_module(self): + sinks = [{"file": "script.py", "line": 1, "method": None, "snippet": "os.system('x')"}] + table = _build_sink_table(sinks) + assert "" in table + + def test_pipe_in_snippet_escaped(self): + sinks = [{"file": "f.py", "line": 1, "method": "fn", "snippet": "a | b"}] + table = _build_sink_table(sinks) + assert "a \\| b" in table + + def test_multiple_rows(self): + sinks = [ + {"file": "a.py", "line": 10, "method": "foo", "snippet": "os.system(x)"}, + {"file": "b.py", "line": 20, "method": "bar", "snippet": "os.system(y)"}, + ] + table = _build_sink_table(sinks) + lines = table.splitlines() + # header + separator + 2 rows = 4 lines + assert len(lines) == 4 + assert "foo" in table + assert "bar" in table + + +# --------------------------------------------------------------------------- +# Integration: build_vulnerability_pattern_context +# --------------------------------------------------------------------------- + + +class TestBuildContextWithRepoSinks: + def test_table_included_when_sinks_found(self, tmp_path): + write(tmp_path / "app.py", "def handler(cmd):\n os.system(cmd)\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" in result + assert "handler" in result + + def test_table_absent_when_no_sinks(self, tmp_path): + write(tmp_path / "app.py", "def safe(x):\n return x\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" not in result + + def test_table_absent_when_no_repo_root(self): + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=None + ) + assert "Dangerous operation locations" not in result + + def test_line_number_in_output(self, tmp_path): + write(tmp_path / "app.py", ( + "def func(x):\n" + " safe()\n" + " os.system(x)\n" + )) + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "| 3 |" in result # 1-indexed line 3 + + def test_both_context_checklist_and_repo_table_present(self, tmp_path): + # code_context method needs >= _MIN_METHOD_BODY_LINES (3) lines to + # appear in the context-level sink checklist. + code_ctx = ( + "def context_method(x):\n" + " y = x\n" + " os.system(y)\n" + ) + write(tmp_path / "app.py", "def repo_method(x):\n os.system(x)\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", code_ctx, repo_root=tmp_path + ) + assert "context_method" in result # from context-level scan + assert "Dangerous operation locations" in result # repo table + assert "repo_method" in result # from repo scan + + def test_path_traversal_repo_table_present(self, tmp_path): + write(tmp_path / "server.py", ( + "def serve(self, path):\n" + " full = os.path.normpath(self.root + path)\n" + " return open(full).read()\n" + )) + result = build_vulnerability_pattern_context( + "path traversal CWE-22", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" in result + assert "serve" in result diff --git a/libs/openant-core/tests/patch/test_test_suggester.py b/libs/openant-core/tests/patch/test_test_suggester.py new file mode 100644 index 00000000..40078ca8 --- /dev/null +++ b/libs/openant-core/tests/patch/test_test_suggester.py @@ -0,0 +1,77 @@ +def load_module(): + import utilities.autopatcher.test_suggester as mod + return mod + + +def test_extract_findings_from_bullets_and_sections(): + mod = load_module() + text = """ +Edge cases: +- Database drivers that use `%s` placeholders (driver mismatch) +- Unicode and binary username encodings + +Potential issues: +- Missing unit tests for edge-case payloads +- Assumes `db.execute` accepts parameterized args as shown +""" + findings = mod.extract_findings(text) + # headings/status labels should not be included + assert not any(f.lower().startswith("still vulnerable") for f in findings) + assert not any(f.lower().startswith("edge cases") for f in findings) + assert not any(f.lower().startswith("potential issues") for f in findings) + + # actionable findings should be present + assert any("unicode and binary" in f.lower() for f in findings) + assert any("database drivers" in f.lower() or "%s" in f for f in findings) + assert len(findings) <= 6 + # ensure deterministic ordering: first finding should be the database driver mismatch + assert findings[0] == "Database drivers that use `%s` placeholders (driver mismatch)" + + +def test_suggest_tests_creates_valid_skeletons(): + mod = load_module() + findings = [ + "Unicode and binary username encodings", + "Database drivers that use %s placeholders", + ] + suggestions = mod.suggest_tests(findings) + assert isinstance(suggestions, list) + assert len(suggestions) == 2 + for s, f in zip(suggestions, findings): + assert s["reason"] == f + assert s["name"].startswith("test_") + assert "pytest.mark.skip" in s["code"] + + +def test_limit_findings_to_six(): + mod = load_module() + # create many bullet lines + text = "\n".join(f"- finding {i}" for i in range(10)) + findings = mod.extract_findings(text) + assert len(findings) == 6 + + +def test_suggest_tests_with_behavior_generates_skeletons_and_dedupe(): + mod = load_module() + # Behavior with function name and two primary behaviors + behavior = { + "function": "authenticate", + "file": "app/auth.py", + "summary": "Does authentication", + "primary_behaviors": ["valid login", "invalid login"], + } + + # No initial findings -> behavior-derived tests should appear + suggestions = mod.suggest_tests([], behavior=behavior) + names = [s["name"] for s in suggestions] + assert "test_authenticate_valid_login" in names + assert "test_authenticate_invalid_login" in names + + # Dedupe: if findings already produce same name, behaviour test not duplicated + findings = ["valid login"] + suggestions2 = mod.suggest_tests(findings, behavior=behavior) + names2 = [s["name"] for s in suggestions2] + # only one test for valid_login should exist + assert names2.count("test_valid_login") + names2.count("test_authenticate_valid_login") >= 1 + # ensure behavior marker in code for at least one suggestion + assert any("# Behavior-focused validation" in s["code"] for s in suggestions2) diff --git a/libs/openant-core/tests/patch/test_trust_package.py b/libs/openant-core/tests/patch/test_trust_package.py new file mode 100644 index 00000000..b973f7cd --- /dev/null +++ b/libs/openant-core/tests/patch/test_trust_package.py @@ -0,0 +1,1545 @@ +"""Unit tests for Trust Package V1 helpers. + +Tests cover the four building blocks: + _classify_finding — categorises single challenger finding strings + _classify_challenger — classifies all findings + produces summary counts + _compute_trust_signals — derives the six trust signals deterministically + _build_recommendation_v1 — produces a V1 deployment decision from signals + _extract_security_gain — extracts the benefit sentence from reviewer text + _build_known_findings — groups classified findings into the Known + Findings section's epistemic categories + +All tests use synthetic inputs; no LLM calls are made. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.pipeline import ( + _classify_finding, + _classify_challenger, + _compute_trust_signals, + _build_recommendation_v1, + _extract_security_gain, + _build_known_findings, + _render_known_findings, + _check_recommendation_consistency, +) + + +# --------------------------------------------------------------------------- +# _classify_finding +# --------------------------------------------------------------------------- + +class TestClassifyFinding: + def test_confirmed_defect_still_vulnerable(self): + assert _classify_finding("The patch is still vulnerable to the original attack") == "confirmed_defect" + + def test_confirmed_defect_does_not_fix(self): + assert _classify_finding("This does not fix the underlying issue") == "confirmed_defect" + + def test_confirmed_defect_bypass(self): + assert _classify_finding("An attacker can bypass this check via the login endpoint") == "confirmed_defect" + + def test_confirmed_defect_attack_remains(self): + assert _classify_finding("The attack vector remains exploitable through direct API access") == "confirmed_defect" + + def test_validation_gap_cannot_verify(self): + assert _classify_finding("Cannot verify this without running the test suite") == "validation_gap" + + def test_validation_gap_without_testing(self): + assert _classify_finding("The fix cannot be confirmed without testing the redirect scenario") == "validation_gap" + + def test_validation_gap_needs_test(self): + assert _classify_finding("This requires testing to validate the fix") == "validation_gap" + + def test_validation_gap_untested(self): + assert _classify_finding("This path is untested and should be verified") == "validation_gap" + + def test_generic_tests_should(self): + assert _classify_finding("Tests should be added to cover this scenario") == "generic" + + def test_generic_consider_adding(self): + assert _classify_finding("Consider adding logging for audit purposes") == "generic" + + def test_plausible_risk_default(self): + assert _classify_finding("Case-insensitive matching may not handle all header variants") == "plausible_risk" + + def test_plausible_risk_specific_path(self): + assert _classify_finding("Custom Retry configurations that override DEFAULT may not benefit") == "plausible_risk" + + def test_empty_string_returns_generic(self): + assert _classify_finding("") == "generic" + + def test_case_insensitive_matching(self): + assert _classify_finding("STILL VULNERABLE to injection") == "confirmed_defect" + + +# --------------------------------------------------------------------------- +# Scope-marker exclusion for "does not fix/address/prevent/close" +# --------------------------------------------------------------------------- + +class TestFindingClassificationScopeMarker: + """Regression: scope-marker exclusion prevents false Confirmed Defects. + + Findings that say "does not fix/address/prevent/close" alongside a version + reference or scope qualifier should be Plausible Risk (scope limitation). + Those without any scope qualifier remain Confirmed Defect (primary failure). + Explicit exploitability language overrides scope markers in both directions. + """ + + # --- Scope-limited findings → Plausible Risk --- + + def test_does_not_address_v1x_is_plausible_risk(self): + assert _classify_finding( + "does not address the urllib3 v1.x branch" + ) == "plausible_risk" + + def test_does_not_prevent_users_who_override_is_plausible_risk(self): + assert _classify_finding( + "does not prevent users who override defaults" + ) == "plausible_risk" + + def test_does_not_close_for_users_running_version_is_plausible_risk(self): + assert _classify_finding( + "does not close the attack vector for users running 1.26.x" + ) == "plausible_risk" + + def test_does_not_fix_older_versions_is_plausible_risk(self): + assert _classify_finding( + "does not fix the issue for older versions of the library" + ) == "plausible_risk" + + def test_does_not_prevent_when_configured_is_plausible_risk(self): + assert _classify_finding( + "does not prevent the attack when configured with custom retry settings" + ) == "plausible_risk" + + def test_does_not_address_unless_is_plausible_risk(self): + assert _classify_finding( + "does not address this unless users explicitly opt in to the new behaviour" + ) == "plausible_risk" + + def test_does_not_fix_separate_fix_needed_is_plausible_risk(self): + assert _classify_finding( + "does not fix the v1 branch; a separate fix would be required" + ) == "plausible_risk" + + def test_does_not_close_legacy_branch_is_plausible_risk(self): + assert _classify_finding( + "does not close the vulnerability on the legacy branch" + ) == "plausible_risk" + + def test_does_not_prevent_only_when_is_plausible_risk(self): + assert _classify_finding( + "does not prevent exploitation only when the application runs in debug mode" + ) == "plausible_risk" + + # --- Primary-failure findings → Confirmed Defect --- + + def test_does_not_fix_sql_injection_is_confirmed_defect(self): + assert _classify_finding( + "does not fix the SQL injection vulnerability in execute_query()" + ) == "confirmed_defect" + + def test_does_not_prevent_exploitation_is_confirmed_defect(self): + assert _classify_finding( + "does not prevent exploitation through the original endpoint" + ) == "confirmed_defect" + + def test_does_not_address_attack_at_line_is_confirmed_defect(self): + assert _classify_finding( + "does not address the attack at line 47 in the vulnerable function" + ) == "confirmed_defect" + + def test_does_not_close_path_traversal_is_confirmed_defect(self): + assert _classify_finding( + "does not close the path traversal vulnerability" + ) == "confirmed_defect" + + def test_does_not_fix_underlying_issue_is_confirmed_defect(self): + assert _classify_finding( + "does not fix the underlying race condition" + ) == "confirmed_defect" + + # --- Explicit exploitability overrides scope markers --- + + def test_still_vulnerable_with_version_still_confirmed_defect(self): + assert _classify_finding( + "still vulnerable in v1.x and v2.x contexts" + ) == "confirmed_defect" + + def test_still_vulnerable_plain_confirmed_defect(self): + assert _classify_finding("still vulnerable to the original attack") == "confirmed_defect" + + def test_validation_gap_unaffected_by_scope_markers(self): + assert _classify_finding("cannot verify without testing") == "validation_gap" + + +# --------------------------------------------------------------------------- +# Narrowed bypass pattern: active exploit vs architectural observation +# --------------------------------------------------------------------------- + +class TestBypassClassification: + """Regression tests for the narrowed bypass pattern. + + Bare 'bypass' (architectural observation, speculation) → plausible_risk. + 'can bypass' / 'can be bypassed' / 'allows bypass' (active exploit) → confirmed_defect. + """ + + # Previously false positives — should now be Plausible Risk + + def test_architectural_bypass_is_plausible_risk(self): + assert _classify_finding( + "Cookies set via other mechanisms (e.g. adapters or poolmanagers) bypass this entirely" + ) == "plausible_risk" + + def test_may_bypass_conditional_is_plausible_risk(self): + assert _classify_finding( + "headers like `cookie`, `COOKIE`, or `CookIe` may bypass the frozenset check " + "if header comparison isn't case-insensitive" + ) == "plausible_risk" + + def test_might_bypass_is_plausible_risk(self): + assert _classify_finding( + "URL-encoded variants might bypass the header comparison in some configurations" + ) == "plausible_risk" + + def test_plain_bypass_verb_is_plausible_risk(self): + assert _classify_finding( + "Requests using connection-level cookies bypass this stripping logic" + ) == "plausible_risk" + + # Active exploit framing — should remain Confirmed Defect + + def test_can_bypass_is_confirmed_defect(self): + assert _classify_finding( + "An attacker can bypass this check via the login endpoint" + ) == "confirmed_defect" + + def test_can_be_bypassed_is_confirmed_defect(self): + assert _classify_finding( + "The vulnerability can be bypassed by sending a crafted request" + ) == "confirmed_defect" + + def test_could_bypass_is_confirmed_defect(self): + assert _classify_finding( + "An attacker could bypass the authentication with a malformed token" + ) == "confirmed_defect" + + def test_allows_bypass_is_confirmed_defect(self): + assert _classify_finding( + "This flaw allows bypass of the session validation" + ) == "confirmed_defect" + + def test_allows_bypassing_is_confirmed_defect(self): + assert _classify_finding( + "The missing check allows bypassing the access control" + ) == "confirmed_defect" + + # Other explicit defect patterns unaffected by this change + + def test_still_vulnerable_unaffected(self): + assert _classify_finding( + "The patch is still vulnerable to the original attack" + ) == "confirmed_defect" + + def test_attack_vector_remains_unaffected(self): + assert _classify_finding( + "The attack vector remains open via the original endpoint" + ) == "confirmed_defect" + + def test_remains_exploitable_unaffected(self): + assert _classify_finding( + "The injection point remains exploitable after this change" + ) == "confirmed_defect" + + +# --------------------------------------------------------------------------- +# _classify_challenger +# --------------------------------------------------------------------------- + +class TestClassifyChallenger: + def _make(self, still_vulnerable=False, edge_cases=None, potential_issues=None): + return { + "still_vulnerable": still_vulnerable, + "edge_cases": edge_cases or [], + "potential_issues": potential_issues or [], + "summary": "Test summary", + } + + def test_empty_challenger_zero_counts(self): + result = _classify_challenger(self._make()) + assert result["confirmed_defect_count"] == 0 + assert result["plausible_risk_count"] == 0 + assert result["validation_gap_count"] == 0 + + def test_confirmed_defect_increments_count(self): + result = _classify_challenger(self._make( + edge_cases=["attack vector remains exploitable"] + )) + assert result["confirmed_defect_count"] == 1 + assert result["plausible_risk_count"] == 0 + + def test_validation_gap_increments_count(self): + result = _classify_challenger(self._make( + potential_issues=["cannot verify without running tests"] + )) + assert result["validation_gap_count"] == 1 + assert result["confirmed_defect_count"] == 0 + + def test_mixed_findings_counted_independently(self): + result = _classify_challenger(self._make( + edge_cases=["still vulnerable via header", "cannot verify without testing"], + potential_issues=["case mismatch may occur"], + )) + assert result["confirmed_defect_count"] == 1 + assert result["validation_gap_count"] == 1 + assert result["plausible_risk_count"] == 1 + + def test_original_fields_preserved(self): + challenger = self._make(still_vulnerable=True) + result = _classify_challenger(challenger) + assert result["still_vulnerable"] is True + assert result["summary"] == "Test summary" + + def test_classified_lists_have_text_and_category(self): + result = _classify_challenger(self._make( + edge_cases=["cannot verify without running tests"] + )) + items = result["classified_edge_cases"] + assert len(items) == 1 + assert "text" in items[0] and "category" in items[0] + assert items[0]["category"] == "validation_gap" + + def test_none_challenger_returns_empty_counts(self): + result = _classify_challenger(None) + assert result.get("confirmed_defect_count", 0) == 0 + + +# --------------------------------------------------------------------------- +# _compute_trust_signals +# --------------------------------------------------------------------------- + +def _clean_applicability(): + return {"applicable": True, "skipped": False, "skipped_reason": None, "error": None, "stderr": ""} + +def _skip_applicability(): + return {"applicable": None, "skipped": True, "skipped_reason": "no .git directory", "error": None, "stderr": ""} + +def _fail_applicability(stderr="error: patch does not apply"): + return {"applicable": False, "skipped": False, "skipped_reason": None, "error": None, "stderr": stderr} + +def _classified(still_vulnerable=False, defects=0, risks=0, gaps=0): + c = { + "still_vulnerable": still_vulnerable, + "confirmed_defect_count": defects, + "plausible_risk_count": risks, + "validation_gap_count": gaps, + "classified_edge_cases": [], + "classified_potential_issues": [], + } + # Populate classified lists to match counts + for _ in range(defects): + c["classified_edge_cases"].append({"text": "still vulnerable", "category": "confirmed_defect"}) + for _ in range(risks): + c["classified_edge_cases"].append({"text": "plausible edge case", "category": "plausible_risk"}) + for _ in range(gaps): + c["classified_potential_issues"].append({"text": "cannot verify without tests", "category": "validation_gap"}) + return c + + +class TestComputeTrustSignals: + + # Patch Integrity + def test_integrity_clean_when_no_issues(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Clean" + + def test_integrity_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Does Not Apply" + + def test_integrity_not_verified_when_skipped(self): + s = _compute_trust_signals([], _skip_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Not Verified" + + def test_integrity_critical_when_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Critical Issues" + + def test_integrity_minor_when_medium_hygiene_only(self): + hygiene = [{"severity": "MEDIUM", "check": "unused_import", "detail": "Unused import"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Minor Issues" + + # Security Improvement + def test_improvement_none_when_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "None" + + def test_improvement_unknown_when_skipped(self): + s = _compute_trust_signals([], _skip_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "Unknown" + + def test_improvement_low_when_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "Low" + + def test_improvement_low_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, defects=1), "None", "low") + assert s["security_improvement"]["value"] == "Low" + + def test_improvement_high_when_not_still_vulnerable(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=False), "None", "low") + assert s["security_improvement"]["value"] == "High" + + def test_improvement_high_when_only_validation_gaps(self): + # still_vulnerable=True but only gaps (no confirmed defects, no plausible risks) + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, gaps=2), "None", "low") + assert s["security_improvement"]["value"] == "High" + + def test_improvement_medium_when_plausible_risks_present(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, risks=1), "None", "low") + assert s["security_improvement"]["value"] == "Medium" + + # Remediation Alignment + def test_alignment_misaligned_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, defects=1), "None", "low") + assert s["remediation_alignment"]["value"] == "Misaligned" + + def test_alignment_aligned_when_not_still_vulnerable(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=False), "None", "low") + assert s["remediation_alignment"]["value"] == "Aligned" + + def test_alignment_likely_aligned_when_only_gaps(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, gaps=2), "None", "low") + assert s["remediation_alignment"]["value"] == "Likely Aligned" + + def test_alignment_partial_when_plausible_risks(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, risks=1), "None", "low") + assert s["remediation_alignment"]["value"] == "Partial" + + # Coverage Confidence + def test_coverage_high_when_no_findings(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["coverage_confidence"]["value"] == "High" + + def test_coverage_medium_when_gaps(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(gaps=2), "None", "low") + assert s["coverage_confidence"]["value"] == "Medium" + + def test_coverage_medium_when_plausible_risks(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(risks=1), "None", "low") + assert s["coverage_confidence"]["value"] == "Medium" + + def test_coverage_low_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(defects=1), "None", "low") + assert s["coverage_confidence"]["value"] == "Low" + + # Test Availability + def test_test_availability_tests_available_for_good(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Good", "low") + assert s["test_availability"]["value"] == "Tests Available" + + def test_test_availability_tests_available_for_some(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Some", "low") + assert s["test_availability"]["value"] == "Tests Available" + + def test_test_availability_no_tests_for_none(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["test_availability"]["value"] == "No Tests Found" + + # Deployment Safety + def test_safety_low_risk_for_low_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["deployment_safety"]["value"] == "Low Risk" + + def test_safety_medium_risk_for_medium_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "medium") + assert s["deployment_safety"]["value"] == "Medium Risk" + + def test_safety_high_risk_for_high_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "high") + assert s["deployment_safety"]["value"] == "High Risk" + + def test_safety_high_risk_for_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["deployment_safety"]["value"] == "High Risk" + + # Composite: urllib3-representative inputs + def test_urllib3_representative_signals(self): + """All six signals for a well-grounded correct urllib3-style patch.""" + classified = _classified(still_vulnerable=True, risks=1, gaps=1) + s = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + assert s["patch_integrity"]["value"] == "Clean" + assert s["security_improvement"]["value"] == "Medium" + assert s["remediation_alignment"]["value"] == "Partial" + assert s["coverage_confidence"]["value"] == "Medium" + assert s["test_availability"]["value"] == "No Tests Found" + assert s["deployment_safety"]["value"] == "Low Risk" + + # Labels contain icons + def test_labels_contain_icon_for_clean(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert "✓" in s["patch_integrity"]["label"] + + def test_labels_contain_icon_for_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert "✗" in s["patch_integrity"]["label"] + + # ---- Language guardrail: "Not Applicable" / "not_applicable" must not + # collapse into the same reassuring labels as a genuine clean finding. ---- + + def test_test_availability_not_applicable_is_not_verified_not_no_tests_found(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Not Applicable", "low") + assert s["test_availability"]["value"] == "Not Verified" + assert s["test_availability"]["value"] != "No Tests Found" + assert s["test_availability"]["value"] != "Tests Available" + + def test_deployment_safety_not_applicable_is_not_verified_not_low_risk(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "not_applicable") + assert s["deployment_safety"]["value"] == "Not Verified" + assert s["deployment_safety"]["value"] != "Low Risk" + + def test_non_python_repo_signals_never_read_as_clean(self): + # Simulates the actual values a non-Python (e.g. curl/C) run produces: + # testing_rating="Not Applicable" from score_test_support, and + # impact_level="not_applicable" from LightweightImpactAnalyzer. + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Not Applicable", "not_applicable") + assert s["test_availability"]["value"] not in ("Tests Available", "No Tests Found") + assert s["deployment_safety"]["value"] not in ("Low Risk", "Medium Risk", "High Risk") + assert s["test_availability"]["value"] == "Not Verified" + assert s["deployment_safety"]["value"] == "Not Verified" + + +# --------------------------------------------------------------------------- +# _build_recommendation_v1 +# --------------------------------------------------------------------------- + +def _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Low Risk"): + return { + "patch_integrity": {"value": integrity, "label": integrity, "notes": ""}, + "security_improvement": {"value": improvement, "label": improvement, "notes": ""}, + "remediation_alignment": {"value": alignment, "label": alignment, "notes": ""}, + "coverage_confidence": {"value": "High", "label": "High", "notes": ""}, + "test_availability": {"value": "No Tests Found", "label": "No Tests Found", "notes": ""}, + "deployment_safety": {"value": safety, "label": safety, "notes": ""}, + } + + +# --------------------------------------------------------------------------- +# Trust Signals v2 — question-style rendering (display only) +# --------------------------------------------------------------------------- + +def _signals_full(**overrides): + """All six keys with a neutral 'good' baseline, so tests only need to + override the one signal they're checking.""" + base = { + "patch_integrity": {"value": "Clean", "label": "Clean", "notes": "Applies cleanly · no hygiene issues"}, + "security_improvement": {"value": "High", "label": "High", "notes": "Adversarial review found no remaining exploit path"}, + "remediation_alignment": {"value": "Aligned", "label": "Aligned", "notes": "Adversarial review confirms fix approach"}, + "coverage_confidence": {"value": "High", "label": "High", "notes": "No gaps identified by adversarial analysis"}, + "test_availability": {"value": "Tests Available", "label": "Tests Available", "notes": "Good — test files cover this module"}, + "deployment_safety": {"value": "Low Risk", "label": "Low Risk", "notes": "Localized change · low regression risk"}, + } + base.update(overrides) + return base + + +class TestTrustSignalsV2Table: + """Rendering-only tests for the redesigned Trust Signals table. None of + these touch _compute_trust_signals or _build_recommendation_v1.""" + + def test_heading_unchanged(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "## Trust Signals\n" in table + + def test_question_rows_present(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + for question in [ + "Does the patch apply?", + "Does it address the vulnerability?", + "Are there unresolved concerns?", + "Do relevant tests already exist?", + "Is deployment risk low?", + ]: + assert question in table + + def test_security_improvement_not_displayed_as_a_row(self): + """Requirement 3: overlapping aggregate signals must not appear as + peer rows. security_improvement must not surface anywhere in the + rendered table, even though it's still computed and still read by + _build_recommendation_v1 elsewhere.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "Security Improvement" not in table + assert "security_improvement" not in table + + def test_old_icon_vocabulary_does_not_leak_through(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + for old_icon in ["✓", "✗", "◑", "○"]: + assert old_icon not in table + + def test_patch_integrity_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Clean": "✅ Good", + "Minor Issues": "⚠️ Needs review", + "Critical Issues": "❌ Blocked", + "Does Not Apply": "❌ Blocked", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + patch_integrity={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Does the patch apply?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_remediation_alignment_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Aligned": "✅ Good", + "Likely Aligned": "⚠️ Needs review", + "Partial": "⚠️ Needs review", + "Misaligned": "❌ Blocked", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + remediation_alignment={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Does it address")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_coverage_confidence_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = {"High": "✅ Good", "Medium": "⚠️ Needs review", "Low": "❌ Blocked"} + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + coverage_confidence={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Are there unresolved")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_test_availability_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Tests Available": "✅ Good", + "No Tests Found": "⚠️ Needs review", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + test_availability={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Do relevant tests already exist?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_deployment_safety_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Low Risk": "✅ Good", + "Medium Risk": "⚠️ Needs review", + "High Risk": "❌ Blocked", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + deployment_safety={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Is deployment risk low?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_coverage_medium_note_points_to_review_results_section(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + coverage_confidence={ + "value": "Medium", "label": "Medium", + "notes": "12 review finding(s) · no deterministic blocker identified", + } + )) + assert "see Review Results section below" in table + assert "see analysis below" not in table + + def test_notes_text_otherwise_preserved(self): + """Requirement 5: the underlying computed notes text is not altered + beyond the one dangling-reference substitution.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "Applies cleanly · no hygiene issues" in table + assert "Adversarial review confirms fix approach" in table + assert "No gaps identified by adversarial analysis" in table + assert "Good — test files cover this module" in table + assert "Localized change · low regression risk" in table + + def test_good_rows_get_no_forward_pointer(self): + """A row that's already ✅ Good has nothing to send the reader to — + except the testing row, which always carries its Existing-Test- + Coverage-vs-Missing-Behavioral-Validation bridge regardless of + status (see test_testing_row_bridge_present_regardless_of_status).""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + rows = [l for l in table.splitlines() if l.startswith("| ") and "Do relevant tests" not in l] + for row in rows: + assert "see" not in row.lower(), f"Unexpected forward pointer in a good row: {row!r}" + + def test_testing_row_bridge_present_regardless_of_status(self): + """The Existing Test Coverage vs Missing Behavioral Validation bridge + must appear whether or not test coverage is good — a "✅ Good" status + here must never look like it also answers whether the new behavior + itself is validated.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + row = [l for l in table.splitlines() if l.startswith("| Do relevant tests already exist?")][0] + assert "Review Results" in row + # Points at Review Results as a whole, not a specific subsection — + # verified against a real challenger run that "no test validates X" + # findings don't reliably classify as validation_gap (they can land + # in Behavior Notes instead), so naming one subsection here would + # risk pointing at an empty section. + assert "Validation Gaps" not in row + + def test_non_good_patch_integrity_points_to_patch_applicability(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + patch_integrity={"value": "Does Not Apply", "label": "Does Not Apply", "notes": "rejected by git apply"} + )) + assert "see Patch Applicability section below" in table + + def test_non_good_test_availability_points_to_test_support(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + test_availability={"value": "No Tests Found", "label": "No Tests Found", "notes": "No test files cover this module"} + )) + assert "see Test Support section below" in table + + def test_non_good_deployment_safety_points_to_impact_surface(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + deployment_safety={"value": "High Risk", "label": "High Risk", "notes": "HIGH impact surface"} + )) + assert "see Impact Surface section below" in table + + def test_non_good_remediation_alignment_points_to_review_results_when_rendered(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table( + _signals_full(remediation_alignment={"value": "Partial", "label": "Partial", "notes": "still_vulnerable flag set"}), + known_findings_rendered=True, + ) + assert "see Review Results section below" in table + + def test_remediation_alignment_pointer_suppressed_when_review_results_empty(self): + """Edge case: remediation_alignment can be non-good ("Likely Aligned") + while no relevant Review Results category is populated. The pointer + must not be shown in that case — never reference a section that + won't exist. (The testing row's own Review Results bridge is + independent of this gate and is checked separately.)""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table( + _signals_full(remediation_alignment={ + "value": "Likely Aligned", "label": "Likely Aligned", + "notes": "Correct mechanism · runtime verification pending", + }), + known_findings_rendered=False, + ) + row = [l for l in table.splitlines() if l.startswith("| Does it address the vulnerability?")][0] + assert "see Review Results" not in row + + +class TestRecommendationV1: + def test_do_not_apply_for_does_not_apply_integrity(self): + rec = _build_recommendation_v1(_signals_for(integrity="Does Not Apply")) + assert rec["decision"] == "Do Not Apply" + + def test_do_not_apply_for_critical_issues(self): + rec = _build_recommendation_v1(_signals_for(integrity="Critical Issues")) + assert rec["decision"] == "Do Not Apply" + + def test_misaligned_alone_is_manual_review_not_do_not_apply(self): + """Recommendation Policy v2: alignment=Misaligned is heuristic-only evidence + (challenger-derived confirmed_defect_count) and must not hard-block by itself.""" + rec = _build_recommendation_v1(_signals_for(alignment="Misaligned")) + assert rec["decision"] == "Manual Review Required" + # Reason must not state the exploit as a confirmed fact. + assert "confirmed exploit path" not in rec["reason"].lower() + + def test_deploy_after_validation_high_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="High", safety="Low Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_after_validation_medium_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Medium", safety="Low Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_after_validation_medium_improvement_medium_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Medium", safety="Medium Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_with_caution_low_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Low", safety="Low Risk")) + assert rec["decision"] == "Deploy With Caution" + + def test_manual_review_required_high_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Low", safety="High Risk")) + assert rec["decision"] == "Manual Review Required" + + def test_recommendation_has_reason(self): + rec = _build_recommendation_v1(_signals_for()) + assert len(rec["reason"]) > 20 + + def test_urllib3_representative_deploy_after_validation(self): + """A well-grounded correct patch with medium coverage → Deploy After Validation.""" + classified = _classified(still_vulnerable=True, risks=1, gaps=1) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1(signals) + assert rec["decision"] == "Deploy After Validation" + + +# --------------------------------------------------------------------------- +# Evaluation case regressions — still_vulnerable guard +# --------------------------------------------------------------------------- + +class TestRecommendationV1EvaluationCases: + """Phase A regression suite for the still_vulnerable hard-block guard. + + Each test represents the definitive signal state observed in the evaluation + run for that case and asserts the expected recommendation before and after + the guard was introduced. Tests named *_unchanged confirm no regression. + """ + + # --- GitPython: the bug case --- + + def test_still_vulnerable_no_defects_produces_manual_review(self): + """still_vulnerable=True with no confirmed defects → Manual Review Required. + + Signals: integrity=Clean, improvement=Medium, alignment=Partial, + safety=Low Risk, still_vulnerable=True, defect_count=0. + The LLM binary verdict alone is insufficient for a hard-block; + the escalation tier asks a human to review the plausible risks. + """ + signals = _signals_for( + integrity="Clean", + improvement="Medium", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + def test_gitpython_reason_names_challenger(self): + """Escalation reason must mention the challenger, not applicability.""" + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert "challenger" in rec["reason"].lower() or "review" in rec["reason"].lower() + + def test_gitpython_before_guard_would_have_passed(self): + """Confirm the pre-fix path: same signals without still_vulnerable → Deploy After Validation.""" + signals = _signals_for( + integrity="Clean", + improvement="Medium", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- urllib3: must not regress --- + + def test_urllib3_unchanged(self): + """urllib3: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- pip: must not regress --- + + def test_pip_unchanged(self): + """pip: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Medium Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- minimist: must not regress --- + + def test_minimist_unchanged(self): + """minimist v3: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- pygeoapi: integrity check takes priority over still_vulnerable --- + + def test_pygeoapi_integrity_fires_before_still_vulnerable(self): + """pygeoapi: Does Not Apply integrity fires before still_vulnerable guard.""" + signals = _signals_for( + integrity="Does Not Apply", + improvement="None", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True) + assert rec["decision"] == "Do Not Apply" + assert "does not apply" in rec["reason"].lower() or "critical" in rec["reason"].lower() + + def test_pygeoapi_do_not_apply_without_still_vulnerable(self): + """pygeoapi: Do Not Apply fires on integrity alone when still_vulnerable=False.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Do Not Apply" + + # --- Guard ordering --- + + def test_still_vulnerable_does_not_fire_when_integrity_fails(self): + """Reason text should reflect integrity failure, not still_vulnerable, when both are true.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Partial") + rec_integrity = _build_recommendation_v1(signals, still_vulnerable=False) + rec_both = _build_recommendation_v1(signals, still_vulnerable=True) + assert rec_integrity["decision"] == rec_both["decision"] == "Do Not Apply" + assert rec_integrity["reason"] == rec_both["reason"] + + def test_still_vulnerable_no_defects_escalates_regardless_of_improvement(self): + """still_vulnerable=True with defect_count=0 escalates even when other signals are clean.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + +# --------------------------------------------------------------------------- +# _extract_security_gain +# --------------------------------------------------------------------------- + +class TestExtractSecurityGain: + def test_extracts_sentence_with_fix_verb(self): + explanation = "The vulnerability exists because the header is not stripped. The patch fixes this by adding Cookie to the removal list." + gain = _extract_security_gain(explanation) + assert "patch" in gain.lower() or "fix" in gain.lower() + assert len(gain) >= 40 + + def test_extracts_sentence_with_prevent_verb(self): + explanation = "Sensitive tokens were logged. This patch prevents token values from appearing in log output." + gain = _extract_security_gain(explanation) + assert "prevent" in gain.lower() + + def test_extracts_sentence_with_add_verb(self): + explanation = "Cookie was not stripped. The fix adds Cookie to DEFAULT_REMOVE_HEADERS_ON_REDIRECT alongside Authorization." + gain = _extract_security_gain(explanation) + assert "add" in gain.lower() or "Cookie" in gain + + def test_skips_short_sentences(self): + # A sentence with a verb but too short should be skipped + explanation = "Bug found. The patch fixes it. Here is why the fix adds significant protection against cross-origin leakage of credentials." + gain = _extract_security_gain(explanation) + assert len(gain) >= 40 + + def test_fallback_to_first_paragraph(self): + explanation = "This change modifies the retry behavior." + gain = _extract_security_gain(explanation) + assert len(gain) > 0 + + def test_empty_explanation_returns_empty(self): + assert _extract_security_gain("") == "" + + +# --------------------------------------------------------------------------- +# _build_known_findings / _render_known_findings +# --------------------------------------------------------------------------- + +class TestBuildKnownFindings: + """_build_known_findings regroups the same four classifier categories + into epistemic-state labels — no new classification, no change to the + counts _compute_trust_signals/_build_recommendation_v1 read. + + plausible_risk/generic findings are further split three ways by the + (optional) finding_calibration stage output — evidence-quality pass — + with a conservative fallback (plausible_risk -> Validation Hypotheses, + generic -> Future Hardening Ideas) when no calibration entry exists for + a given finding, so a calibration failure degrades gracefully instead of + losing information.""" + + def _classified_with(self, defects=(), risks=(), gaps=(), generic=()): + classified_edge = [] + classified_issues = [] + for d in defects: + classified_edge.append({"text": d, "category": "confirmed_defect"}) + for r in risks: + classified_edge.append({"text": r, "category": "plausible_risk"}) + for g in gaps: + classified_issues.append({"text": g, "category": "validation_gap"}) + for gen in generic: + classified_issues.append({"text": gen, "category": "generic"}) + return { + "classified_edge_cases": classified_edge, + "classified_potential_issues": classified_issues, + "confirmed_defect_count": len(defects), + "plausible_risk_count": len(risks), + "validation_gap_count": len(gaps), + } + + def test_confirmed_defects_map_to_potential_remaining_risks(self): + cc = self._classified_with(defects=["confirmed issue"]) + findings = _build_known_findings(cc) + assert findings["potential_remaining_risks"] == ["confirmed issue"] + + def test_validation_gaps_map_to_validation_gaps(self): + cc = self._classified_with(gaps=["gap1"]) + findings = _build_known_findings(cc) + assert findings["validation_gaps"] == ["gap1"] + + def test_plausible_risk_without_calibration_falls_back_to_hypotheses(self): + cc = self._classified_with(risks=["plausible risk"]) + findings = _build_known_findings(cc, finding_calibration=None) + assert findings["validation_hypotheses"] == ["plausible risk"] + assert findings["observed_implementation_notes"] == [] + assert findings["future_hardening_ideas"] == [] + + def test_generic_without_calibration_falls_back_to_hardening(self): + cc = self._classified_with(generic=["tests should be added"]) + findings = _build_known_findings(cc, finding_calibration=None) + assert findings["future_hardening_ideas"] == ["tests should be added"] + + def test_calibration_routes_finding_to_its_assigned_group(self): + cc = self._classified_with(risks=["case normalization detail"]) + calibration = [{ + "original": "case normalization detail", + "group": "observed", + "reworded": "The constructor normalizes header casing via h.lower().", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["observed_implementation_notes"] == [ + "The constructor normalizes header casing via h.lower()." + ] + assert findings["validation_hypotheses"] == [] + + def test_calibration_can_move_a_finding_out_of_its_classifier_bucket(self): + """A plausible_risk-classified finding can still land in Future + Hardening Ideas if calibration judges it unrelated to the advisory — + the classifier bucket only decides which findings are eligible for + calibration, not the final presentation group.""" + cc = self._classified_with(risks=["Proxy-Authorization header not stripped"]) + calibration = [{ + "original": "Proxy-Authorization header not stripped", + "group": "hardening", + "reworded": "Proxy-Authorization is not covered by this advisory; a separate hardening improvement.", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["future_hardening_ideas"] == [ + "Proxy-Authorization is not covered by this advisory; a separate hardening improvement." + ] + assert findings["validation_hypotheses"] == [] + + def test_missing_calibration_entry_for_one_finding_still_falls_back(self): + """Calibration covering only some findings must not drop the rest — + each uncovered finding still gets its conservative default.""" + cc = self._classified_with(risks=["covered finding", "uncovered finding"]) + calibration = [{ + "original": "covered finding", "group": "observed", "reworded": "Covered, reworded.", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["observed_implementation_notes"] == ["Covered, reworded."] + assert findings["validation_hypotheses"] == ["uncovered finding"] + + def test_validation_gaps_capped_at_three(self): + cc = self._classified_with(gaps=["g1", "g2", "g3", "g4", "g5"]) + findings = _build_known_findings(cc) + assert len(findings["validation_gaps"]) <= 3 + + def test_empty_classified_challenger_returns_all_empty(self): + cc = self._classified_with() + findings = _build_known_findings(cc) + assert all(v == [] for v in findings.values()) + + def test_signature_takes_optional_calibration(self): + """Unlike the old _build_known_limitations(cc, coverage_value), this + function takes classified_challenger plus an optional + finding_calibration — gating on coverage is the renderer/caller's + job, and calibration is optional so a calibration failure doesn't + change this function's contract.""" + import inspect + params = inspect.signature(_build_known_findings).parameters + assert list(params) == ["classified_challenger", "finding_calibration"] + assert params["finding_calibration"].default is None + + +class TestRenderKnownFindings: + def _findings(self, risks=(), gaps=(), observed=(), hypotheses=(), hardening=()): + return { + "potential_remaining_risks": list(risks), + "validation_gaps": list(gaps), + "observed_implementation_notes": list(observed), + "validation_hypotheses": list(hypotheses), + "future_hardening_ideas": list(hardening), + } + + def test_empty_findings_render_nothing(self): + assert _render_known_findings(self._findings()) == "" + + def test_bullet_count_disclaimer_present_when_rendered(self): + """Reviewer-experience fix: the number of bullets below this heading + must not read as a count of confirmed defects — a standing + disclaimer states this once, directly under the heading.""" + block = _render_known_findings(self._findings(risks=["r1"])) + assert "not a count of confirmed defects" in block.lower() + + def test_bullet_count_disclaimer_absent_when_empty(self): + assert "confirmed defects" not in _render_known_findings(self._findings()).lower() + + def test_heading_and_subheadings_present(self): + block = _render_known_findings(self._findings( + risks=["r1"], gaps=["g1"], observed=["o1"], hypotheses=["h1"], hardening=["f1"], + )) + assert "## Review Results" in block + assert "### Potential Remaining Risks" in block + assert "### Validation Gaps" in block + assert "### Confirmed Observations" in block + assert "### Validation Questions" in block + assert "### Future Improvements" in block + assert "r1" in block and "g1" in block and "o1" in block and "h1" in block and "f1" in block + + def test_potential_remaining_risks_labeled_as_heuristic_not_confirmed(self): + """Correction: confirmed_defect findings must not be presented as + confirmed facts — the challenger is heuristic LLM analysis.""" + block = _render_known_findings(self._findings(risks=["r1"])) + assert "heuristic" in block.lower() + assert "**Confirmed" not in block + assert "confirmed gap" not in block.lower() + + def test_observed_implementation_notes_labeled_as_repository_backed(self): + block = _render_known_findings(self._findings(observed=["h.lower() normalizes casing"])) + assert "### Confirmed Observations" in block + assert "repository evidence" in block.lower() + + def test_validation_hypotheses_labeled_as_unconfirmed(self): + """Correction: hypotheses must read as conditional reasoning, not + observed behavior.""" + block = _render_known_findings(self._findings(hypotheses=["same-origin redirects may also strip Cookie"])) + assert "### Validation Questions" in block + assert "not directly observed" in block.lower() + assert "not confirmed outcomes" in block.lower() + + def test_hardening_ideas_do_not_reduce_confidence(self): + block = _render_known_findings(self._findings(hardening=["Proxy-Authorization header"])) + assert "### Future Improvements" in block + assert "do not reduce confidence" in block.lower() + + def test_validation_gaps_bridge_to_test_coverage(self): + """Correction: Validation Gaps must clarify it's independent of + whether the repo already has pre-existing tests for this module.""" + block = _render_known_findings(self._findings(gaps=["no test validates the new behavior"])) + assert "Trust Signals" in block + + def test_only_populated_subsections_render(self): + block = _render_known_findings(self._findings(gaps=["g1"])) + assert "### Validation Gaps" in block + assert "### Potential Remaining Risks" not in block + assert "### Confirmed Observations" not in block + assert "### Validation Questions" not in block + assert "### Future Improvements" not in block + + def test_future_hardening_ideas_render_alone(self): + """The section must render even when only hardening ideas are + present — this is the previously-fully-discarded generic bucket.""" + block = _render_known_findings(self._findings(hardening=["consider adding a comment"])) + assert "## Review Results" in block + assert "### Future Improvements" in block + assert "consider adding a comment" in block + + +# --------------------------------------------------------------------------- +# _check_recommendation_consistency (Slice 1 — Decision Consistency) +# --------------------------------------------------------------------------- + +class TestRecommendationConsistency: + """Unit tests for the Slice 1 consistency check. + + Goal: a top-tier recommendation must acknowledge, in its own text, any + already-displayed evidence (test availability, decision-relevant open + findings) that runs against it. This function never changes the decision + itself — these tests only assert on the caveat list it returns. + + The coverage-related caveat is deliberately NOT driven by + coverage_confidence's own value/notes: Coverage Confidence answers "how + thoroughly did we explore?" (Future Hardening Ideas count there, + unchanged) while this caveat answers "should this recommendation be + discounted?" (Future Hardening Ideas must not count here). It computes + its own decision-relevant count from `known_findings` instead. + """ + + def _signals(self, test_availability="Tests Available"): + return { + "test_availability": { + "value": test_availability, "label": test_availability, + "notes": "No test files cover this module" if test_availability == "No Tests Found" else "some notes", + }, + # coverage_confidence is realistic fixture data only — this + # function no longer reads it at all. + "coverage_confidence": { + "value": "Medium", "label": "Medium", + "notes": "11 review finding(s) · no deterministic blocker identified", + }, + } + + def _known_findings(self, risks=0, gaps=0, observed=0, hypotheses=0, hardening=0): + return { + "potential_remaining_risks": [f"risk {i}" for i in range(risks)], + "validation_gaps": [f"gap {i}" for i in range(gaps)], + "observed_implementation_notes": [f"observed {i}" for i in range(observed)], + "validation_hypotheses": [f"hypothesis {i}" for i in range(hypotheses)], + "future_hardening_ideas": [f"hardening {i}" for i in range(hardening)], + } + + # --- Decision gating: only top-tier decisions are checked at all --- + + def test_no_caveat_for_manual_review_required(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=2) + assert _check_recommendation_consistency(signals, "Manual Review Required", findings) == [] + + def test_no_caveat_for_do_not_apply(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=2) + assert _check_recommendation_consistency(signals, "Do Not Apply", findings) == [] + + # --- Test Availability condition --- + + def test_caveat_for_no_tests_found_at_top_tier(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "test coverage" in caveats[0].lower() + + def test_no_caveat_when_tests_available_at_top_tier(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_not_verified_is_not_treated_as_no_tests_found(self): + """Language-guardrail 'Not Verified' means the check didn't run, not + that tests are confirmed absent — must not trip the same caveat.""" + signals = self._signals(test_availability="Not Verified") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + # --- Decision-relevant findings condition --- + + def test_caveat_when_decision_relevant_findings_open(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hypotheses=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "adversarial coverage" in caveats[0].lower() + + def test_no_caveat_when_no_findings_at_all(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_hardening_only_findings_produce_no_caveat(self): + """The core fix: Future Hardening Ideas are real findings (they still + count toward Coverage Confidence and appear in Known Findings) but + must not, on their own, make this caveat fire — they are not reasons + to distrust this deployment recommendation.""" + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hardening=5) + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_hardening_findings_excluded_from_caveat_count(self): + """Mixed case: decision-relevant findings trigger the caveat, but its + wording counts only those — hardening findings present alongside + must not inflate the number shown.""" + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hypotheses=2, hardening=7) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "2 decision-relevant finding(s)" in caveats[0] + assert "7" not in caveats[0] + + def test_confirmed_risks_and_gaps_also_count_as_decision_relevant(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(risks=1, gaps=1, observed=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert "3 decision-relevant finding(s)" in caveats[0] + + # --- Both conditions at once --- + + def test_both_caveats_when_both_weak_at_top_tier(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 2 + + # --- Deploy With Caution is also top-tier --- + + def test_deploy_with_caution_is_also_checked(self): + """Deploy With Caution is reachable through _build_recommendation_v1's + API even though today's _compute_trust_signals output cannot produce + it in practice (see benchmark notes) — the check must still cover it + defensively rather than assume it will never be seen.""" + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy With Caution", findings) + assert len(caveats) == 1 + + # --- Wording reuses already-displayed evidence, not new judgment --- + + def test_caveat_reuses_displayed_notes_text(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert "No test files cover this module" in caveats[0] + + # --- Defensive: missing keys never raise --- + + def test_missing_signal_keys_do_not_raise(self): + assert _check_recommendation_consistency({}, "Deploy After Validation", {}) == [] + + +class TestDecisionRelevantFindingCount: + """Unit tests for the small pure helper the consistency check now uses + instead of reusing coverage_confidence's value.""" + + def test_excludes_future_hardening_ideas(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + findings = { + "potential_remaining_risks": [], "validation_gaps": [], + "observed_implementation_notes": [], "validation_hypotheses": [], + "future_hardening_ideas": ["a", "b", "c"], + } + assert _decision_relevant_finding_count(findings) == 0 + + def test_counts_all_other_categories(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + findings = { + "potential_remaining_risks": ["r"], "validation_gaps": ["g"], + "observed_implementation_notes": ["o"], "validation_hypotheses": ["h1", "h2"], + "future_hardening_ideas": ["ignored"], + } + assert _decision_relevant_finding_count(findings) == 5 + + def test_missing_keys_default_to_empty(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + assert _decision_relevant_finding_count({}) == 0 + + +# --------------------------------------------------------------------------- +# Hybrid blocking policy +# --------------------------------------------------------------------------- + +class TestHybridBlockingPolicy: + """Regression suite for the hybrid blocking policy (Recommendation Policy v2). + + Hard-block → integrity failure only (deterministic evidence) + Escalation → still_vulnerable=True, confirmed_defect_count == 0 + OR confirmed_defect_count > 0 (alignment=Misaligned) — both heuristic-only, + both land at Manual Review Required, never Do Not Apply + Forward → still_vulnerable=False, confirmed_defect_count == 0 + + Benchmark expectations + ---------------------- + urllib3 : still_vulnerable=True, defect_count=0, risks=12 → Manual Review Required + minimist : still_vulnerable=False, defect_count=0 → Deploy After Validation + pip : still_vulnerable=False, defect_count=0 → Deploy After Validation + pygeoapi : integrity=Does Not Apply → Do Not Apply + GitPython : still_vulnerable=True, defect_count=1 → Manual Review Required (via alignment) + """ + + # --- urllib3: still_vulnerable=True, defect_count=0 → escalation --- + + def test_urllib3_still_vulnerable_no_defects_is_manual_review(self): + """urllib3 regression: plausible risks with no confirmed defect → Manual Review Required.""" + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial", safety="Low Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + def test_urllib3_full_signals_manual_review(self): + """End-to-end urllib3 signal path via _compute_trust_signals.""" + classified = _classified(still_vulnerable=True, risks=12) + signals = _compute_trust_signals([], _clean_applicability(), classified, "Good", "high") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + def test_urllib3_escalation_reason_mentions_challenger(self): + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert "challenger" in rec["reason"].lower() + + # --- GitPython v2: defect_count=1 → alignment=Misaligned → hard-block --- + + def test_gitpython_confirmed_defect_is_manual_review_required(self): + """GitPython v2: one confirmed defect drives alignment=Misaligned → Manual Review + Required. Per Recommendation Policy v2, heuristic evidence (challenger-derived + confirmed_defect_count) escalates to human review, it does not hard-block.""" + classified = _classified(still_vulnerable=True, defects=1, risks=13) + signals = _compute_trust_signals([], _clean_applicability(), classified, "Some", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + assert signals["remediation_alignment"]["value"] == "Misaligned" + + def test_gitpython_escalation_survives_still_vulnerable_false(self): + """Contradictory case: LLM says No but a finding classifies as confirmed_defect. + defect_count > 0 drives alignment=Misaligned and escalates to Manual Review + Required regardless of still_vulnerable — still never Do Not Apply on its own.""" + classified = _classified(still_vulnerable=False, defects=1) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + # --- pygeoapi: integrity failure fires before any challenger signal --- + + def test_pygeoapi_integrity_hard_block(self): + """pygeoapi: Does Not Apply integrity blocks regardless of challenger state.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Misaligned") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=1) + assert rec["decision"] == "Do Not Apply" + assert "does not apply" in rec["reason"].lower() or "critical" in rec["reason"].lower() + + # --- minimist / pip: clean path --- + + def test_minimist_clean_path_deploy_after_validation(self): + """minimist: still_vulnerable=False, defect_count=0 → Deploy After Validation.""" + signals = _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Low Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Deploy After Validation" + + def test_pip_clean_path_deploy_after_validation(self): + """pip: still_vulnerable=False, defect_count=0, medium risk → Deploy After Validation.""" + signals = _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Medium Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Deploy After Validation" + + # --- Boundary: still_vulnerable=True with defect_count>0 also hard-blocks --- + + def test_both_still_vulnerable_and_defects_is_manual_review_required(self): + """When still_vulnerable=True AND defect_count>0, alignment=Misaligned fires → + Manual Review Required (heuristic evidence escalates, it does not hard-block).""" + classified = _classified(still_vulnerable=True, defects=2, risks=5) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + # --- Boundary: still_vulnerable=True with only gaps (risk_count=0) --- + + def test_still_vulnerable_gaps_only_is_manual_review(self): + """still_vulnerable=True with only validation gaps (no plausible risks, no defects) + also hits the escalation branch.""" + classified = _classified(still_vulnerable=True, gaps=3) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + +# --------------------------------------------------------------------------- +# Recommendation Policy v2 — explicit policy statement as a regression test +# --------------------------------------------------------------------------- + +class TestPureHeuristicEvidenceNeverBlocks: + """Recommendation Policy v2: pure heuristic evidence must never produce Do Not Apply. + + Do Not Apply may only be produced by deterministic evidence — currently that + means `patch_integrity` (git-apply / static hygiene). Every other input to + `_build_recommendation_v1` (`security_improvement`, `remediation_alignment`, + `still_vulnerable`, `defect_count`) is derived entirely from the adversarial + challenger's free-text output and is therefore heuristic, not deterministic + (see docs/recommendation-policy-v2.md). This test holds `patch_integrity` at + its cleanest, non-blocking value and sweeps every value the heuristic + signals can take — including the worst case on every axis at once — and + asserts none of them, alone or combined, ever reach Do Not Apply. + + This expresses the policy itself, not one input combination: if a future + change reintroduces a path from heuristic evidence to Do Not Apply, this + test fails regardless of which heuristic signal caused it. + """ + + _ALIGNMENT_VALUES = ["Misaligned", "Partial", "Likely Aligned", "Aligned"] + _IMPROVEMENT_VALUES = ["None", "Low", "Medium", "High"] + _SAFETY_VALUES = ["Low Risk", "Medium Risk", "High Risk"] + _DEFECT_COUNTS = (0, 1, 5) + + def test_pure_heuristic_evidence_never_produces_do_not_apply(self): + for alignment in self._ALIGNMENT_VALUES: + for improvement in self._IMPROVEMENT_VALUES: + for safety in self._SAFETY_VALUES: + for still_vulnerable in (True, False): + for defect_count in self._DEFECT_COUNTS: + signals = _signals_for( + integrity="Clean", # the one deterministic control variable + improvement=improvement, + alignment=alignment, + safety=safety, + ) + rec = _build_recommendation_v1( + signals, + still_vulnerable=still_vulnerable, + defect_count=defect_count, + ) + assert rec["decision"] != "Do Not Apply", ( + "Pure heuristic evidence produced Do Not Apply with " + f"integrity=Clean, alignment={alignment!r}, " + f"improvement={improvement!r}, safety={safety!r}, " + f"still_vulnerable={still_vulnerable}, " + f"defect_count={defect_count}" + ) + + def test_integrity_is_still_the_only_path_to_do_not_apply(self): + """Sanity check the sweep isn't vacuous: deterministic integrity failure + must still produce Do Not Apply even with every heuristic signal clean.""" + signals = _signals_for( + integrity="Does Not Apply", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Do Not Apply" diff --git a/libs/openant-core/tests/patch/test_vulnerability_patterns.py b/libs/openant-core/tests/patch/test_vulnerability_patterns.py new file mode 100644 index 00000000..cd3c9a70 --- /dev/null +++ b/libs/openant-core/tests/patch/test_vulnerability_patterns.py @@ -0,0 +1,252 @@ +"""Tests for src/vulnerability_patterns.py + +Validates classification, sink scanning, and context building. +No references to specific project names (pygeoapi, gitpython, etc.). +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.vulnerability_patterns import ( + build_vulnerability_pattern_context, + classify_vuln_class, + find_sink_methods, +) + + +# --------------------------------------------------------------------------- +# classify_vuln_class +# --------------------------------------------------------------------------- + + +class TestClassifyVulnClass: + def test_cwe22_returns_path_traversal(self): + assert classify_vuln_class("This advisory tracks CWE-22") == "PATH_TRAVERSAL" + + def test_cwe23_returns_path_traversal(self): + assert classify_vuln_class("Classified as CWE-23 path traversal") == "PATH_TRAVERSAL" + + def test_path_traversal_keyword(self): + assert classify_vuln_class( + "A path traversal vulnerability in the file serve endpoint." + ) == "PATH_TRAVERSAL" + + def test_directory_traversal_keyword(self): + assert classify_vuln_class( + "directory traversal allows reading /etc/passwd" + ) == "PATH_TRAVERSAL" + + def test_cwe78_returns_command_injection(self): + assert classify_vuln_class( + "CWE-78 OS command injection via user input" + ) == "COMMAND_INJECTION" + + def test_cwe88_returns_command_injection(self): + assert classify_vuln_class( + "Argument injection (CWE-88) in the build runner." + ) == "COMMAND_INJECTION" + + def test_command_injection_keyword(self): + assert classify_vuln_class( + "An OS command injection flaw in the exec handler." + ) == "COMMAND_INJECTION" + + def test_shell_injection_keyword(self): + assert classify_vuln_class( + "Shell injection vulnerability in the build step." + ) == "COMMAND_INJECTION" + + def test_unrelated_cwe_returns_none(self): + assert classify_vuln_class( + "Prototype pollution in merge utility (CWE-1321)" + ) is None + + def test_empty_string_returns_none(self): + assert classify_vuln_class("") is None + + def test_unrecognized_text_returns_none(self): + assert classify_vuln_class("Generic advisory with no class indicators.") is None + + def test_cwe_takes_priority_over_keyword(self): + # CWE-78 is explicit; "path traversal" keyword is also present — CWE wins. + text = "CWE-78 injection triggered via a crafted path traversal string" + assert classify_vuln_class(text) == "COMMAND_INJECTION" + + +# --------------------------------------------------------------------------- +# find_sink_methods +# --------------------------------------------------------------------------- + +_PATH_CONTEXT = ( + "def safe_method(self, path):\n" + " return open(path).read()\n" + "\n" + "def vulnerable_serve(self, dirpath):\n" + " data_path = self.data + dirpath\n" + " return open(data_path).read()\n" + "\n" + "def also_vulnerable(self, filename):\n" + " full = os.path.normpath(self.base + filename)\n" + " return full\n" +) + + +class TestFindSinkMethods: + def test_finds_both_vulnerable_methods(self): + hits = find_sink_methods(_PATH_CONTEXT, "PATH_TRAVERSAL") + assert "vulnerable_serve" in hits + assert "also_vulnerable" in hits + + def test_does_not_include_safe_method(self): + hits = find_sink_methods(_PATH_CONTEXT, "PATH_TRAVERSAL") + assert "safe_method" not in hits + + def test_none_class_returns_empty(self): + assert find_sink_methods(_PATH_CONTEXT, None) == [] + + def test_empty_context_returns_empty(self): + assert find_sink_methods("", "PATH_TRAVERSAL") == [] + + def test_no_sinks_returns_empty(self): + context = ( + "def clean_method(self, x):\n" + " return str(x)\n" + "\n" + "def another_clean(self, y):\n" + " return y.strip()\n" + "\n" + ) + assert find_sink_methods(context, "PATH_TRAVERSAL") == [] + + def test_sink_in_comment_not_counted(self): + context = ( + "def documented_method(self, path):\n" + " # old: os.path.normpath(self.base + path) was used here\n" + " return os.path.realpath(path)\n" + "\n" + "def unrelated(self):\n" + " return 42\n" + ) + hits = find_sink_methods(context, "PATH_TRAVERSAL") + assert "documented_method" not in hits + + def test_command_injection_shell_true(self): + context = ( + "def run_build(self, cmd):\n" + " subprocess.call(cmd, shell=True)\n" + "\n" + "def clean_runner(self, args):\n" + " subprocess.run(args)\n" + "\n" + ) + hits = find_sink_methods(context, "COMMAND_INJECTION") + assert "run_build" in hits + assert "clean_runner" not in hits + + def test_unknown_class_returns_empty(self): + assert find_sink_methods(_PATH_CONTEXT, "SQL_INJECTION") == [] + + +# --------------------------------------------------------------------------- +# build_vulnerability_pattern_context +# --------------------------------------------------------------------------- + + +class TestBuildVulnerabilityPatternContext: + def test_path_traversal_returns_nonempty(self): + result = build_vulnerability_pattern_context( + "Path traversal vulnerability in file server (CWE-22)", "" + ) + assert result != "" + + def test_command_injection_returns_nonempty(self): + result = build_vulnerability_pattern_context( + "OS command injection via user-controlled input (CWE-78)", "" + ) + assert result != "" + + def test_unknown_class_returns_empty(self): + result = build_vulnerability_pattern_context( + "Prototype pollution in lodash merge (CWE-1321)", "" + ) + assert result == "" + + def test_path_traversal_contains_realpath(self): + result = build_vulnerability_pattern_context("path traversal issue", "") + assert "realpath" in result.lower() + + def test_path_traversal_names_normpath_antipattern(self): + result = build_vulnerability_pattern_context("path traversal issue", "") + assert "normpath" in result.lower() + assert "avoid" in result.lower() + + def test_command_injection_contains_shell_true_warning(self): + result = build_vulnerability_pattern_context( + "command injection via shell=True", "" + ) + assert "shell=True" in result or "shell = True" in result + + def test_command_injection_contains_avoid_section(self): + result = build_vulnerability_pattern_context("OS command injection CWE-78", "") + assert "avoid" in result.lower() + + def test_sink_methods_appear_in_output(self): + context = ( + "def serve_file(self, dirpath):\n" + " path = self.base + dirpath\n" + " return open(path).read()\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "serve_file" in result + + def test_sink_checklist_instruction_present(self): + context = ( + "def file_handler(self, path):\n" + " p = self.root + path\n" + " return p\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + lower = result.lower() + assert "checklist" in lower or "every listed" in lower or "must address" in lower + + def test_no_sinks_no_sink_section(self): + context = ( + "def clean_method(self, x):\n" + " return str(x)\n" + "\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "Methods in the provided code" not in result + + def test_repo_root_none_does_not_raise(self): + result = build_vulnerability_pattern_context( + "path traversal CWE-22", "", repo_root=None + ) + assert isinstance(result, str) + + def test_bad_vuln_text_returns_empty(self): + result = build_vulnerability_pattern_context(None, "") + assert result == "" + + def test_output_starts_with_guidance_header(self): + result = build_vulnerability_pattern_context("path traversal CWE-22", "") + assert result.startswith("## Vulnerability class guidance") + + def test_multiple_sink_methods_all_listed(self): + context = ( + "def handler_a(self, dirpath):\n" + " p = self.base + dirpath\n" + " return p\n" + "\n" + "def handler_b(self, name):\n" + " q = os.path.normpath(self.root + name)\n" + " return q\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "handler_a" in result + assert "handler_b" in result diff --git a/libs/openant-core/tests/test_patch_eligible_verdicts.py b/libs/openant-core/tests/test_patch_eligible_verdicts.py new file mode 100644 index 00000000..f685d71a --- /dev/null +++ b/libs/openant-core/tests/test_patch_eligible_verdicts.py @@ -0,0 +1,33 @@ +"""Pins core.verdict_taxonomy.PATCH_ELIGIBLE, the eligibility filter used by +core/patch.py to decide whether a finding may be sent to the patch-trust +pipeline. + +Deliberately a distinct set from DISCLOSURE_ELIGIBLE and DYNAMIC_TESTABLE -- +see the constant's docstring in core/verdict_taxonomy.py for why. +""" + +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_CORE_ROOT)) + +from core.verdict_taxonomy import PATCH_ELIGIBLE, DISCLOSURE_ELIGIBLE, DYNAMIC_TESTABLE + + +def test_patch_eligible_exact_membership(): + assert PATCH_ELIGIBLE == frozenset({"confirmed", "agreed", "vulnerable", "bypassable"}) + + +def test_patch_eligible_excludes_non_vulnerability_verdicts(): + for verdict in ("unverified", "error", "rejected", "safe", "protected", "inconclusive"): + assert verdict not in PATCH_ELIGIBLE + + +def test_patch_eligible_is_its_own_set_not_an_alias(): + """PATCH_ELIGIBLE must differ from both DISCLOSURE_ELIGIBLE (broader -- + includes unverified/error) and DYNAMIC_TESTABLE (narrower -- excludes + bypassable). Guards against someone "simplifying" it into an alias.""" + assert PATCH_ELIGIBLE != DISCLOSURE_ELIGIBLE + assert PATCH_ELIGIBLE != DYNAMIC_TESTABLE + assert "bypassable" in PATCH_ELIGIBLE and "bypassable" not in DYNAMIC_TESTABLE diff --git a/libs/openant-core/utilities/autopatcher/__init__.py b/libs/openant-core/utilities/autopatcher/__init__.py new file mode 100644 index 00000000..6ed1eeab --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/__init__.py @@ -0,0 +1,7 @@ +"""Auto Patcher's patch-generation and trust-scoring engine, merged into OpenAnt. + +Ported from the standalone auto-patcher-mvp project so end users don't need a +separate checkout, interpreter, or virtualenv. Entry point: :func:`pipeline.run`. +See ``core/patch.py`` for the thin OpenAnt-side wrapper (finding lookup, +eligibility, artifact paths) that calls into this package. +""" diff --git a/libs/openant-core/utilities/autopatcher/behavior_summary.py b/libs/openant-core/utilities/autopatcher/behavior_summary.py new file mode 100644 index 00000000..9cd57162 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/behavior_summary.py @@ -0,0 +1,80 @@ +"""Minimal deterministic behavior summary analyzer. + +Produces a tiny BehaviorReport dict with a single function (or file fallback), +one-line summary, and 2-4 primary behaviors to validate. +""" +from __future__ import annotations + +import re +from typing import List + + +def _pick_purpose_and_behaviors(name_tokens: List[str], path_tokens: List[str]): + tokens = set(t.lower() for t in name_tokens + path_tokens) + + # Simple deterministic mapping + if any(t in tokens for t in ("auth", "authenticate", "login", "logout")): + purpose = "authentication" + behaviors = ["valid login", "invalid login", "malformed/injection-style input"] + elif any(t in tokens for t in ("validate", "sanitize", "clean", "normalize")): + purpose = "input validation" + behaviors = ["valid input acceptance", "invalid/malformed input rejection"] + elif any(t in tokens for t in ("db", "query", "execute", "insert", "update", "delete", "cursor")): + purpose = "database operations" + behaviors = ["query correctness", "parameterization vs raw queries"] + elif any(t in tokens for t in ("api", "request", "handler", "route", "view")): + purpose = "request/handler logic" + behaviors = ["happy-path response", "error handling paths"] + else: + purpose = "application logic" + behaviors = ["normal flow", "edge-case handling"] + + # Limit to 4 + return purpose, behaviors[:4] + + +class BehaviorAnalyzer: + """Very small deterministic analyzer for Behavior Summary. + + analyze(patch_diff, repo_context=None) -> dict + """ + + # Accept diff markers like '+' at line start (e.g. '+def foo(') + FUNC_RE = re.compile(r"^[\+\-\s]*def\s+([A-Za-z0-9_]+)\s*\(") + + def analyze(self, patch_diff: str, repo_context=None) -> dict: + # 1) find first changed file + file_path = "unknown" + for line in patch_diff.splitlines(): + if line.startswith("+++ b/"): + file_path = line[6:].strip() + break + + # 2) find first function definition in diff hunks + func_name = "" + for line in patch_diff.splitlines(): + m = self.FUNC_RE.match(line) + if m: + func_name = m.group(1) + break + + # derive tokens + name_tokens = func_name.replace("_", " ").split() if func_name else [] + path_tokens = [p for p in re.split(r"[/_.\\]+", file_path) if p] + + purpose, behaviors = _pick_purpose_and_behaviors(name_tokens, path_tokens) + + if func_name: + summary = f"This patch likely affects {purpose} in {file_path}." + else: + summary = f"This patch likely affects {purpose} in {file_path}." + + # limit behaviors to 2-4 deterministic choices + primary_behaviors = behaviors[:4] + + return { + "function": func_name, + "file": file_path, + "summary": summary, + "primary_behaviors": primary_behaviors, + } diff --git a/libs/openant-core/utilities/autopatcher/confidence_scorer.py b/libs/openant-core/utilities/autopatcher/confidence_scorer.py new file mode 100644 index 00000000..4435b0eb --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/confidence_scorer.py @@ -0,0 +1,63 @@ +""" +Confidence scorer stage. + +Loads the confidence_scorer prompt, sends the full context (vulnerability + +patch + review) to the LLM, and returns a structured confidence assessment. +""" + +from __future__ import annotations + +from pathlib import Path + +from .llm_client import LLMClient + +_PROMPT_PATH = Path(__file__).parent / "prompts" / "confidence_scorer.md" + + +def score_confidence( + vulnerability_text: str, + patch: str, + review: str, + llm: LLMClient, + code_context: str = "", +) -> str: + """ + Assign a confidence score to the generated patch. + + Parameters + ---------- + vulnerability_text: + The original vulnerability description and code context. + patch: + The unified diff patch. + review: + The structured patch review (explanation, affected areas, validation + notes). + llm: + An initialised :class:`LLMClient` instance. + code_context: + Optional repository evidence selected by static analysis. When + provided it is prepended to the user message so the scorer reasons + from the same evidence the patch generator used. + + Returns + ------- + str + The raw LLM response containing the confidence score and reasons. + """ + system_prompt = _PROMPT_PATH.read_text(encoding="utf-8") + context_section = ( + "## Repository evidence (selected by static analysis)\n\n" + + code_context + + "\n\n" + ) if code_context else "" + user_message = ( + context_section + + "## Vulnerability report\n\n" + + vulnerability_text + + "\n\n## Proposed patch\n\n" + + patch + + "\n\n## Patch review\n\n" + + review + ) + return llm.complete(system_prompt, user_message, stage="confidence_scorer") diff --git a/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py b/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py new file mode 100644 index 00000000..82a3088d --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py @@ -0,0 +1,186 @@ +"""Deterministic unified diff hunk header repair. + +LLMs frequently generate unified diffs with arithmetically wrong @@ -a,b +c,d @@ +counts. This module recomputes b and d from the actual hunk body lines, and +recomputes c as old_start + cumulative_prior_net_delta within the same file. +The old start line (a) is preserved as-is — it is the LLM's positional anchor +and requires knowledge of the original file to validate independently. + +Public API: + repair_hunk_headers(patch: str) -> tuple[str, RepairResult] + +RepairResult fields: + normalization_applied : bool — True if any @@ line was changed + hunks_rewritten : int — number of @@ headers with corrected values + files_rewritten : int — number of distinct filenames with ≥1 rewrite +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +_HUNK_RE = re.compile( + r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)" +) +_FENCE_OPEN_RE = re.compile(r"^```") + + +@dataclass +class RepairResult: + normalization_applied: bool = False + hunks_rewritten: int = 0 + files_rewritten: int = 0 + + +def repair_hunk_headers(patch: str) -> tuple[str, RepairResult]: + """Recompute unified diff hunk header counts from body content. + + Returns (repaired_patch, RepairResult). + Never raises — on any unexpected error returns (original_patch, RepairResult()). + """ + meta = RepairResult() + if not patch or not patch.strip(): + return patch, meta + try: + open_fence, clean, close_fence = _strip_md_fences(patch) + repaired, meta = _repair(clean, meta) + return open_fence + repaired + close_fence, meta + except Exception: + return patch, meta + + +# --------------------------------------------------------------------------- +# Internal +# --------------------------------------------------------------------------- + +def _strip_md_fences(patch: str) -> tuple[str, str, str]: + """Extract markdown code fences surrounding a patch string. + + Returns (open_fence, clean_patch, close_fence). + Concatenating the three parts always reconstructs the original string. + """ + lines = patch.splitlines(keepends=True) + open_fence = "" + close_fence = "" + if lines and _FENCE_OPEN_RE.match(lines[0]): + open_fence = lines.pop(0) + if lines and lines[-1].strip() in ("```", "~~~"): + close_fence = lines.pop() + return open_fence, "".join(lines), close_fence + + +def _repair(patch: str, meta: RepairResult) -> tuple[str, RepairResult]: + lines = patch.splitlines(keepends=True) + output: list[str] = [] + + # Per-file accumulator: sum of (new_count - old_count) for prior hunks + file_delta: int = 0 + current_file: str | None = None + files_touched: set[str] = set() + + # Per-hunk state + hunk_orig_header: str | None = None + hunk_old_start: int = 0 + hunk_claimed_new_start: int = 0 + hunk_suffix: str = "" + hunk_body: list[str] = [] + in_hunk: bool = False + + def flush_hunk() -> None: + nonlocal file_delta, in_hunk, hunk_orig_header, hunk_body + + if not in_hunk: + return + + old_count, new_count = _count_body(hunk_body) + + # New-file sentinel: old_start=0 means the old file doesn't exist. + # The delta formula doesn't apply; preserve the original new_start + # (conventionally 1 for a new non-empty file, 0 for an empty one). + if hunk_old_start == 0: + correct_new_start = hunk_claimed_new_start + else: + correct_new_start = hunk_old_start + file_delta + rewritten = ( + f"@@ -{hunk_old_start},{old_count}" + f" +{correct_new_start},{new_count}" + f" @@{hunk_suffix}\n" + ) + + if rewritten != hunk_orig_header: + meta.hunks_rewritten += 1 + meta.normalization_applied = True + if current_file is not None: + files_touched.add(current_file) + + output.append(rewritten) + output.extend(hunk_body) + + file_delta += new_count - old_count + in_hunk = False + hunk_orig_header = None + hunk_body = [] + + for line in lines: + stripped = line.rstrip("\n") + + if stripped.startswith("--- "): + flush_hunk() + file_delta = 0 + # Track filename for metadata (strip a/ prefix when present) + raw = stripped[4:].split("\t")[0].strip() + current_file = raw[2:] if raw.startswith("a/") else raw + output.append(line) + + elif stripped.startswith("+++ "): + output.append(line) + + elif stripped.startswith("@@ "): + flush_hunk() + m = _HUNK_RE.match(stripped) + if not m: + output.append(line) # malformed — pass through unchanged + continue + hunk_old_start = int(m.group(1)) + hunk_claimed_new_start = int(m.group(3)) + hunk_suffix = m.group(5) # text after second @@, e.g. " function setKey" + hunk_orig_header = line + hunk_body = [] + in_hunk = True + + elif in_hunk: + hunk_body.append(line) + + else: + output.append(line) # preamble, diff --git lines, etc. + + flush_hunk() + meta.files_rewritten = len(files_touched) + return "".join(output), meta + + +def _count_body(body: list[str]) -> tuple[int, int]: + """Return (old_count, new_count) by walking hunk body lines. + + old_count = context lines + removed lines + new_count = context lines + added lines + The '\\' No newline marker is excluded from both counts. + """ + old_count = 0 + new_count = 0 + for raw in body: + line = raw.rstrip("\n") + if line.startswith("\\"): + # \\ No newline at end of file — metadata marker, not a content line + continue + if line.startswith("-") and not line.startswith("---"): + old_count += 1 + elif line.startswith("+") and not line.startswith("+++"): + new_count += 1 + else: + # Context: leading space, empty line, or any other non-marker content + old_count += 1 + new_count += 1 + return old_count, new_count diff --git a/libs/openant-core/utilities/autopatcher/diff_parsing.py b/libs/openant-core/utilities/autopatcher/diff_parsing.py new file mode 100644 index 00000000..37638afb --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/diff_parsing.py @@ -0,0 +1,68 @@ +"""Generic unified-diff parsing. + +Extracted from impact_surface.py: this parser understands only the +language-agnostic unified-diff conventions (`--- a/...`, `+++ b/...`, `@@ ... @@` +hunk headers, and ' '/'+'/'-' prefixed body lines). It has no Python-specific +behavior — symbol resolution, AST parsing, and everything else that depends on +a particular language stays in impact_surface.py and consumes this module's +output. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + + +@dataclass(frozen=True) +class DiffHunk: + """One hunk's raw body lines, each still prefixed with its diff marker + (' ', '+', or '-'). `new_start`/`new_count` are kept only for + diagnostic/debugging purposes — symbol resolution does not use them, + by design (see impact_surface.py's module docstring).""" + new_start: int + new_count: int + lines: List[str] = field(default_factory=list) + + +def parse_diff(diff: str) -> Tuple[List[str], Dict[str, List[DiffHunk]]]: + """Parse a unified diff and return list of changed files and hunks per file. + + Unlike a purely line-range-based parse, this keeps each hunk's actual + body lines (context/added/removed), which symbol resolution needs to + relocate the hunk by content rather than by trusting its header. + """ + changed_files: List[str] = [] + file_hunks: Dict[str, List[DiffHunk]] = {} + cur_file: Optional[str] = None + cur_hunk: Optional[DiffHunk] = None + + def flush() -> None: + nonlocal cur_hunk + if cur_hunk is not None and cur_file is not None: + file_hunks[cur_file].append(cur_hunk) + cur_hunk = None + + for line in diff.splitlines(): + if line.startswith("--- "): + flush() + continue + if line.startswith("+++ b/"): + flush() + cur_file = line[6:].strip() + if cur_file not in changed_files: + changed_files.append(cur_file) + file_hunks[cur_file] = [] + continue + if line.startswith("@@") and cur_file is not None: + flush() + m = re.search(r"\+([0-9]+)(?:,([0-9]+))?", line) + if m: + start = int(m.group(1)) + count = int(m.group(2)) if m.group(2) else 1 + cur_hunk = DiffHunk(new_start=start, new_count=count) + continue + if cur_hunk is not None and line[:1] in (" ", "+", "-"): + cur_hunk.lines.append(line) + flush() + return changed_files, file_hunks diff --git a/libs/openant-core/utilities/autopatcher/finding_calibration.py b/libs/openant-core/utilities/autopatcher/finding_calibration.py new file mode 100644 index 00000000..00aac2f9 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/finding_calibration.py @@ -0,0 +1,110 @@ +""" +Finding calibration — an LLM post-processing stage that classifies and +rewords challenger findings for calibrated certainty before they reach the +report. + +Provides `calibrate_findings(vulnerability_text, patch, findings, llm, +code_context)`, which returns one entry per input finding: which of three +epistemic groups it belongs to (Observed / Hypothesis / Hardening), and a +reworded version whose certainty matches that group. + +This is additive to the existing challenger/classifier: it does not change +`_classify_finding`'s categories or counts, and its output is read only by +report presentation (`_build_known_findings` / `_render_known_findings`) — +never by `_compute_trust_signals` or `_build_recommendation_v1`. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import List, Dict + +_PROMPT_PATH = Path(__file__).parent / "prompts" / "finding_calibration.md" + +_VALID_GROUPS = {"observed", "hypothesis", "hardening"} + +# Matches "N. Group: