diff --git a/.gitleaks.toml b/.gitleaks.toml index be082ddd..29aa6db8 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -26,3 +26,13 @@ paths = [ regexes = [ '''(1234567890|LEAKED|EXAMPLE|FAKE)''', ] + +[[allowlists]] +description = "Fake credential in curl CVE-2022-27774 fixture (curl-auth-user; historical commit fb1a038, since replaced)" +condition = "AND" +paths = [ + '''libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774\.md''', +] +regexes = [ + '''alice:s3cr3t''', +] diff --git a/README.md b/README.md index c1b76afa..2ce372b3 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ Providers accept a custom `base_url` for OpenAI-compatible / Anthropic-compatibl #### Adding a new provider adapter -OpenAnt's adapter layer is a small Python recipe — one Python file implementing the `LLMAdapter` Protocol, one factory for the contract-test harness, plus a registry entry — and that alone is enough to run the adapter from a hand-authored config. To also have it offered by the `openant setup llm` wizard and pass its pre-save probe, add a few Go touch-points in `apps/openant-cli/cmd/setup.go` (the supported-provider list, a probe `case`, the per-phase default-model maps) plus a Go probe function. The 12 contract tests run automatically against your adapter once it's wired in. See [`docs/features/llm-providers/HOW_TO_ADD_AN_ADAPTER.md`](docs/features/llm-providers/HOW_TO_ADD_AN_ADAPTER.md) for the full recipe. +OpenAnt's adapter layer is a small Python recipe — one Python file implementing the `LLMAdapter` Protocol, one factory for the contract-test harness, plus a registry entry — and that alone is enough to run the adapter from a hand-authored config. To also have it offered by the `openant setup llm` wizard and pass its pre-save probe, add a few Go touch-points in `apps/openant-cli/cmd/setup.go` (the supported-provider list, a probe `case`, the per-phase default-model maps) plus a Go probe function. The 12 contract tests run automatically against your adapter once it's wired in. ### Python runtime @@ -189,6 +189,10 @@ Or run the full pipeline in one command: openant scan --verify ``` +### 3. Remediate a finding + +Generate a candidate patch and an independent Trust Report for a specific finding — see [Auto Patcher](#auto-patcher) below. + ### Working with multiple projects The pipeline operates on one project at a time. Running `openant init` sets the newly initialized project as the active one, so all subsequent commands target it by default. @@ -212,6 +216,76 @@ openant project show # details of active project openant project switch # switch active project ``` +## Auto Patcher + +Auto Patcher exists to answer one question: **does this AI-generated patch deserve to be trusted?** Generating a candidate patch is only the first step. Auto Patcher focuses on producing the evidence humans need to decide whether that patch should be trusted and deployed. + +Given a specific finding from an OpenAnt scan, it generates a candidate patch and then subjects that patch to independent, adversarial scrutiny, producing a Trust Report that states whether the patch is fit to deploy — backed by the evidence behind that call. It does not autofix your repository: the patch and its Trust Report are written to disk for a human to review, and the target repository is never modified. + +### Why AI-generated patches can't be trusted at face value + +A patch produced by an LLM can look correct — it compiles, it touches the right function, it reads like a competent fix — without actually closing the vulnerability. It may narrow the attack surface without eliminating it, fix the described case while missing an adjacent one, or apply cleanly against one version of a file and silently fail against another. Fluent output is not verified output, and asking the same model that wrote a patch whether the patch is good doesn't close that gap — it just repeats the same blind spot. + +### What Auto Patcher does differently + +Rather than returning a single "here's the fix," every candidate patch goes through a trust-building process: + +``` +Generate a candidate patch + │ + ▼ +Challenge it from an adversarial perspective + │ + ▼ +Collect deterministic evidence — does it apply cleanly? does it introduce obvious defects? + │ + ▼ +Produce a recommendation, backed by the evidence collected above +``` + +The adversarial pass is a distinct reasoning step whose only job is to argue the patch doesn't hold up — not to confirm that it does. The deterministic checks (applying the patch against the real repository, scanning the diff for hygiene issues) never rely on an LLM's opinion of its own work. The final recommendation is computed from all of this evidence by a fixed decision policy, not read off an LLM's self-reported confidence. + +### Philosophy + +- **Never communicate more certainty than the evidence supports.** A check that didn't run is reported as unverified, never as a quiet pass. +- **Recommendations come from deterministic policy, not from an LLM's self-assessment.** Each of the four possible recommendations is computed from evidence gates a human can audit — not a model grading its own patch. +- **Every recommendation ships with the evidence behind it.** The Trust Report separates what was mechanically verified from what is heuristic, adversarial-review judgment, so a reviewer never has to guess which is which. +- **The deployment decision stays with a human.** Auto Patcher never applies a patch to the target repository — it produces a recommendation for someone accountable to act on. + +### Quick start + +Auto Patcher runs against a finding already produced by an OpenAnt scan (`openant scan` / `openant build-output`) whose verdict is patch-eligible — `confirmed`, `agreed`, `vulnerable`, or `bypassable`. + +To find an eligible finding's id, check the `findings` array in your project's `pipeline_output.json` (written by `openant build-output`) — the snippet below requires [`jq`](https://jqlang.org/): + +```bash +jq -r '.findings[] | "\(.id)\t\(.stage2_verdict // .stage1_verdict)"' pipeline_output.json +``` + +Pick an id whose verdict is one of the four above, then pass it to `patch` (`VULN-001` below is a placeholder — use the id you found): + +```bash +LLM_PROVIDER=anthropic ANTHROPIC_API_KEY=sk-ant-... openant patch --finding-id VULN-001 +``` + +`LLM_PROVIDER` (`anthropic` or `openai`) and the matching API key must be set explicitly. This is configured independently of OpenAnt's own `--llm-config` system, and deliberately does not fall back to a mock LLM unless you ask for one. + +### The Trust Report + +Each run writes two files under `patch/` in the project's scan directory: + +- `{finding-id}-vulnerability.md` — the finding as rendered into the input Auto Patcher worked from. +- `{finding-id}-trust-report.md` — the Trust Report. + +The Trust Report leads with a single recommendation — **Deploy After Validation**, **Deploy With Caution**, **Manual Review Required**, or **Do Not Apply** — followed by the evidence behind it: whether the patch applies to the repository, whether adversarial review turned up a remaining exploit path, whether tests already cover the affected code, and what deployment risk the change carries. Each item is marked as either a deterministic check or a heuristic judgment. + +### Known limitations + +- Auto Patcher is an early-stage capability — its own reports are labeled MVP output today. +- Some evidence signals (impact analysis, existing-test discovery) currently run meaningfully only on Python codebases; on other languages they report as not applicable rather than being silently skipped. +- Auto Patcher's LLM access supports Anthropic and OpenAI only; unlike `--llm-config`, it does not support Google. +- This is a decision aid for a human reviewer, not a replacement for manual security review. + ## Roadmap Things on the list, in no particular order: @@ -219,7 +293,7 @@ Things on the list, in no particular order: - **More provider adapters.** Ollama (local models), vLLM, Cohere, Mistral, Groq, Amazon Bedrock, Azure OpenAI — each is a small Python adapter recipe (plus a few Go wizard/probe touch-points if you want it offered by `openant setup llm`) per the contributor guide. Lower the barrier to local / on-prem inference. - **Subscription-based auth.** ChatGPT / Codex, Claude Pro / Max, and Gemini Advanced subscriptions don't currently grant API quota — users have to maintain a separate API-tier key per provider. OAuth-based adapters that ride the consumer subscription would close that gap. - **Cross-provider tool-call quirks.** All three shipped adapters support tool calling, but the long tail (parallel tool calls, strict-mode schema enforcement, retry semantics on partial JSON) behaves differently per provider. Real-world scans surface these — PRs welcome. -- **More languages.** The supported-languages list above is current coverage. Rust, Java, C#, and Swift come up frequently. +- **More languages.** The supported-languages list above is current coverage. Rust, Java, and C# come up frequently. - **Hosted scan service.** Knostic offers free scans for OSS projects today via the form linked above; a self-serve API for trusted partners is a future possibility. PRs welcome on any of these — open an issue first if the scope is non-trivial so we can align before you build. diff --git a/apps/openant-cli/cmd/analyze.go b/apps/openant-cli/cmd/analyze.go index ccd37bb0..b4fdde96 100644 --- a/apps/openant-cli/cmd/analyze.go +++ b/apps/openant-cli/cmd/analyze.go @@ -154,7 +154,7 @@ func runAnalyze(cmd *cobra.Command, args []string) { analyzeLLMConfig, analyzeWorkers, analyzeCheckpoint, analyzeBackoff, ) - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/cmd/buildoutput.go b/apps/openant-cli/cmd/buildoutput.go index fbb74721..a482c5f7 100644 --- a/apps/openant-cli/cmd/buildoutput.go +++ b/apps/openant-cli/cmd/buildoutput.go @@ -98,7 +98,7 @@ func runBuildOutput(cmd *cobra.Command, args []string) { pyArgs = append(pyArgs, "--processing-level", buildOutputProcessingLevel) } - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/cmd/dynamictest.go b/apps/openant-cli/cmd/dynamictest.go index e89c3560..a8a321e1 100644 --- a/apps/openant-cli/cmd/dynamictest.go +++ b/apps/openant-cli/cmd/dynamictest.go @@ -91,7 +91,7 @@ func runDynamicTest(cmd *cobra.Command, args []string) { pyArgs = append(pyArgs, "--llm-config", dynamicTestLLMConfig) } - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/cmd/enhance.go b/apps/openant-cli/cmd/enhance.go index e48efc6b..51ec23fe 100644 --- a/apps/openant-cli/cmd/enhance.go +++ b/apps/openant-cli/cmd/enhance.go @@ -115,7 +115,7 @@ func runEnhance(cmd *cobra.Command, args []string) { pyArgs = append(pyArgs, "--limit", fmt.Sprintf("%d", enhanceLimit)) } - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/cmd/parse.go b/apps/openant-cli/cmd/parse.go index f348132f..e6e4bd5e 100644 --- a/apps/openant-cli/cmd/parse.go +++ b/apps/openant-cli/cmd/parse.go @@ -16,7 +16,7 @@ var parseCmd = &cobra.Command{ Long: `Parse extracts analyzable code units from a repository. The output is a JSON dataset that can be fed into the analyze command. -Supports Python, JavaScript/TypeScript, Go, C/C++, Ruby, and PHP repositories. +Supports Python, JavaScript/TypeScript, Go, C/C++, Ruby, PHP, Zig, and Swift repositories. If no repository path is given, the active project is used (see: openant init).`, Args: cobra.MaximumNArgs(1), @@ -120,7 +120,7 @@ func runParse(cmd *cobra.Command, args []string) { pyArgs := buildParsePyArgs(repoPath, parseOutput, datasetName, parseLanguage, parseLevel, manifestPath, parseFresh) - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/cmd/patch.go b/apps/openant-cli/cmd/patch.go new file mode 100644 index 00000000..cd7eedc4 --- /dev/null +++ b/apps/openant-cli/cmd/patch.go @@ -0,0 +1,224 @@ +package cmd + +import ( + "fmt" + "os" + "regexp" + + "github.com/knostic/open-ant-cli/internal/output" + "github.com/knostic/open-ant-cli/internal/python" + "github.com/spf13/cobra" +) + +var patchCmd = &cobra.Command{ + Use: "patch [pipeline-output-path]", + Short: "Generate and evaluate a candidate remediation for a finding or a known CVE", + Long: `Patch invokes the merged Auto Patcher engine to generate a candidate +remediation and produce a Trust Report judging whether that candidate should +be trusted. Two entry points converge on the same pipeline and the same +Trust Report format: + + openant patch --finding-id remediate an OpenAnt-detected Finding + openant patch --cve CVE-YYYY-NNNN --repo-root remediate a known CVE advisory + +The Trust Report is treated as an opaque artifact: it is written under the +active scan directory but never parsed, scored, or reinterpreted. A +CVE-sourced report additionally discloses that its input was a public +advisory, not an OpenAnt Finding, and that advisory claims are not +repository-verified facts. + +Requires LLM_PROVIDER to be set explicitly (e.g. LLM_PROVIDER=anthropic) so +a run never silently falls back to a mock LLM. Set LLM_PROVIDER=mock only +if you intentionally want a mock run. + +If no path is given for --finding-id, the active project's +pipeline_output.json is used. --cve requires --repo-root, which defaults to +the active project's repo path when not given explicitly.`, + Args: cobra.MaximumNArgs(1), + Run: runPatch, +} + +var ( + patchFindingID string + patchCVE string + patchRepoRoot string + patchOutput string +) + +func init() { + patchCmd.Flags().StringVar(&patchFindingID, "finding-id", "", "ID of the finding to remediate (mutually exclusive with --cve)") + patchCmd.Flags().StringVar(&patchCVE, "cve", "", "CVE identifier to fetch from NVD and remediate (mutually exclusive with --finding-id)") + patchCmd.Flags().StringVar(&patchRepoRoot, "repo-root", "", "Path to the target repository root (defaults to the active project's repo path; required for --cve)") + patchCmd.Flags().StringVarP(&patchOutput, "output", "o", "", "Output directory (default: active scan directory)") +} + +var cveIDPattern = regexp.MustCompile(`^CVE-\d{4}-\d{4,}$`) + +func buildPatchPyArgs(pipelineOutputPath, findingID, repoRoot, outputDir string) []string { + pyArgs := []string{"patch", pipelineOutputPath, "--finding-id", findingID} + if repoRoot != "" { + pyArgs = append(pyArgs, "--repo-root", repoRoot) + } + if outputDir != "" { + pyArgs = append(pyArgs, "--output", outputDir) + } + return pyArgs +} + +func buildPatchCVEPyArgs(cve, repoRoot, outputDir string) []string { + pyArgs := []string{"patch", "--cve", cve, "--repo-root", repoRoot} + if outputDir != "" { + pyArgs = append(pyArgs, "--output", outputDir) + } + return pyArgs +} + +func runPatch(cmd *cobra.Command, args []string) { + if patchFindingID == "" && patchCVE == "" { + output.PrintError("openant patch requires either --finding-id or --cve ") + os.Exit(2) + } + if patchFindingID != "" && patchCVE != "" { + output.PrintError("--finding-id and --cve are mutually exclusive; pass exactly one") + os.Exit(2) + } + + if patchCVE != "" { + runPatchCVE(args) + return + } + runPatchFinding(args) +} + +func runPatchFinding(args []string) { + pipelineOutputPath, ctx, err := resolveFileArg(args, "pipeline_output.json") + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + if _, err := os.Stat(pipelineOutputPath); err != nil { + output.PrintError("pipeline_output.json not found. Run 'openant build-output' first.") + os.Exit(2) + } + + outputDir := patchOutput + if outputDir == "" && ctx != nil { + outputDir = ctx.ScanDir + } + + repoRoot := patchRepoRoot + if repoRoot == "" && ctx != nil { + repoRoot = ctx.RepoPath + } + + // Resolved before ensurePython() so a missing/declined provider fails + // fast, without paying for a venv/dependency bootstrap first. + llmEnv, err := resolvePatchLLMEnv() + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + rt, err := ensurePython() + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + pyArgs := buildPatchPyArgs(pipelineOutputPath, patchFindingID, repoRoot, outputDir) + + // Auto Patcher's LLM calls are independently configured via LLM_PROVIDER / + // OPENAI_API_KEY / ANTHROPIC_API_KEY -- never OpenAnt's own --api-key or + // llm_providers config, so no legacy API key is forwarded here. llmEnv + // carries only what resolvePatchLLMEnv resolved (explicit env passthrough + // needs nothing added; interactive selection adds LLM_PROVIDER + the + // chosen provider's key) into the Python subprocess's environment only. + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, "", llmEnv) + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + printPatchResultAndExit(result) +} + +func runPatchCVE(args []string) { + if len(args) > 0 { + output.PrintError("openant patch --cve does not take a pipeline-output-path argument") + os.Exit(2) + } + if !cveIDPattern.MatchString(patchCVE) { + output.PrintError(fmt.Sprintf("invalid CVE identifier %q: expected format CVE-YYYY-NNNN", patchCVE)) + os.Exit(2) + } + + // resolveProject() is optional here (unlike Finding mode, where a + // missing active project with no positional arg is a hard error) -- + // it's only ever used as a convenience default for --repo-root/--output, + // never required on its own. + ctx, ctxErr := resolveProject() + + repoRoot := patchRepoRoot + if repoRoot == "" && ctxErr == nil { + repoRoot = ctx.RepoPath + } + if repoRoot == "" { + output.PrintError("--cve requires --repo-root (no active project to default to)") + os.Exit(2) + } + if _, err := os.Stat(repoRoot); err != nil { + output.PrintError(fmt.Sprintf("--repo-root does not exist: %s", repoRoot)) + os.Exit(2) + } + + outputDir := patchOutput + if outputDir == "" && ctxErr == nil { + outputDir = ctx.ScanDir + } + + // Resolved before ensurePython() so a missing/declined provider fails + // fast, without paying for a venv/dependency bootstrap first. + llmEnv, err := resolvePatchLLMEnv() + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + rt, err := ensurePython() + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + pyArgs := buildPatchCVEPyArgs(patchCVE, repoRoot, outputDir) + + // Same deliberate omission as Finding mode: Auto Patcher's LLM calls are + // configured independently via LLM_PROVIDER / OPENAI_API_KEY / + // ANTHROPIC_API_KEY, never OpenAnt's own --api-key. llmEnv carries only + // what resolvePatchLLMEnv resolved into the Python subprocess's + // environment only. + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, "", llmEnv) + if err != nil { + output.PrintError(err.Error()) + os.Exit(2) + } + + printPatchResultAndExit(result) +} + +func printPatchResultAndExit(result *python.InvokeResult) { + if result.Envelope.Status == "interrupted" { + os.Exit(130) + } else if jsonOutput { + output.PrintJSON(result.Envelope) + } else if result.Envelope.Status == "success" { + if data, ok := result.Envelope.Data.(map[string]any); ok { + output.PrintPatchSummary(data) + } + } else { + output.PrintErrors(result.Envelope.Errors) + } + + os.Exit(result.ExitCode) +} diff --git a/apps/openant-cli/cmd/patch_llm.go b/apps/openant-cli/cmd/patch_llm.go new file mode 100644 index 00000000..778c90a3 --- /dev/null +++ b/apps/openant-cli/cmd/patch_llm.go @@ -0,0 +1,181 @@ +package cmd + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/charmbracelet/x/term" + "github.com/knostic/open-ant-cli/internal/config" +) + +// isInteractiveTerminal reports whether stdin is a real interactive +// terminal. A package-level var (mirroring internal/python.defaultInvokeTimeout's +// test-override pattern) so tests can force either branch of +// resolvePatchLLMEnv without needing a real TTY -- setup_test.go's +// withScriptedStdin already replaces os.Stdin with a non-tty os.Pipe() for +// scripted input, so the interactive branch needs its own seam to be +// reachable in tests. +var isInteractiveTerminal = func() bool { + return term.IsTerminal(os.Stdin.Fd()) +} + +// patchLLMOption is one entry in the provider menu `openant patch` offers +// when LLM_PROVIDER isn't already set. Deliberately NOT the same set as +// OpenAnt's own scan-pipeline providers: no Google, because +// utilities.autopatcher.llm_client.py's LLM_CONFIG has no google entry -- +// Auto Patcher's Python engine has no code path that would ever use it. +var patchLLMMenu = []struct { + key string + provider string + envName string + label string +}{ + {key: "1", provider: "anthropic", envName: "ANTHROPIC_API_KEY", label: "Anthropic"}, + {key: "2", provider: "openai", envName: "OPENAI_API_KEY", label: "OpenAI"}, + {key: "3", provider: "mock", envName: "", label: "Mock"}, +} + +// resolvePatchLLMEnv resolves the LLM_PROVIDER (+ matching API key) env vars +// to inject into the Auto Patcher Python subprocess ONLY. The returned map +// is meant for python.Invoke's extraEnv parameter -- callers must never +// os.Setenv() these into this process's own environment. +// +// Precedence: +// 1. LLM_PROVIDER already set in the environment -- honored as-is, no +// prompting, no config lookup. Only validated enough to fail fast when a +// named real provider's matching API key is missing; core/patch.py's +// _require_llm_provider() remains the final Python-side backstop +// regardless of what happens here. +// 2. Interactive terminal, no explicit provider -- offer the menu of +// providers Auto Patcher actually supports, reusing this package's +// existing prompt/secret/config helpers (from cmd/setup.go and +// internal/config). Never silently reuses a stored credential -- +// always asks first. +// 3. Non-interactive, no explicit provider -- fail clearly. Never prompts, +// never falls back to mock. +func resolvePatchLLMEnv() (map[string]string, error) { + if provider := os.Getenv("LLM_PROVIDER"); provider != "" { + return validateExplicitPatchProvider(provider) + } + + if !isInteractiveTerminal() { + return nil, fmt.Errorf( + "LLM_PROVIDER is not set and this is not an interactive terminal.\n" + + "Set LLM_PROVIDER=anthropic|openai|mock (and the matching API key,\n" + + "e.g. ANTHROPIC_API_KEY) before running openant patch non-interactively.", + ) + } + + reader := bufio.NewReader(os.Stdin) + return promptForPatchLLMEnv(reader) +} + +// validateExplicitPatchProvider fails fast when a real (non-mock) provider +// is explicitly named but its matching API key is missing from the +// environment. Returns nil, nil on success: the value is already present in +// os.Environ() and reaches the subprocess via the existing environment +// passthrough in python.Invoke -- nothing needs to be added to the +// subprocess-only extraEnv map for this case. +// +// An unrecognized provider name (e.g. a future provider, or a typo) is +// deliberately left unvalidated here: utilities.autopatcher.llm_client.py +// already has its own, unmodified fallback-to-mock-with-warning behavior for +// that case, and this function must not duplicate or override it. +func validateExplicitPatchProvider(provider string) (map[string]string, error) { + switch provider { + case "mock": + return nil, nil + case "anthropic": + if os.Getenv("ANTHROPIC_API_KEY") == "" { + return nil, fmt.Errorf( + "LLM_PROVIDER=anthropic is set but ANTHROPIC_API_KEY is not.\n" + + "Export it, or set LLM_PROVIDER=mock to use mock mode.", + ) + } + return nil, nil + case "openai": + if os.Getenv("OPENAI_API_KEY") == "" { + return nil, fmt.Errorf( + "LLM_PROVIDER=openai is set but OPENAI_API_KEY is not.\n" + + "Export it, or set LLM_PROVIDER=mock to use mock mode.", + ) + } + return nil, nil + default: + return nil, nil + } +} + +// promptForPatchLLMEnv shows the interactive provider menu. Only offers +// providers Auto Patcher's Python engine actually supports -- never Google. +func promptForPatchLLMEnv(reader *bufio.Reader) (map[string]string, error) { + fmt.Fprintln(os.Stderr, "No LLM provider configured for Auto Patcher.") + for _, opt := range patchLLMMenu { + fmt.Fprintf(os.Stderr, "%s) %s\n", opt.key, opt.label) + } + choice, err := promptString(reader, "Choose (1/2/3)", "") + if err != nil { + return nil, err + } + choice = strings.TrimSpace(choice) + + for _, opt := range patchLLMMenu { + if opt.key != choice { + continue + } + if opt.provider == "mock" { + fmt.Fprintln(os.Stderr, "Using mock LLM for this run.") + return map[string]string{"LLM_PROVIDER": "mock"}, nil + } + return promptOrReusePatchProviderKey(reader, opt.provider, opt.envName, opt.label) + } + return nil, fmt.Errorf("invalid choice %q; expected 1, 2, or 3", choice) +} + +// promptOrReusePatchProviderKey offers reuse of an already-configured +// OpenAnt credential for the chosen provider -- visibly, via an explicit +// yes/no confirmation showing the masked key -- before falling back to a +// fresh no-echo prompt. Never reuses a stored credential silently. +func promptOrReusePatchProviderKey(reader *bufio.Reader, provider, envName, label string) (map[string]string, error) { + cfg, _ := config.Load() + if key, ok := existingPatchCredential(cfg, provider); ok { + reuse, err := promptYesNo( + reader, + fmt.Sprintf("Reuse your configured %s key (%s) from OpenAnt setup for Auto Patcher too?", label, config.MaskKey(key)), + true, + ) + if err != nil { + return nil, err + } + if reuse { + return map[string]string{"LLM_PROVIDER": provider, envName: key}, nil + } + } + + key, err := promptSecret(reader, fmt.Sprintf("Enter %s", envName)) + if err != nil { + return nil, err + } + if key == "" { + return nil, fmt.Errorf("%s is required to use %s", envName, label) + } + return map[string]string{"LLM_PROVIDER": provider, envName: key}, nil +} + +// existingPatchCredential looks up a credential OpenAnt already has +// configured for the given provider -- the v2 llm_providers entry first, +// else the legacy v1 api_key field (which is always an Anthropic key). +func existingPatchCredential(cfg *config.Config, provider string) (string, bool) { + if cfg == nil { + return "", false + } + if entry, ok := cfg.GetProvider(provider); ok && entry.APIKey != "" { + return entry.APIKey, true + } + if provider == "anthropic" && cfg.APIKey != "" { + return cfg.APIKey, true + } + return "", false +} diff --git a/apps/openant-cli/cmd/patch_llm_test.go b/apps/openant-cli/cmd/patch_llm_test.go new file mode 100644 index 00000000..dbab175d --- /dev/null +++ b/apps/openant-cli/cmd/patch_llm_test.go @@ -0,0 +1,319 @@ +package cmd + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// forceInteractive overrides isInteractiveTerminal for the duration of the +// test, so scripted (non-tty) stdin from withScriptedStdin can still drive +// the interactive branch of resolvePatchLLMEnv -- mirrors how +// cmd/setup_test.go's withScriptedStdin already accepts that promptSecret +// falls back to its non-tty read path under the same circumstance. +func forceInteractive(t *testing.T, interactive bool) { + t.Helper() + orig := isInteractiveTerminal + isInteractiveTerminal = func() bool { return interactive } + t.Cleanup(func() { isInteractiveTerminal = orig }) +} + +// silenceStderr redirects os.Stderr to /dev/null for the duration of the +// test -- the resolver's menu/prompt text is expected output, not a signal +// worth asserting on here, so this just keeps `go test -v` output readable. +// Mirrors the inline pattern cmd/setup_test.go's TestSetupLLMWizard_HappyPath +// already uses. +func silenceStderr(t *testing.T) { + t.Helper() + orig := os.Stderr + devnull, err := os.Open(os.DevNull) + if err != nil { + t.Fatalf("open devnull: %v", err) + } + os.Stderr = devnull + t.Cleanup(func() { + os.Stderr = orig + devnull.Close() + }) +} + +func clearPatchLLMEnv(t *testing.T) { + t.Helper() + for _, k := range []string{"LLM_PROVIDER", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"} { + orig, had := os.LookupEnv(k) + os.Unsetenv(k) + t.Cleanup(func() { + if had { + os.Setenv(k, orig) + } else { + os.Unsetenv(k) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Explicit LLM_PROVIDER already set. +// --------------------------------------------------------------------------- + +func TestResolvePatchLLMEnv_ExplicitAnthropicWithKey(t *testing.T) { + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "anthropic") + t.Setenv("ANTHROPIC_API_KEY", "sk-ant-test") + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(env) != 0 { + t.Fatalf("expected no extraEnv additions for an already-explicit provider, got %v", env) + } +} + +func TestResolvePatchLLMEnv_ExplicitOpenAIWithKey(t *testing.T) { + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "openai") + t.Setenv("OPENAI_API_KEY", "sk-openai-test") + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(env) != 0 { + t.Fatalf("expected no extraEnv additions for an already-explicit provider, got %v", env) + } +} + +func TestResolvePatchLLMEnv_ExplicitMock(t *testing.T) { + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "mock") + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(env) != 0 { + t.Fatalf("expected no extraEnv additions for explicit mock, got %v", env) + } +} + +func TestResolvePatchLLMEnv_ExplicitRealProviderMissingKeyFailsClearly(t *testing.T) { + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "anthropic") + // ANTHROPIC_API_KEY intentionally left unset. + + _, err := resolvePatchLLMEnv() + if err == nil { + t.Fatal("expected an error when LLM_PROVIDER=anthropic has no matching API key") + } + if !strings.Contains(err.Error(), "ANTHROPIC_API_KEY") { + t.Errorf("error should name the missing env var, got: %v", err) + } +} + +func TestResolvePatchLLMEnv_ExplicitOpenAIMissingKeyFailsClearly(t *testing.T) { + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "openai") + + _, err := resolvePatchLLMEnv() + if err == nil { + t.Fatal("expected an error when LLM_PROVIDER=openai has no matching API key") + } + if !strings.Contains(err.Error(), "OPENAI_API_KEY") { + t.Errorf("error should name the missing env var, got: %v", err) + } +} + +func TestResolvePatchLLMEnv_UnrecognizedExplicitProviderNotGatekept(t *testing.T) { + // e.g. "google" or a typo -- Go must not reject or rewrite it; Python's + // own llm_client.py already has an unmodified fallback for this case. + clearPatchLLMEnv(t) + t.Setenv("LLM_PROVIDER", "google") + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("Go must not gatekeep an unrecognized explicit provider, got error: %v", err) + } + if len(env) != 0 { + t.Fatalf("expected no extraEnv additions, got %v", env) + } +} + +// --------------------------------------------------------------------------- +// Non-interactive, no explicit provider. +// --------------------------------------------------------------------------- + +func TestResolvePatchLLMEnv_NonInteractiveUnsetProviderFailsClearly(t *testing.T) { + clearPatchLLMEnv(t) + forceInteractive(t, false) + + _, err := resolvePatchLLMEnv() + if err == nil { + t.Fatal("expected an error for non-interactive execution with no LLM_PROVIDER") + } + if !strings.Contains(err.Error(), "LLM_PROVIDER") { + t.Errorf("error should mention LLM_PROVIDER, got: %v", err) + } + if !strings.Contains(strings.ToLower(err.Error()), "interactive") { + t.Errorf("error should explain why (not an interactive terminal), got: %v", err) + } +} + +// --------------------------------------------------------------------------- +// Interactive, no explicit provider. +// --------------------------------------------------------------------------- + +func TestResolvePatchLLMEnv_InteractiveFreshProviderSelection(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + withFakeConfigHome(t) // empty config -- nothing to offer reuse of + forceInteractive(t, true) + withScriptedStdin(t, "1\nsk-fresh-anthropic\n") // choose Anthropic, then enter a fresh key + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env["LLM_PROVIDER"] != "anthropic" { + t.Errorf("LLM_PROVIDER = %q, want anthropic", env["LLM_PROVIDER"]) + } + if env["ANTHROPIC_API_KEY"] != "sk-fresh-anthropic" { + t.Errorf("ANTHROPIC_API_KEY = %q, want sk-fresh-anthropic", env["ANTHROPIC_API_KEY"]) + } +} + +func TestResolvePatchLLMEnv_InteractiveMockSelection(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + forceInteractive(t, true) + withScriptedStdin(t, "3\n") // choose Mock + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env["LLM_PROVIDER"] != "mock" { + t.Errorf("LLM_PROVIDER = %q, want mock", env["LLM_PROVIDER"]) + } + if _, hasKey := env["ANTHROPIC_API_KEY"]; hasKey { + t.Errorf("mock selection should not carry an API key, got %v", env) + } +} + +func TestResolvePatchLLMEnv_InteractiveInvalidChoiceFailsClearly(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + forceInteractive(t, true) + withScriptedStdin(t, "9\n") + + _, err := resolvePatchLLMEnv() + if err == nil { + t.Fatal("expected an error for an invalid menu choice") + } +} + +func TestResolvePatchLLMEnv_ReusesExistingCompatibleStoredCredential(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + configPath := withFakeConfigHome(t) + writeConfigJSON(t, configPath, map[string]any{"api_key": "sk-stored-anthropic"}) + forceInteractive(t, true) + withScriptedStdin(t, "1\ny\n") // choose Anthropic, confirm reuse + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env["ANTHROPIC_API_KEY"] != "sk-stored-anthropic" { + t.Errorf("ANTHROPIC_API_KEY = %q, want the stored credential to be reused", env["ANTHROPIC_API_KEY"]) + } +} + +func TestResolvePatchLLMEnv_DeclinedReuseFallsBackToFreshPrompt(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + configPath := withFakeConfigHome(t) + writeConfigJSON(t, configPath, map[string]any{"api_key": "sk-stored-anthropic"}) + forceInteractive(t, true) + withScriptedStdin(t, "1\nn\nsk-fresh-instead\n") // choose Anthropic, decline reuse, enter a fresh key + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env["ANTHROPIC_API_KEY"] != "sk-fresh-instead" { + t.Errorf("ANTHROPIC_API_KEY = %q, want the freshly-entered key since reuse was declined", env["ANTHROPIC_API_KEY"]) + } +} + +func TestResolvePatchLLMEnv_GoogleOnlyStoredConfigIsNeverOfferedOrReused(t *testing.T) { + // Auto Patcher doesn't support Google at all -- a user whose only + // configured OpenAnt credential is Google must land on a fresh-key + // prompt for Anthropic, never see a reuse offer, and never have the + // Google credential surface anywhere. + silenceStderr(t) + clearPatchLLMEnv(t) + configPath := withFakeConfigHome(t) + writeConfigJSON(t, configPath, map[string]any{ + "$schema_version": 2, + "llm_providers": map[string]any{ + "google": map[string]any{"type": "google", "api_key": "sk-google-only"}, + }, + }) + forceInteractive(t, true) + withScriptedStdin(t, "1\nsk-fresh-anthropic\n") // choose Anthropic; no reuse prompt should appear + + env, err := resolvePatchLLMEnv() + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if env["LLM_PROVIDER"] != "anthropic" { + t.Errorf("LLM_PROVIDER = %q, want anthropic", env["LLM_PROVIDER"]) + } + if env["ANTHROPIC_API_KEY"] != "sk-fresh-anthropic" { + t.Errorf("ANTHROPIC_API_KEY = %q, want the freshly-entered key", env["ANTHROPIC_API_KEY"]) + } + for _, v := range env { + if strings.Contains(v, "google") { + t.Fatalf("the Google-only stored credential leaked into the resolved env: %v", env) + } + } +} + +func TestResolvePatchLLMEnv_NeverMutatesProcessEnvironment(t *testing.T) { + silenceStderr(t) + clearPatchLLMEnv(t) + withFakeConfigHome(t) + forceInteractive(t, true) + withScriptedStdin(t, "1\nsk-fresh-anthropic\n") + + if _, err := resolvePatchLLMEnv(); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if v := os.Getenv("LLM_PROVIDER"); v != "" { + t.Fatalf("resolvePatchLLMEnv must never os.Setenv this process's own LLM_PROVIDER; got %q", v) + } + if v := os.Getenv("ANTHROPIC_API_KEY"); v != "" { + t.Fatalf("resolvePatchLLMEnv must never os.Setenv this process's own ANTHROPIC_API_KEY; got %q", v) + } +} + +// writeConfigJSON writes an arbitrary JSON document to path, creating parent +// directories as needed -- used to seed ~/.config/openant/config.json (as +// redirected by withFakeConfigHome) with a specific stored-credential shape. +func writeConfigJSON(t *testing.T, path string, doc map[string]any) { + t.Helper() + data, err := json.MarshalIndent(doc, "", " ") + if err != nil { + t.Fatalf("marshal config fixture: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatalf("mkdir config dir: %v", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write config fixture: %v", err) + } +} diff --git a/apps/openant-cli/cmd/patch_test.go b/apps/openant-cli/cmd/patch_test.go new file mode 100644 index 00000000..c54e47bf --- /dev/null +++ b/apps/openant-cli/cmd/patch_test.go @@ -0,0 +1,153 @@ +package cmd + +import ( + "testing" + + "github.com/spf13/cobra" +) + +func TestBuildPatchPyArgsBaseline(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "", "") + want := []string{"patch", "/scan/pipeline_output.json", "--finding-id", "F-001"} + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchPyArgsWithRepoRootAndOutput(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "/repo", "/scan") + want := []string{ + "patch", "/scan/pipeline_output.json", "--finding-id", "F-001", + "--repo-root", "/repo", + "--output", "/scan", + } + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchPyArgsOmitsRepoRootAndOutputWhenEmpty(t *testing.T) { + args := buildPatchPyArgs("/scan/pipeline_output.json", "F-001", "", "") + if found, _ := findFlag(args, "--repo-root"); found { + t.Errorf("did not expect --repo-root in pyArgs when unset, got %v", args) + } + if found, _ := findFlag(args, "--output"); found { + t.Errorf("did not expect --output in pyArgs when unset, got %v", args) + } +} + +func TestPatchCmdHasFindingIDFlag(t *testing.T) { + flag := patchCmd.Flags().Lookup("finding-id") + if flag == nil { + t.Fatal("patchCmd is missing the --finding-id flag") + } + if flag.DefValue != "" { + t.Errorf("--finding-id default should be empty, got %q", flag.DefValue) + } +} + +func TestPatchCmdIsRegisteredOnRoot(t *testing.T) { + var found *cobra.Command + for _, c := range rootCmd.Commands() { + if c.Name() == "patch" { + found = c + break + } + } + if found == nil { + t.Fatal("patch command not registered on rootCmd") + } + if found.Flags().Lookup("finding-id") == nil { + t.Error("patch subcommand resolved from root is missing --finding-id flag") + } + if found.Flags().Lookup("cve") == nil { + t.Error("patch subcommand resolved from root is missing --cve flag") + } +} + +// --------------------------------------------------------------------------- +// --cve support: argv construction, format validation, and flag registration. +// --------------------------------------------------------------------------- + +func TestBuildPatchCVEPyArgsBaseline(t *testing.T) { + args := buildPatchCVEPyArgs("CVE-2022-25883", "/repo", "") + want := []string{"patch", "--cve", "CVE-2022-25883", "--repo-root", "/repo"} + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchCVEPyArgsWithOutput(t *testing.T) { + args := buildPatchCVEPyArgs("CVE-2022-25883", "/repo", "/scan") + want := []string{ + "patch", "--cve", "CVE-2022-25883", + "--repo-root", "/repo", + "--output", "/scan", + } + if len(args) != len(want) { + t.Fatalf("argv = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Errorf("argv[%d] = %q, want %q (full=%v)", i, args[i], want[i], args) + } + } +} + +func TestBuildPatchCVEPyArgsOmitsOutputWhenEmpty(t *testing.T) { + args := buildPatchCVEPyArgs("CVE-2022-25883", "/repo", "") + if found, _ := findFlag(args, "--output"); found { + t.Errorf("did not expect --output in pyArgs when unset, got %v", args) + } +} + +func TestCveIDPatternAcceptsValidIDs(t *testing.T) { + valid := []string{"CVE-2022-25883", "CVE-1999-0001", "CVE-2023-123456"} + for _, id := range valid { + if !cveIDPattern.MatchString(id) { + t.Errorf("expected %q to match cveIDPattern", id) + } + } +} + +func TestCveIDPatternRejectsInvalidIDs(t *testing.T) { + invalid := []string{ + "CVE-22-25883", // year not 4 digits + "CVE-2022-123", // sequence too short + "cve-2022-25883", // wrong case + "CVE-2022", // missing sequence + "2022-25883", // missing prefix + "CVE-2022-25883x", // trailing garbage + "", + } + for _, id := range invalid { + if cveIDPattern.MatchString(id) { + t.Errorf("expected %q to NOT match cveIDPattern", id) + } + } +} + +func TestPatchCmdHasCVEFlag(t *testing.T) { + flag := patchCmd.Flags().Lookup("cve") + if flag == nil { + t.Fatal("patchCmd is missing the --cve flag") + } + if flag.DefValue != "" { + t.Errorf("--cve default should be empty, got %q", flag.DefValue) + } +} diff --git a/apps/openant-cli/cmd/report.go b/apps/openant-cli/cmd/report.go index 926199c9..63a5ef15 100644 --- a/apps/openant-cli/cmd/report.go +++ b/apps/openant-cli/cmd/report.go @@ -219,7 +219,7 @@ func runReport(cmd *cobra.Command, args []string) { // Other formats delegate to Python pyArgs := buildReportArgs(resultsPath, fmt) - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey(), nil) if err != nil { output.PrintError(fmt + ": " + err.Error()) exitCode = 2 @@ -311,7 +311,7 @@ func runHTMLReport(rt *python.RuntimeInfo, resultsPath string, outputPath string // 1. Call Python report-data to get pre-computed JSON pyArgs := buildReportDataArgs(resultsPath) - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, resolvedAPIKey(), nil) if err != nil { return fmt.Errorf("report-data failed: %w", err) } diff --git a/apps/openant-cli/cmd/root.go b/apps/openant-cli/cmd/root.go index 015d3099..6a061b75 100644 --- a/apps/openant-cli/cmd/root.go +++ b/apps/openant-cli/cmd/root.go @@ -123,7 +123,7 @@ func requireAPIKey() string { fmt.Fprintln(os.Stderr, "Run: openant set-api-key ") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "Or author an `llm_providers` section in ~/.config/openant/config.json") - fmt.Fprintln(os.Stderr, " (see docs/features/llm-providers/HOW_TO_ADD_AN_ADAPTER.md)") + fmt.Fprintln(os.Stderr, " (see the OpenAnt documentation for adding new provider adapters)") fmt.Fprintln(os.Stderr, "") fmt.Fprintln(os.Stderr, "You can get an Anthropic API key at https://console.anthropic.com/settings/keys") os.Exit(2) @@ -146,6 +146,7 @@ func init() { rootCmd.AddCommand(buildOutputCmd) rootCmd.AddCommand(dynamicTestCmd) rootCmd.AddCommand(reportCmd) + rootCmd.AddCommand(patchCmd) rootCmd.AddCommand(projectCmd) rootCmd.AddCommand(configCmd) rootCmd.AddCommand(setAPIKeyCmd) diff --git a/apps/openant-cli/cmd/scan.go b/apps/openant-cli/cmd/scan.go index 9108e50b..085c846b 100644 --- a/apps/openant-cli/cmd/scan.go +++ b/apps/openant-cli/cmd/scan.go @@ -223,7 +223,7 @@ func runScan(cmd *cobra.Command, args []string) { } } - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey(), nil) if err != nil { finalizeScanMetaIfProject(ctx, config.ScanStatusFailed) output.PrintError(err.Error()) diff --git a/apps/openant-cli/cmd/setup.go b/apps/openant-cli/cmd/setup.go index b806ff40..6028ac63 100644 --- a/apps/openant-cli/cmd/setup.go +++ b/apps/openant-cli/cmd/setup.go @@ -328,7 +328,7 @@ func promptNewProvider(reader *bufio.Reader, name string) (config.ProviderEntry, } if !stringSliceContains(supportedProviderTypes, provType) { fmt.Fprintf(os.Stderr, "Unknown provider type %q. The wizard offers: %v.\n", provType, supportedProviderTypes) - fmt.Fprintln(os.Stderr, "To use a provider not listed here, contribute an adapter — see docs/features/llm-providers/HOW_TO_ADD_AN_ADAPTER.md.") + fmt.Fprintln(os.Stderr, "To use a provider not listed here, contribute an adapter — see the OpenAnt documentation for adding new provider adapters.") continue } // Per-provider subscription-vs-API reminder — the wizard needs diff --git a/apps/openant-cli/cmd/verify.go b/apps/openant-cli/cmd/verify.go index 4524c04c..b32c86b8 100644 --- a/apps/openant-cli/cmd/verify.go +++ b/apps/openant-cli/cmd/verify.go @@ -112,7 +112,7 @@ func runVerify(cmd *cobra.Command, args []string) { pyArgs = append(pyArgs, "--llm-config", verifyLLMConfig) } - result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey()) + result, err := python.Invoke(rt.Path, pyArgs, "", quiet, requireAPIKey(), nil) if err != nil { output.PrintError(err.Error()) os.Exit(2) diff --git a/apps/openant-cli/internal/checkpoint/checkpoint.go b/apps/openant-cli/internal/checkpoint/checkpoint.go index 18ff59b9..c1d3265e 100644 --- a/apps/openant-cli/internal/checkpoint/checkpoint.go +++ b/apps/openant-cli/internal/checkpoint/checkpoint.go @@ -65,7 +65,7 @@ func DetectViaPython(pythonPath, scanDir, stepName string) *Info { } // Call Python for accurate counts - result, err := python.Invoke(pythonPath, []string{"checkpoint-status", dir}, "", true, "") + result, err := python.Invoke(pythonPath, []string{"checkpoint-status", dir}, "", true, "", nil) if err != nil || result.Envelope.Status != "success" { // Python failed — fall back to simple file count return DetectFallback(scanDir, stepName) diff --git a/apps/openant-cli/internal/output/formatter.go b/apps/openant-cli/internal/output/formatter.go index cd1ed81a..b5875e6e 100644 --- a/apps/openant-cli/internal/output/formatter.go +++ b/apps/openant-cli/internal/output/formatter.go @@ -335,6 +335,27 @@ func PrintDynamicTestSummary(data map[string]any) { fmt.Println() } +// PrintPatchSummary outputs a formatted summary of a patch-trust run. +func PrintPatchSummary(data map[string]any) { + PrintHeader("Patch Trust Report") + + // finding_id holds either a Finding id or a CVE id (backward-compatible + // field reuse -- see core/patch.py's PatchStepResult); input_type + // discriminates which, so the label printed here matches what's + // actually in it instead of always saying "Finding". + label := "Finding" + if inputType, ok := data["input_type"].(string); ok && inputType == "cve" { + label = "CVE" + } + if id, ok := data["finding_id"].(string); ok { + PrintKeyValue(label, id) + } + if path, ok := data["trust_report_path"].(string); ok { + PrintKeyValue("Report", path) + } + fmt.Println() +} + // PrintBuildOutputSummary outputs a formatted summary of pipeline output generation. func PrintBuildOutputSummary(data map[string]any) { PrintHeader("Pipeline Output") diff --git a/apps/openant-cli/internal/python/invoke.go b/apps/openant-cli/internal/python/invoke.go index b84fcfa7..17279998 100644 --- a/apps/openant-cli/internal/python/invoke.go +++ b/apps/openant-cli/internal/python/invoke.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "os/signal" + "sort" "strings" "sync/atomic" "syscall" @@ -34,11 +35,17 @@ type InvokeResult struct { // Invoke runs `python -m openant ` and returns the parsed JSON result. // -// - stderr is streamed to the terminal in real-time (progress messages) -// - stdout is captured and parsed as JSON -// - Working directory is set to the openant-core lib directory if provided -// - If apiKey is non-empty, it is injected as ANTHROPIC_API_KEY in the subprocess -func Invoke(pythonPath string, args []string, workDir string, quiet bool, apiKey string) (*InvokeResult, error) { +// - stderr is streamed to the terminal in real-time (progress messages) +// - stdout is captured and parsed as JSON +// - Working directory is set to the openant-core lib directory if provided +// - If apiKey is non-empty, it is injected as ANTHROPIC_API_KEY in the subprocess +// - extraEnv overrides/adds arbitrary env vars in the subprocess ONLY -- it is +// merged into a copy of this process's environment and never mutates the +// calling process's own os.Environ() (no os.Setenv is ever called here) +// - stdin is connected to this process's stdin, so a subprocess that needs to +// read interactive input (e.g. Auto Patcher's own legacy provider prompt, +// when invoked directly and not resolved by the Go-side caller first) can +func Invoke(pythonPath string, args []string, workDir string, quiet bool, apiKey string, extraEnv map[string]string) (*InvokeResult, error) { // -P keeps the process working directory off sys.path. `-m openant` otherwise // prepends the CWD, and this engine inherits the user's shell CWD — which in the // standard `git clone X && cd X && openant ...` flow is inside the scanned, @@ -68,13 +75,25 @@ func Invoke(pythonPath string, args []string, workDir string, quiet bool, apiKey cmd.Dir = workDir } - // Pass through environment (Python needs ANTHROPIC_API_KEY, etc.) - // If an API key is provided via flag or config, inject it into the - // subprocess environment so Python picks it up regardless of .env files. - cmd.Env = os.Environ() + // Pass through environment (Python needs ANTHROPIC_API_KEY, etc.), then + // overlay any explicit overrides on top of a COPY of it -- mergeEnv never + // touches os.Environ() itself, so other subcommands and this process's + // own environment are never mutated by a single Invoke call. + overrides := map[string]string{} if apiKey != "" { - cmd.Env = setEnv(cmd.Env, "ANTHROPIC_API_KEY", apiKey) + overrides["ANTHROPIC_API_KEY"] = apiKey } + for k, v := range extraEnv { + overrides[k] = v + } + cmd.Env = mergeEnv(os.Environ(), overrides) + + // Connect stdin so a subprocess that needs to read interactive input can + // do so when this process itself has a real terminal attached. Callers + // that resolve everything up front (e.g. cmd/patch_llm.go) never need + // this, but it must be wired for direct/manual invocations and general + // subprocess correctness. + cmd.Stdin = os.Stdin // Capture stdout (JSON output) stdout, err := cmd.StdoutPipe() @@ -254,16 +273,36 @@ func streamStderr(r io.Reader, quiet bool) { } } -// setEnv sets or replaces an environment variable in a []string env slice. -func setEnv(env []string, key, value string) []string { - prefix := key + "=" - for i, e := range env { - if strings.HasPrefix(e, prefix) { - env[i] = prefix + value - return env +// mergeEnv returns a NEW env slice (os.Environ() format, "KEY=VALUE" pairs) +// combining base with overrides layered on top -- a key present in +// overrides always wins over the same key in base, deterministically. +// Never mutates base or any input; the caller's own environment (and this +// process's os.Environ()) is left untouched, only the returned slice is +// meant for cmd.Env. +func mergeEnv(base []string, overrides map[string]string) []string { + if len(overrides) == 0 { + return base + } + out := make([]string, 0, len(base)+len(overrides)) + for _, kv := range base { + key := kv + if idx := strings.IndexByte(kv, '='); idx >= 0 { + key = kv[:idx] } + if _, replaced := overrides[key]; replaced { + continue + } + out = append(out, kv) + } + keys := make([]string, 0, len(overrides)) + for k := range overrides { + keys = append(keys, k) + } + sort.Strings(keys) // deterministic order, independent of map iteration + for _, k := range keys { + out = append(out, k+"="+overrides[k]) } - return append(env, prefix+value) + return out } // truncate shortens a string to maxLen characters. diff --git a/apps/openant-cli/internal/python/invoke_env_test.go b/apps/openant-cli/internal/python/invoke_env_test.go new file mode 100644 index 00000000..4697cdd1 --- /dev/null +++ b/apps/openant-cli/internal/python/invoke_env_test.go @@ -0,0 +1,235 @@ +package python + +import ( + "os" + "path/filepath" + "reflect" + "runtime" + "strings" + "testing" +) + +// --------------------------------------------------------------------------- +// mergeEnv: pure helper, no subprocess involved. +// --------------------------------------------------------------------------- + +func TestMergeEnv_OverrideWinsOverDuplicate(t *testing.T) { + base := []string{"FOO=old", "BAR=unrelated"} + got := mergeEnv(base, map[string]string{"FOO": "new"}) + + want := map[string]string{"FOO": "new", "BAR": "unrelated"} + assertEnvEquals(t, got, want) +} + +func TestMergeEnv_UnrelatedValuesPreserved(t *testing.T) { + base := []string{"A=1", "B=2", "C=3"} + got := mergeEnv(base, map[string]string{"D": "4"}) + + want := map[string]string{"A": "1", "B": "2", "C": "3", "D": "4"} + assertEnvEquals(t, got, want) +} + +func TestMergeEnv_EmptyOverridesReturnsBaseUnchanged(t *testing.T) { + base := []string{"A=1"} + got := mergeEnv(base, nil) + if !reflect.DeepEqual(got, base) { + t.Fatalf("mergeEnv with no overrides = %v, want unchanged %v", got, base) + } +} + +func TestMergeEnv_DoesNotMutateBaseSlice(t *testing.T) { + base := []string{"FOO=old"} + baseCopy := append([]string(nil), base...) + _ = mergeEnv(base, map[string]string{"FOO": "new"}) + if !reflect.DeepEqual(base, baseCopy) { + t.Fatalf("mergeEnv mutated its base input: got %v, want unchanged %v", base, baseCopy) + } +} + +func TestMergeEnv_DeterministicAcrossCalls(t *testing.T) { + base := []string{"A=1"} + overrides := map[string]string{"Z": "1", "M": "2", "A": "3"} + first := mergeEnv(base, overrides) + second := mergeEnv(base, overrides) + if !reflect.DeepEqual(first, second) { + t.Fatalf("mergeEnv is non-deterministic across identical calls:\n%v\n%v", first, second) + } +} + +func assertEnvEquals(t *testing.T, env []string, want map[string]string) { + t.Helper() + got := map[string]string{} + for _, kv := range env { + idx := strings.IndexByte(kv, '=') + if idx < 0 { + t.Fatalf("malformed env entry %q", kv) + } + got[kv[:idx]] = kv[idx+1:] + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("env = %v, want %v", got, want) + } +} + +// --------------------------------------------------------------------------- +// Invoke: extraEnv reaches the subprocess, overrides win, unrelated env is +// preserved, no global mutation, stdin is connected, secrets never leak into +// returned error text. +// --------------------------------------------------------------------------- + +func writeEnvEchoScript(t *testing.T) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("env-echo test uses a POSIX shell script") + } + dir := t.TempDir() + path := filepath.Join(dir, "echo_env.sh") + // Reads one optional stdin line and echoes both env vars and the stdin + // line back in a success envelope, so a single subprocess run can assert + // on env propagation and stdin connectivity together. + script := "#!/bin/sh\n" + + "read line\n" + + "printf '{\"status\":\"success\",\"data\":{\"my_test_var\":\"%s\",\"my_other_var\":\"%s\",\"stdin_line\":\"%s\"},\"errors\":[]}\\n' " + + "\"$MY_TEST_VAR\" \"$MY_OTHER_VAR\" \"$line\"\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("failed to write script: %v", err) + } + return path +} + +func dataString(t *testing.T, res *InvokeResult, key string) string { + t.Helper() + m, ok := res.Envelope.Data.(map[string]any) + if !ok { + t.Fatalf("envelope data is not a map: %#v", res.Envelope.Data) + } + v, ok := m[key].(string) + if !ok { + t.Fatalf("envelope data[%q] missing or not a string: %#v", key, m) + } + return v +} + +func TestInvoke_ExtraEnvReachesSubprocess(t *testing.T) { + script := writeEnvEchoScript(t) + res, err := Invoke(script, nil, "", true, "", map[string]string{"MY_TEST_VAR": "hello"}) + if err != nil { + t.Fatalf("Invoke returned error: %v", err) + } + if got := dataString(t, res, "my_test_var"); got != "hello" { + t.Fatalf("subprocess saw MY_TEST_VAR=%q, want %q", got, "hello") + } +} + +func TestInvoke_ExtraEnvOverridesExistingProcessValue(t *testing.T) { + t.Setenv("MY_TEST_VAR", "original") + script := writeEnvEchoScript(t) + + res, err := Invoke(script, nil, "", true, "", map[string]string{"MY_TEST_VAR": "override"}) + if err != nil { + t.Fatalf("Invoke returned error: %v", err) + } + if got := dataString(t, res, "my_test_var"); got != "override" { + t.Fatalf("subprocess saw MY_TEST_VAR=%q, want %q (override should win)", got, "override") + } + + // The calling (test) process's own environment must be untouched. + if got := os.Getenv("MY_TEST_VAR"); got != "original" { + t.Fatalf("Invoke mutated this process's own MY_TEST_VAR to %q; must remain %q", got, "original") + } +} + +func TestInvoke_UnrelatedEnvValuesArePreserved(t *testing.T) { + t.Setenv("MY_OTHER_VAR", "unrelated") + script := writeEnvEchoScript(t) + + res, err := Invoke(script, nil, "", true, "", map[string]string{"MY_TEST_VAR": "hello"}) + if err != nil { + t.Fatalf("Invoke returned error: %v", err) + } + if got := dataString(t, res, "my_other_var"); got != "unrelated" { + t.Fatalf("subprocess saw MY_OTHER_VAR=%q, want unrelated value preserved", got) + } +} + +func TestInvoke_NoGlobalEnvironmentMutationFromExtraEnv(t *testing.T) { + os.Unsetenv("MY_TEST_VAR_NEVER_SET") + script := writeEnvEchoScript(t) + + _, err := Invoke(script, nil, "", true, "", map[string]string{"MY_TEST_VAR_NEVER_SET": "subprocess-only"}) + if err != nil { + t.Fatalf("Invoke returned error: %v", err) + } + if _, ok := os.LookupEnv("MY_TEST_VAR_NEVER_SET"); ok { + t.Fatalf("Invoke leaked an extraEnv-only variable into this process's own environment via os.Setenv") + } + for _, kv := range os.Environ() { + if strings.HasPrefix(kv, "MY_TEST_VAR_NEVER_SET=") { + t.Fatalf("this process's os.Environ() unexpectedly contains MY_TEST_VAR_NEVER_SET") + } + } +} + +func TestInvoke_StdinIsConnected(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stdin-pipe test uses os.Pipe") + } + script := writeEnvEchoScript(t) + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + if _, err := w.WriteString("hello-from-test-stdin\n"); err != nil { + t.Fatalf("write to pipe: %v", err) + } + w.Close() + origStdin := os.Stdin + os.Stdin = r + t.Cleanup(func() { + os.Stdin = origStdin + r.Close() + }) + + res, err := Invoke(script, nil, "", true, "", nil) + if err != nil { + t.Fatalf("Invoke returned error: %v", err) + } + if got := dataString(t, res, "stdin_line"); got != "hello-from-test-stdin" { + t.Fatalf("subprocess read stdin line %q, want %q -- Invoke's cmd.Stdin is not connected", got, "hello-from-test-stdin") + } +} + +func writeFailingScript(t *testing.T) string { + t.Helper() + if runtime.GOOS == "windows" { + t.Skip("failing-script test uses a POSIX shell script") + } + dir := t.TempDir() + path := filepath.Join(dir, "fail.sh") + // Exits 0 with no stdout at all -- Invoke's own empty-stdout handling + // (see TestInvoke_EmptyStdoutSurfacesErrorCode) turns this into an error + // envelope without this script ever needing to reference its env itself. + if err := os.WriteFile(path, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("failed to write script: %v", err) + } + return path +} + +func TestInvoke_SecretExtraEnvValueNeverAppearsInErrorText(t *testing.T) { + script := writeFailingScript(t) + const secret = "sk-super-secret-test-value" + + res, err := Invoke(script, nil, "", true, "", map[string]string{"ANTHROPIC_API_KEY": secret}) + if err != nil { + if strings.Contains(err.Error(), secret) { + t.Fatalf("Invoke's returned error contains the secret extraEnv value: %v", err) + } + return + } + for _, e := range res.Envelope.Errors { + if strings.Contains(e, secret) { + t.Fatalf("Invoke's error envelope contains the secret extraEnv value: %v", e) + } + } +} diff --git a/apps/openant-cli/internal/python/invoke_race_test.go b/apps/openant-cli/internal/python/invoke_race_test.go index 7e271d46..13ad2894 100644 --- a/apps/openant-cli/internal/python/invoke_race_test.go +++ b/apps/openant-cli/internal/python/invoke_race_test.go @@ -37,7 +37,7 @@ func TestInvoke_InterruptedFlagHasNoRace(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - _, _ = Invoke(hang, []string{"parse", "."}, "", true, "") + _, _ = Invoke(hang, []string{"parse", "."}, "", true, "", nil) }() // Let Invoke start the subprocess and install its signal.Notify handler diff --git a/apps/openant-cli/internal/python/invoke_test.go b/apps/openant-cli/internal/python/invoke_test.go index 6a86d57a..01a47f61 100644 --- a/apps/openant-cli/internal/python/invoke_test.go +++ b/apps/openant-cli/internal/python/invoke_test.go @@ -79,7 +79,7 @@ func TestInvoke_HangingSubprocessIsBoundedByTimeout(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - _, _ = Invoke(hang, []string{"parse", "."}, "", true, "") + _, _ = Invoke(hang, []string{"parse", "."}, "", true, "", nil) }() select { @@ -142,7 +142,7 @@ func writeEmptyStdoutScript(t *testing.T) string { // empty-stdout return path. A revert to `ExitCode: exitCode` would yield 0 here. func TestInvoke_EmptyStdoutSurfacesErrorCode(t *testing.T) { script := writeEmptyStdoutScript(t) - res, err := Invoke(script, []string{"parse", "."}, "", true, "") + res, err := Invoke(script, []string{"parse", "."}, "", true, "", nil) if err != nil { t.Fatalf("Invoke returned error: %v", err) } @@ -210,7 +210,7 @@ func TestInvoke_LateInterruptDoesNotDiscardEnvelope(t *testing.T) { } ch := make(chan outcome, 1) go func() { - r, e := Invoke(script, []string{"scan", "."}, "", true, "") + r, e := Invoke(script, []string{"scan", "."}, "", true, "", nil) ch <- outcome{r, e} }() diff --git a/docs/auto-patcher/recommendation-policy.md b/docs/auto-patcher/recommendation-policy.md new file mode 100644 index 00000000..92ab7aec --- /dev/null +++ b/docs/auto-patcher/recommendation-policy.md @@ -0,0 +1,252 @@ +# The Trust Report and Recommendation Policy + +This document explains how Auto Patcher's Trust Report is produced, and — more +importantly — how its final recommendation is decided. It is written for +security engineers who need to know exactly what a recommendation is and is +not based on before acting on it. + +For what Auto Patcher is and how to run it, see the +[Auto Patcher section of the README](../../README.md#auto-patcher). This document +goes one level deeper: it describes the evidence-to-decision machinery behind +the report that command produces. + +Everything below is grounded in the current implementation of +`utilities/autopatcher/pipeline.py` in `libs/openant-core`. Function names are +cited so this document can be re-verified against source at any time; treat +any mismatch you find as this document being stale, not the code. + +## Purpose + +A Trust Report does not tell you a patch is correct. It tells you what was +checked, what that checking found, and — given exactly that evidence and +nothing else — what a fixed, auditable policy recommends. The recommendation +is a starting point for review, not a substitute for it. Nothing in this +system applies a patch to a repository; every run writes its output to disk +for a human to read. + +## Philosophy + +The recommendation policy (`_compute_trust_signals`, `_build_recommendation_v1` +in `pipeline.py`) is built around a small set of invariants, documented +in-code directly above `_compute_trust_signals`. Restated here: + +- **No positive inference from missing evidence.** If a check didn't run — + timed out, was skipped, raised an exception — that reads as "not verified" + or "unknown," never as a passing result. A check that never ran must never + look identical to a check that ran and passed. +- **Whitelists, not blacklists.** Every gate in the recommendation policy is + phrased as "value is in this specific set of known-good values," never as + "value is not the one known-bad value." A blacklist silently admits any + future or unrecognized value as if it were good; a whitelist doesn't. +- **Heuristic evidence is never treated as proof.** Output from the + adversarial LLM challenger is classified and counted, but it never on its + own reaches the strongest recommendation or the strongest rejection — + see [Recommendation Policy](#recommendation-policy). +- **The report never communicates more certainty than the evidence supports.** + This governs both the policy's gates and the report's own wording — e.g. a + "Minor Issues" patch-integrity result is deliberately excluded from the + "positive" whitelist even though it doesn't hard-block, because it is real + observed evidence of a defect, not an absence of evidence. + +## The trust pipeline + +Auto Patcher's evidence-to-decision flow has four stages: + +``` +Evidence + │ (deterministic checks + an adversarial LLM challenge, + │ the challenge classified by a deterministic rule) + ▼ +Trust Signals + │ (six named values, computed by fixed rules from the evidence above) + ▼ +Recommendation Policy + │ (a fixed decision tree over four of those signals) + ▼ +Recommendation + (one of four labels, with a fixed reason string and, for the + top two labels, an evidence-check caveat where warranted) +``` + +Each stage is described in its own section below. The next section describes +what feeds Trust Signals; note that a stage further down the pipeline can only +see what a prior stage already computed — nothing is recomputed or +re-inferred at the Recommendation stage. + +## Evidence + +Two categories of evidence feed the Trust Signals. It matters which is which, +because the policy treats them differently (see Philosophy, above). + +### Deterministic evidence + +Produced by code with no LLM in the loop: + +| Source | What it checks | Module | +|---|---|---| +| Patch Hygiene | Diff-shape defects: empty hunks, duplicate constants, unused imports | `patch_hygiene.check_patch` | +| Patch Applicability | Whether the diff applies to the target repository (`git apply --check`, read-only) | `patch_applicability.check_applicability` | +| Test Support | Whether existing repository tests already cover the changed file/module | `testing_support.discover_tests` / `tests_for_file` / `score_test_support` | +| Impact Surface | AST-based usage analysis of changed symbols — **Python-only**; reports "not applicable" for other languages | `impact_surface.LightweightImpactAnalyzer` | + +### Heuristic evidence: the adversarial Challenger + +One LLM output feeds the Trust Signals: the **Challenger** +(`patch_challenger.challenge_patch`). It is a separate reasoning pass whose +only instruction is to argue that the patch does *not* hold — not to confirm +that it does. It returns: + +- `still_vulnerable` — a boolean the LLM asserts directly, parsed from its + response. This is a raw model judgment, not a derived value. +- `edge_cases` / `potential_issues` — free-text findings. + +Those free-text findings are then run through a **deterministic** classifier, +`_classify_finding` (pattern-matched against explicit-exploit phrasing, +version/scope qualifiers, validation-gap language, and generic-observation +language), sorting each into one of four categories: + +- `confirmed_defect` — an unambiguous claim the fix doesn't hold, with no + scope/version qualifier attached. +- `plausible_risk` — a claim that reads as a scope or version limitation + rather than a primary fix failure. +- `validation_gap` — the challenger states something wasn't tested/verified, + not that it's broken. +- `generic` — a stylistic or non-security observation. + +`_classify_challenger` then aggregates these into counts +(`confirmed_defect_count`, `plausible_risk_count`, `validation_gap_count`). +**These counts, plus the raw `still_vulnerable` boolean, are the only pieces +of Challenger output the Trust Signals or the Recommendation Policy read.** +The challenger's prose itself is presentational (rendered in Review Results), +not a policy input. + +## Trust Signals + +`_compute_trust_signals` computes six named signals from the evidence above. +Five are rendered as their own row in the report's Trust Signals table; one +is computed but not separately displayed (its rationale, from the code's own +history, is that an earlier report design showed it and found it a +"peer-displayed duplicate" of two other rows — it was dropped from *display*, +not from *computation*, because the policy still depends on it). + +| Signal | Possible values | Computed from | Shown as its own row? | +|---|---|---|---| +| `patch_integrity` | Clean · Minor Issues · Not Verified · Does Not Apply · Critical Issues | Hygiene findings + Applicability result | Yes — "Does the patch apply?" | +| `security_improvement` | None · Unknown · Low · Medium · High | Applicability + Hygiene + classified Challenger counts | **No** | +| `remediation_alignment` | Aligned · Likely Aligned · Partial · Misaligned | Classified Challenger counts + `still_vulnerable` | Yes — "Does it address the vulnerability?" | +| `coverage_confidence` | High · Medium · Low | Classified Challenger counts | Yes — "Are there unresolved concerns?" | +| `test_availability` | Tests Available · No Tests Found · Not Verified | Test Support rating | Yes — "Do relevant tests already exist?" | +| `deployment_safety` | Low Risk · Medium Risk · High Risk · Not Verified | Impact Surface result | Yes — "Is deployment risk low?" | + +Each signal also carries a short human-readable `notes` string explaining the +specific evidence behind its value (e.g. which hygiene check fired, how many +review findings remain open). + +## Recommendation Policy + +`_build_recommendation_v1` turns evidence into exactly one of four labels. +There is no fifth value and no numeric score anywhere in this function. + +**The four recommendations:** + +| Recommendation | Meaning | +|---|---| +| 🟢 **Deploy After Validation** | All mandatory gates passed; run the listed validation actions, then deploy. | +| 🟡 **Deploy With Caution** | Limited or uncertain security improvement, but no blocking evidence. | +| 🟠 **Manual Review Required** | Evidence is inconclusive, heuristic-only, or partially contradictory. | +| 🔴 **Do Not Apply** | A deterministic check failed: the patch has critical hygiene issues or does not apply to the repository. | + +**Decision order** (each check is evaluated in sequence; the first match +wins): + +1. `patch_integrity` is a hard blocker (Critical Issues / Does Not Apply) → + **Do Not Apply**. This is the only path to this label, and it is reached + only through deterministic evidence — heuristic Challenger findings alone + can never produce it. +2. `remediation_alignment` is `Misaligned` (i.e. `confirmed_defect_count > 0`) + → **Manual Review Required**. +3. `still_vulnerable` is true but `confirmed_defect_count == 0` (an unresolved + heuristic claim with no confirmed defect behind it) → **Manual Review + Required**. +4. Only if `patch_integrity == Clean` **and** `security_improvement` is + `High`/`Medium` **and** `deployment_safety` is `Low Risk`/`Medium Risk` (an + explicit three-way whitelist, not "didn't hit a worse case") → + **Deploy After Validation**. +5. `security_improvement == Low` and `deployment_safety == Low Risk` → + **Deploy With Caution**. +6. `deployment_safety == High Risk` → **Manual Review Required**. +7. Anything else — including `Unknown`/`Not Verified` on any axis — → + **Manual Review Required** (the catch-all; nothing falls through to a + stronger label by default). + +A secondary, non-decision-changing step, `_check_recommendation_consistency`, +runs only when the decision is Deploy After Validation or Deploy With +Caution. It checks `test_availability` and the count of open Review Results +findings, and — if either is unfavorable — appends an "Evidence check" +caveat sentence to the report. It never changes which of the four labels is +shown; it only makes sure a confident-sounding label doesn't sit next to +undisclosed weak evidence. + +> Note: an older function, `build_recommendation`, also exists in +> `pipeline.py` with a different, three-label vocabulary (Safe to deploy / +> Deploy with caution / Do not deploy yet) that reads a numeric confidence +> score. It is exercised only by its own unit test and is not called by the +> report-building path (`_build_report` calls `_build_recommendation_v1` +> exclusively). It should not be treated as describing current behavior. + +## What does NOT affect the recommendation + +The report contains more evidence than the recommendation policy uses. This +section exists so that evidence you see in a Trust Report is not mistaken for +evidence that shaped its recommendation. + +- **Confidence score.** The confidence-scorer stage still runs, and its + output is still deterministically discounted (0.4× if the Challenger found + the patch still vulnerable, 0.7× if it found edge cases/issues, otherwise + unchanged). But this number is never read by `_compute_trust_signals` or + `_build_recommendation_v1`, and it is not rendered anywhere in the Trust + Report. It is computed and then discarded. +- **Finding calibration.** The calibration pass rewords and regroups + `plausible_risk`/`generic` Challenger findings for presentation in Review + Results (e.g. splitting them into "Confirmed Observations" vs. "Future + Improvements"). By its own design, it changes wording and grouping only — + it does not change the confirmed/plausible/gap/generic classification or + counts that the Trust Signals and Recommendation Policy read. +- **Deterministic static signals** (constraint/remediation-signal scripts, + when available for the target repository). Rendered as their own + "Deterministic Signals" table in the report's Appendices. Not read by + `_compute_trust_signals` or the recommendation policy. +- **Behavior Summary.** A diff-only, language-agnostic summary of what the + patch appears to do. Feeds the report's Validation Actions suggestions, not + the Trust Signals. +- **Repository Context (grounding).** Explains which repository locations + were used to inform patch generation and review. Purely explanatory; not an + input to any signal. +- **`coverage_confidence`.** Computed and rendered as its own row, but not + read by `_build_recommendation_v1` at all — it is derived from the same + Challenger counts that `remediation_alignment` already uses, presented as a + separate lens ("how much did we look"), not consulted as a separate gate. +- **`test_availability`.** Rendered as its own row and does feed the + secondary consistency-caveat check, but is not one of the four signals the + primary decision (`_build_recommendation_v1`) gates on. + +## Current limitations + +- `security_improvement` is a required input to the strongest recommendation + (Deploy After Validation) but is not shown as its own row in the Trust + Signals table — a reader relying on the visible table alone cannot see one + of the gates behind that label without this document. +- Impact Surface and Test Support are the two deterministic signals most + central to `deployment_safety` and `test_availability`; both currently run + meaningfully only on Python codebases. On other languages they resolve to + "not applicable," which the policy treats as "not verified," never as a + clean result — but it does mean fewer of the six signals carry real signal + on non-Python repositories today. +- The confidence-scorer stage consumes an LLM call and produces output that, + per the above, is discarded before reaching the report or the policy. This + is current behavior, not a documentation gap — flagged here because it is + easy to assume otherwise from the pipeline's stage log output. +- This document describes the recommendation policy as implemented in + `_build_recommendation_v1` today. The file also contains an unused, + differently-worded legacy function (`build_recommendation`); if it is ever + wired back in, this document must be updated accordingly. diff --git a/libs/openant-core/core/patch.py b/libs/openant-core/core/patch.py new file mode 100644 index 00000000..1ed431f0 --- /dev/null +++ b/libs/openant-core/core/patch.py @@ -0,0 +1,397 @@ +""" +Patch-trust wrapper. + +Loads a Finding from pipeline_output.json, checks it is eligible for +remediation, renders it into a vulnerability description, and runs it +through the merged Auto Patcher engine (``utilities.autopatcher``) to +produce a Trust Report. Mirrors core/dynamic_tester.py's shape: a thin +wrapper around a heavier ``utilities.*`` engine. + +Also supports patching directly from a known CVE identifier +(``run_patch_cve``), which shares ``run_patch``'s artifact-writing tail +(``_run_engine_and_write_artifacts``) rather than duplicating it -- both +entry points converge on the same ``utilities.autopatcher.pipeline.run()`` +call. + +The Trust Report is treated as an opaque artifact: this module never parses +its Recommendation or Trust Signals, only the path it was written to. +""" + +import os +from dataclasses import dataclass, asdict +from datetime import datetime, timezone +from pathlib import Path + +from core.reporter import _coerce_to_str +from core.verdict_taxonomy import PATCH_ELIGIBLE +from utilities.file_io import read_json, normalize_results + + +@dataclass +class PatchStepResult: + """Result of `openant patch`. + + input_type/input_id are additive fields distinguishing what finding_id + actually holds. finding_id itself is kept as-is (not renamed) for + backward compatibility: for a CVE-mode run it holds the CVE id, same as + it always has; input_type/input_id make that explicit rather than + leaving it implicit in a field name that predates CVE mode. + """ + finding_id: str + vulnerability_path: str + trust_report_path: str + input_type: str = "finding" # "finding" | "cve" + input_id: str | None = None # mirrors finding_id's value, named accurately regardless of mode + + def to_dict(self) -> dict: + return asdict(self) + + +def find_finding_by_id(findings: list, finding_id: str) -> dict: + """Return the finding dict with the given id, or raise ValueError.""" + for f in findings: + if isinstance(f, dict) and f.get("id") == finding_id: + return f + raise ValueError(f"no finding with id {finding_id!r} in pipeline_output.json") + + +def effective_verdict(finding: dict) -> str: + """The verdict that governs eligibility: stage2_verdict if set, else stage1_verdict.""" + return finding.get("stage2_verdict") or finding.get("stage1_verdict") or "" + + +def check_eligible(finding: dict) -> None: + """Raise ValueError if finding's effective verdict is not patch-eligible. + + Deliberately an explicit allowlist (PATCH_ELIGIBLE), not a denylist, so + an empty, unknown, or future verdict value fails closed. + """ + verdict = effective_verdict(finding) + if verdict not in PATCH_ELIGIBLE: + raise ValueError( + f"finding {finding.get('id')} has verdict {verdict!r}, which is not " + f"eligible for remediation (eligible: {', '.join(sorted(PATCH_ELIGIBLE))})" + ) + + +def render_vulnerability_markdown(finding: dict) -> str: + """Render a Finding into deterministic Markdown for the patch engine's input. + + suggested_fix and rejection_reason are never read here: feeding OpenAnt's + own suggested fix into the patch engine could bias its independently + generated candidate patch. + """ + location = finding.get("location") or {} + lines = [ + f"# {finding.get('name', '')}", + "", + "## Vulnerability description", + "", + f"- **Finding ID:** {finding.get('id', '')}", + f"- **CWE:** CWE-{finding.get('cwe_id', '')} ({finding.get('cwe_name', '')})", + f"- **Location:** {location.get('file', '')} ({location.get('function', '')})", + f"- **Verdict:** {effective_verdict(finding)}", + ] + + description = finding.get("description") + if description: + lines += ["", _coerce_to_str(description)] + + vulnerable_code = finding.get("vulnerable_code") + if vulnerable_code: + lines += ["", "## Vulnerable code", "", "```", _coerce_to_str(vulnerable_code), "```"] + + # impact/steps_to_reproduce are documented as lists but some models emit a + # single string; iterating a string directly yields one bullet per + # character, so a lone string is normalized to a single-item list first. + impact = finding.get("impact") or [] + if isinstance(impact, str): + impact = [impact] + if impact: + lines += ["", "## Impact", ""] + lines += [f"- {_coerce_to_str(item)}" for item in impact] + + steps = finding.get("steps_to_reproduce") or [] + if isinstance(steps, str): + steps = [steps] + if steps: + lines += ["", "## Attack scenario", ""] + lines += [f"{i + 1}. {_coerce_to_str(step)}" for i, step in enumerate(steps)] + + return "\n".join(lines) + "\n" + + +def _find_openant_root() -> Path | None: + """Walk up from this file to the OpenAnt repo root (contains libs/openant-core), + for the Run Metadata report's commit row. Returns None if not found (e.g. + installed as a wheel outside a git checkout) -- collect_git_info already + degrades to 'unknown' in that case.""" + here = Path(__file__).resolve() + for parent in here.parents: + if (parent / "libs" / "openant-core").is_dir(): + return parent + return None + + +def _require_llm_provider() -> None: + if not os.environ.get("LLM_PROVIDER"): + raise RuntimeError( + "LLM_PROVIDER is not set. The patch engine requires an explicit " + "provider (e.g. LLM_PROVIDER=anthropic) so a run triggered by " + "OpenAnt never silently falls back to mock mode. Set " + "LLM_PROVIDER=mock only if you intentionally want a mock run." + ) + + +def _run_engine_and_write_artifacts( + vulnerability_text: str, + repo_root: str | None, + output_dir: str, + artifact_label: str, + input_type: str = "finding", + advisory_id: str | None = None, + advisory_source: str | None = None, +) -> PatchStepResult: + """Shared tail of run_patch()/run_patch_cve(): removes any stale trust + report from a previous failed run, writes {artifact_label}-vulnerability.md, + invokes the Auto Patcher engine (utilities.autopatcher.pipeline.run), + and writes {artifact_label}-trust-report.md. + + This is run_patch()'s pre-existing tail, unchanged in behavior, with + finding_id generalized to a caller-supplied artifact_label so + run_patch_cve() can reuse it unmodified (naming its two artifacts after + the CVE id instead). + + input_type/advisory_id/advisory_source are additive: run_patch() doesn't + pass them, so its RunMetadata/PatchStepResult output is unchanged from + before these parameters existed. run_patch_cve() passes + input_type="cve" so the written Trust Report honestly discloses its + provenance (see run_metadata.render_metadata_section). + + Raises: + RuntimeError: if LLM_PROVIDER is unset. (Also checked by run_patch() + itself before this is reached, preserving its exact existing + error-ordering relative to the pipeline_output-not-found check; + checked again here so run_patch_cve(), which has no equivalent + earlier check, still gets the guarantee.) + """ + _require_llm_provider() + + patch_dir = os.path.join(output_dir, "patch") + os.makedirs(patch_dir, exist_ok=True) + + # Remove any trust report left behind by a previous failed run for this + # artifact_label *before* doing any work that can fail. The trust report + # is only written on success, at the very end of this function -- if a + # stale one from an earlier run were left in place, a failed run could + # look like it succeeded with a report that doesn't match the fresh + # vulnerability.md written below. + trust_report_path = os.path.join(patch_dir, f"{artifact_label}-trust-report.md") + if os.path.exists(trust_report_path): + os.remove(trust_report_path) + + vulnerability_path = os.path.join(patch_dir, f"{artifact_label}-vulnerability.md") + with open(vulnerability_path, "w", encoding="utf-8") as f: + f.write(vulnerability_text) + + from utilities.autopatcher import llm_client as _llm + from utilities.autopatcher import run_metadata as _rm + from utilities.autopatcher.pipeline import run as _run_pipeline + + # Reset per-run LLM call metadata -- module-level state that would + # otherwise leak a stale stage entry from a prior run in this process. + _llm.clear_call_metadata() + + timestamp = datetime.now(timezone.utc) + openant_root = _find_openant_root() + patcher_commit = _rm.collect_git_info(openant_root) if openant_root else "unknown" + repo_commit = _rm.collect_git_info(Path(repo_root)) if repo_root else "-" + + # Run-scoped directory for the Repository Understanding investigation's + # parser artifacts (candidate_enrichment.build_investigation_context) -- + # outside the target repo, under this run's own output directory, keyed + # by artifact_label so it doesn't collide with another run's artifacts. + # Only needed (and only created) when there's a repository to parse. + investigation_dir = None + if repo_root: + investigation_dir = os.path.join(patch_dir, f"{artifact_label}-investigation") + os.makedirs(investigation_dir, exist_ok=True) + + api_key = os.environ.get("OPENAI_API_KEY", "") + report_body = _run_pipeline( + vulnerability_text=vulnerability_text, + api_key=api_key, + repo_root=repo_root, + investigation_output_dir=investigation_dir, + ) + + provider = _llm._cached_provider or os.environ.get("LLM_PROVIDER", "unknown") + if provider and provider != "unknown": + model = _llm._cached_model.get(provider, "unknown") + else: + model = "unknown" + if provider == "mock": + model = "mock" + + llm_mode = "MOCK" if _llm.LLMClient(api_key=api_key).is_mock else "LIVE" + + call_metadata = _llm.get_call_metadata() + stage_stop_reasons = {stage: info.get("stop_reason") for stage, info in call_metadata.items()} + max_tokens_configured = next( + ( + info.get("max_tokens_configured") + for info in call_metadata.values() + if info.get("max_tokens_configured") is not None + ), + None, + ) + + meta = _rm.RunMetadata( + timestamp=timestamp.strftime("%Y-%m-%d %H:%M:%S UTC"), + input_source=vulnerability_path, + repo_root=repo_root or "", + repo_commit=repo_commit, + llm_provider=provider, + llm_model=model, + llm_mode=llm_mode, + output_path=trust_report_path, + patcher_commit=patcher_commit, + max_tokens_configured=max_tokens_configured, + stage_stop_reasons=stage_stop_reasons, + input_type=input_type, + advisory_id=advisory_id, + advisory_source=advisory_source, + ) + full_report = report_body + "\n---\n\n" + _rm.render_metadata_section(meta) + + with open(trust_report_path, "w", encoding="utf-8") as f: + f.write(full_report) + + return PatchStepResult( + finding_id=artifact_label, + vulnerability_path=vulnerability_path, + trust_report_path=trust_report_path, + input_type=input_type, + input_id=artifact_label, + ) + + +def run_patch( + pipeline_output_path: str, + finding_id: str, + output_dir: str, + repo_root: str | None = None, +) -> PatchStepResult: + """Generate and evaluate a candidate remediation for one finding. + + Requires LLM_PROVIDER to be set explicitly in the environment: Auto + Patcher's engine silently falls back to a deterministic mock LLM when + LLM_PROVIDER is unset and stdin is non-interactive (always true when + invoked from OpenAnt's Go CLI). An OpenAnt-triggered run must never + silently produce a mock Trust Report that looks real -- LLM_PROVIDER=mock + is allowed, it just must have been explicitly chosen. + + Writes two artifacts under ``{output_dir}/patch/``: + {finding_id}-vulnerability.md -- the rendered input (for transparency) + {finding_id}-trust-report.md -- the engine's opaque Trust Report + + Raises: + RuntimeError: if LLM_PROVIDER is unset. + FileNotFoundError: if pipeline_output_path doesn't exist. + ValueError: if finding_id is unknown or ineligible. + """ + _require_llm_provider() + + if not os.path.exists(pipeline_output_path): + raise FileNotFoundError(f"pipeline_output.json not found: {pipeline_output_path}") + + pipeline_data = read_json(pipeline_output_path) + if "findings" in pipeline_data: + normalize_results(pipeline_data, "findings") + findings = pipeline_data.get("findings", []) + + finding = find_finding_by_id(findings, finding_id) + check_eligible(finding) + + vulnerability_text = render_vulnerability_markdown(finding) + + # Normalize repo_root once, here at the entry point, before it reaches + # InvestigationCase / ground_repository / parsing -- an unresolved path + # (e.g. macOS's /var/... symlink to /private/var/...) can otherwise + # degrade repository-grounding candidate paths to bare filenames. + if repo_root: + repo_root = str(Path(repo_root).resolve()) + + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + case = case_from_vulnerability_text( + vulnerability_text, repo_root=Path(repo_root) if repo_root else None + ) + projection = case.to_context_projection() + + return _run_engine_and_write_artifacts( + vulnerability_text=projection.vulnerability_text, + repo_root=str(projection.repo_root) if projection.repo_root else None, + output_dir=output_dir, + artifact_label=finding_id, + ) + + +def run_patch_cve( + cve_id: str, + repo_root: str, + output_dir: str, +) -> PatchStepResult: + """Generate and evaluate a candidate remediation seeded from a public CVE + advisory instead of an OpenAnt Finding. + + Fetches the CVE from NVD, builds an InvestigationCase from it + (utilities.autopatcher.investigation_adapters.case_from_cve), and + projects that case down to the same (vulnerability_text, repo_root) + contract the engine already accepts -- + utilities.autopatcher.pipeline.run() itself is untouched, invoked + identically to the Finding-mode path via the same + _run_engine_and_write_artifacts tail. + + Unlike run_patch(), repo_root is required and checked to exist on disk + before any network call: there is no pipeline_output.json fallback here, + and fetching NVD data is pointless if repo grounding will fail anyway. + + Writes the same two artifacts as run_patch(), named after cve_id instead + of finding_id: {output_dir}/patch/{cve_id}-vulnerability.md and + {cve_id}-trust-report.md. The written Trust Report additionally + discloses its CVE provenance (see run_metadata.render_metadata_section) + and PatchStepResult.input_type/input_id make that explicit in the + returned result too -- finding_id itself still holds cve_id, kept for + backward compatibility with existing consumers of that field. + + Raises: + ValueError: repo_root is missing or not a directory. + RuntimeError: if LLM_PROVIDER is unset. + CVENotFoundError: NVD has no record for cve_id. + CVEFetchError: network/HTTP/parse failure while contacting NVD. + """ + if not repo_root or not os.path.isdir(repo_root): + raise ValueError(f"--repo-root does not exist: {repo_root!r}") + + # Normalize once, here at the entry point, before InvestigationCase / + # ground_repository / parsing ever see it -- see run_patch()'s matching + # comment for why. + repo_root = str(Path(repo_root).resolve()) + + from utilities.autopatcher.cve_fetcher import fetch_cve + from utilities.autopatcher.investigation_adapters import case_from_cve + + cve = fetch_cve(cve_id) + case = case_from_cve(cve, repo_root=Path(repo_root)) + projection = case.to_context_projection() + + return _run_engine_and_write_artifacts( + vulnerability_text=projection.vulnerability_text, + repo_root=str(projection.repo_root) if projection.repo_root else repo_root, + output_dir=output_dir, + artifact_label=cve_id, + input_type="cve", + advisory_id=cve_id, + advisory_source="NVD", + ) diff --git a/libs/openant-core/core/verdict_taxonomy.py b/libs/openant-core/core/verdict_taxonomy.py index 7d29e659..5801fa77 100644 --- a/libs/openant-core/core/verdict_taxonomy.py +++ b/libs/openant-core/core/verdict_taxonomy.py @@ -100,3 +100,17 @@ "agreed", "vulnerable", }) + +# --- Patch-eligibility filter ------------------------------------------------- +# Findings eligible to be sent to the patch-trust pipeline (``core/patch.py``). +# Deliberately its OWN set, not an alias of DISCLOSURE_ELIGIBLE or +# DYNAMIC_TESTABLE: sending a finding to an external-generation patch pipeline +# is a stricter bar than including it in a report, and "bypassable" (a real, +# exploitable-if-bypassed finding) is worth patching even though it isn't +# worth an active dynamic-test reproduction attempt. +PATCH_ELIGIBLE = frozenset({ + "confirmed", + "agreed", + "vulnerable", + "bypassable", +}) diff --git a/libs/openant-core/openant/cli.py b/libs/openant-core/openant/cli.py index 5b78f872..40df7d56 100644 --- a/libs/openant-core/openant/cli.py +++ b/libs/openant-core/openant/cli.py @@ -581,6 +581,65 @@ def cmd_dynamic_test(args): return 2 +def cmd_patch(args): + """Generate and evaluate a candidate remediation for a Finding or a known CVE.""" + from core.patch import run_patch, run_patch_cve + from core.schemas import success, error + from core.step_report import step_context + + finding_id = getattr(args, "finding_id", None) + cve = getattr(args, "cve", None) + + if bool(finding_id) == bool(cve): + _output_json(error("exactly one of --finding-id or --cve is required")) + return 2 + + if cve and not args.repo_root: + _output_json(error("--cve requires --repo-root")) + return 2 + + if finding_id and not args.pipeline_output: + _output_json(error("pipeline_output is required when using --finding-id")) + return 2 + + output_dir = args.output or tempfile.mkdtemp(prefix="openant_patch_") + + try: + if cve: + with step_context("patch", output_dir, inputs={"cve": cve}) as ctx: + result = run_patch_cve( + cve_id=cve, + repo_root=args.repo_root, + output_dir=output_dir, + ) + ctx.outputs = { + "vulnerability_path": result.vulnerability_path, + "trust_report_path": result.trust_report_path, + } + else: + with step_context("patch", output_dir, inputs={ + "pipeline_output_path": os.path.abspath(args.pipeline_output), + "finding_id": finding_id, + }) as ctx: + result = run_patch( + pipeline_output_path=args.pipeline_output, + finding_id=finding_id, + output_dir=output_dir, + repo_root=args.repo_root, + ) + ctx.outputs = { + "vulnerability_path": result.vulnerability_path, + "trust_report_path": result.trust_report_path, + } + + _output_json(success(result.to_dict())) + return 0 + + except Exception as e: + _output_json(error(str(e))) + return 2 + + def _default_report_output(results_path: str, fmt: str) -> str: """Derive a sensible default output path based on format.""" reports_dir = os.path.join(os.path.dirname(os.path.abspath(results_path)), "final-reports") @@ -1543,6 +1602,28 @@ def build_parser() -> argparse.ArgumentParser: ) dt_p.set_defaults(func=cmd_dynamic_test) + # --------------------------------------------------------------- + # patch — generate and evaluate a candidate remediation for a finding + # --------------------------------------------------------------- + patch_p = subparsers.add_parser( + "patch", help="Generate and evaluate a candidate remediation for a finding or a known CVE" + ) + patch_p.add_argument( + "pipeline_output", nargs="?", default=None, + help="Path to pipeline_output.json (required unless --cve is given)", + ) + patch_p.add_argument( + "--finding-id", help="ID of the finding to remediate (mutually exclusive with --cve)" + ) + patch_p.add_argument( + "--cve", help="CVE identifier to fetch from NVD and remediate (mutually exclusive with --finding-id)" + ) + patch_p.add_argument( + "--repo-root", help="Path to the target repository root (required when using --cve)" + ) + patch_p.add_argument("--output", "-o", help="Output directory (default: temp dir)") + patch_p.set_defaults(func=cmd_patch) + # --------------------------------------------------------------- # report — generate reports from results # --------------------------------------------------------------- diff --git a/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md b/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md new file mode 100644 index 00000000..65810708 --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/curl-cve-2022-27774.md @@ -0,0 +1,69 @@ +# curl — Credential Leak on Redirect (CVE-2022-27774) + +## Vulnerability description + +**Type:** Insufficiently Protected Credentials (CWE-522) +**Severity:** Medium (CVSS 5.0, NVD) +**Component:** libcurl — HTTP(S) redirect handling + +When libcurl performs a request that includes authentication credentials +(for example HTTP Basic, Digest, Bearer, or NTLM authentication supplied via +`-u` / `CURLOPT_USERPWD` or an equivalent option) and the server responds +with a redirect, libcurl may continue sending those same credentials to the +destination of the redirect — even when that destination is a different +host, a different port on the same host, or a different protocol/scheme +than the original request. + +Because libcurl automatically follows redirects when configured to do so +(`CURLOPT_FOLLOWLOCATION`, or `-L` on the command line), a malicious or +compromised server can respond to an authenticated request with a redirect +that points somewhere entirely different from the original destination. If +libcurl carries the original request's credentials along to that new +destination, an attacker who controls or can influence the redirect target +can obtain the user's authentication credentials, even though the user +never intended to send them there. + +This is a credential-leak issue, not a request-forgery issue: the redirect +target does not need to be part of the original service at all. Any server +able to issue a redirect response to an authenticated client can potentially +be used to exfiltrate that client's credentials to a destination of its +choosing. + +## Attack scenario + +An application uses libcurl (or the `curl` command-line tool) to make an +authenticated HTTP request to a trusted service, for example: + +``` +curl -L -u alice:changeme https://trusted-service.example/api/resource +``` + +The trusted service is compromised, or a man-in-the-middle is able to +influence its responses. Instead of returning the requested resource, it +responds with a redirect (HTTP 301, 302, 303, 307, or 308) to a URL under +the attacker's control — one that may differ from the original request in +host, port, or scheme. + +Because `-L` / `CURLOPT_FOLLOWLOCATION` causes libcurl to follow the +redirect automatically, and credentials set on the original request are not +necessarily cleared when the destination changes, libcurl may re-send +`alice`'s username and password to the attacker's server while following +that redirect — despite the user never having configured curl to send +credentials to that destination. + +## Affected versions + +curl versions 4.9 up to and including 7.82.0 are affected. Versions prior +to 4.9 are not affected. The issue is fixed starting in curl 7.83.0. + +## Additional context + +- This affects any authentication scheme where libcurl retains credential + state across a request (Basic, Digest, Bearer, NTLM, etc.) when redirect + following is enabled. +- The concern is specifically about credentials libcurl itself carries + forward across a redirect — not about credentials a user has separately + and explicitly configured to be sent to a specific destination. +- No source code is provided with this advisory. The patch generator should + produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md b/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md new file mode 100644 index 00000000..4b73535a --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/node-semver-cve-2022-25883.md @@ -0,0 +1,62 @@ +# node-semver — Regular Expression Denial of Service via `new Range` (CVE-2022-25883) + +## Vulnerability description + +**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333 +(Inefficient Regular Expression Complexity) +**Severity:** High (CVSS 7.5 — AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H) +**Component:** `semver` — version range parsing (`new Range()`) + +Versions of the package `semver` before 7.5.2 on the 7.x branch, before +6.3.1 on the 6.x branch, and all other versions before 5.7.2 are +vulnerable to Regular Expression Denial of Service (ReDoS) via the +function `new Range`, when untrusted user data is provided as a range. + +`semver`'s range-parsing logic relies on regular expressions to tokenize +and validate version range strings. When a range string originates from an +untrusted source — for example, a dependency specifier, a request +parameter, or any other externally influenced input passed to +`new Range()` — a specially crafted string can cause the underlying +regular expression engine to take an extreme amount of time to process, +due to catastrophic backtracking. Because Node.js executes JavaScript on a +single thread, a call that hangs while backtracking blocks the entire +process, denying service to any other work sharing that thread. + +## Attack scenario + +An application accepts a version-range string from an untrusted source and +passes it to `semver`, for example: + +``` +const semver = require('semver') +semver.satisfies(userSuppliedVersion, untrustedRangeString) +``` + +Common real-world sources of an untrusted range string include a request +parameter, a header, a configuration value sourced from an external +system, or a package specifier resolved from an untrusted registry or +lockfile. If an attacker can control or influence the value passed as the +range, they can supply a string engineered to trigger catastrophic +backtracking in `semver`'s internal range-parsing regular expressions. The +resulting call to `new Range()` — directly, or indirectly via functions +such as `satisfies()` or `validRange()` that construct a `Range` +internally — can take an extreme amount of time to return, consuming CPU +and blocking further request processing. This is a denial-of-service +condition that requires no authentication or special privilege from the +attacker. + +## Affected versions + +- `semver` `>= 7.0.0, < 7.5.2` (7.x branch) +- `semver` `>= 6.0.0, < 6.3.1` (6.x branch) +- `semver` `>= 2.0.0-alpha, < 5.7.2` (all other versions) + +Fixed starting in 7.5.2 (7.x branch), 6.3.1 (6.x branch), and 5.7.2 (all +other versions). + +## Additional context + +- CVE: CVE-2022-25883. GHSA: GHSA-c2qf-rxjj-qqgw. +- No source code is provided with this advisory. The patch generator + should produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md b/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md new file mode 100644 index 00000000..2db1649b --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/urllib3-cve-2023-43804.md @@ -0,0 +1,63 @@ +# urllib3 — Cookie HTTP header isn't stripped on cross-origin redirects (CVE-2023-43804) + +## Vulnerability description + +**Type:** Exposure of Sensitive Information to an Unauthorized Actor — +CWE-200. +**Component:** `urllib3` — HTTP redirect handling. + +The `urllib3` library fails to strip the `Cookie` HTTP header during +cross-origin redirects. `urllib3` does not provide built-in cookie-jar +management, so it is up to the calling application to attach a `Cookie` +header to a request when needed. When such a request receives an HTTP +redirect (3xx) response pointing to a different origin than the one the +request was originally sent to, `urllib3` automatically follows the +redirect and, prior to the fix, forwarded the `Cookie` header unchanged to +the new destination. + +## Attack scenario + +An application manually attaches a `Cookie` header to an outgoing request, +for example to carry a session token or other authentication material: + +```python +resp = http.request("GET", "https://example.com/start", headers={"Cookie": "session=..."}) +``` + +If `https://example.com` is malicious, compromised, or simply +misconfigured, it can respond with a redirect to a different origin (e.g. +`https://attacker.example`). `urllib3` automatically follows the redirect +and forwards the `Cookie` header, unmodified, to that different origin — +an unintended disclosure of potentially sensitive authentication data to +a third party the application never intended to share it with. + +All of the following conditions must hold for this to be exploitable: + +- an affected `urllib3` version is in use; +- the calling application manually sets a `Cookie` header on the request + (`urllib3` itself does not manage cookies automatically); +- HTTP redirect-following is not disabled; +- the server issues a redirect to a different origin. + +## Affected versions + +- `urllib3` `>= 2.0.0, < 2.0.6` +- `urllib3` `< 1.26.17` + +Fixed starting in `2.0.6` (2.x line) and `1.26.17` (1.26.x line). + +## Remediation options (as published) + +1. Update to `urllib3` `v1.26.17` or `v2.0.6` or later. +2. Disable automatic redirect-following (e.g. pass `redirects=False`). +3. Avoid setting the `Cookie` header directly on requests that may be + redirected cross-origin. + +## Additional context + +- CVE: CVE-2023-43804. GHSA: + [GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f). + Published 2023-10-02. +- No source code is provided with this advisory. The patch generator + should produce a best-effort fix based on the description above; use + `--repo-root` to enable grounding against the actual codebase. diff --git a/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md b/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md new file mode 100644 index 00000000..06d3d6ee --- /dev/null +++ b/libs/openant-core/tests/patch/fixtures/examples/vulnerability.md @@ -0,0 +1,51 @@ +# SQL Injection Vulnerability — Example Input + +## Vulnerability description + +**Type:** SQL Injection (CWE-89) +**Severity:** Critical +**Component:** `app/auth.py` — `authenticate()` function + +A classic SQL injection vulnerability exists in the authentication module. +User-supplied `username` and `password` values are interpolated directly into a +SQL query string without sanitisation or parameterisation. An attacker can +craft a malicious input (e.g. `' OR '1'='1`) to bypass authentication entirely +or extract arbitrary data from the database. + +## Vulnerable code + +```python +# app/auth.py (lines 38-46) + +import sqlite3 + +db = sqlite3.connect("users.db") + +def authenticate(username: str, password: str) -> bool: + query = f"SELECT * FROM users WHERE username='{username}' AND password='{password}'" + cursor = db.execute(query) + return cursor.fetchone() is not None +``` + +## Attack scenario + +An attacker sends the following credentials via the login form: + +- **username:** `admin' --` +- **password:** *(anything)* + +The resulting query becomes: + +```sql +SELECT * FROM users WHERE username='admin' --' AND password='anything' +``` + +The `--` comment character causes the password check to be ignored, granting +the attacker full access to the `admin` account. + +## Additional context + +- Database: SQLite 3.x (driver: `sqlite3` from the Python standard library) +- Python version: 3.11 +- Framework: Flask 3.0 (no ORM; raw SQL is used throughout `app/`) +- There are no existing parameterised query helpers in the codebase diff --git a/libs/openant-core/tests/patch/test_behavior_recommendation.py b/libs/openant-core/tests/patch/test_behavior_recommendation.py new file mode 100644 index 00000000..749381c5 --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_recommendation.py @@ -0,0 +1,29 @@ +""" +Pipeline-level test: ensure Recommendation reason includes behavior mention +when behavior summary exists (mock mode). +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_recommendation_includes_behavior_phrase(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Recommendation" in report + start = report.find("## Recommendation") + assert start != -1 + block = report[start:] + + # Should include the word 'affects' from the appended phrase + assert "affects" in block + + # And include either the function name or file name from behavior + assert ("authenticate" in block) or ("app/auth.py" in block) diff --git a/libs/openant-core/tests/patch/test_behavior_suggested_tests.py b/libs/openant-core/tests/patch/test_behavior_suggested_tests.py new file mode 100644 index 00000000..26b436bd --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_suggested_tests.py @@ -0,0 +1,33 @@ +""" +Integration test: ensure Suggested Tests block contains behavior-derived skeletons +when a behavior summary exists (mock LLM mode). +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_pipeline_suggested_tests_include_behavior(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Suggested Tests" in report + start = report.find("## Suggested Tests") + assert start != -1 + after = report.find("## Confidence score", start) + block = report[start:after if after != -1 else start + 1000] + + # Suggested Tests no longer inlines pytest skeleton code (presentation + # shortening) — a behavior-derived suggestion is now identified by its + # test name and reason instead of the removed "# Behavior-focused + # validation" code-body marker. + assert "Based on finding:" in block + + # At least one behavior-derived test name present + assert "test_" in block diff --git a/libs/openant-core/tests/patch/test_behavior_summary.py b/libs/openant-core/tests/patch/test_behavior_summary.py new file mode 100644 index 00000000..7ecdf7b3 --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_summary.py @@ -0,0 +1,74 @@ +from pathlib import Path +import sys + + +from utilities.autopatcher.behavior_summary import BehaviorAnalyzer +from utilities.autopatcher.pipeline import TargetRepoContext + + +def write_file(p: Path, text: str): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8") + + +def test_behavior_auth_case(tmp_path): + repo = tmp_path / "repo_auth" + repo.mkdir() + auth = repo / "app" / "auth.py" + write_file(auth, "def authenticate(username, password):\n return db.query(username)\n") + + diff = """+++ b/app/auth.py +@@ -1,1 +1,3 @@ ++def authenticate(username, password): ++ return db.query(username) +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "authenticate" + assert r["file"] == "app/auth.py" + assert "authentication" in r["summary"] + assert any("login" in b for b in r["primary_behaviors"]) + + +def test_behavior_db_case(tmp_path): + repo = tmp_path / "repo_db" + repo.mkdir() + q = repo / "db" / "queries.py" + write_file(q, "def update_user(u):\n cursor.execute(\"UPDATE...\")\n") + + diff = """+++ b/db/queries.py +@@ -1,1 +1,3 @@ ++def update_user(u): ++ cursor.execute("UPDATE users SET ...") +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "update_user" + assert r["file"] == "db/queries.py" + assert "database" in r["summary"] or "database" in " ".join(r["primary_behaviors"]) + + +def test_behavior_fallback(tmp_path): + repo = tmp_path / "repo_fallback" + repo.mkdir() + f = repo / "utils" / "misc.py" + write_file(f, "# no function here\npass\n") + + diff = """+++ b/utils/misc.py +@@ -1 +1 @@ ++some minor change +""" + + ctx = TargetRepoContext(repo) + ba = BehaviorAnalyzer() + r = ba.analyze(diff, repo_context=ctx) + + assert r["function"] == "" + assert r["file"] == "utils/misc.py" + assert isinstance(r["primary_behaviors"], list) diff --git a/libs/openant-core/tests/patch/test_behavior_validation_plan.py b/libs/openant-core/tests/patch/test_behavior_validation_plan.py new file mode 100644 index 00000000..9efe298f --- /dev/null +++ b/libs/openant-core/tests/patch/test_behavior_validation_plan.py @@ -0,0 +1,42 @@ +""" +Test that Validation Actions includes a behavior-derived validation action +when a behavior summary is present (mock LLM mode). + +Note: this used to check a separate "Validation Plan" section further down +the report. That section was removed as a terminology/navigation cleanup — +it repeated the same (already capped-at-3) items shown in "Validation +Actions" near the top, with only a "Reason:" line added; "Validation +Actions" now includes Reason too, so nothing unique was lost. +""" +from __future__ import annotations + +import sys +from pathlib import Path + + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +def test_validation_plan_includes_behavior(): + from utilities.autopatcher.pipeline import run + + vuln_file = EXAMPLES_DIR / "vulnerability.md" + report = run(vulnerability_text=vuln_file.read_text(encoding="utf-8"), api_key="") + + assert "## Validation Actions" in report + start = report.find("## Validation Actions") + assert start != -1 + # Patch Hygiene now precedes Validation Actions (promoted next to the + # diff) — Review Results is the next heading that follows it. + after = report.find("## Review Results", start) + block = report[start:after if after != -1 else start + 800] + + # Expect the behavior-driven validation action to be present + assert "Validate behavior" in block + + # Behavior summary sentence should be present (BehaviorAnalyzer uses + # "This patch likely affects ...") — now carried via the Reason field. + assert "This patch likely affects" in block + + # At least one primary behavior item should appear (non-brittle check) + assert any(x in block for x in ("valid login", "invalid login", "valid input acceptance", "query correctness", "happy-path response")) diff --git a/libs/openant-core/tests/patch/test_candidate_enrichment.py b/libs/openant-core/tests/patch/test_candidate_enrichment.py new file mode 100644 index 00000000..37db8317 --- /dev/null +++ b/libs/openant-core/tests/patch/test_candidate_enrichment.py @@ -0,0 +1,513 @@ +"""Unit tests for candidate_enrichment.py (Phase 2A: Candidate Enrichment). + +Deterministic only -- no LLM calls, no vulnerability verdicts. Enrichment +attaches CandidateEnrichment metadata directly onto the existing +RepositoryCandidate objects (RepositoryCandidate.enrichment), never a +second candidate model. +""" + +from __future__ import annotations + +import ast +import inspect + +from utilities.agentic_enhancer.reachability_analyzer import ReachabilityAnalyzer +from utilities.agentic_enhancer.repository_index import RepositoryIndex +from utilities.autopatcher.candidate_enrichment import ( + InvestigationContext, + _resolve_containing_function, + build_investigation_context, + enrich_candidates, +) +from utilities.autopatcher.candidate_selection import CandidateSelection +from utilities.autopatcher.repository_grounding_models import ( + DiscoveryEvidence, + RepositoryCandidate, +) + + +def _evidence(pass_name: str, tier: int, hit_line: "int | None" = 0) -> DiscoveryEvidence: + return DiscoveryEvidence( + pass_name=pass_name, tier=tier, matched_tokens=None, + total_occurrences=None, hit_line=hit_line, resolution_strategy=None, + ) + + +def _candidate(path: str, pass_name: str, tier: int, hit_line: "int | None" = 0) -> RepositoryCandidate: + return RepositoryCandidate( + path=path, evidence=[_evidence(pass_name, tier, hit_line)], best_tier=tier + ) + + +def _selection(*candidates: RepositoryCandidate, max_candidates: int = 3) -> CandidateSelection: + return CandidateSelection( + generated=list(candidates), + excluded_by_policy=[], + eligible=list(candidates), + selected=list(candidates)[:max_candidates], + excluded_by_cap=list(candidates)[max_candidates:], + max_candidates=max_candidates, + ) + + +def _func(name: str, start: int, end: int, code: str = "") -> dict: + return { + "name": name, "startLine": start, "endLine": end, + "unitType": "function", "className": None, "code": code or f"def {name}(): pass\n", + } + + +def _context( + functions: dict, + call_graph: "dict | None" = None, + reverse_call_graph: "dict | None" = None, + entry_points: "set | None" = None, +) -> InvestigationContext: + index = RepositoryIndex({"functions": functions}) + call_graph = call_graph or {} + reverse_call_graph = reverse_call_graph or {} + reachability = ReachabilityAnalyzer(functions, reverse_call_graph, entry_points or set()) + return InvestigationContext( + index=index, call_graph=call_graph, + reverse_call_graph=reverse_call_graph, reachability=reachability, + ) + + +class TestResolveContainingFunction: + def test_containment_succeeds_when_hit_line_falls_inside_function(self): + candidate = _candidate("app/auth.py", "symbol_search", 2, hit_line=12) + functions_in_file = [ + {"id": "app/auth.py:helper", "name": "helper", "startLine": 1, "endLine": 5}, + {"id": "app/auth.py:authenticate", "name": "authenticate", "startLine": 8, "endLine": 20}, + ] + resolved, note = _resolve_containing_function(functions_in_file, candidate) + assert resolved["id"] == "app/auth.py:authenticate" + assert note is None + + def test_no_containing_function_falls_back_to_nearest_with_explicit_note(self): + candidate = _candidate("app/auth.py", "symbol_search", 2, hit_line=100) + functions_in_file = [ + {"id": "app/auth.py:a", "name": "a", "startLine": 1, "endLine": 5}, + {"id": "app/auth.py:b", "name": "b", "startLine": 50, "endLine": 60}, + ] + resolved, note = _resolve_containing_function(functions_in_file, candidate) + assert resolved["id"] == "app/auth.py:b" + assert note is not None and "nearest" in note + + def test_file_with_no_functions_resolves_to_none_with_honest_note(self): + candidate = _candidate("app/config.py", "symbol_search", 2, hit_line=3) + resolved, note = _resolve_containing_function([], candidate) + assert resolved is None + assert note is not None and "no parsed functions" in note + + def test_strongest_evidence_used_when_multiple_tiers_present(self): + candidate = RepositoryCandidate( + path="app/auth.py", + evidence=[ + _evidence("symbol_search", 2, hit_line=100), + _evidence("symbol_definition", 3, hit_line=10), + ], + best_tier=3, + ) + functions_in_file = [ + {"id": "app/auth.py:a", "name": "a", "startLine": 1, "endLine": 20}, + {"id": "app/auth.py:b", "name": "b", "startLine": 90, "endLine": 120}, + ] + resolved, note = _resolve_containing_function(functions_in_file, candidate) + # tier-3 evidence (hit_line=10) must win over tier-2 (hit_line=100) + assert resolved["id"] == "app/auth.py:a" + assert note is None + + +class TestEnrichCandidatesWithContext: + def test_callees_and_callers_populated_from_call_graph(self, tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text("def authenticate():\n pass\n", encoding="utf-8") + + func_id = "app/auth.py:authenticate" + callee_id = "app/db.py:query" + caller_id = "app/routes.py:handler" + functions = {func_id: _func("authenticate", 1, 2)} + context = _context( + functions, + call_graph={func_id: [callee_id]}, + reverse_call_graph={func_id: [caller_id]}, + ) + candidate = _candidate("app/auth.py", "symbol_definition", 3, hit_line=1) + selection = _selection(candidate) + + enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=context) + + assert candidate.enrichment.callees == [callee_id] + assert candidate.enrichment.callers_by_call_graph == [caller_id] + + def test_reachable_entry_point_sets_true_and_path(self, tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text("def authenticate():\n pass\n", encoding="utf-8") + + func_id = "app/auth.py:authenticate" + functions = {func_id: _func("authenticate", 1, 2)} + context = _context(functions, entry_points={func_id}) + candidate = _candidate("app/auth.py", "symbol_definition", 3, hit_line=1) + selection = _selection(candidate) + + enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=context) + + assert candidate.enrichment.is_reachable_from_entry_point is True + assert candidate.enrichment.entry_point_path == [func_id] + + def test_unreachable_sets_false_and_no_path(self, tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text("def authenticate():\n pass\n", encoding="utf-8") + + func_id = "app/auth.py:authenticate" + functions = {func_id: _func("authenticate", 1, 2)} + context = _context(functions, entry_points=set()) + candidate = _candidate("app/auth.py", "symbol_definition", 3, hit_line=1) + selection = _selection(candidate) + + enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=context) + + assert candidate.enrichment.is_reachable_from_entry_point is False + assert candidate.enrichment.entry_point_path is None + + +class TestEnrichCandidatesWithoutContext: + def test_none_context_still_enriches_every_candidate(self, tmp_path): + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text("def authenticate():\n pass\n", encoding="utf-8") + candidate = _candidate("app/auth.py", "symbol_search", 2, hit_line=0) + selection = _selection(candidate) + + result = enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=None) + + assert len(result) == 1 + assert candidate.enrichment is not None + assert candidate.enrichment.resolved_function is None + assert "no investigation context" in candidate.enrichment.resolution_note + assert candidate.enrichment.callees == [] + assert candidate.enrichment.is_reachable_from_entry_point is None + + +class TestFailureIsolation: + def test_one_candidate_failing_does_not_affect_others(self, tmp_path): + (tmp_path / "a.py").write_text("def a(): pass\n", encoding="utf-8") + (tmp_path / "b.py").write_text("def b(): pass\n", encoding="utf-8") + + func_id_a = "a.py:a" + func_id_b = "b.py:b" + functions = {func_id_a: _func("a", 1, 2), func_id_b: _func("b", 1, 2)} + context = _context(functions) + + good = _candidate("b.py", "symbol_search", 2, hit_line=1) + bad = _candidate("a.py", "symbol_search", 2, hit_line=1) + selection = _selection(bad, good, max_candidates=2) + + original = context.index.list_functions_in_file + + def _boom(path): + if path == "a.py": + raise RuntimeError("simulated failure") + return original(path) + + context.index.list_functions_in_file = _boom + + result = enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=context) + + assert len(result) == 2 + assert bad.enrichment.enrichment_errors # recorded, not raised + assert "simulated failure" in bad.enrichment.enrichment_errors[0] + assert good.enrichment.enrichment_errors == [] + assert good.enrichment.resolved_function is not None + + +class TestOrderingAndIdentity: + def test_returned_order_matches_selection_selected_order(self, tmp_path): + for name in ("a", "b", "c"): + (tmp_path / f"{name}.py").write_text(f"def {name}(): pass\n", encoding="utf-8") + a = _candidate("a.py", "symbol_search", 2, hit_line=0) + b = _candidate("b.py", "symbol_search", 2, hit_line=0) + c = _candidate("c.py", "symbol_search", 2, hit_line=0) + selection = _selection(a, b, c, max_candidates=3) + + result = enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=None) + + assert [cand.path for cand in result] == ["a.py", "b.py", "c.py"] + + def test_same_object_identity_is_returned(self, tmp_path): + (tmp_path / "a.py").write_text("def a(): pass\n", encoding="utf-8") + candidate = _candidate("a.py", "symbol_search", 2, hit_line=0) + selection = _selection(candidate) + result = enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=None) + assert result[0] is candidate + + def test_evidence_and_best_tier_never_mutated(self, tmp_path): + (tmp_path / "a.py").write_text("def a(): pass\n", encoding="utf-8") + candidate = _candidate("a.py", "symbol_search", 2, hit_line=0) + evidence_before = list(candidate.evidence) + best_tier_before = candidate.best_tier + selection = _selection(candidate) + + enrich_candidates(selection, repo_root=tmp_path, vulnerability_text="x", context=None) + + assert candidate.evidence == evidence_before + assert candidate.best_tier == best_tier_before + + +class TestBackwardCompatibility: + def test_unenriched_candidate_has_none_enrichment_by_default(self): + candidate = RepositoryCandidate(path="a.py", evidence=[], best_tier=1) + assert candidate.enrichment is None + + def test_existing_style_construction_still_valid(self): + # Mirrors repo_locator.py's own construction call shape. + candidate = RepositoryCandidate( + path="a.py", + evidence=[_evidence("explicit_path", 4, hit_line=0)], + best_tier=4, + ) + assert candidate.enrichment is None + + +class TestSinkMatches: + def test_no_resolvable_vuln_class_gives_none_not_empty_list(self, tmp_path): + (tmp_path / "a.py").write_text("def a(): pass\n", encoding="utf-8") + candidate = _candidate("a.py", "symbol_search", 2, hit_line=0) + selection = _selection(candidate) + # CWE-200-shaped text -- resolves no covered vulnerability class. + vulnerability_text = "Cookie header retained across cross-origin redirects" + + enrich_candidates(selection, repo_root=tmp_path, vulnerability_text=vulnerability_text, context=None) + + assert candidate.enrichment.sink_matches is None + + +class TestNoLLMPath: + def test_module_imports_no_llm_machinery(self): + from utilities.autopatcher import candidate_enrichment + + source = inspect.getsource(candidate_enrichment) + tree = ast.parse(source) + + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + + assert not any("llm" in name.lower() for name in imported), imported + + +class TestExtractLiteralConstants: + def test_module_level_bare_literal(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("TIMEOUT = 30\n") + assert result["TIMEOUT"]["outcome"] == "literal" + assert result["TIMEOUT"]["ast_literal_kind"] == "Constant" + assert result["TIMEOUT"]["value"] == 30 + assert result["TIMEOUT"]["class_name"] is None + + def test_class_level_wrapper_call_literal_urllib3_shape(self): + """The exact shape of CVE-2023-43804's actual fix: a class-level + frozenset(...) call wrapping a bare list literal.""" + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + text = ( + "class Retry:\n" + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Authorization\"])\n" + ) + result = _extract_literal_constants(text) + entry = result["Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT"] + assert entry["outcome"] == "literal" + assert entry["ast_literal_kind"] == "frozenset_call" + assert entry["value"] == frozenset({"Authorization"}) + assert entry["class_name"] == "Retry" + assert entry["name"] == "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" + + def test_detects_the_actual_cve_2023_43804_value_change(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + before = _extract_literal_constants( + "class Retry:\n DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Authorization\"])\n" + ) + after = _extract_literal_constants( + "class Retry:\n DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Cookie\", \"Authorization\"])\n" + ) + key = "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" + assert before[key]["value"] == frozenset({"Authorization"}) + assert after[key]["value"] == frozenset({"Authorization", "Cookie"}) + assert before[key]["value"] != after[key]["value"] + + def test_call_rhs_that_is_not_a_wrapper_is_non_literal(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("BACKEND = default_backend()\n") + assert result["BACKEND"]["outcome"] == "non_literal" + assert result["BACKEND"]["value"] is None + + def test_attribute_and_name_rhs_are_non_literal(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("X = some.attr\nY = other_name\n") + assert result["X"]["outcome"] == "non_literal" + assert result["Y"]["outcome"] == "non_literal" + + def test_augmented_assignment_is_recorded_without_a_fabricated_value(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("FLAGS = {1}\nFLAGS |= {2}\n") + # The AugAssign itself is recorded, but note it overwrites the + # plain Assign's entry for the same name -- both are direct + # children of Module, encountered in source order. + assert result["FLAGS"]["outcome"] == "augmented_assign" + assert result["FLAGS"]["value"] is None + + def test_annotation_only_has_no_value(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("x: int\n") + assert result["x"]["outcome"] == "annotation_only" + + def test_multi_target_and_destructuring_assignments_are_excluded(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("a = b = {1}\nc, d = (1, 2)\n") + assert result == {} + + def test_function_body_assignment_is_out_of_scope(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("def f():\n LOCAL = {1}\n return LOCAL\n") + assert result == {} + + def test_set_and_list_literals_with_same_elements_are_distinguished(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("A = {1, 2}\nB = [1, 2]\n") + assert result["A"]["ast_literal_kind"] != result["B"]["ast_literal_kind"] + assert result["A"]["value"] != result["B"]["value"] # frozenset({1,2}) != (1,2) + + def test_unparseable_file_returns_empty_dict_not_a_guess(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + assert _extract_literal_constants("def f(:\n") == {} + + def test_zero_arg_wrapper_call(self): + from utilities.autopatcher.candidate_enrichment import _extract_literal_constants + + result = _extract_literal_constants("EMPTY = frozenset()\n") + assert result["EMPTY"]["outcome"] == "literal" + assert result["EMPTY"]["value"] == frozenset() + + +class TestScopeConstantsInEnrichment: + """_enrich_one's module/class-level scoping rule for scope_constants, + tested through enrich_candidates() with a hand-built InvestigationContext + (no file I/O, matching this test file's existing style).""" + + def _ctx_with_constants(self, functions, constants): + return InvestigationContext( + index=RepositoryIndex({"functions": functions}), + call_graph={}, + reverse_call_graph={}, + reachability=ReachabilityAnalyzer(functions, {}, set()), + constants=constants, + ) + + def test_module_level_constant_always_in_scope(self): + functions = {"a.py:f": {"name": "f", "startLine": 1, "endLine": 2, "unitType": "function", "className": None, "code": ""}} + constants = {"a.py": {"TIMEOUT": {"qualified_name": "TIMEOUT", "class_name": None, "name": "TIMEOUT", "outcome": "literal", "ast_literal_kind": "Constant", "value": 30, "line": 1, "end_line": 1}}} + candidate = _candidate("a.py", "symbol_search", 2, hit_line=1) + enrich_candidates(_selection(candidate), "/nonexistent", "irrelevant vuln text with no sinks", self._ctx_with_constants(functions, constants)) + names = [e["qualified_name"] for e in candidate.enrichment.scope_constants] + assert "TIMEOUT" in names + + def test_class_level_constant_scoped_to_resolved_functions_class(self): + functions = { + "a.py:Retry.increment": {"name": "increment", "startLine": 5, "endLine": 6, "unitType": "function", "className": "Retry", "code": ""}, + "a.py:Other.method": {"name": "method", "startLine": 8, "endLine": 9, "unitType": "function", "className": "Other", "code": ""}, + } + constants = {"a.py": { + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": {"qualified_name": "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "class_name": "Retry", "name": "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "outcome": "literal", "ast_literal_kind": "frozenset_call", "value": frozenset({"Authorization"}), "line": 1, "end_line": 1}, + "Other.UNRELATED": {"qualified_name": "Other.UNRELATED", "class_name": "Other", "name": "UNRELATED", "outcome": "literal", "ast_literal_kind": "Constant", "value": 1, "line": 2, "end_line": 2}, + }} + # hit_line=5 falls inside Retry.increment, resolving className="Retry". + candidate = _candidate("a.py", "symbol_search", 2, hit_line=5) + enrich_candidates(_selection(candidate), "/nonexistent", "irrelevant vuln text", self._ctx_with_constants(functions, constants)) + names = {e["qualified_name"] for e in candidate.enrichment.scope_constants} + assert "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in names + assert "Other.UNRELATED" not in names + + def test_no_resolved_function_includes_every_class_level_constant(self): + """No narrowing signal available -- never silently drop a + class-level constant just because no function was resolved.""" + functions = {} # nothing resolvable + constants = {"a.py": { + "Retry.X": {"qualified_name": "Retry.X", "class_name": "Retry", "name": "X", "outcome": "literal", "ast_literal_kind": "Constant", "value": 1, "line": 1, "end_line": 1}, + "Other.Y": {"qualified_name": "Other.Y", "class_name": "Other", "name": "Y", "outcome": "literal", "ast_literal_kind": "Constant", "value": 2, "line": 2, "end_line": 2}, + }} + candidate = _candidate("a.py", "symbol_search", 2, hit_line=1) + enrich_candidates(_selection(candidate), "/nonexistent", "irrelevant vuln text", self._ctx_with_constants(functions, constants)) + names = {e["qualified_name"] for e in candidate.enrichment.scope_constants} + assert names == {"Retry.X", "Other.Y"} + + def test_module_level_fallback_resolution_includes_every_class_level_constant(self): + """Regression, found by running against the real urllib3 repo: a + class-body constant (between methods, contained by no real + function's line range) resolves via _resolve_containing_function's + own whole-file "module_level" catch-all unit -- which carries + className=None. That None must NOT be read as "the real scope is + module-only" (it would incorrectly exclude every class-level + constant, including the one the hit_line actually sits inside); + it must be treated exactly like resolved_function being None.""" + functions = { + "a.py:__module__": {"name": "__module__", "startLine": 1, "endLine": 20, "unitType": "module_level", "className": None, "code": ""}, + } + constants = {"a.py": { + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": {"qualified_name": "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "class_name": "Retry", "name": "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "outcome": "literal", "ast_literal_kind": "frozenset_call", "value": frozenset({"Authorization"}), "line": 5, "end_line": 5}, + }} + candidate = _candidate("a.py", "symbol_search", 2, hit_line=5) + enrich_candidates(_selection(candidate), "/nonexistent", "irrelevant vuln text", self._ctx_with_constants(functions, constants)) + assert candidate.enrichment.resolved_function["unitType"] == "module_level" + names = {e["qualified_name"] for e in candidate.enrichment.scope_constants} + assert "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in names + + +class TestRealIntegration: + def test_select_then_enrich_against_a_real_small_repo(self, tmp_path): + from utilities.autopatcher.candidate_selection import select_candidates + from utilities.autopatcher.repo_locator import ground_repository + + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text( + "def authenticate(u, p):\n" + " return check_password(u, p)\n" + "\n" + "def check_password(u, p):\n" + " return True\n", + encoding="utf-8", + ) + vuln_text = "Vulnerability in app/auth.py — authenticate() is exploitable" + + grounding = ground_repository(vuln_text, tmp_path) + selection = select_candidates(grounding, max_candidates=3) + assert selection.selected, "grounding must find the candidate for this test to be meaningful" + + context = build_investigation_context(tmp_path, tmp_path / "_investigation") + assert context is not None, "a real Python file must produce a usable investigation context" + + result = enrich_candidates(selection, tmp_path, vuln_text, context) + + assert result + enriched = result[0].enrichment + assert enriched is not None + assert enriched.enrichment_errors == [] + # The real parser must resolve authenticate() as the containing + # function, and the real call graph must show it calling + # check_password() -- proving the whole chain end to end, not just + # that it didn't crash. + assert enriched.resolved_function is not None + assert enriched.resolved_function["name"] == "authenticate" + assert any("check_password" in callee for callee in enriched.callees) diff --git a/libs/openant-core/tests/patch/test_candidate_selection.py b/libs/openant-core/tests/patch/test_candidate_selection.py new file mode 100644 index 00000000..cca0bfbf --- /dev/null +++ b/libs/openant-core/tests/patch/test_candidate_selection.py @@ -0,0 +1,195 @@ +"""Unit tests for candidate_selection.py (Phase 1: Candidate Selection). + +No LLM calls, no OpenAnt investigation -- this phase only selects a bounded, +deterministically-ordered subset of RepositoryGroundingResult.candidates. +All fixtures use the real repository-grounding dataclasses directly. +""" + +from __future__ import annotations + +import pytest + +from utilities.autopatcher.repository_grounding_models import ( + DiscoveryEvidence, + RepositoryCandidate, + RepositoryGroundingResult, +) +from utilities.autopatcher.candidate_selection import ( + DEFAULT_MAX_CANDIDATES, + select_candidates, +) + + +EXPLICIT = ("explicit_path", 4) +SYMBOL_DEF = ("symbol_definition", 3) +SYMBOL_SEARCH = ("symbol_search", 2) +CWE = ("cwe_keywords", 1) + + +def _evidence(pass_name: str, tier: int) -> DiscoveryEvidence: + return DiscoveryEvidence( + pass_name=pass_name, tier=tier, matched_tokens=None, + total_occurrences=None, hit_line=0, resolution_strategy=None, + ) + + +def _candidate(path: str, pass_name: str, tier: int) -> RepositoryCandidate: + return RepositoryCandidate(path=path, evidence=[_evidence(pass_name, tier)], best_tier=tier) + + +def _grounding(*candidates: RepositoryCandidate) -> RepositoryGroundingResult: + return RepositoryGroundingResult( + rendered_context="", candidates=list(candidates), decisions=[], + extraction_signals={}, budget=None, + ) + + +class TestOrderingByTier: + def test_explicit_path_and_symbol_definition_outrank_weaker_tiers(self): + strong_a = _candidate("app/auth.py", *EXPLICIT) + strong_b = _candidate("app/session.py", *SYMBOL_DEF) + weak = _candidate("app/utils.py", *SYMBOL_SEARCH) + selection = select_candidates(_grounding(weak, strong_b, strong_a), max_candidates=3) + assert [c.path for c in selection.selected] == [ + "app/auth.py", "app/session.py", "app/utils.py", + ] + + def test_symbol_search_remains_eligible_below_stronger_evidence(self): + strong = _candidate("app/auth.py", *EXPLICIT) + weak = _candidate("app/utils.py", *SYMBOL_SEARCH) + selection = select_candidates(_grounding(strong, weak), max_candidates=5) + assert weak in selection.eligible + assert weak in selection.selected # capacity available + + def test_cwe_fallback_does_not_crowd_out_stronger_candidates_when_cap_reached(self): + strong = [_candidate(f"app/strong_{i}.py", *EXPLICIT) for i in range(3)] + weak = _candidate("app/weak.py", *CWE) + selection = select_candidates(_grounding(*strong, weak), max_candidates=3) + assert weak not in selection.selected + assert weak in selection.excluded_by_cap + + def test_cwe_fallback_selected_when_no_stronger_candidate_exists(self): + weak = _candidate("app/weak.py", *CWE) + selection = select_candidates(_grounding(weak), max_candidates=3) + assert weak in selection.selected + + +class TestCapBehavior: + def test_more_eligible_than_cap_keeps_strongest_and_excludes_tail(self): + candidates = [ + _candidate("app/a.py", *EXPLICIT), + _candidate("app/b.py", *SYMBOL_DEF), + _candidate("app/c.py", *SYMBOL_SEARCH), + _candidate("app/d.py", *CWE), + ] + selection = select_candidates(_grounding(*candidates), max_candidates=2) + assert [c.path for c in selection.selected] == ["app/a.py", "app/b.py"] + assert [c.path for c in selection.excluded_by_cap] == ["app/c.py", "app/d.py"] + assert len(selection.selected) <= selection.max_candidates + + def test_max_candidates_one_selects_single_strongest(self): + candidates = [_candidate("app/a.py", *SYMBOL_DEF), _candidate("app/b.py", *EXPLICIT)] + selection = select_candidates(_grounding(*candidates), max_candidates=1) + assert [c.path for c in selection.selected] == ["app/b.py"] + + def test_max_candidates_zero_selects_nothing_and_is_not_an_error(self): + selection = select_candidates( + _grounding(_candidate("app/a.py", *EXPLICIT)), max_candidates=0 + ) + assert selection.selected == [] + assert selection.used_fallback is True + + def test_negative_max_candidates_raises(self): + with pytest.raises(ValueError): + select_candidates(_grounding(), max_candidates=-1) + + +class TestDeterministicOrdering: + def test_tied_tier_breaks_by_path_and_is_stable_across_repeated_calls(self): + a = _candidate("app/b.py", *SYMBOL_DEF) + b = _candidate("app/a.py", *SYMBOL_DEF) + grounding = _grounding(a, b) + first = [c.path for c in select_candidates(grounding, max_candidates=5).selected] + second = [c.path for c in select_candidates(grounding, max_candidates=5).selected] + assert first == second == ["app/a.py", "app/b.py"] + + +class TestBookkeepingInvariants: + def test_generated_eligible_selected_excluded_by_cap_partition_correctly(self): + candidates = [_candidate(f"app/{i}.py", *SYMBOL_SEARCH) for i in range(5)] + selection = select_candidates(_grounding(*candidates), max_candidates=2) + + generated_paths = {c.path for c in selection.generated} + eligible_paths = {c.path for c in selection.eligible} + excluded_by_policy_paths = {c.path for c in selection.excluded_by_policy} + selected_paths = {c.path for c in selection.selected} + excluded_by_cap_paths = {c.path for c in selection.excluded_by_cap} + + assert generated_paths == excluded_by_policy_paths | eligible_paths + assert eligible_paths == selected_paths | excluded_by_cap_paths + assert len(selection.selected) <= selection.max_candidates + + def test_empty_grounding_result_is_safe_and_uses_fallback(self): + selection = select_candidates(_grounding(), max_candidates=DEFAULT_MAX_CANDIDATES) + assert selection.generated == [] + assert selection.eligible == [] + assert selection.selected == [] + assert selection.excluded_by_cap == [] + assert selection.excluded_by_policy == [] + assert selection.used_fallback is True + + +class TestNoSideEffects: + def test_does_not_mutate_grounding_result_or_its_candidates(self): + candidates = [_candidate("app/a.py", *EXPLICIT), _candidate("app/b.py", *CWE)] + grounding = _grounding(*candidates) + before = list(grounding.candidates) + + select_candidates(grounding, max_candidates=1) + + assert grounding.candidates == before + assert grounding.candidates == candidates + + def test_module_makes_no_llm_or_environment_calls(self): + import ast + import inspect + + from utilities.autopatcher import candidate_selection + + source = inspect.getsource(candidate_selection) + tree = ast.parse(source) + + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + + assert not any("llm" in name.lower() for name in imported), imported + assert "os" not in imported + + # No os.environ / os.getenv attribute access anywhere in the module body. + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and node.attr in ("environ", "getenv"): + pytest.fail(f"unexpected os.{node.attr} usage in candidate_selection.py") + + +class TestRealGroundingIntegration: + def test_selects_from_real_ground_repository_output(self, tmp_path): + from utilities.autopatcher.repo_locator import ground_repository + + (tmp_path / "app").mkdir() + (tmp_path / "app" / "auth.py").write_text( + "def authenticate(u, p):\n pass\n", encoding="utf-8" + ) + (tmp_path / "app" / "other.py").write_text( + "def unrelated():\n pass\n", encoding="utf-8" + ) + vuln_text = "Vulnerability in app/auth.py — authenticate() is exploitable" + + grounding = ground_repository(vuln_text, tmp_path) + selection = select_candidates(grounding, max_candidates=DEFAULT_MAX_CANDIDATES) + + assert any("auth.py" in c.path for c in selection.selected) + assert len(selection.selected) <= DEFAULT_MAX_CANDIDATES diff --git a/libs/openant-core/tests/patch/test_cve_converter.py b/libs/openant-core/tests/patch/test_cve_converter.py new file mode 100644 index 00000000..4aa0f0db --- /dev/null +++ b/libs/openant-core/tests/patch/test_cve_converter.py @@ -0,0 +1,248 @@ +"""Unit tests for cve_converter. + +Ported from the standalone Auto Patcher project's test_cve_converter.py. The +rendered Markdown template is byte-for-byte identical to the reference +implementation's -- no wording changes were introduced in this commit. The +"rendering parity with the reference implementation" requirement is covered +by pinning assertions to exact strings the reference implementation is known +to produce (its module isn't importable here -- it lives in a sibling repo, +not a dependency of this one). +""" + +from __future__ import annotations + +from utilities.autopatcher.cve_converter import ( + cve_to_vuln_text, + extract_affected_products, + extract_cvss, + extract_cwes, + extract_description, + first_sentence, +) + +FULL_CVE = { + "id": "CVE-2021-12345", + "descriptions": [ + { + "lang": "en", + "value": ( + "A SQL injection vulnerability exists in the authenticate() function. " + "An attacker can bypass authentication." + ), + } + ], + "metrics": { + "cvssMetricV31": [ + { + "source": "nvd@nist.gov", + "type": "Primary", + "cvssData": {"baseScore": 9.8, "baseSeverity": "CRITICAL"}, + } + ] + }, + "weaknesses": [ + {"source": "nvd@nist.gov", "type": "Primary", "description": [{"lang": "en", "value": "CWE-89"}]} + ], + "configurations": [ + { + "nodes": [ + { + "operator": "OR", + "cpeMatch": [ + { + "vulnerable": True, + "criteria": "cpe:2.3:a:example:example-lib:*:*:*:*:*:*:*:*", + } + ], + } + ] + } + ], + "references": [ + {"url": "https://github.com/example/security/advisories/GHSA-1234", "source": "nvd@nist.gov"} + ], +} + +SPARSE_CVE = {"id": "CVE-0000-00000"} + + +def _cve_with_n_cpe_matches(n: int) -> dict: + matches = [ + {"vulnerable": True, "criteria": f"cpe:2.3:a:example:lib-{i}:*:*:*:*:*:*:*:*"} + for i in range(n) + ] + return {"id": "CVE-0001-00001", "configurations": [{"nodes": [{"cpeMatch": matches}]}]} + + +class TestCveToVulnText: + def test_returns_string(self): + result = cve_to_vuln_text(FULL_CVE) + assert isinstance(result, str) + assert len(result) > 50 + + def test_deterministic_rendering(self): + assert cve_to_vuln_text(FULL_CVE) == cve_to_vuln_text(FULL_CVE) + + def test_includes_advisory_line_recognized_by_pipeline(self): + # pipeline.py's _ADVISORY_LINE_RE/_CVE_ID_RE match a literal + # "**Advisory:** " line -- this is the existing, unmodified + # mechanism that surfaces the CVE id in the pipeline's own report. + result = cve_to_vuln_text(FULL_CVE) + assert "**Advisory:** CVE-2021-12345" in result + + def test_severity_present(self): + assert "CRITICAL" in cve_to_vuln_text(FULL_CVE) + + def test_includes_cwe(self): + assert "CWE-89" in cve_to_vuln_text(FULL_CVE) + + def test_includes_description(self): + assert "authenticate()" in cve_to_vuln_text(FULL_CVE) + + def test_includes_affected_product_cpe(self): + result = cve_to_vuln_text(FULL_CVE) + assert "cpe:2.3:a:example:example-lib" in result + + def test_affected_products_section_header_matches_reference_implementation(self): + # No wording change from the reference implementation in this commit + # -- the non-verification caveat lives in extract_affected_products' + # docstring, not in the rendered Markdown. + result = cve_to_vuln_text(FULL_CVE) + assert "## Affected products\n" in result + + def test_includes_no_code_snippet_note(self): + assert "No source code snippet is available" in cve_to_vuln_text(FULL_CVE) + + def test_cvss_score_present(self): + assert "9.8" in cve_to_vuln_text(FULL_CVE) + + def test_references_included(self): + result = cve_to_vuln_text(FULL_CVE) + assert "github.com/example/security" in result + + def test_summary_derived_from_first_sentence(self): + result = cve_to_vuln_text(FULL_CVE) + assert result.startswith( + "# A SQL injection vulnerability exists in the authenticate() function." + ) + + +class TestCveToVulnTextMissingOptionalFields: + def test_handles_sparse_cve_without_crash(self): + result = cve_to_vuln_text(SPARSE_CVE) + assert isinstance(result, str) + assert "CVE-0000-00000" in result + + def test_sparse_cve_reports_no_description_available(self): + assert "No description available" in cve_to_vuln_text(SPARSE_CVE) + + def test_sparse_cve_reports_unknown_cwe(self): + assert "**Type:** Unknown" in cve_to_vuln_text(SPARSE_CVE) + + def test_sparse_cve_reports_na_cvss_and_unknown_severity(self): + result = cve_to_vuln_text(SPARSE_CVE) + assert "**Severity:** UNKNOWN (CVSS: N/A)" in result + + def test_sparse_cve_reports_products_not_specified(self): + assert "- (not specified)" in cve_to_vuln_text(SPARSE_CVE) + + def test_sparse_cve_reports_no_references(self): + assert "- (none)" in cve_to_vuln_text(SPARSE_CVE) + + def test_missing_id_falls_back_to_unknown(self): + result = cve_to_vuln_text({}) + assert "**Advisory:** unknown" in result + + +class TestExtractDescription: + def test_extracts_english_description(self): + assert "SQL injection" in extract_description(FULL_CVE) + + def test_missing_description_returns_empty_string(self): + assert extract_description({}) == "" + + def test_ignores_non_english_description(self): + cve = {"descriptions": [{"lang": "fr", "value": "une vulnerabilite"}]} + assert extract_description(cve) == "" + + +class TestExtractCwes: + def test_extract_cwes_deduped(self): + cve = dict(FULL_CVE, weaknesses=FULL_CVE["weaknesses"] * 2) + assert extract_cwes(cve) == ["CWE-89"] + + def test_missing_weaknesses_returns_empty_list(self): + assert extract_cwes({}) == [] + + +class TestExtractCvss: + def test_extract_cvss_prefers_v31(self): + score, severity = extract_cvss(FULL_CVE) + assert score == "9.8" + assert severity == "CRITICAL" + + def test_extract_cvss_falls_back_when_absent(self): + assert extract_cvss({}) == ("N/A", "UNKNOWN") + + def test_extract_cvss_falls_back_to_v2_when_v31_and_v30_absent(self): + cve = { + "metrics": { + "cvssMetricV2": [ + {"cvssData": {"baseScore": 5.0}, "baseSeverity": "MEDIUM"} + ] + } + } + score, severity = extract_cvss(cve) + assert score == "5.0" + assert severity == "MEDIUM" + + +class TestExtractAffectedProducts: + def test_returns_criteria_strings(self): + products = extract_affected_products(FULL_CVE) + assert products == ["cpe:2.3:a:example:example-lib:*:*:*:*:*:*:*:*"] + + def test_caps_at_five_products(self): + cve = _cve_with_n_cpe_matches(8) + products = extract_affected_products(cve) + assert len(products) == 5 + + def test_deduplicates_identical_criteria(self): + cve = { + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + {"vulnerable": True, "criteria": "cpe:2.3:a:x:y:*"}, + {"vulnerable": True, "criteria": "cpe:2.3:a:x:y:*"}, + ] + } + ] + } + ] + } + assert extract_affected_products(cve) == ["cpe:2.3:a:x:y:*"] + + def test_ignores_non_vulnerable_matches(self): + cve = { + "configurations": [ + {"nodes": [{"cpeMatch": [{"vulnerable": False, "criteria": "cpe:2.3:a:x:y:*"}]}]} + ] + } + assert extract_affected_products(cve) == [] + + def test_missing_configurations_returns_empty_list(self): + assert extract_affected_products({}) == [] + + +class TestFirstSentence: + def test_extracts_first_sentence(self): + assert first_sentence("First one. Second one.") == "First one." + + def test_empty_string_returns_empty_string(self): + assert first_sentence("") == "" + + def test_truncates_long_sentence_to_120_chars(self): + long_text = "A" * 200 + "." + assert len(first_sentence(long_text)) <= 120 diff --git a/libs/openant-core/tests/patch/test_cve_fetcher.py b/libs/openant-core/tests/patch/test_cve_fetcher.py new file mode 100644 index 00000000..caeea748 --- /dev/null +++ b/libs/openant-core/tests/patch/test_cve_fetcher.py @@ -0,0 +1,199 @@ +"""Unit tests for cve_fetcher. All tests use mocked HTTP -- no network required. + +Ported from the standalone Auto Patcher project's test_cve_fetcher.py, adapted +to assert on the split CVENotFoundError/CVEFetchError exception types instead +of a single bare ValueError. +""" + +from __future__ import annotations + +import json +import urllib.error +from unittest import mock + +import pytest + +from utilities.autopatcher.cve_fetcher import CVEFetchError, CVENotFoundError, fetch_cve + +FIXTURE_CVE = { + "id": "CVE-2021-12345", + "descriptions": [ + {"lang": "en", "value": "A SQL injection vulnerability exists in the authenticate() function."} + ], + "metrics": { + "cvssMetricV31": [ + { + "source": "nvd@nist.gov", + "type": "Primary", + "cvssData": {"baseScore": 9.8, "baseSeverity": "CRITICAL"}, + } + ] + }, + "weaknesses": [ + {"source": "nvd@nist.gov", "type": "Primary", "description": [{"lang": "en", "value": "CWE-89"}]} + ], + "references": [{"url": "https://example.com/advisory", "source": "nvd@nist.gov"}], +} + +FIXTURE_ENVELOPE = { + "resultsPerPage": 1, + "startIndex": 0, + "totalResults": 1, + "vulnerabilities": [{"cve": FIXTURE_CVE}], +} + + +def _make_response(data: dict): + """Build a mock context-manager response for urlopen.""" + body = json.dumps(data).encode("utf-8") + cm = mock.MagicMock() + cm.__enter__ = mock.Mock(return_value=cm) + cm.__exit__ = mock.Mock(return_value=False) + cm.read = mock.Mock(return_value=body) + return cm + + +class TestFetchCveSuccess: + def test_returns_the_unwrapped_cve_object(self): + with mock.patch("urllib.request.urlopen", return_value=_make_response(FIXTURE_ENVELOPE)): + result = fetch_cve("CVE-2021-12345") + assert isinstance(result, dict) + assert result == FIXTURE_CVE + assert result["id"] == "CVE-2021-12345" + + def test_passes_timeout_through_to_urlopen(self): + captured_kwargs = {} + + def fake_urlopen(req, **kwargs): + captured_kwargs.update(kwargs) + return _make_response(FIXTURE_ENVELOPE) + + with mock.patch("urllib.request.urlopen", fake_urlopen): + fetch_cve("CVE-2021-12345", timeout=42) + + assert captured_kwargs.get("timeout") == 42 + + +class TestFetchCveNotFound: + def test_http_404_raises_cve_not_found_error(self): + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="https://services.nvd.nist.gov/rest/json/cves/2.0", + code=404, + msg="Not Found", + hdrs=None, + fp=None, + ), + ): + with pytest.raises(CVENotFoundError, match="HTTP 404"): + fetch_cve("CVE-bad") + + def test_empty_vulnerabilities_list_raises_cve_not_found_error(self): + empty_envelope = {"resultsPerPage": 0, "totalResults": 0, "vulnerabilities": []} + with mock.patch("urllib.request.urlopen", return_value=_make_response(empty_envelope)): + with pytest.raises(CVENotFoundError, match="no matching record"): + fetch_cve("CVE-9999-99999") + + def test_missing_vulnerabilities_key_raises_cve_not_found_error(self): + with mock.patch("urllib.request.urlopen", return_value=_make_response({"resultsPerPage": 0})): + with pytest.raises(CVENotFoundError): + fetch_cve("CVE-9999-99999") + + def test_cve_not_found_error_is_a_value_error(self): + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="x", code=404, msg="Not Found", hdrs=None, fp=None + ), + ): + with pytest.raises(ValueError): + fetch_cve("CVE-bad") + + +class TestFetchCveFetchFailures: + def test_other_http_errors_raise_cve_fetch_error(self): + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="https://services.nvd.nist.gov/rest/json/cves/2.0", + code=503, + msg="Service Unavailable", + hdrs=None, + fp=None, + ), + ): + with pytest.raises(CVEFetchError, match="HTTP 503"): + fetch_cve("CVE-2021-12345") + + def test_url_error_raises_cve_fetch_error(self): + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.URLError("name resolution failed"), + ): + with pytest.raises(CVEFetchError, match="name resolution failed"): + fetch_cve("CVE-2021-12345") + + def test_timeout_raises_cve_fetch_error(self): + with mock.patch("urllib.request.urlopen", side_effect=TimeoutError("timed out")): + with pytest.raises(CVEFetchError, match="timed out"): + fetch_cve("CVE-2021-12345") + + def test_malformed_json_raises_cve_fetch_error(self): + cm = mock.MagicMock() + cm.__enter__ = mock.Mock(return_value=cm) + cm.__exit__ = mock.Mock(return_value=False) + cm.read = mock.Mock(return_value=b"not json{{") + with mock.patch("urllib.request.urlopen", return_value=cm): + with pytest.raises(CVEFetchError, match="unparseable"): + fetch_cve("CVE-2021-12345") + + def test_vulnerabilities_entry_missing_cve_key_raises_cve_fetch_error(self): + envelope = {"vulnerabilities": [{"not_cve": {}}]} + with mock.patch("urllib.request.urlopen", return_value=_make_response(envelope)): + with pytest.raises(CVEFetchError, match="malformed"): + fetch_cve("CVE-2021-12345") + + def test_cve_fetch_error_is_a_value_error(self): + with mock.patch("urllib.request.urlopen", side_effect=urllib.error.URLError("boom")): + with pytest.raises(ValueError): + fetch_cve("CVE-2021-12345") + + +class TestApiKeyHeader: + def test_includes_api_key_header_when_set(self, monkeypatch): + monkeypatch.setenv("NVD_API_KEY", "nvd_test_key") + captured = [] + + def fake_urlopen(req, timeout=None): + captured.append(req) + return _make_response(FIXTURE_ENVELOPE) + + with mock.patch("urllib.request.urlopen", fake_urlopen): + fetch_cve("CVE-2021-12345") + + assert captured, "urlopen was not called" + assert captured[0].get_header("Apikey") == "nvd_test_key" + + def test_no_api_key_header_without_env_var(self, monkeypatch): + monkeypatch.delenv("NVD_API_KEY", raising=False) + captured = [] + + def fake_urlopen(req, timeout=None): + captured.append(req) + return _make_response(FIXTURE_ENVELOPE) + + with mock.patch("urllib.request.urlopen", fake_urlopen): + fetch_cve("CVE-2021-12345") + + assert captured[0].get_header("Apikey") is None + + def test_api_key_value_never_appears_in_a_raised_error_message(self, monkeypatch): + monkeypatch.setenv("NVD_API_KEY", "super-secret-key-value") + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError(url="x", code=500, msg="err", hdrs=None, fp=None), + ): + with pytest.raises(CVEFetchError) as exc_info: + fetch_cve("CVE-2021-12345") + assert "super-secret-key-value" not in str(exc_info.value) diff --git a/libs/openant-core/tests/patch/test_diff_hunk_repair.py b/libs/openant-core/tests/patch/test_diff_hunk_repair.py new file mode 100644 index 00000000..463c9065 --- /dev/null +++ b/libs/openant-core/tests/patch/test_diff_hunk_repair.py @@ -0,0 +1,590 @@ +"""Tests for diff_hunk_repair.repair_hunk_headers. + +Covers: + - Minimist-exact corrupt header regression (primary case) + - Already-correct patch is returned unchanged (no-op guarantee) + - Multi-hunk with net-positive first hunk: offset propagation + - Multi-hunk where both count AND offset are wrong (cascaded error) + - Hunk suffix text preserved verbatim + - No-newline marker excluded from counts + - Count-omitted single-line form normalized to explicit count + - Multi-file: file_delta resets between files + - Multi-file: hunk-body content resembling a "--- "/"+++ " header must not + be misparsed as one (F-36, F-41, F-45) + - Non-diff / empty input passthrough + - New-file hunk (--- /dev/null) handled + - RepairResult metadata reflects actual changes +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.diff_hunk_repair import repair_hunk_headers, RepairResult + + +# --------------------------------------------------------------------------- +# Fixtures / shared patch strings +# --------------------------------------------------------------------------- + +# Exact minimist patch from the evaluation run — corrupt hunk headers. +# Hunk 1: claims old=7/new=9 (net +2). Actual body: old=8/new=8 (net 0). +# Hunk 2: claims old_start=79/new_start=81. Correct: 79/79 (delta carried wrong). +_MINIMIST_CORRUPT = """\ +--- a/index.js ++++ b/index.js +@@ -69,7 +69,9 @@ + function setKey (obj, keys, value) { + var o = obj; + for (var i = 0; i < keys.length-1; i++) { + var key = keys[i]; +- if (key === '__proto__') return; ++ if (key === '__proto__' || key === 'constructor' || key === 'prototype') return; + if (o[key] === undefined) o[key] = {}; + if (o[key] === Object.prototype || o[key] === Number.prototype + || o[key] === String.prototype) o[key] = {}; +@@ -79,7 +81,7 @@ + + var key = keys[keys.length - 1]; +- if (key === '__proto__') return; ++ if (key === '__proto__' || key === 'constructor' || key === 'prototype') return; + if (o === Object.prototype || o === Number.prototype + || o === String.prototype) o = {}; + if (o === Array.prototype) o = []; +""" + +# A correctly formed patch — headers must survive unchanged. +# Hunk adds 1 line (net +1): old=4 context+1 removed=5, new=4 context+2 added=6. +_CORRECT_PATCH = """\ +--- a/example.py ++++ b/example.py +@@ -10,5 +10,6 @@ + def foo(): + a = 1 + b = 2 +- return a ++ c = a + b ++ return c + +""" + + +# --------------------------------------------------------------------------- +# Primary regression: minimist corrupt headers +# --------------------------------------------------------------------------- + +class TestMinimistCorruptHeaders: + def test_first_hunk_old_count_corrected(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -69,8 +69,8 @@"), ( + f"Expected @@ -69,8 +69,8 @@, got: {hunk_lines[0]}" + ) + + def test_second_hunk_new_start_corrected(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + # file_delta after hunk1 = 0 (net zero), so new_start = 79 + 0 = 79 + assert hunk_lines[1].startswith("@@ -79,6 +79,6 @@"), ( + f"Expected @@ -79,6 +79,6 @@, got: {hunk_lines[1]}" + ) + + def test_body_content_unchanged(self): + repaired, _ = repair_hunk_headers(_MINIMIST_CORRUPT) + orig_body = [l for l in _MINIMIST_CORRUPT.splitlines() if not l.startswith("@@")] + repr_body = [l for l in repaired.splitlines() if not l.startswith("@@")] + assert orig_body == repr_body + + def test_metadata_reports_two_hunks_rewritten(self): + _, meta = repair_hunk_headers(_MINIMIST_CORRUPT) + assert meta.normalization_applied is True + assert meta.hunks_rewritten == 2 + assert meta.files_rewritten == 1 + + +# --------------------------------------------------------------------------- +# No-op guarantee: already-correct patch must be returned byte-for-byte +# --------------------------------------------------------------------------- + +class TestNoOpGuarantee: + def test_correct_patch_is_unchanged(self): + repaired, meta = repair_hunk_headers(_CORRECT_PATCH) + assert repaired == _CORRECT_PATCH + + def test_correct_patch_metadata_zero(self): + _, meta = repair_hunk_headers(_CORRECT_PATCH) + assert meta.normalization_applied is False + assert meta.hunks_rewritten == 0 + assert meta.files_rewritten == 0 + + +# --------------------------------------------------------------------------- +# Multi-hunk: net-positive first hunk propagates offset to second hunk +# --------------------------------------------------------------------------- + +class TestOffsetPropagation: + def test_net_positive_first_hunk_shifts_second_hunk_new_start(self): + # Hunk 1 adds 2 lines (net +2). Hunk 2's new_start must shift by +2. + patch = ( + "--- a/foo.py\n" + "+++ b/foo.py\n" + "@@ -1,3 +1,5 @@\n" # correct: old=3, new=5 (net +2) + " line1\n" + "+added1\n" + "+added2\n" + " line2\n" + " line3\n" + "@@ -10,3 +12,3 @@\n" # correct: old_start=10, new_start=12 (+2 from prior) + " lineA\n" + " lineB\n" + " lineC\n" + ) + repaired, meta = repair_hunk_headers(patch) + assert meta.normalization_applied is False, ( + "Patch with correct headers must not be modified" + ) + + def test_wrong_second_hunk_offset_is_corrected(self): + # Hunk 1 is correct (net +2). Hunk 2 has wrong new_start (still +10 not +12). + patch = ( + "--- a/foo.py\n" + "+++ b/foo.py\n" + "@@ -1,3 +1,5 @@\n" + " line1\n" + "+added1\n" + "+added2\n" + " line2\n" + " line3\n" + "@@ -10,3 +10,3 @@\n" # wrong: new_start should be 12 (10+2) + " lineA\n" + " lineB\n" + " lineC\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[1].startswith("@@ -10,3 +12,3 @@"), ( + f"Expected @@ -10,3 +12,3 @@, got: {hunk_lines[1]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Hunk suffix text preserved verbatim +# --------------------------------------------------------------------------- + +class TestSuffixPreserved: + def test_function_name_suffix_preserved(self): + patch = ( + "--- a/index.js\n" + "+++ b/index.js\n" + "@@ -69,7 +69,9 @@ function setKey\n" + " function setKey (obj, keys, value) {\n" + " var o = obj;\n" + " for (var i = 0; i < keys.length-1; i++) {\n" + " var key = keys[i];\n" + "- if (key === '__proto__') return;\n" + "+ if (key === '__proto__' || key === 'constructor') return;\n" + " if (o[key] === undefined) o[key] = {};\n" + " if (o[key] === Object.prototype || o[key] === Number.prototype\n" + " || o[key] === String.prototype) o[key] = {};\n" + ) + repaired, _ = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].endswith("@@ function setKey"), ( + f"Suffix not preserved: {hunk_lines[0]!r}" + ) + + +# --------------------------------------------------------------------------- +# No-newline marker excluded from counts +# --------------------------------------------------------------------------- + +class TestNoNewlineMarker: + def test_no_newline_marker_not_counted(self): + # 1 context + 1 removed + 1 added + marker = old=2, new=2 + patch = ( + "--- a/file.py\n" + "+++ b/file.py\n" + "@@ -5,99 +5,99 @@\n" # intentionally wrong counts + " context\n" + "-old line\n" + "+new line\n" + "\\ No newline at end of file\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -5,2 +5,2 @@"), ( + f"Got: {hunk_lines[0]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Count-omitted single-line form: @@ -5 +5 @@ → @@ -5,1 +5,1 @@ +# --------------------------------------------------------------------------- + +class TestCountOmittedForm: + def test_omitted_count_normalized_to_explicit(self): + patch = ( + "--- a/x.py\n" + "+++ b/x.py\n" + "@@ -5 +5 @@\n" # count omitted — means 1 + "-old\n" + "+new\n" + ) + repaired, _ = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -5,1 +5,1 @@"), ( + f"Got: {hunk_lines[0]}" + ) + + +# --------------------------------------------------------------------------- +# Multi-file: file_delta resets between files +# --------------------------------------------------------------------------- + +class TestMultiFileDeltaReset: + def test_delta_does_not_leak_across_files(self): + # File 1: net +1. File 2's hunk new_start must NOT be shifted by +1. + patch = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,2 +1,3 @@\n" # correct: old=2, new=3, net+1 + " context\n" + "-removed\n" + "+added1\n" + "+added2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,3 +10,3 @@\n" # correct: new_start=10, not 11 + " a\n" + " b\n" + " c\n" + ) + repaired, meta = repair_hunk_headers(patch) + assert meta.normalization_applied is False, ( + "Correct headers across two files must not be modified" + ) + + def test_wrong_count_in_second_file_corrected_without_delta_from_first(self): + # File 1: net +1 (correct). File 2: wrong counts, but no delta from file 1. + patch = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,2 +1,3 @@\n" + " context\n" + "-removed\n" + "+added1\n" + "+added2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,99 +10,99 @@\n" # wrong counts; no delta from file1 + " a\n" + " b\n" + " c\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[1].startswith("@@ -10,3 +10,3 @@"), ( + f"Got: {hunk_lines[1]}" + ) + assert meta.hunks_rewritten == 1 + assert meta.files_rewritten == 1 + + def test_body_content_resembling_header_does_not_corrupt_second_file(self): + """F-36 regression: an added line whose own text starts with "++ " + (raw line "+++ ...") must not be mistaken for a real "+++ " file + header — doing so would corrupt file_delta/current_file bookkeeping + and cascade into file2's header being rewritten incorrectly.""" + patch = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,3 +1,4 @@\n" + " def f():\n" + "- old = 1\n" + "+++ marker line inside file1's hunk\n" + "+ new = 2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,3 +10,3 @@\n" + " a\n" + " b\n" + " c\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert len(hunk_lines) == 2, f"Expected 2 hunk headers, got: {hunk_lines}" + assert hunk_lines[1].startswith("@@ -10,3 +10,3 @@"), ( + f"file2's correct header was corrupted: {hunk_lines[1]}" + ) + + +# --------------------------------------------------------------------------- +# Hunk-body content that resembles a file header ("--- "/"+++ " prefix) +# --------------------------------------------------------------------------- + +class TestHunkBodyContentResemblingFileHeader: + def test_plus_plus_plus_body_content_not_treated_as_header(self): + """F-41 regression: a removed/added line whose own text starts with + "++ " produces the raw line "+++ ...", indistinguishable from a real + file header by a naive startswith check. It must stay inside the + hunk body, not be hoisted in front of the (possibly rewritten) hunk + header.""" + patch = ( + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -1,3 +1,4 @@\n" + " def f():\n" + "- old = 1\n" + "+++ this added line of code starts with plus plus plus\n" + "+ new = 2\n" + ) + repaired, meta = repair_hunk_headers(patch) + lines = repaired.splitlines(keepends=True) + idx_marker = next( + i for i, l in enumerate(lines) if l.startswith("+++ this added line") + ) + idx_header = next(i for i, l in enumerate(lines) if l.startswith("@@ ")) + assert idx_marker > idx_header, ( + "The '+++ ' body line was hoisted before the hunk header" + ) + assert meta.files_rewritten <= 1 + + def test_dash_dash_dash_body_content_not_treated_as_header(self): + """F-45 regression: same as above, for a removed line whose own text + starts with "-- " (raw line "--- ...").""" + patch = ( + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -1,3 +1,3 @@\n" + " def f():\n" + "--- this removed line of code starts with dash dash dash\n" + "+ return new\n" + ) + repaired, meta = repair_hunk_headers(patch) + lines = repaired.splitlines(keepends=True) + idx_marker = next( + i for i, l in enumerate(lines) if l.startswith("--- this removed line") + ) + idx_header = next(i for i, l in enumerate(lines) if l.startswith("@@ ")) + assert idx_marker > idx_header, ( + "The '--- ' body line was hoisted before the hunk header" + ) + assert meta.files_rewritten <= 1 + + +# --------------------------------------------------------------------------- +# Non-diff / empty input passthrough +# --------------------------------------------------------------------------- + +class TestPassthrough: + def test_empty_string(self): + repaired, meta = repair_hunk_headers("") + assert repaired == "" + assert meta.normalization_applied is False + + def test_whitespace_only(self): + repaired, meta = repair_hunk_headers(" \n \n") + assert meta.normalization_applied is False + + def test_non_diff_content(self): + text = "# just a comment\nnot a diff at all\n" + repaired, meta = repair_hunk_headers(text) + assert repaired == text + assert meta.normalization_applied is False + + def test_never_raises_on_garbage(self): + for garbage in ["@@@@", "@@ broken @@\n+line", "---\n+++\n@@bad"]: + repaired, meta = repair_hunk_headers(garbage) + assert isinstance(repaired, str) + assert isinstance(meta, RepairResult) + + +# --------------------------------------------------------------------------- +# New-file hunk (--- /dev/null) +# --------------------------------------------------------------------------- + +class TestNewFileHunk: + def test_new_file_hunk_counts_only_added_lines(self): + patch = ( + "--- /dev/null\n" + "+++ b/newfile.py\n" + "@@ -0,0 +1,99 @@\n" # wrong new count + "+line1\n" + "+line2\n" + "+line3\n" + ) + repaired, meta = repair_hunk_headers(patch) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + # old=0 (no context, no removed), new=3 + assert hunk_lines[0].startswith("@@ -0,0 +1,3 @@"), ( + f"Got: {hunk_lines[0]}" + ) + assert meta.hunks_rewritten == 1 + + +# --------------------------------------------------------------------------- +# Fenced patches: ``` must not be absorbed into hunk body +# --------------------------------------------------------------------------- + +# Regression patch: LLM-generated fenced diff with inverted +/- order and +# wrong hunk header counts — the exact pattern seen in the minimist v2 eval. +# Before the fix, the closing ``` was absorbed into hunk-2's body, inflating +# its count by 1 so that the "repaired" header still said @@ -81,8 +81,8 @@ +# (matching the over-counted body), causing git apply to fail with +# "corrupt patch at line 21" even after repair reported "2 hunks fixed". +_FENCED_MINIMIST_V2 = """\ +```diff +--- a/index.js ++++ b/index.js +@@ -69,6 +69,8 @@ + function setKey (obj, keys, value) { + var o = obj; + for (var i = 0; i < keys.length-1; i++) { + var key = keys[i]; ++ if (key === 'constructor' || key === '__proto__') return; +- if (key === '__proto__') return; + if (o[key] === undefined) o[key] = {}; + if (o[key] === Object.prototype || o[key] === Number.prototype +@@ -81,6 +81,6 @@ + } + + var key = keys[keys.length - 1]; ++ if (key === 'constructor' || key === '__proto__') return; +- if (key === '__proto__') return; + if (o === Object.prototype || o === Number.prototype + || o === String.prototype) o = {}; + if (o === Array.prototype) o = []; +```""" + +def _make_minimist_fixture(tmp_path: Path) -> Path: + """Init a minimal git repo whose index.js matches, line-for-line, what + _FENCED_MINIMIST_V2's hunk headers target (lines 69-75 and 81-87) — so + the repaired patch can be checked against a real `git apply` without + depending on an externally-managed minimist checkout. + """ + lines = [f"// filler line {i}" for i in range(1, 69)] + lines += [ + " function setKey (obj, keys, value) {", + " var o = obj;", + " for (var i = 0; i < keys.length-1; i++) {", + " var key = keys[i];", + " if (key === '__proto__') return;", + " if (o[key] === undefined) o[key] = {};", + " if (o[key] === Object.prototype || o[key] === Number.prototype", + ] + lines += [f"// filler line {i}" for i in range(76, 81)] + lines += [ + " }", + "", + " var key = keys[keys.length - 1];", + " if (key === '__proto__') return;", + " if (o === Object.prototype || o === Number.prototype", + " || o === String.prototype) o = {};", + " if (o === Array.prototype) o = [];", + ] + index_js = "\n".join(lines) + "\n" + + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "index.js").write_text(index_js, encoding="utf-8") + subprocess.run(["git", "add", "index.js"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=tmp_path, capture_output=True, check=True, + ) + return tmp_path + + +class TestFencedPatch: + def test_both_hunk_headers_corrected(self): + repaired, meta = repair_hunk_headers(_FENCED_MINIMIST_V2) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert hunk_lines[0].startswith("@@ -69,7 +69,7 @@"), hunk_lines[0] + assert hunk_lines[1].startswith("@@ -81,7 +81,7 @@"), hunk_lines[1] + + def test_metadata_two_hunks_rewritten(self): + _, meta = repair_hunk_headers(_FENCED_MINIMIST_V2) + assert meta.hunks_rewritten == 2 + assert meta.files_rewritten == 1 + + def test_closing_fence_not_counted_as_context(self): + """``` must not inflate hunk-2 counts — old bug produced @@ -81,8 +81,8 @@.""" + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + hunk_lines = [l for l in repaired.splitlines() if l.startswith("@@")] + assert not hunk_lines[1].startswith("@@ -81,8"), ( + f"Closing fence was counted as context: {hunk_lines[1]}" + ) + + def test_fences_preserved_in_output(self): + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + lines = repaired.splitlines() + assert lines[0].startswith("```"), "Opening fence should be preserved" + assert lines[-1].strip() == "```", "Closing fence should be preserved" + + def test_context_line_of_triple_backticks_survives_unfenced_patch(self): + """F-38 regression: a legitimate context line whose content is ``` + (e.g. an unchanged closing Markdown code fence) must not be mistaken + for the LLM's own wrapper fence and stripped, even when the patch + has no surrounding ``` wrapper at all.""" + patch = ( + "--- a/README.md\n" + "+++ b/README.md\n" + "@@ -1,3 +1,3 @@\n" + " intro\n" + "-old code\n" + "+new code\n" + " ```\n" + ) + repaired, meta = repair_hunk_headers(patch) + assert repaired.splitlines(keepends=True)[-1] == " ```\n", ( + "Legitimate context line of ``` was stripped as a fake wrapper fence" + ) + assert meta.normalization_applied is False + + def test_correct_fenced_patch_is_noop(self): + """A fenced patch with already-correct headers must not be modified.""" + fenced_correct = ( + "```diff\n" + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -10,5 +10,6 @@\n" + " def foo():\n" + " a = 1\n" + " b = 2\n" + "- return a\n" + "+ c = a + b\n" + "+ return c\n" + "\n" + "```" + ) + repaired, meta = repair_hunk_headers(fenced_correct) + assert meta.normalization_applied is False + assert repaired == fenced_correct + + def test_repaired_patch_applies_cleanly(self, tmp_path): + """End-to-end: repaired fenced patch must pass git apply --check.""" + import re as _re + repo = _make_minimist_fixture(tmp_path) + repaired, _ = repair_hunk_headers(_FENCED_MINIMIST_V2) + # Strip fences the same way patch_applicability does + lines = repaired.splitlines() + if lines and _re.match(r"^```", lines[0]): + lines = lines[1:] + if lines and lines[-1].strip() == "```": + lines = lines[:-1] + stripped = "\n".join(lines).strip() + result = subprocess.run( + ["git", "apply", "--check", "--whitespace=nowarn", "-"], + input=stripped + "\n", + cwd=str(repo), + capture_output=True, + text=True, + ) + assert result.returncode == 0, f"git apply failed: {result.stderr.strip()}" diff --git a/libs/openant-core/tests/patch/test_diff_parsing.py b/libs/openant-core/tests/patch/test_diff_parsing.py new file mode 100644 index 00000000..88439032 --- /dev/null +++ b/libs/openant-core/tests/patch/test_diff_parsing.py @@ -0,0 +1,150 @@ +from utilities.autopatcher.diff_parsing import DiffHunk, parse_diff + + +def test_empty_input_returns_no_files_or_hunks(): + changed_files, file_hunks = parse_diff("") + assert changed_files == [] + assert file_hunks == {} + + +def test_non_diff_input_returns_no_files_or_hunks(): + changed_files, file_hunks = parse_diff("hello\nworld\nthis is not a diff\n") + assert changed_files == [] + assert file_hunks == {} + + +def test_one_changed_file_one_hunk(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,3 @@\n" + "+def foo():\n" + "+ pass\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py"] + assert list(file_hunks.keys()) == ["a.py"] + hunks = file_hunks["a.py"] + assert len(hunks) == 1 + assert hunks[0] == DiffHunk( + new_start=1, new_count=3, lines=["+def foo():", "+ pass"] + ) + + +def test_multiple_changed_files(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+a\n" + "--- a/b.py\n" + "+++ b/b.py\n" + "@@ -1,1 +1,1 @@\n" + "+b\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py", "b.py"] + assert file_hunks["a.py"][0].lines == ["+a"] + assert file_hunks["b.py"][0].lines == ["+b"] + + +def test_multiple_hunks_in_one_file(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "-old\n" + "+new\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["a.py"] + hunks = file_hunks["a.py"] + assert len(hunks) == 2 + assert hunks[0].new_start == 1 and hunks[0].new_count == 2 + assert hunks[1].new_start == 7 and hunks[1].new_count == 2 + + +def test_hunk_header_metadata_with_explicit_count(): + diff = "+++ b/a.py\n@@ -1,5 +10,20 @@\n context\n" + _, file_hunks = parse_diff(diff) + hunk = file_hunks["a.py"][0] + assert hunk.new_start == 10 + assert hunk.new_count == 20 + + +def test_hunk_header_metadata_defaults_count_to_one_when_omitted(): + # "@@ -1 +1 @@" form: no ",N" count suffix on the new-file side. + diff = "+++ b/a.py\n@@ -1 +1 @@\n+x\n" + _, file_hunks = parse_diff(diff) + hunk = file_hunks["a.py"][0] + assert hunk.new_start == 1 + assert hunk.new_count == 1 + + +def test_added_removed_and_context_lines_are_preserved_with_markers(): + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,3 +1,3 @@\n" + " context line\n" + "-removed line\n" + "+added line\n" + ) + _, file_hunks = parse_diff(diff) + assert file_hunks["a.py"][0].lines == [ + " context line", + "-removed line", + "+added line", + ] + + +def test_lines_without_a_recognized_marker_are_dropped(): + # A line inside a hunk that isn't prefixed with ' ', '+', or '-' (e.g. a + # "\ No newline at end of file" marker) is not appended to hunk.lines. + diff = ( + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+added\n" + "\\ No newline at end of file\n" + ) + _, file_hunks = parse_diff(diff) + assert file_hunks["a.py"][0].lines == ["+added"] + + +def test_minus_a_and_plus_b_paths_used_correctly(): + # Only "+++ b/..." sets the tracked filename; "--- a/..." is recognized + # only as a hunk-flush boundary, and its own path is never used. + diff = ( + "--- a/old_name.py\n" + "+++ b/new_name.py\n" + "@@ -1,1 +1,1 @@\n" + "+x\n" + ) + changed_files, file_hunks = parse_diff(diff) + assert changed_files == ["new_name.py"] + assert "old_name.py" not in file_hunks + + +def test_dash_a_line_flushes_pending_hunk_before_next_file(): + # A trailing hunk for the first file must be flushed when the second + # file's "--- a/" line appears, not silently merged into the next file. + diff = ( + "--- a/a.py\n" + "+++ b/a.py\n" + "@@ -1,1 +1,1 @@\n" + "+from a\n" + "--- a/b.py\n" + "+++ b/b.py\n" + "@@ -1,1 +1,1 @@\n" + "+from b\n" + ) + _, file_hunks = parse_diff(diff) + assert len(file_hunks["a.py"]) == 1 + assert file_hunks["a.py"][0].lines == ["+from a"] + assert len(file_hunks["b.py"]) == 1 + assert file_hunks["b.py"][0].lines == ["+from b"] diff --git a/libs/openant-core/tests/patch/test_evidence_fusion.py b/libs/openant-core/tests/patch/test_evidence_fusion.py new file mode 100644 index 00000000..1f57c06d --- /dev/null +++ b/libs/openant-core/tests/patch/test_evidence_fusion.py @@ -0,0 +1,764 @@ +"""Unit tests for evidence_fusion.py (deterministic Evidence Fusion). + +No LLM calls, no I/O, no parsing, no vulnerability verdicts. Fusion +preserves candidate identity: RepositoryUnderstanding.candidate_evidence +holds the exact same RepositoryCandidate objects candidate_selection.py/ +candidate_enrichment.py already produced -- never copies, never wraps. +""" + +from __future__ import annotations + +import ast +import inspect + +from utilities.autopatcher.candidate_selection import CandidateSelection +from utilities.autopatcher.evidence_fusion import ( + DEFAULT_MAX_CHARS, + CandidateRelationship, + RepositoryUnderstanding, + fuse_evidence, + render_repository_understanding, +) +from utilities.autopatcher.repository_grounding_models import ( + CandidateEnrichment, + DiscoveryEvidence, + RepositoryCandidate, +) + + +def _evidence(pass_name: str, tier: int, hit_line: "int | None" = 0) -> DiscoveryEvidence: + return DiscoveryEvidence( + pass_name=pass_name, tier=tier, matched_tokens=None, + total_occurrences=None, hit_line=hit_line, resolution_strategy=None, + ) + + +def _enrichment( + resolved_function: "dict | None" = None, + callees: "list[str] | None" = None, + callers_by_call_graph: "list[str] | None" = None, + callers_by_text_search: "list[dict] | None" = None, + is_reachable_from_entry_point: "bool | None" = None, + enrichment_errors: "list[str] | None" = None, + resolution_note: "str | None" = "_unset_", + entry_point_path: "list[str] | None" = None, + related_tests: "list[dict] | None" = None, + test_support_rating: "tuple | None" = None, + sink_matches: "list[dict] | None" = None, +) -> CandidateEnrichment: + if resolution_note == "_unset_": + resolution_note = None if resolved_function else "no function resolved" + return CandidateEnrichment( + functions_in_file=[resolved_function] if resolved_function else [], + resolved_function=resolved_function, + resolution_note=resolution_note, + callees=callees or [], + callers_by_call_graph=callers_by_call_graph or [], + callers_by_text_search=callers_by_text_search or [], + is_reachable_from_entry_point=is_reachable_from_entry_point, + entry_point_path=entry_point_path, + related_tests=related_tests or [], + test_support_rating=test_support_rating, + sink_matches=sink_matches, + enrichment_errors=enrichment_errors or [], + ) + + +def _candidate( + path: str, + pass_name: str, + tier: int, + hit_line: "int | None" = 0, + enrichment: "CandidateEnrichment | None" = None, +) -> RepositoryCandidate: + candidate = RepositoryCandidate( + path=path, evidence=[_evidence(pass_name, tier, hit_line)], best_tier=tier + ) + candidate.enrichment = enrichment + return candidate + + +def _selection(*candidates: RepositoryCandidate, max_candidates: int = 3) -> CandidateSelection: + return CandidateSelection( + generated=list(candidates), + excluded_by_policy=[], + eligible=list(candidates), + selected=list(candidates)[:max_candidates], + excluded_by_cap=list(candidates)[max_candidates:], + max_candidates=max_candidates, + ) + + +class TestCandidateIdentityPreserved: + def test_candidate_evidence_holds_exact_same_objects(self): + a = _candidate("a.py", "explicit_path", 4) + b = _candidate("b.py", "symbol_search", 2) + selection = _selection(a, b) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert understanding.candidate_evidence[0] is a + assert understanding.candidate_evidence[1] is b + + +class TestRelationshipDetection: + def test_relationship_detected_from_callees(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment( + resolved_function={"id": "a.py:foo", "name": "foo", "startLine": 1, "endLine": 3}, + callees=["b.py:bar"], + ), + ) + b = _candidate( + "b.py", "symbol_search", 2, + enrichment=_enrichment( + resolved_function={"id": "b.py:bar", "name": "bar", "startLine": 1, "endLine": 3}, + ), + ) + selection = _selection(a, b) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert understanding.relationships == [ + CandidateRelationship(from_path="a.py", to_path="b.py", kind="calls", detail="b.py:bar") + ] + + def test_same_edge_from_both_directions_deduplicated(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment(callees=["b.py:bar"]), + ) + b = _candidate( + "b.py", "symbol_search", 2, + enrichment=_enrichment(callers_by_call_graph=["a.py:foo"]), + ) + selection = _selection(a, b) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert len(understanding.relationships) == 1 + assert understanding.relationships[0].from_path == "a.py" + assert understanding.relationships[0].to_path == "b.py" + + def test_relationship_ignored_when_target_not_selected(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment(callees=["c.py:not_selected"]), + ) + selection = _selection(a) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert understanding.relationships == [] + + def test_callers_by_text_search_never_asserts_a_relationship(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment( + callers_by_text_search=[{"id": "b.py:bar", "name": "bar", "file": "b.py", "matches": []}], + ), + ) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(a, b) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert understanding.relationships == [] + + +class TestFusionNotes: + def test_divergence_note_for_strong_tier_unresolved(self): + strong_unresolved = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + weak_unresolved = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(strong_unresolved, weak_unresolved) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + joined = " ".join(understanding.fusion_notes) + assert "a.py" in joined + assert "b.py" not in joined + + def test_investigation_context_unavailable_note(self): + a = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + selection = _selection(a) + + understanding = fuse_evidence(selection, investigation_context_available=False) + + assert any("investigation context unavailable" in n for n in understanding.fusion_notes) + assert understanding.investigation_context_available is False + assert understanding.candidate_evidence == [a] + + def test_enrichment_errors_surfaced_for_weak_tier_candidate(self): + weak_with_error = _candidate( + "a.py", "symbol_search", 2, + enrichment=_enrichment( + resolved_function={"id": "a.py:foo", "name": "foo", "startLine": 1, "endLine": 3}, + enrichment_errors=["graph enrichment failed: RuntimeError: boom"], + ), + ) + selection = _selection(weak_with_error) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + joined = " ".join(understanding.fusion_notes) + assert "a.py" in joined + assert "boom" in joined + + def test_candidate_never_double_reported_when_strong_and_erroring(self): + strong_with_error = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment(enrichment_errors=["boom"]), + ) + selection = _selection(strong_with_error) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + a_notes = [n for n in understanding.fusion_notes if n.startswith("a.py:")] + assert len(a_notes) == 1 + + +class TestEmptySelection: + def test_empty_selection_returns_empty_understanding(self): + selection = _selection() + + understanding = fuse_evidence(selection, investigation_context_available=True) + + assert understanding.candidate_evidence == [] + assert understanding.relationships == [] + assert understanding.fusion_notes == [] + + +class TestPurity: + def test_fuse_evidence_does_not_mutate_inputs(self): + enrichment = _enrichment( + resolved_function={"id": "a.py:foo", "name": "foo", "startLine": 1, "endLine": 3}, + callees=["b.py:bar"], + ) + a = _candidate("a.py", "explicit_path", 4, enrichment=enrichment) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(a, b) + + evidence_before = list(a.evidence) + best_tier_before = a.best_tier + enrichment_before = a.enrichment + selected_before = list(selection.selected) + + fuse_evidence(selection, investigation_context_available=True) + + assert a.evidence == evidence_before + assert a.best_tier == best_tier_before + assert a.enrichment is enrichment_before + assert selection.selected == selected_before + + +class TestConsumerPattern: + def test_reachable_paths_derivable_directly_from_candidate_evidence(self): + reachable = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment(is_reachable_from_entry_point=True), + ) + unreachable = _candidate( + "b.py", "symbol_search", 2, + enrichment=_enrichment(is_reachable_from_entry_point=False), + ) + selection = _selection(reachable, unreachable) + + understanding = fuse_evidence(selection, investigation_context_available=True) + + reachable_paths = [ + c.path + for c in understanding.candidate_evidence + if c.enrichment and c.enrichment.is_reachable_from_entry_point + ] + assert reachable_paths == ["a.py"] + + +class TestNoLLMPath: + def test_module_imports_no_llm_machinery(self): + from utilities.autopatcher import evidence_fusion + + source = inspect.getsource(evidence_fusion) + tree = ast.parse(source) + + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + + assert not any("llm" in name.lower() for name in imported), imported + + +# --------------------------------------------------------------------------- +# render_repository_understanding -- deterministic Markdown rendering. +# --------------------------------------------------------------------------- + +def _heavy_candidate(i: int) -> RepositoryCandidate: + """A candidate whose rendered block is large (8 callees) -- used to + exercise the character-budget/truncation behavior without needing a + real repository.""" + return _candidate( + f"file_{i}.py", "symbol_search", 2, hit_line=i, + enrichment=_enrichment( + resolved_function={"id": f"file_{i}.py:fn", "name": "fn", "startLine": 1, "endLine": 50}, + callees=[f"callee_{i}_{j}.py:x" for j in range(8)], + ), + ) + + +class TestRenderHeadingAndOrdering: + def test_heading_and_candidate_order_preserved(self): + strong = _candidate("a.py", "explicit_path", 4) + weak = _candidate("b.py", "symbol_search", 2) + # candidate_selection.py already orders by (-best_tier, path) before + # this renderer ever sees the list -- simulate that ordering here + # rather than re-deriving it, since the renderer must trust it. + selection = _selection(strong, weak) + + understanding = fuse_evidence(selection, investigation_context_available=True) + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert rendered.startswith("## Repository Understanding") + assert rendered.index("`a.py`") < rendered.index("`b.py`") + + +class TestRenderCandidateFacts: + def test_path_and_grounding_evidence_rendered(self): + a = _candidate("app/auth.py", "explicit_path", 4, hit_line=10) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "`app/auth.py`" in rendered + assert "best tier 4" in rendered + assert "explicit_path (tier 4)" in rendered + + def test_resolved_function_details_rendered(self): + a = _candidate( + "app/auth.py", "symbol_definition", 3, + enrichment=_enrichment( + resolved_function={ + "id": "app/auth.py:authenticate", "name": "authenticate", + "startLine": 10, "endLine": 20, + }, + ), + ) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "Resolved near grounding evidence" in rendered + assert "`authenticate`" in rendered + assert "lines 10-20" in rendered + + def test_unresolved_function_uses_honest_wording(self): + a = _candidate( + "app/config.py", "symbol_search", 2, + enrichment=_enrichment(resolution_note="no function contains hit_line 3"), + ) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "No function was resolved" in rendered + assert "no function contains hit_line 3" in rendered + assert "Resolved near grounding evidence" not in rendered + + def test_reachability_true_false_none_are_distinguishable(self): + reachable = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment( + is_reachable_from_entry_point=True, + entry_point_path=["a.py:entry", "a.py:foo"], + ), + ) + unreachable = _candidate( + "b.py", "symbol_search", 2, + enrichment=_enrichment(is_reachable_from_entry_point=False), + ) + not_evaluated = _candidate( + "c.py", "cwe_keywords", 1, + enrichment=_enrichment(is_reachable_from_entry_point=None), + ) + selection = _selection(reachable, unreachable, not_evaluated, max_candidates=3) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "detected as reachable by current entry-point heuristics (path:" in rendered + assert "detected as not reachable by current entry-point heuristics" in rendered + assert "not evaluated (no investigation context available)" in rendered + + def test_enrichment_errors_visible_when_present(self): + a = _candidate( + "a.py", "symbol_search", 2, + enrichment=_enrichment(enrichment_errors=["graph enrichment failed: RuntimeError: boom"]), + ) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "Enrichment errors:" in rendered + assert "boom" in rendered + + def test_missing_enrichment_is_stated_honestly(self): + a = RepositoryCandidate( + path="a.py", + evidence=[_evidence("cwe_keywords", 1)], + best_tier=1, + ) # enrichment left as default None -- never enriched + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "Enrichment: not attempted for this candidate" in rendered + + +class TestRenderCandidateRoles: + def test_single_candidate_is_labeled_primary_only(self): + a = _candidate("a.py", "symbol_search", 2) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "**Primary evidence**" in rendered + assert "**Supporting evidence**" not in rendered + assert "**Additional candidate**" not in rendered + + def test_second_candidate_connected_by_call_graph_is_labeled_supporting(self): + primary = _candidate( + "primary.py", "explicit_path", 4, + enrichment=_enrichment(callees=["other.py:helper"]), + ) + other = _candidate("other.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(primary, other) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + primary_block = rendered.split("### `primary.py`", 1)[1].split("### `other.py`", 1)[0] + other_block = rendered.split("### `other.py`", 1)[1].split("###", 1)[0] + + assert "**Primary evidence**" in primary_block + assert "**Supporting evidence**" in other_block + assert "**Additional candidate**" not in other_block + + def test_relationship_detected_from_either_direction_still_labels_supporting(self): + # The edge is recorded via the OTHER candidate's callers_by_call_graph + # naming the primary, not via the primary's own callees -- role + # assignment must not care which side recorded it. + primary = _candidate("primary.py", "explicit_path", 4, enrichment=_enrichment()) + other = _candidate( + "other.py", "symbol_search", 2, + enrichment=_enrichment(callers_by_call_graph=["primary.py:entry"]), + ) + selection = _selection(primary, other) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + other_block = rendered.split("### `other.py`", 1)[1].split("###", 1)[0] + assert "**Supporting evidence**" in other_block + + def test_second_candidate_with_no_relationship_is_labeled_independent(self): + primary = _candidate("primary.py", "explicit_path", 4, enrichment=_enrichment()) + unrelated = _candidate("unrelated.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(primary, unrelated) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + unrelated_block = rendered.split("### `unrelated.py`", 1)[1].split("###", 1)[0] + + assert "**Additional candidate**" in unrelated_block + assert "**Supporting evidence**" not in unrelated_block + + def test_text_search_only_connection_does_not_count_as_supporting(self): + # callers_by_text_search is regex-based and noisier than the call + # graph -- _find_call_relationships deliberately never builds a + # relationship from it, so role assignment must not either. + primary = _candidate( + "primary.py", "explicit_path", 4, + enrichment=_enrichment( + callers_by_text_search=[ + {"id": "other.py:maybe", "name": "maybe", "file": "other.py", "matches": []} + ], + ), + ) + other = _candidate("other.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(primary, other) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + other_block = rendered.split("### `other.py`", 1)[1].split("###", 1)[0] + assert "**Additional candidate**" in other_block + assert "**Supporting evidence**" not in other_block + + def test_only_relationships_touching_the_primary_count_as_supporting(self): + # b and c are connected to EACH OTHER but neither is connected to + # the primary (a) -- that must not make either one "supporting"; + # the signal is specifically "connected to the primary." + a = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment(callees=["c.py:helper"])) + c = _candidate("c.py", "cwe_keywords", 1, enrichment=_enrichment()) + selection = _selection(a, b, c, max_candidates=3) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + b_block = rendered.split("### `b.py`", 1)[1].split("### `c.py`", 1)[0] + c_block = rendered.split("### `c.py`", 1)[1].split("###", 1)[0] + + assert "**Additional candidate**" in b_block + assert "**Additional candidate**" in c_block + + def test_role_labels_do_not_change_candidate_order(self): + a = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(a, b) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert rendered.index("`a.py`") < rendered.index("`b.py`") + + +class TestRenderRelationshipsAndNotes: + def test_relationship_rendered_once_not_duplicated_in_notes(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment( + resolved_function={"id": "a.py:foo", "name": "foo", "startLine": 1, "endLine": 3}, + callees=["b.py:bar"], + ), + ) + b = _candidate( + "b.py", "symbol_search", 2, + enrichment=_enrichment( + resolved_function={"id": "b.py:bar", "name": "bar", "startLine": 1, "endLine": 3}, + ), + ) + selection = _selection(a, b) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "### Structural relationships" in rendered + assert "direct call-graph relationship" in rendered + # the raw fusion-note form of this same relationship must not also + # be echoed under Notes -- stated once, not twice. + assert "a.py calls b.py (via b.py:bar)" not in rendered + + def test_callers_by_text_search_never_rendered_as_a_relationship(self): + a = _candidate( + "a.py", "explicit_path", 4, + enrichment=_enrichment( + callers_by_text_search=[{"id": "b.py:bar", "name": "bar", "file": "b.py", "matches": []}], + ), + ) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(a, b) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + section = rendered.split("### Structural relationships", 1)[1].split("###", 1)[0] + assert "None detected." in section + assert "b.py:bar" not in section + + def test_divergence_note_visible_in_notes_section(self): + strong_unresolved = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + selection = _selection(strong_unresolved) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + section = rendered.split("### Notes", 1)[1].split("###", 1)[0] + assert "strong grounding" in section + assert "a.py" in section + + def test_investigation_context_unavailable_is_explicit_and_not_duplicated(self): + a = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=False) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert "### Investigation context" in rendered + assert "Investigation context unavailable" in rendered + notes_section = rendered.split("### Notes", 1)[1].split("### Investigation context", 1)[0] + assert "investigation context unavailable" not in notes_section + + +class TestRenderEmptyUnderstanding: + def test_empty_understanding_renders_a_valid_honest_section(self): + selection = _selection() + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding) + + assert rendered.startswith("## Repository Understanding") + assert "No repository candidates were selected" in rendered + assert "### Structural relationships" in rendered + assert "None detected." in rendered + assert "### Notes" in rendered + assert "None." in rendered + assert "### Investigation context" in rendered + assert len(rendered) <= DEFAULT_MAX_CHARS + + +class TestRenderCharacterBudget: + def test_representative_two_candidate_understanding_fits_default_budget(self): + """A realistically-enriched, two-candidate understanding -- the + common case, since candidate_selection.py caps selection at 3 -- + must fit under DEFAULT_MAX_CHARS without omitting either + candidate.""" + retry = _candidate( + "src/urllib3/util/retry.py", "explicit_path", 4, hit_line=92, + enrichment=_enrichment( + resolved_function={ + "id": "src/urllib3/util/retry.py:Retry.increment", + "name": "increment", "startLine": 92, "endLine": 134, + }, + callees=[ + "src/urllib3/util/retry.py:Retry.is_retry", + "src/urllib3/util/retry.py:Retry.get_backoff_time", + ], + callers_by_call_graph=["src/urllib3/connectionpool.py:HTTPConnectionPool.urlopen"], + is_reachable_from_entry_point=True, + entry_point_path=[ + "src/urllib3/poolmanager.py:PoolManager.urlopen", + "src/urllib3/connectionpool.py:HTTPConnectionPool.urlopen", + "src/urllib3/util/retry.py:Retry.increment", + ], + related_tests=[ + {"path": "test/test_retry.py", "proximity": "same-module", "reason": "imports urllib3.util.retry"}, + ], + test_support_rating=("Good", 0.05, {}), + ), + ) + connectionpool = _candidate( + "src/urllib3/connectionpool.py", "symbol_search", 2, hit_line=700, + enrichment=_enrichment( + resolution_note="no function contains hit_line 700; used nearest function by start line", + callers_by_text_search=[ + {"id": "src/urllib3/util/retry.py:Retry.increment", "name": "increment", + "file": "src/urllib3/util/retry.py", "matches": []}, + ], + sink_matches=[ + {"file": "src/urllib3/connectionpool.py", "line": 705, "method": "urlopen", + "snippet": 'headers.pop("Authorization", None)'}, + ], + ), + ) + selection = _selection(retry, connectionpool) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding) # default budget + + assert len(rendered) <= DEFAULT_MAX_CHARS + assert "omitted" not in rendered + assert "truncated" not in rendered + assert "`src/urllib3/util/retry.py`" in rendered + assert "`src/urllib3/connectionpool.py`" in rendered + + def test_budget_respected_with_no_truncation_needed(self): + a = _candidate("a.py", "explicit_path", 4) + b = _candidate("b.py", "symbol_search", 2) + selection = _selection(a, b) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=4_000) + + assert len(rendered) <= 4_000 + assert "omitted" not in rendered + assert "truncated" not in rendered + + def test_budget_never_exceeded_with_many_heavy_candidates(self): + candidates = [_heavy_candidate(i) for i in range(10)] + selection = _selection(*candidates, max_candidates=10) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=500) + + assert len(rendered) <= 500 + + def test_budget_never_exceeded_at_a_pathologically_small_budget(self): + a = _candidate("a.py", "explicit_path", 4, enrichment=_enrichment()) + selection = _selection(a) + understanding = fuse_evidence(selection, investigation_context_available=True) + + rendered = render_repository_understanding(understanding, max_chars=10) + + assert len(rendered) <= 10 + + def test_truncation_is_explicit_and_deterministic(self): + candidates = [_heavy_candidate(i) for i in range(3)] + selection = _selection(*candidates, max_candidates=3) + understanding = fuse_evidence(selection, investigation_context_available=True) + + budget = 900 + rendered_1 = render_repository_understanding(understanding, max_chars=budget) + rendered_2 = render_repository_understanding(understanding, max_chars=budget) + + assert rendered_1 == rendered_2, "rendering must be a pure, deterministic function" + assert len(rendered_1) <= budget + assert f"omitted to stay within the {budget}-character budget" in rendered_1 + omission_text = rendered_1[rendered_1.index("omitted to stay within") :] + assert any(f"file_{i}.py" in omission_text for i in range(3)) + + +class TestRenderPurity: + def test_render_does_not_mutate_understanding_or_candidates(self): + enrichment = _enrichment( + resolved_function={"id": "a.py:foo", "name": "foo", "startLine": 1, "endLine": 3}, + callees=["b.py:bar"], + ) + a = _candidate("a.py", "explicit_path", 4, enrichment=enrichment) + b = _candidate("b.py", "symbol_search", 2, enrichment=_enrichment()) + selection = _selection(a, b) + understanding = fuse_evidence(selection, investigation_context_available=True) + + candidate_evidence_before = list(understanding.candidate_evidence) + relationships_before = list(understanding.relationships) + fusion_notes_before = list(understanding.fusion_notes) + evidence_before = list(a.evidence) + best_tier_before = a.best_tier + enrichment_before = a.enrichment + + render_repository_understanding(understanding, max_chars=100) # tiny budget, forces truncation path + render_repository_understanding(understanding, max_chars=4_000) + + assert understanding.candidate_evidence == candidate_evidence_before + assert understanding.relationships == relationships_before + assert understanding.fusion_notes == fusion_notes_before + assert a.evidence == evidence_before + assert a.best_tier == best_tier_before + assert a.enrichment is enrichment_before + + +class TestRenderNoRuntimeIntegration: + def test_module_introduces_no_io_network_or_environment_access(self): + from utilities.autopatcher import evidence_fusion + + source = inspect.getsource(evidence_fusion) + tree = ast.parse(source) + + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + imported.add(node.module) + + disallowed = {"os", "sys", "socket", "subprocess", "requests", "urllib", "pathlib"} + assert not (imported & disallowed), imported diff --git a/libs/openant-core/tests/patch/test_f01_repo_root_fail_closed.py b/libs/openant-core/tests/patch/test_f01_repo_root_fail_closed.py new file mode 100644 index 00000000..2e3b30b0 --- /dev/null +++ b/libs/openant-core/tests/patch/test_f01_repo_root_fail_closed.py @@ -0,0 +1,181 @@ +"""Regression tests for F-01: pipeline.run() must not fall back to +Path.cwd() when no repository root is provided. + +Repository-dependent evidence (Impact Surface, Test Support) must be +treated as unavailable -- routed through the existing F-23/F-24 trust +policy ("Not Verified"/"unavailable") -- rather than silently analyzing +whatever directory the process happens to run in. + +Every end-to-end test here deliberately runs from a temp cwd seeded with +decoy files, so a regression that reintroduces a Path.cwd() fallback would +make these tests fail regardless of the *real* working directory the test +suite happens to run from. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from utilities.autopatcher.pipeline import _compute_trust_signals, run + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" +_VULN_TEXT = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + +_NOT_EVALUATED_TEXT = "Not evaluated — no repository root was provided." + + +def _seed_decoy_cwd(tmp_path: Path) -> None: + """Populate tmp_path with files that would only appear in a report if + something fell back to scanning the process's cwd -- e.g. a + Path.cwd() fallback re-added to pipeline.py. + + `authenticate` matches a symbol name from the vulnerability fixture + (fixtures/examples/vulnerability.md references app/auth.py's + `authenticate()`), so a cwd-based Impact Surface scan would find and + quote this file as "usage evidence". + """ + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_decoy_should_never_appear.py").write_text( + "def test_decoy(): pass\n", encoding="utf-8" + ) + (tmp_path / "decoy_module_should_never_appear.py").write_text( + "def authenticate(username, password):\n return True\n", + encoding="utf-8", + ) + + +@pytest.fixture +def decoy_cwd(tmp_path, monkeypatch): + """A cwd containing files that must never leak into a repo_root=None + report. Fixture (not a bare tmp_path use) so every test in this file + runs from a *different* real directory each time -- the fix must hold + regardless of what the actual process cwd is.""" + _seed_decoy_cwd(tmp_path) + monkeypatch.chdir(tmp_path) + return tmp_path + + +def _run_without_repo_root(monkeypatch) -> str: + monkeypatch.setenv("LLM_PROVIDER", "mock") + return run(vulnerability_text=_VULN_TEXT, api_key="", repo_root=None) + + +class TestNoCwdFallback: + """The core regression: repo_root=None must never scan the process cwd.""" + + def test_decoy_files_never_appear_in_report(self, decoy_cwd, monkeypatch): + report = _run_without_repo_root(monkeypatch) + assert "test_decoy_should_never_appear.py" not in report + assert "decoy_module_should_never_appear.py" not in report + + def test_decoy_cwd_path_never_appears_in_report(self, decoy_cwd, monkeypatch): + report = _run_without_repo_root(monkeypatch) + assert str(decoy_cwd) not in report + + def test_result_independent_of_cwd_identity(self, tmp_path, monkeypatch): + """Running from two different decoy cwds must produce the same + repository-dependent sections -- proving neither run is reading + cwd content into the report.""" + cwd_a = tmp_path / "a" + cwd_b = tmp_path / "b" + cwd_a.mkdir() + cwd_b.mkdir() + _seed_decoy_cwd(cwd_a) + _seed_decoy_cwd(cwd_b) + (cwd_b / "extra_marker_file.py").write_text("MARKER = 1\n", encoding="utf-8") + + monkeypatch.chdir(cwd_a) + report_a = _run_without_repo_root(monkeypatch) + monkeypatch.chdir(cwd_b) + report_b = _run_without_repo_root(monkeypatch) + + assert "extra_marker_file.py" not in report_b + for report in (report_a, report_b): + # Repository Context + Impact Surface + Test Support + assert report.count(_NOT_EVALUATED_TEXT) == 3 + + +class TestExplicitNotEvaluatedMessaging: + """F-01 item 4: sections must state the gap explicitly, not omit it.""" + + def test_impact_surface_states_not_evaluated(self, decoy_cwd, monkeypatch): + report = _run_without_repo_root(monkeypatch) + idx = report.find("## Impact Surface") + assert idx != -1, "Impact Surface section must still be present" + section = report[idx: idx + 300] + assert _NOT_EVALUATED_TEXT in section + + def test_test_support_states_not_evaluated(self, decoy_cwd, monkeypatch): + report = _run_without_repo_root(monkeypatch) + idx = report.find("### Test Support") + assert idx != -1, "Test Support section must still be present" + section = report[idx: idx + 300] + assert _NOT_EVALUATED_TEXT in section + # Must not show the populated-section fields with fabricated values. + assert "Total test files found:" not in section + + def test_trust_signals_table_marks_both_signals_not_verified(self, decoy_cwd, monkeypatch): + report = _run_without_repo_root(monkeypatch) + assert "No repository root was provided" in report + + def test_repository_context_states_not_evaluated_not_zero_selection(self, decoy_cwd, monkeypatch): + """Repository Context must say grounding was never attempted, not + reuse _render_repository_context_section's zero-selection sentence + ("No repository locations were identified...") -- that sentence + also covers "grounding ran and found nothing", so reusing it here + would read as if a repository search happened and came up empty.""" + report = _run_without_repo_root(monkeypatch) + idx = report.find("## Repository Context") + assert idx != -1, "Repository Context section must still be present" + section = report[idx: idx + 300] + assert _NOT_EVALUATED_TEXT in section + assert "No repository locations were identified" not in section + + def test_recommendation_never_deploy_after_validation(self, decoy_cwd, monkeypatch): + """Deployment Safety is forced to Not Verified when no repo_root is + given (impact analysis is skipped, not run against the wrong root), + which the existing F-37 whitelist gate already excludes from + Deploy After Validation -- this must hold even though decoy_cwd + contains files that would otherwise look like real evidence.""" + report = _run_without_repo_root(monkeypatch) + assert "**Deploy After Validation**" not in report + + +class TestComputeTrustSignalsNotVerifiedRating: + """Unit-level coverage for the new testing_rating="Not Verified" branch + added to _compute_trust_signals (mirrors test_trust_package.py's style).""" + + @staticmethod + def _applicability_clean(): + return {"applicable": True, "skipped": False, "skipped_reason": None, "error": None, "stderr": ""} + + @staticmethod + def _classified(): + return {"still_vulnerable": False, "confirmed_defect_count": 0, "plausible_risk_count": 0, "validation_gap_count": 0} + + def test_test_availability_not_verified_for_missing_repo_root(self): + signals = _compute_trust_signals( + [], self._applicability_clean(), self._classified(), "Not Verified", "unavailable" + ) + assert signals["test_availability"]["value"] == "Not Verified" + assert signals["test_availability"]["notes"] == "No repository root was provided" + + def test_test_availability_not_verified_distinct_from_no_tests_found(self): + """"Not Verified" (no repo_root -- nothing was searched) must never + collapse into "No Tests Found" (a search ran and found nothing) -- + those are different claims about different amounts of evidence.""" + not_verified = _compute_trust_signals( + [], self._applicability_clean(), self._classified(), "Not Verified", "unavailable" + ) + no_tests_found = _compute_trust_signals( + [], self._applicability_clean(), self._classified(), "None", "low" + ) + assert not_verified["test_availability"]["value"] != no_tests_found["test_availability"]["value"] + + def test_deployment_safety_not_verified_when_impact_unavailable(self): + signals = _compute_trust_signals( + [], self._applicability_clean(), self._classified(), "Not Verified", "unavailable" + ) + assert signals["deployment_safety"]["value"] == "Not Verified" diff --git a/libs/openant-core/tests/patch/test_finding_calibration.py b/libs/openant-core/tests/patch/test_finding_calibration.py new file mode 100644 index 00000000..aabedfb2 --- /dev/null +++ b/libs/openant-core/tests/patch/test_finding_calibration.py @@ -0,0 +1,106 @@ +"""Unit tests for the finding calibration stage (evidence-quality pass). + +Covers the two things that must be robust against imperfect LLM output: +_parse_response's fallback behavior (never drop a finding, never invent an +invalid group), and calibrate_findings's LLM-call contract (empty input never +calls the LLM; mock mode returns something parseable). +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +from utilities.autopatcher.finding_calibration import _parse_response, calibrate_findings + + +class TestParseResponse: + def test_well_formed_response_parses_in_order(self): + resp = ( + "1. Group: Observed\n" + " Reworded: The constructor normalizes casing via h.lower().\n\n" + "2. Group: Hypothesis\n" + " Reworded: Same-origin redirects may also strip Cookie if stripping is not scoped.\n" + ) + findings = ["finding one", "finding two"] + result = _parse_response(resp, findings) + assert len(result) == 2 + assert result[0] == { + "original": "finding one", "group": "observed", + "reworded": "The constructor normalizes casing via h.lower().", + } + assert result[1]["group"] == "hypothesis" + + def test_missing_block_falls_back_to_original_as_hypothesis(self): + """Fewer blocks than findings must not drop the uncovered finding.""" + resp = "1. Group: Observed\n Reworded: Reworded first finding.\n" + findings = ["first finding", "second finding with no block"] + result = _parse_response(resp, findings) + assert len(result) == 2 + assert result[1] == { + "original": "second finding with no block", + "group": "hypothesis", + "reworded": "second finding with no block", + } + + def test_invalid_group_name_falls_back_to_hypothesis(self): + resp = "1. Group: Definitely\n Reworded: Some reworded text.\n" + result = _parse_response(resp, ["original text"]) + assert result[0]["group"] == "hypothesis" + assert result[0]["reworded"] == "original text" + + def test_empty_reworded_falls_back_to_original(self): + resp = "1. Group: Observed\n Reworded: \n" + result = _parse_response(resp, ["original text"]) + assert result[0]["reworded"] == "original text" + assert result[0]["group"] == "hypothesis" + + def test_completely_unparseable_response_falls_back_for_every_finding(self): + result = _parse_response("not a structured response at all", ["a", "b", "c"]) + assert len(result) == 3 + assert all(r["group"] == "hypothesis" for r in result) + assert [r["reworded"] for r in result] == ["a", "b", "c"] + + def test_empty_findings_list_returns_empty(self): + assert _parse_response("anything", []) == [] + + def test_reworded_text_collapses_internal_whitespace(self): + resp = "1. Group: Hardening\n Reworded: Line one\n continues on line two.\n" + result = _parse_response(resp, ["x"]) + assert "\n" not in result[0]["reworded"] + + def test_group_name_case_insensitive(self): + resp = "1. Group: HARDENING\n Reworded: Some text.\n" + result = _parse_response(resp, ["x"]) + assert result[0]["group"] == "hardening" + + +class TestCalibrateFindings: + def test_empty_findings_returns_empty_without_calling_llm(self): + llm = mock.MagicMock() + result = calibrate_findings("vuln text", "patch", [], llm, code_context="ctx") + assert result == [] + llm.complete.assert_not_called() + + def test_calls_llm_with_stage_label_and_parses_result(self): + llm = mock.MagicMock() + llm.complete.return_value = "1. Group: Observed\n Reworded: Reworded finding.\n" + result = calibrate_findings("vuln text", "patch", ["a finding"], llm, code_context="ctx") + _, kwargs = llm.complete.call_args + assert kwargs.get("stage") == "finding_calibration" + assert result == [{"original": "a finding", "group": "observed", "reworded": "Reworded finding."}] + + def test_user_message_includes_code_context_and_findings(self): + llm = mock.MagicMock() + llm.complete.return_value = "1. Group: Hardening\n Reworded: x\n" + calibrate_findings("VULN_MARKER", "PATCH_MARKER", ["FINDING_MARKER"], llm, code_context="CONTEXT_MARKER") + args, kwargs = llm.complete.call_args + user_message = args[1] if len(args) > 1 else kwargs.get("user_message") + assert "VULN_MARKER" in user_message + assert "PATCH_MARKER" in user_message + assert "FINDING_MARKER" in user_message + assert "CONTEXT_MARKER" in user_message diff --git a/libs/openant-core/tests/patch/test_impact_surface.py b/libs/openant-core/tests/patch/test_impact_surface.py new file mode 100644 index 00000000..cf71d3f4 --- /dev/null +++ b/libs/openant-core/tests/patch/test_impact_surface.py @@ -0,0 +1,525 @@ +import os +from pathlib import Path +import json + +from utilities.autopatcher.impact_surface import LightweightImpactAnalyzer +from utilities.autopatcher.pipeline import enhance_findings_with_impact, TargetRepoContext + + +def write_file(p: Path, text: str): + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(text, encoding="utf-8") + + +def make_diff_for(path: str) -> str: + # minimal unified diff header for file + return f"+++ b/{path}\n@@ -1,1 +1,3 @@\n+def placeholder():\n" + + +def test_low_impact_same_file(tmp_path): + # repo with a.py only; changed symbol used only in same file + repo = tmp_path / "repo" + repo.mkdir() + a = repo / "a.py" + write_file(a, "def foo(x):\n return x\n\nprint(foo(1))\n") + + diff = """+++ b/a.py +@@ -1,3 +1,5 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "low" + assert isinstance(report.impact_summary, str) + + +def test_medium_impact_two_files(tmp_path): + repo = tmp_path / "repo2" + repo.mkdir() + a = repo / "a.py" + b = repo / "b.py" + c = repo / "c.py" + write_file(a, "def foo(x):\n return x\n") + write_file(b, "from a import foo\nprint(foo(2))\n") + write_file(c, "x = foo(3)\n") + + diff = """+++ b/a.py +@@ -1,1 +1,3 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "medium" + assert len(report.affected_files) == 2 + + +def test_high_impact_three_files_or_entrypoint(tmp_path): + repo = tmp_path / "repo3" + repo.mkdir() + a = repo / "a.py" + b = repo / "b.py" + c = repo / "c.py" + d = repo / "api" / "routes.py" + write_file(a, "def foo(x):\n return x\n") + write_file(b, "print(foo(2))\n") + write_file(c, "print(foo(3))\n") + write_file(d, "from a import foo\n# route uses foo\n") + + diff = """+++ b/a.py +@@ -1,1 +1,3 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + # either >=3 external files or entrypoint hit should mark high + assert report.impact_level == "high" + + +def test_enhance_findings_with_impact(): + challenger = {} + # simulate impact report dicts + high = {"impact_level": "high"} + med = {"impact_level": "medium"} + low = {"impact_level": "low"} + + # high + c1 = dict(challenger) + enhance_findings_with_impact(c1, high) + assert "impact_annotations" in c1 and any("propagate" in s for s in c1["impact_annotations"]) + + # medium + c2 = dict(challenger) + enhance_findings_with_impact(c2, med) + assert "impact_annotations" in c2 and len(c2["impact_annotations"]) > 0 + + # low + c3 = dict(challenger) + enhance_findings_with_impact(c3, low) + assert "impact_annotations" not in c3 + + +def test_constant_hunk_does_not_extract_init(tmp_path): + """Regression: constant-only hunk must not fall back to __init__ and produce HIGH impact. + + Reproduces the urllib3 cookie-redirect case: a frozenset constant is + modified, no def appears in the hunk, and several other files define + __init__. Without the fix the upward scan grabs __init__, the search + matches every class file, and impact is incorrectly HIGH. + """ + repo = tmp_path / "repo_constant" + repo.mkdir() + + # The patched file: class with __init__ above a class-level constant. + write_file( + repo / "retry.py", + "\n".join([ + "class Retry:", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + ]), + ) + + # Five other files that each define __init__ — would inflate count if searched. + for i in range(5): + write_file( + repo / f"module{i}.py", + f"class Foo{i}:\n def __init__(self):\n pass\n", + ) + + # Diff touches only the constant; no def in the hunk. + diff = ( + "--- a/retry.py\n" + "+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert "__init__" not in report.changed_symbols, ( + "constant-only hunk must not fall back to __init__" + ) + assert report.impact_level == "low", ( + f"expected low impact for constant change, got {report.impact_level!r}; " + f"affected_files={report.affected_files}" + ) + + +def test_sensitive_auth_bumps_low_to_medium(tmp_path): + # changed auth file with no external usages should be bumped to medium + repo = tmp_path / "repo_sensitive" + (repo / "app").mkdir(parents=True) + auth = repo / "app" / "auth.py" + write_file(auth, "def authenticate(user, pwd):\n return True\n") + + diff = """+++ b/app/auth.py +@@ -1,1 +1,3 @@ ++def authenticate(user, pwd): ++ return True +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "medium" + assert "authenticate" in (report.changed_symbols or []) + + +def test_non_python_repo_reports_not_applicable_not_low(tmp_path): + # A C change with zero extractable Python symbols must not be scored + # "low impact" (which reads as a reassuring, verified-clean finding). + # It must be explicitly "not_applicable" instead. + repo = tmp_path / "repo_c" + repo.mkdir() + write_file(repo / "lib" / "http.c", "int Curl_follow(void) { return 0; }\n") + + diff = """+++ b/lib/http.c +@@ -1,1 +1,3 @@ ++int Curl_follow(void) { ++ return 0; ++} +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx, repo_language="c") + + assert report.impact_level == "not_applicable" + assert report.impact_level != "low" + assert report.changed_symbols == [] + assert report.usage_matches == [] + assert report.affected_files == [] + # Diff parsing itself is language-agnostic and must still work. + assert report.changed_files == ["lib/http.c"] + assert "Not Applicable" in report.impact_summary + + +def test_python_repo_impact_unchanged_by_default(tmp_path): + # Default repo_language="python" must reproduce prior behavior exactly. + repo = tmp_path / "repo_default" + repo.mkdir() + a = repo / "a.py" + write_file(a, "def foo(x):\n return x\n\nprint(foo(1))\n") + + diff = """+++ b/a.py +@@ -1,3 +1,5 @@ ++def foo(x): ++ return x +""" + + analyzer = LightweightImpactAnalyzer() + ctx = TargetRepoContext(repo) + report = analyzer.analyze(diff, repo_context=ctx) + + assert report.impact_level == "low" + + +# --------------------------------------------------------------------------- +# Symbol-resolution robustness (the urllib3 hunk-drift incident and its +# resilience requirements: diff line offsets, nearby comments, whitespace-only +# edits, insertion/deletion before the target, equivalent patch formatting). +# --------------------------------------------------------------------------- + +_RETRY_FILE = "\n".join([ + "class Retry:", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + "", + " def new(self, **kw):", + " return Retry(**kw)", +]) + +# Other files that each define __init__ — if the analyzer ever falls back to +# that name, usage search inflates across all of them, exactly reproducing +# the false-HIGH-impact incident. +_OTHER_INIT_FILES = {f"module{i}.py": f"class Foo{i}:\n def __init__(self):\n pass\n" for i in range(5)} + + +def _write_retry_repo(tmp_path, retry_text=_RETRY_FILE, extra_files=None): + repo = tmp_path / "repo" + repo.mkdir() + write_file(repo / "retry.py", retry_text) + for name, content in {**_OTHER_INIT_FILES, **(extra_files or {})}.items(): + write_file(repo / name, content) + return repo + + +class TestShiftedHunkHeaders: + """The core incident: a hunk header claiming the wrong line number for + byte-identical content must not change which symbol is resolved.""" + + def test_correct_header_resolves_constant(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert report.impact_level == "low" + + def test_wildly_shifted_header_still_resolves_the_same_constant(self, tmp_path): + """Same file, same content change, header claims line 50 instead of + line 2 — the actual incident, reproduced deterministically.""" + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -50,2 +50,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert "__init__" not in report.changed_symbols + assert report.impact_level == "low" + + def test_shifted_header_on_a_function_change_still_resolves_correctly(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -999,2 +999,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + def test_two_runs_with_different_headers_same_content_agree(self, tmp_path): + """Direct stability check: semantically identical patches, differing + only in hunk header line number, must produce the identical report.""" + repo = _write_retry_repo(tmp_path) + body = ( + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + diff_a = f"--- a/retry.py\n+++ b/retry.py\n@@ -189,2 +189,2 @@\n{body}" + diff_b = f"--- a/retry.py\n+++ b/retry.py\n@@ -196,2 +196,2 @@\n{body}" + + report_a = LightweightImpactAnalyzer().analyze(diff_a, repo_context=TargetRepoContext(repo)) + report_b = LightweightImpactAnalyzer().analyze(diff_b, repo_context=TargetRepoContext(repo)) + + assert report_a.changed_symbols == report_b.changed_symbols + assert report_a.impact_level == report_b.impact_level + assert report_a.affected_files == report_b.affected_files + + +class TestSymbolKindRendering: + """Release-polish change #9: the low-impact summary's changed-symbol + mention must only append "()" for a function/method — a constant, + class attribute, or field must never render as if it were callable + (e.g. `DEFAULT_REMOVE_HEADERS_ON_REDIRECT()` for a changed constant).""" + + def test_constant_change_does_not_render_as_callable(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.impact_level == "low" + assert "DEFAULT_REMOVE_HEADERS()" not in report.impact_summary + assert "`DEFAULT_REMOVE_HEADERS`" in report.impact_summary + + def test_function_change_still_renders_as_callable(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -999,2 +999,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + assert report.impact_level == "low" + assert "`new()`" in report.impact_summary + + +class TestWhitespaceOnlyChanges: + def test_reindentation_only_produces_no_symbol_and_low_impact(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == [] + assert report.impact_level == "low" + + def test_trailing_whitespace_only_produces_no_symbol(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,1 +7,1 @@\n" + "- def new(self, **kw):\n" + "+ def new(self, **kw): \n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == [] + + +class TestInsertionBeforeTarget: + def test_lines_inserted_above_target_do_not_break_resolution(self, tmp_path): + """The header's claimed line number is stale (as if computed before + 20 lines were inserted above the target) — content match must still + find the real, shifted location.""" + padded = "\n".join([f"# padding line {i}" for i in range(20)]) + "\n" + _RETRY_FILE + repo = _write_retry_repo(tmp_path, retry_text=padded) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" # stale: correct pre-padding location + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + + +class TestNearbyComments: + def test_misleading_def_in_comment_is_ignored(self, tmp_path): + """A comment that looks like a function definition must never be + mistaken for real code — ast parsing never sees comments at all.""" + text = "\n".join([ + "class Retry:", + " # def __init__(self): pass -- old implementation, removed", + " DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])", + "", + " def __init__(self, retries=3):", + " self.retries = retries", + ]) + repo = _write_retry_repo(tmp_path, retry_text=text) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,3 +1,3 @@\n" + " class Retry:\n" + " # def __init__(self): pass -- old implementation, removed\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + assert "__init__" not in report.changed_symbols + + +class TestConstantAssignmentChanges: + def test_class_level_constant(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_REMOVE_HEADERS"] + + def test_module_level_constant(self, tmp_path): + text = "DEFAULT_TIMEOUT = 30\n\ndef connect():\n return DEFAULT_TIMEOUT\n" + repo = _write_retry_repo(tmp_path, retry_text=text) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,1 +1,1 @@\n" + "-DEFAULT_TIMEOUT = 30\n" + "+DEFAULT_TIMEOUT = 60\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["DEFAULT_TIMEOUT"] + + +class TestFunctionChanges: + def test_change_inside_method_body_resolves_to_method_name(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + def test_change_to_signature_line_resolves_to_function_name(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -7,1 +7,1 @@\n" + "- def new(self, **kw):\n" + "+ def new(self, **kw, strict=False):\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert report.changed_symbols == ["new"] + + +class TestMultipleHunks: + def test_two_hunks_in_same_file_resolve_two_distinct_symbols(self, tmp_path): + repo = _write_retry_repo(tmp_path) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + "@@ -7,2 +7,2 @@\n" + " def new(self, **kw):\n" + "- return Retry(**kw)\n" + "+ return Retry(**kw, extra=True)\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert set(report.changed_symbols) == {"DEFAULT_REMOVE_HEADERS", "new"} + + def test_hunks_across_two_files_resolve_independently(self, tmp_path): + repo = _write_retry_repo(tmp_path, extra_files={ + "helpers.py": "def build_url():\n return ''\n", + }) + diff = ( + "--- a/retry.py\n+++ b/retry.py\n" + "@@ -1,2 +1,2 @@\n" + " class Retry:\n" + "- DEFAULT_REMOVE_HEADERS = frozenset(['Authorization'])\n" + "+ DEFAULT_REMOVE_HEADERS = frozenset(['Authorization', 'Cookie'])\n" + "--- a/helpers.py\n+++ b/helpers.py\n" + "@@ -1,2 +1,2 @@\n" + "-def build_url():\n" + "- return ''\n" + "+def build_url():\n" + "+ return 'https://'\n" + ) + report = LightweightImpactAnalyzer().analyze(diff, repo_context=TargetRepoContext(repo)) + assert set(report.changed_symbols) == {"DEFAULT_REMOVE_HEADERS", "build_url"} + assert set(report.changed_files) == {"retry.py", "helpers.py"} diff --git a/libs/openant-core/tests/patch/test_investigation_adapters.py b/libs/openant-core/tests/patch/test_investigation_adapters.py new file mode 100644 index 00000000..75e1fc5e --- /dev/null +++ b/libs/openant-core/tests/patch/test_investigation_adapters.py @@ -0,0 +1,144 @@ +"""Characterization tests for the InvestigationCase adapters. + +Ported from the standalone Auto Patcher project's test_investigation_adapters.py. +TestCaseFromVulnerabilityText covers case_from_vulnerability_text (file mode, +i.e. core/patch.py's rendered Finding markdown). TestCaseFromCve covers +case_from_cve, added to support patching directly from a known CVE +identifier. The GHSA adapter and the evidence-collection classes tested +functionality that was excluded during the merge into OpenAnt (see +utilities/autopatcher/investigation_adapters.py's module docstring). + +These tests prove the one guarantee that matters here: an InvestigationCase +built from either input projects back to byte-identical text -- for +case_from_vulnerability_text, core/patch.py's render_vulnerability_markdown() +output; for case_from_cve, cve_converter.cve_to_vuln_text()'s output. Neither +must be altered en route to the patch engine. +""" + +from __future__ import annotations + +from pathlib import Path + +EXAMPLE_FILE = Path(__file__).parent / "fixtures" / "examples" / "vulnerability.md" + +FIXTURE_CVE = { + "id": "CVE-2021-12345", + "descriptions": [ + {"lang": "en", "value": "A SQL injection vulnerability exists in the authenticate() function."} + ], + "metrics": { + "cvssMetricV31": [ + { + "source": "nvd@nist.gov", + "type": "Primary", + "cvssData": {"baseScore": 9.8, "baseSeverity": "CRITICAL"}, + } + ] + }, + "weaknesses": [ + {"source": "nvd@nist.gov", "type": "Primary", "description": [{"lang": "en", "value": "CWE-89"}]} + ], + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "vulnerable": True, + "criteria": "cpe:2.3:a:example:example-lib:*:*:*:*:*:*:*:*", + } + ] + } + ] + } + ], + "references": [{"url": "https://example.com/advisory", "source": "nvd@nist.gov"}], +} + + +class TestCaseFromVulnerabilityText: + def test_round_trips_example_file_byte_identical(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + original = EXAMPLE_FILE.read_text(encoding="utf-8") + case = case_from_vulnerability_text(original, repo_root=Path("/tmp/repo")) + projection = case.to_context_projection() + + assert projection.vulnerability_text == original + assert projection.repo_root == Path("/tmp/repo") + + def test_repo_root_optional(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + case = case_from_vulnerability_text("# Some vuln\n\nDetails.") + assert case.to_context_projection().repo_root is None + + def test_framing_summary_extracted(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + case = case_from_vulnerability_text("# SQL Injection Vulnerability\n\nDetails.") + assert case.framing.summary == "SQL Injection Vulnerability" + + def test_raw_artifact_preserves_source_type_and_text(self): + from utilities.autopatcher.investigation_adapters import case_from_vulnerability_text + + text = "# Vuln\n\nDetails." + case = case_from_vulnerability_text(text) + assert case.raw_artifact.source_type == "vulnerability_text" + assert case.raw_artifact.raw_text == text + + +class TestCaseFromCve: + def test_projection_byte_identical_to_cve_to_vuln_text(self): + from utilities.autopatcher.cve_converter import cve_to_vuln_text + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve(FIXTURE_CVE, repo_root=Path("/tmp/repo")) + projection = case.to_context_projection() + + assert projection.vulnerability_text == cve_to_vuln_text(FIXTURE_CVE) + assert projection.repo_root == Path("/tmp/repo") + + def test_repo_root_optional(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve(FIXTURE_CVE) + assert case.to_context_projection().repo_root is None + + def test_raw_artifact_preserves_source_type_url_and_structured_payload(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve(FIXTURE_CVE) + assert case.raw_artifact.source_type == "cve" + assert case.raw_artifact.source_url == "https://nvd.nist.gov/vuln/detail/CVE-2021-12345" + assert case.raw_artifact.structured_payload == FIXTURE_CVE + + def test_raw_artifact_source_url_none_when_id_missing(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve({}) + assert case.raw_artifact.source_url is None + + def test_framing_extracted_from_cve(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve(FIXTURE_CVE) + assert case.framing.cwes == ["CWE-89"] + assert case.framing.severity == "CRITICAL" + assert case.framing.packages == [{"cpe": "cpe:2.3:a:example:example-lib:*:*:*:*:*:*:*:*"}] + assert "SQL injection" in case.framing.summary or "authenticate" in case.framing.summary + + def test_evidence_stays_empty(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve(FIXTURE_CVE, repo_root=Path("/tmp/repo")) + assert case.evidence == [] + + def test_handles_sparse_cve_without_crash(self): + from utilities.autopatcher.investigation_adapters import case_from_cve + + case = case_from_cve({"id": "CVE-0000-00000"}) + projection = case.to_context_projection() + assert "CVE-0000-00000" in projection.vulnerability_text + assert case.framing.cwes == [] + assert case.framing.packages == [] diff --git a/libs/openant-core/tests/patch/test_investigation_integration.py b/libs/openant-core/tests/patch/test_investigation_integration.py new file mode 100644 index 00000000..3be92798 --- /dev/null +++ b/libs/openant-core/tests/patch/test_investigation_integration.py @@ -0,0 +1,332 @@ +"""Integration tests for the live Repository Understanding wiring. + +Covers the orchestration boundary approved for this phase: + + ground_repository() -> select_candidates() -> build_investigation_context() + -> enrich_candidates() -> fuse_evidence() -> render_repository_understanding() + +now running once inside pipeline.run() (reusing the single existing +ground_repository() call) and reaching generate_patch()'s code_context. +core/patch.py-level coverage (repo_root normalization, the run-scoped +investigation directory) lives in test_run_patch_cve.py and +test_patch_wrapper_contract.py instead, alongside each entry point's other +contract tests. + +Hermetic: LLM_PROVIDER=mock, no network, real (but tiny) on-disk repos under +tmp_path so ground_repository()/parse_repository() have real, deterministic +work to do. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest import mock + +import utilities.autopatcher.candidate_enrichment as _ce_mod +import utilities.autopatcher.candidate_selection as _cs_mod +import utilities.autopatcher.evidence_fusion as _ef_mod +import utilities.autopatcher.patch_generator as _pg_mod +from utilities.autopatcher.pipeline import run as pipeline_run + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" +_VULN_TEXT = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + +def _write_auth_repo(root: Path) -> None: + """A tiny real repo matching fixtures/examples/vulnerability.md's + explicit `app/auth.py` / `authenticate()` reference -- gives + ground_repository() a real, strong-tier (explicit_path) candidate to + select, and the parser a real function to resolve.""" + auth = root / "app" / "auth.py" + auth.parent.mkdir(parents=True) + auth.write_text( + "import sqlite3\n\n" + "db = sqlite3.connect(\"users.db\")\n\n" + "def authenticate(username, password):\n" + " query = f\"SELECT * FROM users WHERE username='{username}'\"\n" + " return db.execute(query).fetchone() is not None\n", + encoding="utf-8", + ) + + +def _capture_generate_patch(): + """Patch pipeline.generate_patch to record (vulnerability_text, + code_context) while still delegating to the real (mock-LLM) + implementation, mirroring test_pipeline.py's TestPipelineCodeContext + convention.""" + captured: list[dict] = [] + original = _pg_mod.generate_patch + + def _capturing(vtext, llm, code_context="", retry_hint=""): + captured.append({"vulnerability_text": vtext, "code_context": code_context}) + return original(vtext, llm, code_context=code_context, retry_hint=retry_hint) + + return mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=_capturing), captured + + +class TestInvestigationRunsOnce: + """Each investigation stage must run exactly once per pipeline.run() + call -- no duplicate grounding, no duplicate parsing, no repeated + candidate selection.""" + + def test_each_stage_called_exactly_once(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + calls = {"select": 0, "build_context": 0, "enrich": 0, "fuse": 0, "render": 0} + original_select = _cs_mod.select_candidates + original_build = _ce_mod.build_investigation_context + original_enrich = _ce_mod.enrich_candidates + original_fuse = _ef_mod.fuse_evidence + original_render = _ef_mod.render_repository_understanding + + def _select(*a, **kw): + calls["select"] += 1 + return original_select(*a, **kw) + + def _build(*a, **kw): + calls["build_context"] += 1 + return original_build(*a, **kw) + + def _enrich(*a, **kw): + calls["enrich"] += 1 + return original_enrich(*a, **kw) + + def _fuse(*a, **kw): + calls["fuse"] += 1 + return original_fuse(*a, **kw) + + def _render(*a, **kw): + calls["render"] += 1 + return original_render(*a, **kw) + + with ( + mock.patch.object(_cs_mod, "select_candidates", side_effect=_select), + mock.patch.object(_ce_mod, "build_investigation_context", side_effect=_build), + mock.patch.object(_ce_mod, "enrich_candidates", side_effect=_enrich), + mock.patch.object(_ef_mod, "fuse_evidence", side_effect=_fuse), + mock.patch.object(_ef_mod, "render_repository_understanding", side_effect=_render), + ): + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + assert calls == {"select": 1, "build_context": 1, "enrich": 1, "fuse": 1, "render": 1} + + def test_no_investigation_when_no_repo_root(self, monkeypatch): + """repo_root=None must never trigger selection/enrichment/fusion -- + matches F-01's no-cwd-fallback guarantee.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + + with mock.patch.object(_cs_mod, "select_candidates") as m_select: + pipeline_run(vulnerability_text=_VULN_TEXT, api_key="", repo_root=None) + + m_select.assert_not_called() + + def test_no_new_llm_call_is_introduced(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + patcher, captured = _capture_generate_patch() + with patcher: + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + # Exactly the one, existing patch-generation call -- investigation + # adds repository facts to its input, not a second model call. + assert len(captured) == 1 + + +class TestContextComposition: + def test_repository_understanding_appended_after_existing_context(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + patcher, captured = _capture_generate_patch() + with patcher: + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + assert captured, "generate_patch was never called" + assert captured[0]["vulnerability_text"] == _VULN_TEXT, ( + "vulnerability_text must reach generate_patch unchanged" + ) + + ctx = captured[0]["code_context"] + assert "## Repository Understanding" in ctx + # The real candidate (app/auth.py, found by ground_repository()) must + # be the one rendered -- proves selection/enrichment/fusion ran + # against real repo data, not a stub. + assert "### `app/auth.py`" in ctx + # Appended, not prepended -- existing repo/pattern context precedes it. + idx = ctx.index("## Repository Understanding") + assert idx > 0 and ctx[:idx].strip() != "" + + def test_existing_repo_code_context_still_present(self, tmp_path, monkeypatch): + """Raw repository code context (ground_repository()'s own rendered + snippet of app/auth.py) must still reach code_context unchanged -- + Repository Understanding must complement it, not replace it. + + (Vulnerability-class guidance is not asserted here: classify_vuln_class + only recognizes PATH_TRAVERSAL/COMMAND_INJECTION today, so this + fixture's CWE-89/SQL-injection text never produces that block, + independent of this change.) + """ + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + patcher, captured = _capture_generate_patch() + with patcher: + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + ctx = captured[0]["code_context"] + assert "auth.py" in ctx + assert 'def authenticate(username, password):' in ctx + + +class TestFailureDegradesGracefully: + def test_enrichment_failure_falls_back_to_existing_context(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + def _boom(*a, **kw): + raise RuntimeError("simulated enrichment failure") + + patcher, captured = _capture_generate_patch() + with patcher, mock.patch.object(_ce_mod, "enrich_candidates", side_effect=_boom): + report = pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + assert report # the run completes; a failure here must not abort it + assert captured, "generate_patch must still be called" + ctx = captured[0]["code_context"] + assert "## Repository Understanding" not in ctx + # existing repo context must still make it through untouched + assert "auth.py" in ctx + + +class TestRetentionForLaterReporting: + def test_repository_understanding_retained_on_pipeline_result(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + import utilities.autopatcher.pipeline as _pipeline_module + + captured = {} + original_build_report = _pipeline_module._build_report + + def _capturing_build_report(result): + captured["result"] = result + return original_build_report(result) + + with mock.patch.object(_pipeline_module, "_build_report", side_effect=_capturing_build_report): + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + result = captured["result"] + assert result.repository_understanding is not None + assert result.repository_understanding.candidate_evidence + assert result.repository_understanding.candidate_evidence[0].path.endswith("auth.py") + assert len(result.repository_understanding.candidate_evidence) <= 3 # DEFAULT_MAX_CANDIDATES + + +class TestOnlySelectedCandidatesAreEnriched: + def test_enrich_candidates_receives_select_candidates_own_selected_list(self, tmp_path, monkeypatch): + """Orchestration-wiring check: enrich_candidates() must be called + with select_candidates()'s own bounded `.selected` output, not + `.generated`/`.eligible` -- the actual bounding (<= 3) is already + unit-tested in test_candidate_selection.py.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + investigation_dir = tmp_path / "investigation" + investigation_dir.mkdir() + + captured = {} + original_select = _cs_mod.select_candidates + original_enrich = _ce_mod.enrich_candidates + + def _select(*a, **kw): + selection = original_select(*a, **kw) + captured["selection"] = selection + return selection + + def _enrich(selection, *a, **kw): + captured["enrich_selection_arg"] = selection + return original_enrich(selection, *a, **kw) + + with ( + mock.patch.object(_cs_mod, "select_candidates", side_effect=_select), + mock.patch.object(_ce_mod, "enrich_candidates", side_effect=_enrich), + ): + pipeline_run( + vulnerability_text=_VULN_TEXT, + api_key="", + repo_root=str(repo_root), + investigation_output_dir=str(investigation_dir), + ) + + assert captured["enrich_selection_arg"] is captured["selection"] + assert len(captured["selection"].selected) <= 3 + + +class TestBackwardCompatibility: + def test_run_without_investigation_output_dir_still_works(self): + """The pre-existing call signature -- no repo_root, no + investigation_output_dir -- must remain valid for direct callers + that predate this phase.""" + report = pipeline_run(vulnerability_text=_VULN_TEXT, api_key="") + assert isinstance(report, str) and report.strip() + + def test_run_with_repo_root_but_no_investigation_output_dir_degrades(self, tmp_path, monkeypatch): + """A caller that passes repo_root but not investigation_output_dir + (every existing test in test_pipeline*.py) must still get a report; + candidate enrichment runs in degraded (context=None) mode.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + report = pipeline_run(vulnerability_text=_VULN_TEXT, api_key="", repo_root=str(repo_root)) + assert isinstance(report, str) and report.strip() diff --git a/libs/openant-core/tests/patch/test_investigation_models.py b/libs/openant-core/tests/patch/test_investigation_models.py new file mode 100644 index 00000000..6db8d006 --- /dev/null +++ b/libs/openant-core/tests/patch/test_investigation_models.py @@ -0,0 +1,80 @@ +"""Unit tests for the canonical Investigation Engine data model (Slice 1).""" + +from __future__ import annotations + +import sys +from pathlib import Path + + + +class TestModelConstruction: + def test_raw_artifact_defaults(self): + from utilities.autopatcher.investigation_models import RawArtifact + a = RawArtifact(source_type="free_text", raw_text="hello") + assert a.source_url is None + assert a.structured_payload is None + + def test_problem_claims_defaults_are_independent(self): + from utilities.autopatcher.investigation_models import ProblemClaims + a = ProblemClaims() + b = ProblemClaims() + a.cwes.append("CWE-89") + assert b.cwes == [] # default_factory must not share state across instances + + def test_hypothesis_defaults(self): + from utilities.autopatcher.investigation_models import Hypothesis + h = Hypothesis(id="h1", statement="SQL injection in authenticate()") + assert h.status == "open" + assert h.rejection_reason is None + assert h.confirming_evidence_needed == [] + + def test_evidence_item_defaults(self): + from utilities.autopatcher.investigation_models import EvidenceItem + e = EvidenceItem(evidence_type="location", source_module="repo_locator", content="app/auth.py") + assert e.supports == [] + assert e.refutes == [] + + def test_context_projection_fields(self): + from utilities.autopatcher.investigation_models import ContextProjection + p = ContextProjection(vulnerability_text="# Vuln", repo_root=Path("/tmp/repo")) + assert p.vulnerability_text == "# Vuln" + assert p.repo_root == Path("/tmp/repo") + + +class TestInvestigationCase: + def _make_case(self): + from utilities.autopatcher.investigation_models import InvestigationCase, ProblemClaims, RawArtifact + return InvestigationCase( + raw_artifact=RawArtifact(source_type="vulnerability_text", raw_text="# Vuln\n"), + framing=ProblemClaims(summary="Vuln"), + rendered_text="# Vuln\n", + repo_root=Path("/tmp/repo"), + ) + + def test_defaults_are_empty(self): + case = self._make_case() + assert case.hypotheses == [] + assert case.evidence == [] + assert case.leading_hypothesis_id is None + assert case.open_questions == [] + + def test_to_context_projection_carries_text_and_repo_root(self): + case = self._make_case() + projection = case.to_context_projection() + assert projection.vulnerability_text == "# Vuln\n" + assert projection.repo_root == Path("/tmp/repo") + + def test_to_context_projection_repo_root_optional(self): + from utilities.autopatcher.investigation_models import InvestigationCase, ProblemClaims, RawArtifact + case = InvestigationCase( + raw_artifact=RawArtifact(source_type="vulnerability_text", raw_text="x"), + framing=ProblemClaims(), + rendered_text="x", + ) + assert case.to_context_projection().repo_root is None + + def test_default_lists_are_independent_across_cases(self): + case_a = self._make_case() + case_b = self._make_case() + case_a.open_questions.append("is this the right package?") + assert case_b.open_questions == [] diff --git a/libs/openant-core/tests/patch/test_language_support.py b/libs/openant-core/tests/patch/test_language_support.py new file mode 100644 index 00000000..9a2e3a7d --- /dev/null +++ b/libs/openant-core/tests/patch/test_language_support.py @@ -0,0 +1,70 @@ +"""Tests for src/language_support.py — dominant-language detection used to +gate Python-only deterministic signals on non-Python repositories.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +from utilities.autopatcher.language_support import detect_language, is_python_repo + + +def write(path: Path, content: str = "") -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +def test_detects_python_repo(tmp_path: Path): + write(tmp_path / "src" / "foo.py", "def foo():\n pass\n") + write(tmp_path / "tests" / "test_foo.py", "def test_foo():\n pass\n") + + assert detect_language(tmp_path) == "python" + assert is_python_repo(tmp_path) is True + + +def test_detects_c_repo(tmp_path: Path): + write(tmp_path / "lib" / "http.c", "int Curl_follow(void) { return 0; }\n") + write(tmp_path / "lib" / "http.h", "int Curl_follow(void);\n") + + assert detect_language(tmp_path) == "c" + assert is_python_repo(tmp_path) is False + + +def test_detects_javascript_repo(tmp_path: Path): + write(tmp_path / "index.js", "module.exports = function() {};\n") + write(tmp_path / "lib" / "parse.js", "function parse() {}\n") + + assert detect_language(tmp_path) == "javascript" + assert is_python_repo(tmp_path) is False + + +def test_mixed_repo_uses_dominant_extension(tmp_path: Path): + # A handful of stray .py maintenance scripts must not flip a C repo to "python". + write(tmp_path / "lib" / "a.c", "int a(void) { return 0; }\n") + write(tmp_path / "lib" / "b.c", "int b(void) { return 0; }\n") + write(tmp_path / "lib" / "c.c", "int c(void) { return 0; }\n") + write(tmp_path / "scripts" / "release.py", "print('release')\n") + + assert detect_language(tmp_path) == "c" + + +def test_empty_repo_is_unknown(tmp_path: Path): + assert detect_language(tmp_path) == "unknown" + assert is_python_repo(tmp_path) is False + + +def test_missing_repo_root_is_unknown(): + assert detect_language(None) == "unknown" + assert detect_language(Path("/definitely/does/not/exist/xyz")) == "unknown" + + +def test_ignored_dirs_excluded_from_detection(tmp_path: Path): + write(tmp_path / "lib" / "a.c", "int a(void) { return 0; }\n") + write( + tmp_path / ".venv" / "lib" / "python3.14" / "site-packages" / "pkg" / "mod.py", + "x = 1\n", + ) + + assert detect_language(tmp_path) == "c" diff --git a/libs/openant-core/tests/patch/test_llm_client.py b/libs/openant-core/tests/patch/test_llm_client.py new file mode 100644 index 00000000..52049fbb --- /dev/null +++ b/libs/openant-core/tests/patch/test_llm_client.py @@ -0,0 +1,467 @@ +import os +import sys +import types + +import pytest + +# Ensure `src` is on path so imports like `from llm_client import ...` +# work when running tests directly. + +import utilities.autopatcher.llm_client as llm_client +from utilities.autopatcher.llm_client import LLMClient, call_llm, _mock_response +from utilities.autopatcher.llm_config import DEFAULT_MAX_TOKENS + + +# --------------------------------------------------------------------------- +# LLMClient.is_mock — must reflect the active LLM_PROVIDER, not just OPENAI_API_KEY +# --------------------------------------------------------------------------- + +def test_is_mock_false_when_anthropic_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert not client.is_mock + + +def test_is_mock_false_when_openai_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setattr(llm_client, "_cached_provider", None) + client = LLMClient(api_key="sk-fake") + assert not client.is_mock + + +def test_is_mock_true_when_mock_provider(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + client = LLMClient() + assert client.is_mock + + +def test_is_mock_true_when_no_provider_no_key(monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert client.is_mock + + +def test_is_mock_false_when_cached_provider_anthropic(monkeypatch): + # Simulates a session where the user already selected anthropic interactively. + monkeypatch.delenv("LLM_PROVIDER", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", "anthropic") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + client = LLMClient() + assert not client.is_mock + + +# --------------------------------------------------------------------------- + + +def test_mock_provider_returns_string(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + res = call_llm("test prompt for mock") + assert isinstance(res, str) + assert res + + +def test_anthropic_explicit_provider_raises_on_api_error(monkeypatch): + """LLM_PROVIDER=anthropic + API failure must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + class FakeAnthropic: + def __init__(self, api_key=None): + pass + + class messages: + @staticmethod + def create(model, max_tokens, messages): + raise RuntimeError("simulated anthropic failure") + + fake_mod = types.SimpleNamespace(Anthropic=FakeAnthropic) + monkeypatch.setitem(sys.modules, "anthropic", fake_mod) + + with pytest.raises(RuntimeError, match="Anthropic API call failed"): + call_llm("prompt that triggers anthropic") + + +def test_openai_explicit_provider_raises_on_api_error(monkeypatch): + """LLM_PROVIDER=openai + API failure must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + class FakeOpenAI: + def __init__(self, api_key=None): + pass + + class chat: + class completions: + @staticmethod + def create(model, messages, temperature, max_tokens): + raise RuntimeError("simulated openai failure") + + fake_mod = types.SimpleNamespace(OpenAI=FakeOpenAI) + monkeypatch.setitem(sys.modules, "openai", fake_mod) + + with pytest.raises(RuntimeError, match="OpenAI API call failed"): + call_llm("prompt that triggers openai") + + +def test_anthropic_explicit_provider_raises_on_missing_key(monkeypatch): + """LLM_PROVIDER=anthropic with no key must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("builtins.input", lambda *a, **k: "") + + with pytest.raises(RuntimeError, match="ANTHROPIC_API_KEY is not set"): + call_llm("prompt with no anthropic key") + + +def test_openai_explicit_provider_raises_on_missing_key(monkeypatch): + """LLM_PROVIDER=openai with no key must raise, not fall back to mock.""" + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("builtins.input", lambda *a, **k: "") + + with pytest.raises(RuntimeError, match="OPENAI_API_KEY is not set"): + call_llm("prompt with no openai key") + + +class _FakeInteractiveStdin: + def isatty(self): + return True + + +def _fake_anthropic_ok(captured=None): + class FakeContentBlock: + text = "ok" + + class FakeResponse: + content = [FakeContentBlock()] + stop_reason = "end_turn" + + class FakeAnthropic: + def __init__(self, api_key=None): + if captured is not None: + captured["api_key"] = api_key + + class messages: + @staticmethod + def create(model, max_tokens, messages): + if captured is not None: + captured["model"] = model + return FakeResponse() + + return types.SimpleNamespace(Anthropic=FakeAnthropic) + + +# --------------------------------------------------------------------------- +# Interactive prompt text must reach stderr, never stdout -- stdout is +# reserved for the JSON envelope openant/cli.py writes, and Go's +# python.Invoke parses it as pure JSON. The selected/typed value must still +# flow through correctly regardless of which channel displays the prompt. +# --------------------------------------------------------------------------- + +def test_provider_menu_prompt_text_goes_to_stderr_not_stdout(monkeypatch, capsys): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr("sys.stdin", _FakeInteractiveStdin()) + monkeypatch.setattr("builtins.input", lambda: "3") # 3) Mock + + res = call_llm("prompt") + + assert res == _mock_response("prompt") + captured = capsys.readouterr() + assert "Choose (1/2/3): " in captured.err + assert "Choose (1/2/3): " not in captured.out + + +def test_model_menu_prompt_text_goes_to_stderr_not_stdout(monkeypatch, capsys): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_model", {}) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("sys.stdin", _FakeInteractiveStdin()) + monkeypatch.setattr("builtins.input", lambda: "1") # first model in the menu + + captured_call: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_ok(captured_call)) + + call_llm("prompt") + + captured = capsys.readouterr() + assert "Select Anthropic model:" in captured.err + assert "Choose (1-" in captured.err + assert "Select Anthropic model:" not in captured.out + assert "Choose (1-" not in captured.out + # the typed choice ("1") actually selected a real model, not a fallback default + assert captured_call["model"] + + +def test_api_key_prompt_text_goes_to_stderr_not_stdout(monkeypatch, capsys): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", "anthropic") + monkeypatch.setattr(llm_client, "_cached_model", {"anthropic": "claude-haiku-4-5-20251001"}) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr("sys.stdin", _FakeInteractiveStdin()) + monkeypatch.setattr("builtins.input", lambda: "sk-typed-key") + + captured_call: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_ok(captured_call)) + + call_llm("prompt") + + captured = capsys.readouterr() + assert "Enter ANTHROPIC_API_KEY: " in captured.err + assert "Enter ANTHROPIC_API_KEY: " not in captured.out + # the typed value actually reached the API client, not just the prompt + assert captured_call["api_key"] == "sk-typed-key" + + +def test_non_interactive_defaults_to_mock(monkeypatch): + # Ensure no provider env var + monkeypatch.delenv("LLM_PROVIDER", raising=False) + + # Ensure input() would raise if called + monkeypatch.setattr("builtins.input", lambda *a, **k: (_ for _ in ()).throw(RuntimeError("input called"))) + + # Simulate non-interactive stdin + class FakeStdin: + def isatty(self): + return False + + monkeypatch.setattr("sys.stdin", FakeStdin()) + + res = call_llm("no interactive") + assert res == _mock_response("no interactive") + + +# --------------------------------------------------------------------------- +# Configurable max_tokens (LLM_MAX_TOKENS override + DEFAULT_MAX_TOKENS) +# --------------------------------------------------------------------------- + +def _fake_anthropic_module(captured, stop_reason="end_turn", text="ok"): + class FakeContentBlock: + def __init__(self, text): + self.text = text + + class FakeResponse: + def __init__(self): + self.content = [FakeContentBlock(text)] + self.stop_reason = stop_reason + + class FakeAnthropic: + def __init__(self, api_key=None): + pass + + class messages: + @staticmethod + def create(model, max_tokens, messages): + captured["max_tokens"] = max_tokens + return FakeResponse() + + return types.SimpleNamespace(Anthropic=FakeAnthropic) + + +def _fake_openai_module(captured, finish_reason="stop", text="ok"): + class FakeMessage: + def __init__(self, content): + self.content = content + + class FakeChoice: + def __init__(self, content, finish_reason): + self.message = FakeMessage(content) + self.finish_reason = finish_reason + + class FakeResponse: + def __init__(self): + self.choices = [FakeChoice(text, finish_reason)] + + class FakeOpenAI: + def __init__(self, api_key=None): + pass + + class chat: + class completions: + @staticmethod + def create(model, messages, temperature, max_tokens): + captured["max_tokens"] = max_tokens + return FakeResponse() + + return types.SimpleNamespace(OpenAI=FakeOpenAI) + + +def test_anthropic_uses_default_max_tokens_when_env_unset(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.delenv("LLM_MAX_TOKENS", raising=False) + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == DEFAULT_MAX_TOKENS + assert DEFAULT_MAX_TOKENS == 4096 + + +def test_anthropic_honors_llm_max_tokens_override(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "8000") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == 8000 + + +def test_invalid_llm_max_tokens_falls_back_to_default(monkeypatch, capsys): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "not-a-number") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "anthropic", _fake_anthropic_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == DEFAULT_MAX_TOKENS + # stderr, not stdout: progress/warning prints were redirected to stderr + # during the merge into OpenAnt (stdout carries only the JSON envelope). + assert "Invalid LLM_MAX_TOKENS" in capsys.readouterr().err + + +def test_openai_receives_explicit_max_tokens(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setenv("LLM_MAX_TOKENS", "2048") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + + captured: dict = {} + monkeypatch.setitem(sys.modules, "openai", _fake_openai_module(captured)) + + call_llm("prompt") + assert captured["max_tokens"] == 2048 + + +# --------------------------------------------------------------------------- +# Per-stage call metadata (stop_reason / finish_reason capture) +# --------------------------------------------------------------------------- + +def test_anthropic_call_metadata_captures_max_tokens_stop_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + captured: dict = {} + monkeypatch.setitem( + sys.modules, "anthropic", _fake_anthropic_module(captured, stop_reason="max_tokens") + ) + + call_llm("prompt", stage="patch_generation") + meta = llm_client.get_call_metadata() + assert meta["patch_generation"]["stop_reason"] == "max_tokens" + assert meta["patch_generation"]["provider"] == "anthropic" + + +def test_openai_call_metadata_captures_finish_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "openai") + monkeypatch.setenv("OPENAI_API_KEY", "fake-key") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_cached_api_keys", {}) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + captured: dict = {} + monkeypatch.setitem( + sys.modules, "openai", _fake_openai_module(captured, finish_reason="length") + ) + + call_llm("prompt", stage="patch_review") + meta = llm_client.get_call_metadata() + assert meta["patch_review"]["stop_reason"] == "length" + + +def test_mock_call_populates_metadata_with_mock_stop_reason(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + call_llm("prompt", stage="confidence_scorer") + meta = llm_client.get_call_metadata() + assert meta["confidence_scorer"]["stop_reason"] == "mock" + assert meta["confidence_scorer"]["max_tokens_configured"] is None + + +def test_complete_passes_stage_through_to_call_metadata(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + client = LLMClient(api_key="") + client.complete("system prompt", "user message", stage="challenger") + meta = llm_client.get_call_metadata() + assert "challenger" in meta + + +def test_clear_call_metadata_empties_the_store(monkeypatch): + monkeypatch.setattr(llm_client, "_call_metadata", { + "patch_generation": {"provider": "anthropic", "model": "x", "max_tokens_configured": 1000, "stop_reason": "max_tokens"}, + }) + llm_client.clear_call_metadata() + assert llm_client.get_call_metadata() == {} + + +def test_stale_metadata_does_not_leak_into_a_new_run(monkeypatch): + # Simulates two sequential runs in the same process (e.g. a batch + # runner) without the reset: a stage from run 1 must not survive into + # run 2's metadata once clear_call_metadata() is called between them. + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + # "Run 1": patch_generation stage recorded. + call_llm("prompt for run 1", stage="patch_generation") + assert "patch_generation" in llm_client.get_call_metadata() + + # Reset, as main.py now does before each run. + llm_client.clear_call_metadata() + assert llm_client.get_call_metadata() == {} + + # "Run 2": only a different stage is called. + call_llm("prompt for run 2", stage="challenger") + meta = llm_client.get_call_metadata() + assert "challenger" in meta + assert "patch_generation" not in meta, "stale stage from a previous run leaked into the new run's metadata" + + +def test_complete_default_stage_is_unknown_and_does_not_break(monkeypatch): + # Existing callers that don't pass `stage` must keep working unchanged. + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setattr(llm_client, "_call_metadata", {}) + + client = LLMClient(api_key="") + result = client.complete("system prompt", "user message") + assert isinstance(result, str) + meta = llm_client.get_call_metadata() + assert "unknown" in meta diff --git a/libs/openant-core/tests/patch/test_patch_applicability.py b/libs/openant-core/tests/patch/test_patch_applicability.py new file mode 100644 index 00000000..ba205e9b --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_applicability.py @@ -0,0 +1,615 @@ +"""Tests for patch_applicability.check_applicability.""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +from pathlib import Path +from unittest import mock + +import pytest + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + +_FENCED_DIFF = """\ +```diff +--- a/auth.py ++++ b/auth.py +@@ -1,2 +1,2 @@ + def authenticate(u, p): +- return True ++ return check_credentials(u, p) +```""" + +_INNER_DIFF = """\ +--- a/auth.py ++++ b/auth.py +@@ -1,2 +1,2 @@ + def authenticate(u, p): +- return True ++ return check_credentials(u, p) +""" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_git_repo(tmp_path: Path) -> Path: + """Init a minimal git repo with one committed file.""" + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "auth.py").write_text( + "def authenticate(u, p):\n return True\n", encoding="utf-8" + ) + subprocess.run(["git", "add", "auth.py"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=tmp_path, capture_output=True, check=True, + ) + return tmp_path + + +def _mock_git_run(returncode: int, stderr: str = ""): + cm = mock.MagicMock() + cm.returncode = returncode + cm.stderr = stderr + return cm + + +# --------------------------------------------------------------------------- +# Skip conditions +# --------------------------------------------------------------------------- + +class TestSkipConditions: + def test_no_repo_root_skipped(self): + from utilities.autopatcher.patch_applicability import check_applicability + r = check_applicability(_FENCED_DIFF, None) + assert r["skipped"] is True + assert r["applicable"] is None + assert "no repo_root" in r["skipped_reason"] + + def test_not_git_repo_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is True + assert "git" in r["skipped_reason"].lower() + + def test_empty_patch_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + r = check_applicability("", tmp_path) + assert r["skipped"] is True + assert "empty" in r["skipped_reason"].lower() + + def test_fences_only_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + r = check_applicability("```diff\n```", tmp_path) + assert r["skipped"] is True + + def test_git_not_found_skipped(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + side_effect=FileNotFoundError): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is True + assert "git" in r["skipped_reason"].lower() + assert r["error"] is None + + +# --------------------------------------------------------------------------- +# Error state (not skipped) +# --------------------------------------------------------------------------- + +class TestErrorState: + def test_timeout_is_error_not_skip(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=subprocess.TimeoutExpired("git", 10), + ): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is None + assert r["error"] is not None + assert "timed out" in r["error"].lower() + + def test_unexpected_exception_is_error_not_skip(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=RuntimeError("something broke"), + ): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["skipped"] is False + assert r["error"] is not None + assert r["applicable"] is None + + +# --------------------------------------------------------------------------- +# Success / failure from git (mocked) +# --------------------------------------------------------------------------- + +class TestApplicabilityResult: + def test_applicable_true_on_returncode_0(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["applicable"] is True + assert r["skipped"] is False + assert r["error"] is None + assert r["exit_code"] == 0 + + def test_applicable_false_on_nonzero_returncode(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, "error: auth.py: does not exist in index")): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["applicable"] is False + assert r["exit_code"] == 1 + assert "does not exist" in r["stderr"] + + def test_fences_stripped_before_passing_to_git(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + captured = [] + def _capture(cmd, **kwargs): + captured.append(kwargs.get("input", "")) + return _mock_git_run(0) + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", side_effect=_capture): + check_applicability(_FENCED_DIFF, tmp_path) + assert captured + assert "```diff" not in captured[0] + assert "```" not in captured[0].strip().splitlines()[-1] + + def test_plain_diff_no_fences_also_accepted(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)) as m: + check_applicability(_INNER_DIFF, tmp_path) + assert m.called + + +# --------------------------------------------------------------------------- +# stderr truncation +# --------------------------------------------------------------------------- + +class TestStderrTruncation: + def test_truncated_at_20_lines(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + long_stderr = "\n".join(f"error line {i}" for i in range(50)) + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, long_stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + lines = r["stderr"].splitlines() + assert len(lines) <= 21 # 20 data + 1 truncation marker + assert "truncated" in r["stderr"].lower() + + def test_truncated_at_2000_chars(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + long_stderr = "x" * 3000 + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, long_stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert len(r["stderr"]) <= 2_100 # 2000 + marker + + def test_short_stderr_not_truncated(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + (tmp_path / ".git").mkdir() + stderr = "error: auth.py: patch does not apply" + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, stderr)): + r = check_applicability(_FENCED_DIFF, tmp_path) + assert r["stderr"] == stderr + + +# --------------------------------------------------------------------------- +# Integration: real git (skipped if git unavailable) +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not shutil.which("git"), reason="git not available") +class TestRealGitApply: + def test_correct_patch_applies(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + " def authenticate(u, p):\n" + "- return True\n" + "+ return check_credentials(u, p)\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is True + assert r["error"] is None + + def test_nonexistent_file_fails(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + patch = ( + "```diff\n" + "--- a/nonexistent.py\n" + "+++ b/nonexistent.py\n" + "@@ -1,1 +1,1 @@\n" + "-old_line()\n" + "+new_line()\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is False + assert r["stderr"] # git names the problem + + def test_wrong_context_fails(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + _make_git_repo(tmp_path) + # The file has "def authenticate" but the patch removes "def wrong_name" + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + "-def wrong_name(u, p):\n" + "+def authenticate(u, p):\n" + " return True\n" + "```" + ) + r = check_applicability(patch, tmp_path) + assert r["skipped"] is False + assert r["applicable"] is False + + +# --------------------------------------------------------------------------- +# apply_patch: skip/error conditions (mirrors check_applicability's, minus +# the "skipped" flag — apply_patch just reports applied=False + error) +# --------------------------------------------------------------------------- + +class TestApplyPatchSkipConditions: + def test_no_workspace_root(self): + from utilities.autopatcher.patch_applicability import apply_patch + r = apply_patch(_FENCED_DIFF, None) + assert r.applied is False + assert "no workspace_root" in r.error + assert r.error_kind == "invalid_input" + + def test_not_git_repo(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is False + assert "git" in r.error.lower() + assert r.error_kind == "not_git_repository" + + def test_empty_patch(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + r = apply_patch("", tmp_path) + assert r.applied is False + assert "empty" in r.error.lower() + assert r.error_kind == "invalid_input" + + def test_git_not_found(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + side_effect=FileNotFoundError): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is False + assert "git" in r.error.lower() + assert r.error_kind == "git_not_found" + + +class TestApplyPatchErrorState: + def test_timeout(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=subprocess.TimeoutExpired("git", 10), + ): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is False + assert "timed out" in r.error.lower() + assert r.error_kind == "timeout" + + def test_unexpected_exception(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + with mock.patch( + "utilities.autopatcher.patch_applicability.run_utf8", + side_effect=RuntimeError("something broke"), + ): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is False + assert r.error is not None + assert r.error_kind == "unexpected_error" + + +class TestApplyPatchResult: + def test_applied_true_on_returncode_0(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is True + assert r.error is None + assert r.error_kind is None + assert r.exit_code == 0 + + def test_applied_false_on_nonzero_returncode(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(1, "error: auth.py: does not exist in index")): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert r.applied is False + assert r.exit_code == 1 + assert r.error_kind == "apply_rejected" + assert "does not exist" in r.stderr + + def test_no_check_flag_passed_to_git(self, tmp_path): + """apply_patch must NOT pass --check — it mutates the tree.""" + from utilities.autopatcher.patch_applicability import apply_patch + (tmp_path / ".git").mkdir() + captured = [] + def _capture(cmd, **kwargs): + captured.append(cmd) + return _mock_git_run(0) + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", side_effect=_capture): + apply_patch(_FENCED_DIFF, tmp_path) + assert captured + assert "--check" not in captured[0] + + def test_result_is_frozen_dataclass(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch, PatchApplicationResult + (tmp_path / ".git").mkdir() + with mock.patch("utilities.autopatcher.patch_applicability.run_utf8", + return_value=_mock_git_run(0)): + r = apply_patch(_FENCED_DIFF, tmp_path) + assert isinstance(r, PatchApplicationResult) + with pytest.raises(Exception): # dataclasses.FrozenInstanceError + r.applied = False + + +# --------------------------------------------------------------------------- +# apply_patch: real git, mutating a disposable copy (never repo_root itself) +# --------------------------------------------------------------------------- + +@pytest.mark.skipif(not shutil.which("git"), reason="git not available") +class TestRealGitApplyPatch: + def test_correct_patch_applies_and_mutates_file(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + _make_git_repo(tmp_path) + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + " def authenticate(u, p):\n" + "- return True\n" + "+ return check_credentials(u, p)\n" + "```" + ) + r = apply_patch(patch, tmp_path) + assert r.applied is True + assert r.error is None + assert r.error_kind is None + assert "check_credentials" in (tmp_path / "auth.py").read_text(encoding="utf-8") + + def test_bad_patch_does_not_apply_and_leaves_file_untouched(self, tmp_path): + from utilities.autopatcher.patch_applicability import apply_patch + _make_git_repo(tmp_path) + original = (tmp_path / "auth.py").read_text(encoding="utf-8") + patch = ( + "```diff\n" + "--- a/nonexistent.py\n" + "+++ b/nonexistent.py\n" + "@@ -1,1 +1,1 @@\n" + "-old_line()\n" + "+new_line()\n" + "```" + ) + r = apply_patch(patch, tmp_path) + assert r.applied is False + assert r.error_kind == "apply_rejected" + assert (tmp_path / "auth.py").read_text(encoding="utf-8") == original + + def test_compose_with_temporary_repo_copy_never_touches_real_repo(self, tmp_path): + """The intended composition: copy, then apply to the copy only.""" + from utilities.autopatcher.patch_applicability import apply_patch + from utilities.autopatcher.patch_workspace import temporary_repo_copy + real_repo = _make_git_repo(tmp_path) + original = (real_repo / "auth.py").read_text(encoding="utf-8") + patch = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + " def authenticate(u, p):\n" + "- return True\n" + "+ return check_credentials(u, p)\n" + "```" + ) + with temporary_repo_copy(real_repo) as workspace_root: + r = apply_patch(patch, workspace_root) + assert r.applied is True + assert "check_credentials" in (workspace_root / "auth.py").read_text(encoding="utf-8") + # Real repo must be untouched while the copy is mutated. + assert (real_repo / "auth.py").read_text(encoding="utf-8") == original + # And still untouched after the workspace is torn down. + assert (real_repo / "auth.py").read_text(encoding="utf-8") == original + + +# --------------------------------------------------------------------------- +# Pipeline-level: section renders correctly +# --------------------------------------------------------------------------- + +class TestFenceStrippingPreservesTrailingBlankContext: + """Regression coverage for the byte-preserving _strip_fences fix. + + Historical bug: splitlines() + "\\n".join(...) followed by .strip() + silently deleted a trailing single-space context line, turning a valid + hunk into one whose body no longer matched its @@ header's declared + count — surfacing as `git apply`'s generic "corrupt patch at line N" + instead of the real applicability result. + """ + + def _repo_with_blank_line_before_brace(self, tmp_path: Path) -> Path: + """A hermetic git repo whose file has a real blank line before `}`.""" + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "redirect.c").write_text( + "int follow(int x) {\n" + " do_thing(x);\n" + "\n" + " return 0;\n" + "}\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "redirect.c"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=tmp_path, capture_output=True, check=True) + return tmp_path + + def _fenced_patch_with_trailing_blank_context(self) -> str: + """Fenced diff whose final hunk's last line is a single-space context + line (representing the real blank line before `}` in redirect.c).""" + body = ( + "--- a/redirect.c\n" + "+++ b/redirect.c\n" + "@@ -1,5 +1,8 @@\n" + " int follow(int x) {\n" + " do_thing(x);\n" + "+ if(x) {\n" + "+ clear_creds(x);\n" + "+ }\n" + " \n" + " return 0;\n" + " }\n" + ) + # No trailing newline after the closing fence, matching how + # patch_generator._extract_diff_block assembles the string. + return "```diff\n" + body + "```" + + def test_strip_fences_supports_tilde_closing_fence(self): + """~~~ is accepted as a closing fence, mirroring + diff_hunk_repair._strip_md_fences's same ("```", "~~~") check.""" + from utilities.autopatcher.patch_applicability import _strip_fences + patch = "```diff\n--- a/x\n+++ b/x\n@@ -1,1 +1,1 @@\n-a\n+b\n~~~" + stripped = _strip_fences(patch) + assert "~~~" not in stripped + assert "```" not in stripped + assert stripped == "--- a/x\n+++ b/x\n@@ -1,1 +1,1 @@\n-a\n+b\n" + + def test_strip_fences_preserves_trailing_backtick_context_line(self): + """F-38 regression: a legitimate context line whose content is ``` + must not be mistaken for the LLM's own wrapper fence and stripped, + even when the patch has no surrounding ``` wrapper at all.""" + from utilities.autopatcher.patch_applicability import _strip_fences + patch = ( + "--- a/x\n" + "+++ b/x\n" + "@@ -1,2 +1,2 @@\n" + "-a\n" + "+b\n" + " ```\n" + ) + stripped = _strip_fences(patch) + assert stripped.splitlines(keepends=True)[-1] == " ```\n" + + def test_strip_fences_preserves_single_space_last_line(self): + from utilities.autopatcher.patch_applicability import _strip_fences + patch = self._fenced_patch_with_trailing_blank_context() + stripped = _strip_fences(patch) + lines = stripped.splitlines(keepends=True) + # The blank context line (single space) two lines before the final + # ' }\n' context line must survive untouched, not be deleted. + assert lines[-3] == " \n" + assert lines[-2] == " return 0;\n" + assert lines[-1] == " }\n" + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_no_longer_reported_as_corrupt(self, tmp_path): + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + patch = self._fenced_patch_with_trailing_blank_context() + r = check_applicability(patch, tmp_path) + assert "corrupt patch" not in (r["stderr"] or "").lower() + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_well_formed_patch_with_trailing_blank_context_applies(self, tmp_path): + """The patch above is genuinely well-formed against redirect.c — with + the fence-stripping bug fixed it must apply cleanly, not merely + avoid the word 'corrupt'.""" + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + patch = self._fenced_patch_with_trailing_blank_context() + r = check_applicability(patch, tmp_path) + assert r["applicable"] is True, r["stderr"] + + @pytest.mark.skipif(not shutil.which("git"), reason="git not available") + def test_genuine_context_mismatch_still_fails_normally(self, tmp_path): + """A real context mismatch (not a fence-stripping artifact) must + still be reported as a normal, non-corrupt applicability failure.""" + from utilities.autopatcher.patch_applicability import check_applicability + self._repo_with_blank_line_before_brace(tmp_path) + bad_patch = ( + "```diff\n" + "--- a/redirect.c\n" + "+++ b/redirect.c\n" + "@@ -1,5 +1,8 @@\n" + " int follow(int x) {\n" + " do_thing_that_does_not_exist(x);\n" + "+ if(x) {\n" + "+ clear_creds(x);\n" + "+ }\n" + " \n" + " return 0;\n" + " }\n" + "```" + ) + r = check_applicability(bad_patch, tmp_path) + assert r["applicable"] is False + assert "corrupt patch" not in (r["stderr"] or "").lower() + assert r["stderr"] # git still names the problem + + +class TestPipelineApplicabilitySection: + def test_section_present_in_report(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + assert "## Patch Applicability" in report + + def test_skipped_when_no_repo_root(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="", repo_root=None) + start = report.find("## Patch Applicability") + end = report.find("---", start + 1) + section = report[start:end] + assert "Skipped" in section + + def test_hygiene_before_applicability(self): + from utilities.autopatcher.pipeline import run + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + idx_hygiene = report.find("## Patch Hygiene") + idx_applicability = report.find("## Patch Applicability") + assert idx_hygiene < idx_applicability diff --git a/libs/openant-core/tests/patch/test_patch_challenger.py b/libs/openant-core/tests/patch/test_patch_challenger.py new file mode 100644 index 00000000..5d559832 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_challenger.py @@ -0,0 +1,124 @@ +"""Unit tests for patch_challenger.challenge_patch, focused on the +code_context parameter (mirrors generate_patch/score_confidence grounding).""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + + +_CHALLENGER_RESPONSE = """\ +Still vulnerable: No + +Edge cases: +- Some edge case + +Potential issues: +- Some potential issue + +Summary: +- A concise paragraph summarising the adversarial findings. +""" + + +class TestCodeContextParameter: + """code_context is optional and, when provided, must reach the LLM call + the same way generate_patch/score_confidence already do.""" + + def test_default_omits_repository_evidence_section(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm) + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" not in user_message + + def test_code_context_included_when_provided(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" in user_message + assert "def foo(): pass" in user_message + + def test_empty_code_context_omits_section(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="") + + _system, user_message = llm.complete.call_args[0] + assert "## Repository evidence" not in user_message + + def test_code_context_precedes_vulnerability_report(self): + """Same ordering as score_confidence: repository evidence first, so + the model reads the real code before the advisory framing.""" + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert user_message.index("## Repository evidence") < user_message.index("## Vulnerability report") + + def test_backward_compatible_without_code_context_kwarg(self): + """Existing positional-only call sites must keep working unchanged.""" + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + result = challenge_patch("some vuln", "some diff", llm) + + assert result["still_vulnerable"] is False + assert result["edge_cases"] == ["Some edge case"] + assert result["potential_issues"] == ["Some potential issue"] + + +class TestChallengePatchBasicBehavior: + """Baseline behavior, unrelated to code_context, that the new parameter + must not disturb.""" + + def test_returns_expected_keys(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + result = challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert set(result.keys()) == {"still_vulnerable", "edge_cases", "potential_issues", "summary"} + + def test_still_vulnerable_yes_parsed_true(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE.replace( + "Still vulnerable: No", "Still vulnerable: Yes" + ) + + result = challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert result["still_vulnerable"] is True + + def test_stage_argument_is_challenger(self): + from utilities.autopatcher.patch_challenger import challenge_patch + + llm = mock.MagicMock() + llm.complete.return_value = _CHALLENGER_RESPONSE + + challenge_patch("some vuln", "some diff", llm, code_context="ctx") + + assert llm.complete.call_args.kwargs.get("stage") == "challenger" diff --git a/libs/openant-core/tests/patch/test_patch_generator.py b/libs/openant-core/tests/patch/test_patch_generator.py new file mode 100644 index 00000000..d74aec64 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_generator.py @@ -0,0 +1,549 @@ +"""Unit tests for patch_generator._extract_diff_block and generate_patch.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +```""" + +_PROSE_PREAMBLE = """\ +The vulnerability is in retry.py. Here is the minimal fix: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +``` + +Let me know if you need a more defensive approach.""" + +_TWO_ALTERNATIVES = """\ +Option A — minimal fix: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +``` + +Option B — more defensive: + +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization", "X-Csrf-Token"]) +```""" + +_MULTI_FILE_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/connectionpool.py ++++ b/src/urllib3/connectionpool.py +@@ -1,3 +1,4 @@ ++# Cookie stripping is handled via DEFAULT_REMOVE_HEADERS_ON_REDIRECT. +```""" + +_PATCH_TAG = """\ +```patch +--- a/src/utils.py ++++ b/src/utils.py +@@ -1,1 +1,1 @@ +-old_line() ++new_line() +```""" + +_UDIFF_TAG = """\ +```udiff +--- a/src/utils.py ++++ b/src/utils.py +@@ -1,1 +1,1 @@ +-old_line() ++new_line() +```""" + +_NO_DIFF_BLOCK = "The vulnerability requires manual intervention. No automated patch is possible." + +_WINDOWS_ENDINGS = "```diff\r\n--- a/foo.py\r\n+++ b/foo.py\r\n@@ -1 +1 @@\r\n-old\r\n+new\r\n```" + +# F-29 regression fixtures: a diff body that itself contains a nested +# Markdown fence must not be truncated at the inner fence. + +_NESTED_FENCE_DIFF = """\ +```diff +diff --git a/README.md b/README.md +--- a/README.md ++++ b/README.md +@@ -10,4 +10,9 @@ Some intro text + ## Usage + ++```bash ++openant patch --check ++``` ++ ++See docs for more. +diff --git a/lib/auth.py b/lib/auth.py +--- a/lib/auth.py ++++ b/lib/auth.py +@@ -50,7 +50,7 @@ def verify_token(token): +- if token == expected: ++ if hmac.compare_digest(token, expected): + return True +```""" + +_QUAD_FENCE_DIFF = """\ +````diff +diff --git a/README.md b/README.md +--- a/README.md ++++ b/README.md +@@ -1,2 +1,5 @@ + # Title ++```bash ++echo hello ++``` +````""" + +_TILDE_FENCE_DIFF = """\ +~~~diff +--- a/src/utils.py ++++ b/src/utils.py +@@ -1,1 +1,1 @@ +-old_line() ++new_line() +~~~""" + +# Context lines (unchanged, single leading space) reproducing a fence from +# the patched file's own content -- must not be mistaken for the outer +# fence's closer, since a genuine closer is never diff-prefixed. +_CONTEXT_LINE_FENCE_DIFF = """\ +```diff +--- a/README.md ++++ b/README.md +@@ -10,7 +10,7 @@ Some intro + ## Usage + + ```bash + echo hi + ``` + +-See docs. ++See documentation. +```""" + +# Recognised opener, real applicable-looking hunk content, but the fence is +# never closed -- e.g. the model's response was cut off mid-generation. +_UNCLOSED_FENCE_DIFF = ( + "```diff\n" + "--- a/auth.py\n" + "+++ b/auth.py\n" + "@@ -1,2 +1,2 @@\n" + " def authenticate(u, p):\n" + "- return True\n" + "+ return check_credentials(u, p)\n" +) + +# The first ```diff opener is never closed, but a second, independently +# well-formed ```diff block follows later in the same response. The first +# block must be treated as malformed on its own -- neither merged with the +# second block's content nor skipped in favour of it. +_UNCLOSED_FIRST_THEN_VALID_SECOND_DIFF = ( + "```diff\n" + "--- a/first.py\n" + "+++ b/first.py\n" + "@@ -1,1 +1,1 @@\n" + "-old_first\n" + "+new_first\n" + "\n" + "Some trailing prose, still inside the unclosed first block.\n" + "\n" + "```diff\n" + "--- a/second.py\n" + "+++ b/second.py\n" + "@@ -1,1 +1,1 @@\n" + "-old_second\n" + "+new_second\n" + "```\n" +) + + +# --------------------------------------------------------------------------- +# Tests for _extract_diff_block +# --------------------------------------------------------------------------- + +class TestExtractDiffBlock: + def test_clean_diff_preserved(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_CLEAN_DIFF) + assert result.startswith("```diff\n") + assert result.strip().endswith("```") + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result + + def test_prose_preamble_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PROSE_PREAMBLE) + assert result.startswith("```diff\n") + assert "The vulnerability" not in result + assert "Let me know" not in result + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result + + def test_prose_postamble_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PROSE_PREAMBLE) + assert "Let me know" not in result + + def test_first_of_two_alternatives_returned(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_TWO_ALTERNATIVES) + assert result.count("```diff") == 1 + assert "Option A" not in result + assert "Option B" not in result + assert "X-Csrf-Token" not in result + assert '"Cookie", "Authorization"' in result + + def test_multi_file_diff_preserved(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_MULTI_FILE_DIFF) + assert "retry.py" in result + assert "connectionpool.py" in result + + def test_patch_tag_normalised_to_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_PATCH_TAG) + assert result.startswith("```diff\n") + assert "+new_line()" in result + + def test_udiff_tag_normalised_to_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_UDIFF_TAG) + assert result.startswith("```diff\n") + assert "+new_line()" in result + + def test_no_fenced_block_returns_raw_stripped(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_NO_DIFF_BLOCK) + assert result == _NO_DIFF_BLOCK.strip() + + def test_empty_string_returns_empty(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + assert _extract_diff_block("") == "" + + def test_windows_line_endings_accepted(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_WINDOWS_ENDINGS) + assert result.startswith("```diff\n") + assert "+new" in result + + def test_output_always_starts_with_diff_fence(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + for raw in (_CLEAN_DIFF, _PROSE_PREAMBLE, _TWO_ALTERNATIVES, _PATCH_TAG): + result = _extract_diff_block(raw) + assert result.startswith("```diff\n"), f"Failed for input starting: {raw[:40]!r}" + + +# --------------------------------------------------------------------------- +# F-29 regression: nested/embedded fences must not truncate the diff +# --------------------------------------------------------------------------- + +class TestExtractDiffBlockNestedFences: + def test_nested_markdown_fence_does_not_truncate_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_NESTED_FENCE_DIFF) + assert result.count("```diff") == 1 + assert "openant patch --check" in result + # The security-relevant change after the inner fenced block must survive. + assert "lib/auth.py" in result + assert "hmac.compare_digest(token, expected)" in result + + def test_quad_backtick_outer_fence_survives_triple_backtick_content(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_QUAD_FENCE_DIFF) + assert result.startswith("```diff\n") + assert "echo hello" in result + assert "README.md" in result + + def test_tilde_fenced_diff_normalised_to_diff(self): + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_TILDE_FENCE_DIFF) + assert result.startswith("```diff\n") + assert "+new_line()" in result + assert "~~~" not in result + + def test_context_line_fence_not_mistaken_for_closer(self): + """A single-space-prefixed context line reproducing a bare fence + (as unified-diff grammar requires for unchanged content) must not + be mistaken for the outer fence's closer.""" + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_CONTEXT_LINE_FENCE_DIFF) + assert result.count("```diff") == 1 + assert "-See docs." in result + assert "+See documentation." in result + + def test_unclosed_recognised_fence_returns_empty_string(self): + """A recognised opener with no matching closer is malformed + structured output -- it must not be returned as partial content or + silently repackaged as a complete-looking diff.""" + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_UNCLOSED_FENCE_DIFF) + assert result == "" + + def test_unclosed_first_block_not_merged_with_valid_second_block(self): + """An unclosed first opener followed by an independently + well-formed second block must not be merged or skipped past -- the + first block is malformed on its own, so the whole extraction fails + closed (""), regardless of the second block's validity.""" + from utilities.autopatcher.patch_generator import _extract_diff_block + result = _extract_diff_block(_UNCLOSED_FIRST_THEN_VALID_SECOND_DIFF) + assert result == "" + + +# --------------------------------------------------------------------------- +# Integration: generate_patch extracts clean output from messy LLM response +# --------------------------------------------------------------------------- + +class TestGeneratePatchExtraction: + def test_extracts_diff_from_messy_response(self): + from utilities.autopatcher.patch_generator import generate_patch + messy = ( + "I'll fix this by adding Cookie to the frozenset:\n\n" + "```diff\n" + "--- a/src/urllib3/util/retry.py\n" + "+++ b/src/urllib3/util/retry.py\n" + "@@ -187,7 +187,7 @@\n" + '- DEFAULT_REMOVE = frozenset(["Authorization"])\n' + '+ DEFAULT_REMOVE = frozenset(["Cookie", "Authorization"])\n' + "```\n\n" + "Alternatively you could also strip X-Csrf-Token." + ) + llm = mock.MagicMock() + llm.complete.return_value = messy + result = generate_patch("some vuln", llm) + assert result.startswith("```diff\n") + assert "Alternatively" not in result + assert result.count("```diff") == 1 + + def test_fallback_when_no_fenced_block(self): + from utilities.autopatcher.patch_generator import generate_patch + plain = "No automated patch available for this vulnerability." + llm = mock.MagicMock() + llm.complete.return_value = plain + result = generate_patch("some vuln", llm) + assert result == plain.strip() + + +# --------------------------------------------------------------------------- +# F-29 integration regression: an unclosed recognised fence must not become +# an applicable patch after the real downstream repair/hygiene/applicability +# chain runs on it -- proven against the actual pipeline code, not asserted +# as a property of the extractor's output alone. +# --------------------------------------------------------------------------- + +class TestUnclosedFenceCannotBecomeApplicable: + def test_unclosed_fence_never_reaches_applicable_true(self, tmp_path): + from utilities.autopatcher.patch_generator import _extract_diff_block + from utilities.autopatcher.diff_hunk_repair import repair_hunk_headers + from utilities.autopatcher.patch_hygiene import check_patch + from utilities.autopatcher.patch_applicability import check_applicability + + (tmp_path / ".git").mkdir() + + # Same fence-open-but-never-closed input as the unit test above, run + # through the real downstream chain in the same order pipeline.py + # uses: repair -> hygiene -> applicability. + patch = _extract_diff_block(_UNCLOSED_FENCE_DIFF) + patch, _repair_meta = repair_hunk_headers(patch) + check_patch(patch) # hygiene is best-effort; must not raise + result = check_applicability(patch, tmp_path) + + assert result["applicable"] is not True + assert result["skipped"] is True + assert "empty" in (result["skipped_reason"] or "").lower() + + def test_unclosed_first_block_with_valid_second_block_never_reaches_applicable_true(self, tmp_path): + """An unclosed first opener followed by an independently + well-formed second block must not, via the real downstream chain, + end up applicable=True with either block's content -- and it must + not merge the two into some other syntactically-valid patch either.""" + from utilities.autopatcher.patch_generator import _extract_diff_block + from utilities.autopatcher.diff_hunk_repair import repair_hunk_headers + from utilities.autopatcher.patch_hygiene import check_patch + from utilities.autopatcher.patch_applicability import check_applicability + + (tmp_path / ".git").mkdir() + + patch = _extract_diff_block(_UNCLOSED_FIRST_THEN_VALID_SECOND_DIFF) + patch, _repair_meta = repair_hunk_headers(patch) + check_patch(patch) # hygiene is best-effort; must not raise + result = check_applicability(patch, tmp_path) + + assert result["applicable"] is not True + assert result["skipped"] is True + assert "empty" in (result["skipped_reason"] or "").lower() + + +# --------------------------------------------------------------------------- +# AUTOPATCHER_DEBUG prompt dump +# --------------------------------------------------------------------------- + +class TestDebugDump: + def test_debug_file_written_when_env_set(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + # Redirect debug output to tmp_path + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("SQL injection in auth.py", llm, code_context="def auth(): pass") + + debug_dir = tmp_path / "reports" / "debug" + assert debug_dir.exists(), "reports/debug/ must be created" + files = list(debug_dir.glob("prompt_*.txt")) + assert len(files) == 1, f"Expected one debug file, found {files}" + content = files[0].read_text() + assert "SQL injection" in content + assert "def auth(): pass" in content + + def test_no_debug_file_when_env_unset(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.delenv("AUTOPATCHER_DEBUG", raising=False) + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("some vuln", llm) + + debug_dir = tmp_path / "reports" / "debug" + assert not debug_dir.exists(), "No debug directory should be created without the env var" + + def test_debug_file_does_not_contain_api_key(self, tmp_path, monkeypatch): + from utilities.autopatcher.patch_generator import generate_patch + + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("vuln text", llm, code_context="some code") + + debug_dir = tmp_path / "reports" / "debug" + files = list(debug_dir.glob("prompt_*.txt")) + assert files + content = files[0].read_text() + # user_message contains only vuln text + code context, never the API key + assert "sk-" not in content + assert "OPENAI_API_KEY" not in content + + +# --------------------------------------------------------------------------- +# retry_hint parameter +# --------------------------------------------------------------------------- + +class TestRetryHint: + def test_retry_hint_appended_to_user_message(self): + from utilities.autopatcher.patch_generator import generate_patch + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch( + "some vuln", + llm, + code_context="def foo(): pass", + retry_hint="Please fix the context lines.", + ) + + _system, user_message = llm.complete.call_args[0] + assert "## Retry instruction" in user_message + assert "Please fix the context lines." in user_message + + def test_no_retry_section_when_hint_empty(self): + from utilities.autopatcher.patch_generator import generate_patch + + llm = mock.MagicMock() + llm.complete.return_value = "```diff\n--- a/f.py\n+++ b/f.py\n```" + + generate_patch("some vuln", llm, code_context="def foo(): pass") + + _system, user_message = llm.complete.call_args[0] + assert "## Retry instruction" not in user_message + + +# --------------------------------------------------------------------------- +# Base prompt content — no repository-specific contamination +# --------------------------------------------------------------------------- + +class TestPromptGeneratorMarkdownIsGeneric: + """The base system prompt is sent verbatim to every patch-generation + call, for every target repository. It must never contain terms specific + to one repository (e.g. GitPython internals leaked into the universal + template) -- those are hard errors for a completely unrelated repo like + urllib3, not real patch-generation rules.""" + + # Exact, proven-leaked terms only -- not a broad denylist of every + # benchmark repo name, and not generic words (`HEAD`, `file`, `git`, + # `repository`, `path`) that would produce fragile false positives. + _LEAKED_GITPYTHON_TERMS = [ + "for_git_dir", + "repo.common_dir", + "ORIG_HEAD", + "FETCH_HEAD", + "MERGE_HEAD", + "LockedFD", + "assure_directory_exists", + ] + + def _prompt_text(self) -> str: + from utilities.autopatcher.patch_generator import _PROMPT_PATH + return _PROMPT_PATH.read_text(encoding="utf-8") + + def test_no_gitpython_specific_terms_in_base_prompt(self): + prompt = self._prompt_text() + leaked = [term for term in self._LEAKED_GITPYTHON_TERMS if term in prompt] + assert leaked == [], f"repository-specific terms leaked into the universal prompt: {leaked}" + + def test_prompt_still_requires_a_unified_diff(self): + prompt = self._prompt_text() + assert "unified diff" in prompt.lower() + + def test_prompt_still_references_repository_code_context(self): + prompt = self._prompt_text() + assert "repository code context" in prompt.lower() + + def test_prompt_still_discourages_unrelated_changes(self): + prompt = self._prompt_text() + assert "unrelated" in prompt.lower() + + def test_prompt_still_honors_an_authoritative_patch_plan(self): + prompt = self._prompt_text() + assert "Patch Plan" in prompt + assert "authoritative" in prompt.lower() diff --git a/libs/openant-core/tests/patch/test_patch_hygiene.py b/libs/openant-core/tests/patch/test_patch_hygiene.py new file mode 100644 index 00000000..a13f9583 --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_hygiene.py @@ -0,0 +1,434 @@ +"""Unit tests for patch_hygiene.check_patch and its three sub-checks.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +# --------------------------------------------------------------------------- +# Fixtures — raw diff strings (without fences, as _parse_file_patches sees them) +# --------------------------------------------------------------------------- + +_CLEAN_PATCH = """\ +--- a/app/auth.py ++++ b/app/auth.py +@@ -42,7 +42,7 @@ + def authenticate(username: str, password: str) -> bool: +- query = f"SELECT * FROM users WHERE username='{username}'" ++ query = "SELECT * FROM users WHERE username=?" + cursor = db.execute(query) + return cursor.fetchone() is not None +""" + +_EMPTY_HUNK = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/_collections.py ++++ b/src/urllib3/_collections.py +@@ -1,3 +1,3 @@ + # no actual changes here +""" + +_DUPLICATE_CONST = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,8 +187,9 @@ + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_CORRECT_CONST_REPLACEMENT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,7 +187,7 @@ +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_UNUSED_IMPORT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,5 +1,6 @@ ++import re + class Retry: +- DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +""" + +_USED_IMPORT = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,5 +1,7 @@ ++import re ++PATTERN = re.compile(r'https?://') + class Retry: +- OLD_VAL = "x" ++ NEW_VAL = "y" +""" + +_NEW_FILE_WITH_CONST = """\ +--- /dev/null ++++ b/src/urllib3/util/headers.py +@@ -0,0 +1,3 @@ ++SENSITIVE_HEADERS = frozenset(["Cookie", "Authorization"]) ++ ++def strip_sensitive(headers): pass +""" + +# A genuinely new constant added to an existing file — no other line in the +# diff (added, removed, or unchanged context) mentions this name at all. +_NEW_CONSTANT_EXISTING_FILE = """\ +--- a/app/config.py ++++ b/app/config.py +@@ -10,6 +10,7 @@ + import os + + TIMEOUT = 30 ++MAX_REQUEST_BYTES = 1048576 + + def load(): + pass +""" + +# Simulates the dirty urllib3 output we observed in the live run +_URLLIB3_DIRTY = """\ +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -187,8 +187,9 @@ + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) ++ DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) +--- a/src/urllib3/_collections.py ++++ b/src/urllib3/_collections.py +@@ -1,3 +1,3 @@ + # unchanged +--- a/src/urllib3/connectionpool.py ++++ b/src/urllib3/connectionpool.py +@@ -1,4 +1,5 @@ ++import re + class HTTPConnectionPool: + pass +""" + +# The mock patch used throughout existing tests — should be clean +_MOCK_PATCH_INNER = """\ +--- a/app/auth.py ++++ b/app/auth.py +@@ -42,7 +42,7 @@ + def authenticate(username: str, password: str) -> bool: +- query = f"SELECT * FROM users WHERE username='{username}' AND password='{password}'" ++ query = "SELECT * FROM users WHERE username=? AND password=?" + cursor = db.execute(query) ++ # Use parameterized queries to prevent SQL injection + return cursor.fetchone() is not None +""" + +# An added line whose own text starts with "++ " — the raw diff line is +# "+++ ...", indistinguishable from a real "+++ " file header by a naive +# startswith check. +_PLUS_PLUS_PLUS_BODY_CONTENT = ( + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -1,3 +1,4 @@\n" + " def f():\n" + "- old = 1\n" + "+++ this added line of code starts with plus plus plus\n" + "+ new = 2\n" +) + +# A removed line whose own text starts with "-- " — the raw diff line is +# "--- ...", indistinguishable from a real "--- " file header. +_DASH_DASH_DASH_BODY_CONTENT = ( + "--- a/example.py\n" + "+++ b/example.py\n" + "@@ -1,3 +1,3 @@\n" + " def f():\n" + "--- this removed line of code starts with dash dash dash\n" + "+ return new\n" +) + +# A body-content header lookalike in file1's hunk, followed by a genuinely +# new file2 (--- /dev/null). Exercises whether is_new_file for file1 gets +# contaminated by file2's /dev/null header. +_IS_NEW_FILE_SHIFT = ( + "--- a/existing.py\n" + "+++ b/existing.py\n" + "@@ -1,2 +1,2 @@\n" + " def f():\n" + "--- this removed line of code starts with dash dash dash\n" + "--- /dev/null\n" + "+++ b/new_module.py\n" + "@@ -0,0 +1,2 @@\n" + "+def g():\n" + "+ pass\n" +) + +# Two files where file1's hunk contains a body-content header lookalike. +# The false match must not split file1 into a phantom extra "file" or +# corrupt the file1/file2 boundary. +_MULTI_FILE_HEADER_LOOKALIKE = ( + "--- a/file1.py\n" + "+++ b/file1.py\n" + "@@ -1,3 +1,4 @@\n" + " def f():\n" + "- old = 1\n" + "+++ marker line inside file1's hunk\n" + "+ new = 2\n" + "--- a/file2.py\n" + "+++ b/file2.py\n" + "@@ -10,3 +10,3 @@\n" + " a\n" + " b\n" + " c\n" +) + + +# --------------------------------------------------------------------------- +# check_patch — public API +# --------------------------------------------------------------------------- + +class TestCheckPatchSafety: + def test_empty_string_returns_empty(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch("") == [] + + def test_none_like_string_returns_empty(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(" ") == [] + + def test_fenced_block_accepted(self): + from utilities.autopatcher.patch_hygiene import check_patch + fenced = "```diff\n" + _CLEAN_PATCH + "\n```" + result = check_patch(fenced) + assert isinstance(result, list) + + def test_clean_patch_no_findings(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(_CLEAN_PATCH) == [] + + def test_mock_patch_no_findings(self): + from utilities.autopatcher.patch_hygiene import check_patch + assert check_patch(_MOCK_PATCH_INNER) == [] + + +# --------------------------------------------------------------------------- +# _parse_file_patches — hunk-body content resembling a file header must not +# be misparsed as one (F-36, F-41, F-44, F-45) +# --------------------------------------------------------------------------- + +class TestParseFilePatches: + def test_plus_plus_plus_body_content_kept_in_single_file(self): + """F-41 regression.""" + from utilities.autopatcher.patch_hygiene import _parse_file_patches + fps = _parse_file_patches(_PLUS_PLUS_PLUS_BODY_CONTENT) + assert len(fps) == 1, ( + f"Expected exactly one file section, got {len(fps)}: " + f"{[fp.filename for fp in fps]}" + ) + assert fps[0].filename == "example.py" + # line[1:] strips only the single leading '+' marker, so the + # line's own "++ " content prefix is legitimately retained. + assert "++ this added line of code starts with plus plus plus" in fps[0].added_lines + + def test_dash_dash_dash_body_content_kept_in_single_file(self): + """F-45 regression.""" + from utilities.autopatcher.patch_hygiene import _parse_file_patches + fps = _parse_file_patches(_DASH_DASH_DASH_BODY_CONTENT) + assert len(fps) == 1, ( + f"Expected exactly one file section, got {len(fps)}: " + f"{[fp.filename for fp in fps]}" + ) + assert fps[0].filename == "example.py" + # line[1:] strips only the single leading '-' marker, so the + # line's own "-- " content prefix is legitimately retained. + assert ( + "-- this removed line of code starts with dash dash dash" + in fps[0].removed_lines + ) + + def test_is_new_file_not_shifted_across_files(self): + """F-44 regression: is_new_file must reflect each file's own from_path, + not whatever from_path a later file's header happened to leave behind + by the time this file's data is flushed.""" + from utilities.autopatcher.patch_hygiene import _parse_file_patches + fps = _parse_file_patches(_IS_NEW_FILE_SHIFT) + by_name = {fp.filename: fp for fp in fps} + assert "existing.py" in by_name, ( + f"existing.py missing from parsed files: {[fp.filename for fp in fps]}" + ) + assert "new_module.py" in by_name, ( + f"new_module.py missing from parsed files: {[fp.filename for fp in fps]}" + ) + assert by_name["existing.py"].is_new_file is False, ( + "existing.py's is_new_file was contaminated by the /dev/null " + "header belonging to the next file" + ) + assert by_name["new_module.py"].is_new_file is True + + def test_multi_file_boundary_not_corrupted_by_header_lookalike(self): + """F-36 regression: a header lookalike inside file1's hunk must not + split file1 into a phantom extra "file" or corrupt the file1/file2 + boundary.""" + from utilities.autopatcher.patch_hygiene import _parse_file_patches + fps = _parse_file_patches(_MULTI_FILE_HEADER_LOOKALIKE) + assert [fp.filename for fp in fps] == ["file1.py", "file2.py"], ( + f"Multi-file boundary corrupted: {[fp.filename for fp in fps]}" + ) + + +# --------------------------------------------------------------------------- +# Check A — empty / no-op hunks +# --------------------------------------------------------------------------- + +class TestEmptyHunkCheck: + def test_empty_hunk_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_EMPTY_HUNK) + empty = [f for f in findings if f["check"] == "empty_hunk"] + assert len(empty) == 1 + assert "_collections.py" in empty[0]["detail"] + assert empty[0]["severity"] == "HIGH" + + def test_file_with_changes_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_EMPTY_HUNK) + empty = [f for f in findings if f["check"] == "empty_hunk"] + details = " ".join(f["detail"] for f in empty) + assert "retry.py" not in details + + def test_multiple_empty_hunks_all_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + empty = [f for f in findings if f["check"] == "empty_hunk"] + filenames = " ".join(f["detail"] for f in empty) + assert "_collections.py" in filenames + + +# --------------------------------------------------------------------------- +# Check B — duplicate assignment +# --------------------------------------------------------------------------- + +class TestDuplicateAssignmentCheck: + def test_duplicate_constant_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_DUPLICATE_CONST) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert len(dups) == 1 + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in dups[0]["detail"] + assert dups[0]["severity"] == "MEDIUM" + + def test_duplicate_detail_only_states_what_is_visible(self): + # The wording must not claim file-wide duplication, execution order, + # runtime shadowing, or that the patch is ineffective — only that + # both lines are visible in the diff and warrant a human look. + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_DUPLICATE_CONST) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + detail = dups[0]["detail"].lower() + assert "verify manually" in detail or "verify" in detail + assert "likely duplicates it" not in detail + + def test_correct_replacement_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_CORRECT_CONST_REPLACEMENT) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert dups == [] + + def test_new_file_constant_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_NEW_FILE_WITH_CONST) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert dups == [], "new-file constants should not be flagged as duplicates" + + def test_new_constant_in_existing_file_not_flagged(self): + # Regression for F-25: a genuinely new constant added to an existing + # file must not be flagged just because the file already has other + # constants — the name must co-occur as unchanged context to trigger. + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_NEW_CONSTANT_EXISTING_FILE) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert dups == [], "a genuinely new constant should not be flagged as a duplicate" + + def test_dirty_urllib3_flags_duplicate(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + dups = [f for f in findings if f["check"] == "duplicate_assignment"] + assert any("DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in f["detail"] for f in dups) + assert all(f["severity"] == "MEDIUM" for f in dups) + + +# --------------------------------------------------------------------------- +# Check C — unused imports +# --------------------------------------------------------------------------- + +class TestUnusedImportCheck: + def test_unused_import_detected(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_UNUSED_IMPORT) + imps = [f for f in findings if f["check"] == "unused_import"] + assert len(imps) == 1 + assert "re" in imps[0]["detail"] + assert imps[0]["severity"] == "MEDIUM" + + def test_used_import_not_flagged(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_USED_IMPORT) + imps = [f for f in findings if f["check"] == "unused_import"] + assert imps == [] + + def test_dirty_urllib3_flags_unused_re_import(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + imps = [f for f in findings if f["check"] == "unused_import"] + assert any("re" in f["detail"] for f in imps) + + +# --------------------------------------------------------------------------- +# Composite: dirty patch triggers all three checks +# --------------------------------------------------------------------------- + +class TestDirtyPatch: + def test_all_three_checks_fire_on_dirty_urllib3_patch(self): + from utilities.autopatcher.patch_hygiene import check_patch + findings = check_patch(_URLLIB3_DIRTY) + checks_found = {f["check"] for f in findings} + assert "empty_hunk" in checks_found + assert "duplicate_assignment" in checks_found + assert "unused_import" in checks_found + + def test_finding_structure_valid(self): + from utilities.autopatcher.patch_hygiene import check_patch + for finding in check_patch(_URLLIB3_DIRTY): + assert "severity" in finding + assert finding["severity"] in ("HIGH", "MEDIUM") + assert "check" in finding + assert "detail" in finding + assert isinstance(finding["detail"], str) and finding["detail"] + + +# --------------------------------------------------------------------------- +# Pipeline-level: hygiene section appears in report +# --------------------------------------------------------------------------- + +class TestPipelineHygieneSection: + def test_hygiene_section_present_in_report(self): + from utilities.autopatcher.pipeline import run + EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + assert "## Patch Hygiene" in report + + def test_clean_mock_patch_shows_no_issues(self): + from utilities.autopatcher.pipeline import run + EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="") + start = report.find("## Patch Hygiene") + end = report.find("---", start) + section = report[start:end] + assert "No obvious hygiene issues detected." in section diff --git a/libs/openant-core/tests/patch/test_patch_workspace.py b/libs/openant-core/tests/patch/test_patch_workspace.py new file mode 100644 index 00000000..9fa13eac --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_workspace.py @@ -0,0 +1,102 @@ +"""Tests for patch_workspace.temporary_repo_copy.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + + +def _make_git_repo(tmp_path: Path) -> Path: + """Init a minimal git repo with one committed file.""" + subprocess.run(["git", "init"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=tmp_path, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=tmp_path, capture_output=True) + (tmp_path / "auth.py").write_text( + "def authenticate(u, p):\n return True\n", encoding="utf-8" + ) + subprocess.run(["git", "add", "auth.py"], cwd=tmp_path, capture_output=True, check=True) + subprocess.run( + ["git", "commit", "-m", "init"], + cwd=tmp_path, capture_output=True, check=True, + ) + return tmp_path + + +class TestTemporaryRepoCopy: + def test_copy_contains_same_files(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + with temporary_repo_copy(repo) as workspace_root: + assert (workspace_root / "auth.py").exists() + assert (workspace_root / "auth.py").read_text(encoding="utf-8") == (repo / "auth.py").read_text(encoding="utf-8") + + def test_copy_is_a_different_path(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + with temporary_repo_copy(repo) as workspace_root: + assert workspace_root != repo + assert str(workspace_root) != str(repo) + + def test_git_directory_included(self, tmp_path): + """`.git` must survive the copy so git-based tooling (apply_patch) + works against the copy unmodified.""" + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + with temporary_repo_copy(repo) as workspace_root: + assert (workspace_root / ".git").exists() + + def test_original_repo_never_modified(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + original_content = (repo / "auth.py").read_text(encoding="utf-8") + with temporary_repo_copy(repo) as workspace_root: + (workspace_root / "auth.py").write_text("mutated in the copy\n", encoding="utf-8") + (workspace_root / "new_file.py").write_text("new\n", encoding="utf-8") + # Original repo untouched, both during and after the context. + assert (repo / "auth.py").read_text(encoding="utf-8") == original_content + assert not (repo / "new_file.py").exists() + + def test_cleanup_removes_temp_dir_on_normal_exit(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + captured_root = None + with temporary_repo_copy(repo) as workspace_root: + captured_root = workspace_root + assert captured_root.exists() + assert not captured_root.exists() + assert not captured_root.parent.exists() + + def test_cleanup_removes_temp_dir_on_exception(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + captured_root = None + with pytest.raises(RuntimeError): + with temporary_repo_copy(repo) as workspace_root: + captured_root = workspace_root + raise RuntimeError("boom") + assert captured_root is not None + assert not captured_root.exists() + + def test_ignores_pycache_and_node_modules(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + (repo / "__pycache__").mkdir() + (repo / "__pycache__" / "x.pyc").write_text("junk", encoding="utf-8") + (repo / "node_modules").mkdir() + (repo / "node_modules" / "pkg.js").write_text("junk", encoding="utf-8") + with temporary_repo_copy(repo) as workspace_root: + assert not (workspace_root / "__pycache__").exists() + assert not (workspace_root / "node_modules").exists() + assert (workspace_root / "auth.py").exists() + + def test_multiple_concurrent_copies_are_independent(self, tmp_path): + from utilities.autopatcher.patch_workspace import temporary_repo_copy + repo = _make_git_repo(tmp_path) + with temporary_repo_copy(repo) as workspace_root_1, temporary_repo_copy(repo) as workspace_root_2: + assert workspace_root_1 != workspace_root_2 + (workspace_root_1 / "auth.py").write_text("copy one\n", encoding="utf-8") + (workspace_root_2 / "auth.py").write_text("copy two\n", encoding="utf-8") + assert (workspace_root_1 / "auth.py").read_text(encoding="utf-8") == "copy one\n" + assert (workspace_root_2 / "auth.py").read_text(encoding="utf-8") == "copy two\n" diff --git a/libs/openant-core/tests/patch/test_patch_wrapper_contract.py b/libs/openant-core/tests/patch/test_patch_wrapper_contract.py new file mode 100644 index 00000000..bf0be38e --- /dev/null +++ b/libs/openant-core/tests/patch/test_patch_wrapper_contract.py @@ -0,0 +1,566 @@ +"""Contract tests for the openant-patch integration (core/patch.py + cmd_patch). + +These are the tests unique to THIS integration -- not ported from the +standalone Auto Patcher project, which had no notion of "a Go CLI triggered +this and must never get a report indistinguishable from a real one." + +Covers the specific safety/scope properties called out in the migration +plan: + - a run never silently falls back to Auto Patcher's mock mode + - suggested_fix/rejection_reason never reach the patch engine's input + - the eligibility gate is an explicit allowlist (fails closed) + - the envelope contract matches every other openant subcommand + - the Trust Report is written as an artifact whose content this + integration never inspects beyond existence + +All hermetic: LLM_PROVIDER=mock, no network, no real repo, no Docker. +""" + +import json +import os +from unittest import mock + +import pytest + +from core.patch import ( + PatchStepResult, + _require_llm_provider, + check_eligible, + effective_verdict, + find_finding_by_id, + render_vulnerability_markdown, + run_patch, +) + + +FIXTURE_FINDING_ELIGIBLE = { + "id": "F-001", + "name": "SQL Injection", + "location": {"file": "app.py", "function": "handle_request"}, + "cwe_id": 89, + "cwe_name": "SQL Injection", + "stage1_verdict": "vulnerable", + "stage2_verdict": "confirmed", + "description": "User input reaches a raw SQL query unsanitized.", + "vulnerable_code": "cursor.execute(query)", + "impact": ["Full database read access"], + "steps_to_reproduce": ["Send a crafted id"], + "suggested_fix": "DO-NOT-LEAK-THIS-SUGGESTED-FIX", + "rejection_reason": "DO-NOT-LEAK-THIS-REJECTION-REASON", +} + +FIXTURE_FINDING_REJECTED = { + **FIXTURE_FINDING_ELIGIBLE, + "id": "F-002", + "stage2_verdict": "rejected", +} + + +def _write_pipeline_output(tmp_path, findings): + path = tmp_path / "pipeline_output.json" + path.write_text(json.dumps({"findings": findings})) + return str(path) + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + +def test_find_finding_by_id_hit(): + f = find_finding_by_id([FIXTURE_FINDING_ELIGIBLE], "F-001") + assert f["id"] == "F-001" + + +def test_find_finding_by_id_miss(): + with pytest.raises(ValueError, match="F-999"): + find_finding_by_id([FIXTURE_FINDING_ELIGIBLE], "F-999") + + +@pytest.mark.parametrize( + "stage1,stage2,eligible", + [ + ("vulnerable", "confirmed", True), + ("vulnerable", "agreed", True), + ("vulnerable", "vulnerable", True), + ("bypassable", "bypassable", True), + ("vulnerable", "unverified", False), + ("vulnerable", "error", False), + ("vulnerable", "rejected", False), + ("safe", "safe", False), + ("vulnerable", "protected", False), + ("vulnerable", "inconclusive", False), + ("", "", False), + ("vulnerable", "Confirmed", False), # case-sensitive: fails closed, not normalized + ("vulnerable", "quarantined", False), # unknown future value: fails closed + ("vulnerable", "", True), # stage2 empty falls back to eligible stage1 + ("safe", "", False), # stage2 empty falls back to ineligible stage1 + ], +) +def test_check_eligible_allowlist(stage1, stage2, eligible): + finding = {"id": "X", "stage1_verdict": stage1, "stage2_verdict": stage2} + if eligible: + check_eligible(finding) # must not raise + else: + with pytest.raises(ValueError): + check_eligible(finding) + + +def test_effective_verdict_prefers_stage2(): + assert effective_verdict({"stage1_verdict": "vulnerable", "stage2_verdict": "confirmed"}) == "confirmed" + + +def test_effective_verdict_falls_back_to_stage1_when_stage2_empty(): + assert effective_verdict({"stage1_verdict": "vulnerable", "stage2_verdict": ""}) == "vulnerable" + + +def test_render_vulnerability_markdown_includes_key_fields(): + rendered = render_vulnerability_markdown(FIXTURE_FINDING_ELIGIBLE) + for expected in ("F-001", "SQL Injection", "app.py", "handle_request", + "CWE-89", "confirmed", "cursor.execute(query)", + "Full database read access", "Send a crafted id"): + assert expected in rendered + + +def test_render_vulnerability_markdown_never_includes_suggested_fix_or_rejection_reason(): + """Regression guard: OpenAnt's own suggested_fix/rejection_reason must + never reach the patch engine's input -- they could bias the + independently-generated candidate patch.""" + rendered = render_vulnerability_markdown(FIXTURE_FINDING_ELIGIBLE) + assert "DO-NOT-LEAK-THIS-SUGGESTED-FIX" not in rendered + assert "DO-NOT-LEAK-THIS-REJECTION-REASON" not in rendered + + +# --------------------------------------------------------------------------- +# F-31 / F-35: impact/steps_to_reproduce may arrive as a string instead of a +# list, and description/vulnerable_code may arrive as a non-string value. +# --------------------------------------------------------------------------- + +def test_render_vulnerability_markdown_accepts_string_impact_and_steps(): + """F-31: a string impact/steps_to_reproduce must render as one bullet, + not one bullet per character.""" + finding = { + **FIXTURE_FINDING_ELIGIBLE, + "impact": "Full database read access", + "steps_to_reproduce": "Send a crafted id", + } + rendered = render_vulnerability_markdown(finding) + assert "- Full database read access" in rendered + assert "1. Send a crafted id" in rendered + # the character-by-character bullet regression would produce this + assert "- F\n- u\n- l\n- l" not in rendered + + +def test_render_vulnerability_markdown_accepts_list_impact_and_steps(): + """Existing list-shaped inputs keep rendering as before.""" + finding = { + **FIXTURE_FINDING_ELIGIBLE, + "impact": ["Full database read access", "Data exfiltration"], + "steps_to_reproduce": ["Send a crafted id", "Read the response"], + } + rendered = render_vulnerability_markdown(finding) + assert "- Full database read access" in rendered + assert "- Data exfiltration" in rendered + assert "1. Send a crafted id" in rendered + assert "2. Read the response" in rendered + + +def test_render_vulnerability_markdown_coerces_non_string_description_and_code(): + """F-35: a non-string description/vulnerable_code must not raise.""" + finding = { + **FIXTURE_FINDING_ELIGIBLE, + "description": {"summary": "User input reaches a raw SQL query"}, + "vulnerable_code": ["cursor.execute(query)", "conn.commit()"], + } + rendered = render_vulnerability_markdown(finding) # must not raise + assert "User input reaches a raw SQL query" in rendered + assert "cursor.execute(query)" in rendered + + +def test_render_vulnerability_markdown_empty_impact_and_steps_omit_sections(): + """No behavior regression: absent/empty fields still omit their sections.""" + finding = {**FIXTURE_FINDING_ELIGIBLE, "impact": [], "steps_to_reproduce": []} + rendered = render_vulnerability_markdown(finding) + assert "## Impact" not in rendered + assert "## Attack scenario" not in rendered + + +# --------------------------------------------------------------------------- +# run_patch end-to-end (mock mode) +# --------------------------------------------------------------------------- + +def test_run_patch_happy_path(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE, FIXTURE_FINDING_REJECTED]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + assert isinstance(result, PatchStepResult) + assert result.finding_id == "F-001" + assert os.path.exists(result.vulnerability_path) + assert os.path.exists(result.trust_report_path) + assert result.vulnerability_path == str(tmp_path / "patch" / "F-001-vulnerability.md") + assert result.trust_report_path == str(tmp_path / "patch" / "F-001-trust-report.md") + + +def test_run_patch_without_repo_root_never_scans_process_cwd(tmp_path, monkeypatch): + """F-01: this module's own docstring calls these tests "hermetic ... + no real repo" -- that was only true of the *inputs*. Before the fix, + repo_root=None made the engine fall back to Path.cwd() and scan + whatever directory pytest happened to be invoked from (this repo's own + hundreds of test files), embedding real absolute paths into the + on-disk Trust Report artifact. Assert that no longer happens.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + report_text = open(result.trust_report_path, encoding="utf-8").read() + assert "Not evaluated — no repository root was provided." in report_text + openant_core_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + assert openant_core_root not in report_text + + +def test_run_patch_never_leaks_suggested_fix_into_artifact(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + vuln_text = open(result.vulnerability_path, encoding="utf-8").read() + assert "DO-NOT-LEAK-THIS-SUGGESTED-FIX" not in vuln_text + assert "DO-NOT-LEAK-THIS-REJECTION-REASON" not in vuln_text + + +def test_run_patch_mock_mode_is_self_disclosing(tmp_path, monkeypatch): + """The core safety property: a Go-triggered run must never produce a + mock report indistinguishable from a real one. LLM_PROVIDER=mock is + allowed, but the resulting Trust Report must say so, loudly.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + report_text = open(result.trust_report_path, encoding="utf-8").read() + assert "MOCK MODE" in report_text + assert "LLM mode | MOCK" in report_text + + +def test_run_patch_requires_llm_provider(tmp_path, monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + with pytest.raises(RuntimeError, match="LLM_PROVIDER"): + run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + +def test_run_patch_never_leaves_stale_trust_report_after_failed_rerun(tmp_path, monkeypatch): + """F-39: reusing an output directory after a failed run must never leave + a stale trust report from an earlier successful run sitting next to a + freshly-written vulnerability.md.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + first = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + assert os.path.exists(first.trust_report_path) + + import utilities.autopatcher.pipeline as _pipeline_module + + def _boom(**kwargs): + raise RuntimeError("simulated pipeline failure") + + monkeypatch.setattr(_pipeline_module, "run", _boom) + + with pytest.raises(RuntimeError, match="simulated pipeline failure"): + run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + assert not os.path.exists(first.trust_report_path), ( + "a failed rerun must not leave the previous run's trust report behind" + ) + assert os.path.exists(first.vulnerability_path) + + +def test_run_patch_rejects_ineligible_finding(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_REJECTED]) + + with pytest.raises(ValueError, match="not eligible"): + run_patch(po_path, "F-002", str(tmp_path), repo_root=None) + + +def test_run_patch_rejects_unknown_finding(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + with pytest.raises(ValueError, match="no finding"): + run_patch(po_path, "does-not-exist", str(tmp_path), repo_root=None) + + +def test_run_patch_missing_pipeline_output(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + + with pytest.raises(FileNotFoundError): + run_patch(str(tmp_path / "nope.json"), "F-001", str(tmp_path), repo_root=None) + + +# --------------------------------------------------------------------------- +# _require_llm_provider: the Python-side backstop the Go resolver's +# guarantee relies on. A correctly-resolved interactive run (Go sets +# LLM_PROVIDER in the subprocess env before Python starts) must not trigger +# this at all -- these tests confirm that directly rather than only +# indirectly through run_patch(). +# --------------------------------------------------------------------------- + +def test_require_llm_provider_does_not_raise_when_set(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + _require_llm_provider() # must not raise + + +def test_require_llm_provider_does_not_raise_for_mock(monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + _require_llm_provider() # must not raise + + +def test_require_llm_provider_raises_when_unset(monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + with pytest.raises(RuntimeError, match="LLM_PROVIDER"): + _require_llm_provider() + + +def test_run_patch_input_type_and_input_id_default_to_finding(tmp_path, monkeypatch): + """Regression guard: these fields are additive -- Finding-mode's existing + PatchStepResult contract must not regress when they were introduced for + CVE mode.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + assert result.input_type == "finding" + assert result.input_id == "F-001" + + +# --------------------------------------------------------------------------- +# Repository Understanding integration: repo_root normalization and the +# run-scoped investigation directory, mirroring test_run_patch_cve.py's +# equivalent coverage for CVE mode. +# --------------------------------------------------------------------------- + +def test_run_patch_repo_root_is_resolved_before_reaching_pipeline_run(tmp_path, monkeypatch): + """repo_root must be normalized (resolved) once, here at the entry + point, before InvestigationCase / ground_repository / parsing ever see + it -- same guarantee as run_patch_cve().""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + real_repo = tmp_path / "real_repo" + real_repo.mkdir() + link_repo = tmp_path / "link_repo" + link_repo.symlink_to(real_repo) + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + import utilities.autopatcher.pipeline as _pipeline_module + + captured = {} + original_run = _pipeline_module.run + + def _capturing_run(*, vulnerability_text, api_key, repo_root=None, investigation_output_dir=None): + captured["repo_root"] = repo_root + return original_run( + vulnerability_text=vulnerability_text, api_key=api_key, + repo_root=repo_root, investigation_output_dir=investigation_output_dir, + ) + + with mock.patch.object(_pipeline_module, "run", side_effect=_capturing_run): + run_patch(po_path, "F-001", str(tmp_path), repo_root=str(link_repo)) + + assert captured["repo_root"] == str(real_repo.resolve()) + assert captured["repo_root"] != str(link_repo) + + +def test_run_patch_without_repo_root_creates_no_investigation_directory(tmp_path, monkeypatch): + """Finding mode without a repository must preserve current behavior -- + no InvestigationCase repo_root, no investigation directory created.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + result = run_patch(po_path, "F-001", str(tmp_path), repo_root=None) + + assert not (tmp_path / "patch" / "F-001-investigation").exists() + vuln_text_written = open(result.vulnerability_path, encoding="utf-8").read() + assert vuln_text_written == render_vulnerability_markdown(FIXTURE_FINDING_ELIGIBLE) + + +def test_run_patch_investigation_directory_created_when_repo_root_given(tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + + run_patch(po_path, "F-001", str(tmp_path), repo_root=str(repo_root)) + + expected = tmp_path / "patch" / "F-001-investigation" + assert expected.is_dir() + assert not str(expected).startswith(str(repo_root)) + + +# --------------------------------------------------------------------------- +# Full CLI dispatch contract (openant/cli.py's cmd_patch) -- the exact +# envelope shape internal/python.Invoke() parses on the Go side. +# --------------------------------------------------------------------------- + +class _Args: + def __init__(self, **kw): + self.__dict__.update(kw) + + +def test_cmd_patch_success_envelope_shape(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + args = _Args(pipeline_output=po_path, finding_id="F-001", repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 0 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "success" + assert envelope["errors"] == [] + assert envelope["data"]["finding_id"] == "F-001" + assert os.path.exists(envelope["data"]["trust_report_path"]) + assert os.path.exists(envelope["data"]["vulnerability_path"]) + + +def test_cmd_patch_error_envelope_shape_for_unknown_finding(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + po_path = _write_pipeline_output(tmp_path, [FIXTURE_FINDING_ELIGIBLE]) + args = _Args(pipeline_output=po_path, finding_id="nope", repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert envelope["data"] == {} + assert len(envelope["errors"]) == 1 + + +# --------------------------------------------------------------------------- +# cmd_patch: --cve dispatch, mutual exclusion with --finding-id, and the +# full CVE-mode CLI contract (mocked NVD fetch, no network, no real repo). +# --------------------------------------------------------------------------- + +FIXTURE_CVE_FOR_CLI = { + "id": "CVE-2021-12345", + "descriptions": [{"lang": "en", "value": "A test vulnerability for CLI-contract coverage."}], +} + + +def test_cmd_patch_rejects_both_finding_id_and_cve(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + args = _Args( + pipeline_output=None, finding_id="F-001", cve="CVE-2021-12345", + repo_root=str(tmp_path), output=str(tmp_path / "out"), + ) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert envelope["data"] == {} + assert "exactly one" in envelope["errors"][0] + + +def test_cmd_patch_rejects_neither_finding_id_nor_cve(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + args = _Args(pipeline_output=None, finding_id=None, cve=None, repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert "exactly one" in envelope["errors"][0] + + +def test_cmd_patch_cve_mode_requires_repo_root(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + args = _Args(pipeline_output=None, finding_id=None, cve="CVE-2021-12345", repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert "--repo-root" in envelope["errors"][0] + + +def test_cmd_patch_finding_id_mode_requires_pipeline_output(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + args = _Args(pipeline_output=None, finding_id="F-001", cve=None, repo_root=None, output=str(tmp_path / "out")) + + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert "pipeline_output" in envelope["errors"][0] + + +def test_cmd_patch_success_envelope_shape_cve_mode(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + args = _Args( + pipeline_output=None, finding_id=None, cve="CVE-2021-12345", + repo_root=str(repo_root), output=str(tmp_path / "out"), + ) + + with mock.patch("utilities.autopatcher.cve_fetcher.fetch_cve", return_value=FIXTURE_CVE_FOR_CLI): + exit_code = cmd_patch(args) + + assert exit_code == 0 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "success" + assert envelope["errors"] == [] + assert envelope["data"]["finding_id"] == "CVE-2021-12345" + assert envelope["data"]["input_type"] == "cve" + assert envelope["data"]["input_id"] == "CVE-2021-12345" + assert os.path.exists(envelope["data"]["trust_report_path"]) + assert os.path.exists(envelope["data"]["vulnerability_path"]) + + +def test_cmd_patch_cve_mode_error_envelope_for_unknown_cve(tmp_path, monkeypatch, capsys): + from openant.cli import cmd_patch + from utilities.autopatcher.cve_fetcher import CVENotFoundError + + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + args = _Args( + pipeline_output=None, finding_id=None, cve="CVE-9999-99999", + repo_root=str(repo_root), output=str(tmp_path / "out"), + ) + + with mock.patch( + "utilities.autopatcher.cve_fetcher.fetch_cve", + side_effect=CVENotFoundError("no such CVE"), + ): + exit_code = cmd_patch(args) + + assert exit_code == 2 + envelope = json.loads(capsys.readouterr().out) + assert envelope["status"] == "error" + assert envelope["data"] == {} + assert len(envelope["errors"]) == 1 diff --git a/libs/openant-core/tests/patch/test_pipeline.py b/libs/openant-core/tests/patch/test_pipeline.py new file mode 100644 index 00000000..f8a9a731 --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline.py @@ -0,0 +1,1543 @@ +""" +Unit tests for the Auto Patcher MVP. + +All tests run in mock mode (no OPENAI_API_KEY required). +""" + +from __future__ import annotations + +import os +import sys +import textwrap +from pathlib import Path +from unittest import mock + +import pytest + + +import utilities.autopatcher as _ap_pkg +PROMPTS_DIR = Path(_ap_pkg.__file__).parent / "prompts" +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" + + +# --------------------------------------------------------------------------- +# llm_client +# --------------------------------------------------------------------------- + +class TestLLMClientMock: + def test_mock_mode_when_no_key(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + assert client.is_mock is True + + def test_mock_patch_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("patch generator system prompt", "fix this") + assert "diff" in response.lower() or "---" in response or "+++" in response + + def test_mock_review_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("review and explain impact of patch", "review this") + # Should contain review-like content + assert len(response) > 20 + + def test_mock_score_response(self): + from utilities.autopatcher.llm_client import LLMClient + client = LLMClient(api_key="") + response = client.complete("confidence score assessment", "score this") + assert "confidence" in response.lower() or "score" in response.lower() + + def test_live_mode_when_key_provided(self): + import utilities.autopatcher.llm_client as llm_client + from utilities.autopatcher.llm_client import LLMClient + # Clear cached provider and LLM_PROVIDER so is_mock falls back to key check. + with mock.patch.object(llm_client, "_cached_provider", None), \ + mock.patch.dict(os.environ, {}, clear=False): + os.environ.pop("LLM_PROVIDER", None) + client = LLMClient(api_key="sk-fake-key-for-testing") + assert client.is_mock is False + + def test_env_var_key_used(self): + import utilities.autopatcher.llm_client as llm_client + from utilities.autopatcher.llm_client import LLMClient + with mock.patch.object(llm_client, "_cached_provider", None), \ + mock.patch.dict(os.environ, {"OPENAI_API_KEY": "sk-env-key"}): + os.environ.pop("LLM_PROVIDER", None) + client = LLMClient() + assert client.is_mock is False + + +# --------------------------------------------------------------------------- +# Pipeline LLM mode log line +# --------------------------------------------------------------------------- + +class TestPipelineLLMModeLog: + """The early [pipeline] LLM mode: ... log must never include a model name + that hasn't been resolved yet. MOCK stays MOCK; LIVE carries no model. + + Progress logs go to stderr, not stdout: OpenAnt's Go CLI parses stdout + as a single final JSON envelope (internal/python.Invoke()), so any + engine progress print()s ported from the standalone Auto Patcher + project were redirected to stderr during the merge -- see + utilities/autopatcher/pipeline.py and llm_client.py.""" + + def test_mock_mode_log(self, monkeypatch, capsys): + import utilities.autopatcher.llm_client as llm_client + monkeypatch.setenv("LLM_PROVIDER", "mock") + monkeypatch.setattr(llm_client, "_cached_provider", None) + from utilities.autopatcher.pipeline import run + run("XSS in login form") + captured = capsys.readouterr() + assert "[pipeline] LLM mode: MOCK" in captured.err + + def test_live_mode_log_has_no_model_name(self, monkeypatch, capsys): + # With Anthropic configured, the early log should say LIVE with no model. + import utilities.autopatcher.llm_client as llm_client + monkeypatch.setenv("LLM_PROVIDER", "anthropic") + monkeypatch.setattr(llm_client, "_cached_provider", None) + monkeypatch.setenv("ANTHROPIC_API_KEY", "fake-key") + + # Stub out the actual Anthropic call so the test stays offline. + import types + class FakeAnthropic: + def __init__(self, api_key=None): pass + class messages: + @staticmethod + def create(model, max_tokens, messages): + import types + msg = types.SimpleNamespace(content=[types.SimpleNamespace(text="```diff\n--- a/f\n+++ b/f\n@@ -1,1 +1,1 @@\n-old\n+new\n```")]) + return types.SimpleNamespace(content=[msg.content[0]]) + monkeypatch.setitem(sys.modules, "anthropic", types.SimpleNamespace(Anthropic=FakeAnthropic)) + + from utilities.autopatcher.pipeline import run + run("path traversal in upload handler") + captured = capsys.readouterr() + assert "[pipeline] LLM mode: LIVE" in captured.err + # Must not include a specific model name in the early log. + assert "gpt-4o" not in captured.err.split("[pipeline] LLM mode:")[1].split("\n")[0] + assert "claude" not in captured.err.split("[pipeline] LLM mode:")[1].split("\n")[0].lower() + + +# --------------------------------------------------------------------------- +# patch_generator +# --------------------------------------------------------------------------- + +class TestPatchGenerator: + def test_returns_string(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_generator import generate_patch + llm = LLMClient(api_key="") + result = generate_patch("SQL injection in auth.py", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_patch_contains_diff_markers(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_generator import generate_patch + llm = LLMClient(api_key="") + result = generate_patch("SQL injection vulnerability", llm) + assert "---" in result or "+++" in result or "diff" in result.lower() + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "patch_generator.md" + assert prompt.exists(), "prompts/patch_generator.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# patch_reviewer +# --------------------------------------------------------------------------- + +class TestPatchReviewer: + def test_returns_string(self): + from utilities.autopatcher.llm_client import LLMClient + from utilities.autopatcher.patch_reviewer import review_patch + llm = LLMClient(api_key="") + result = review_patch("vuln description", "some patch diff", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "patch_reviewer.md" + assert prompt.exists(), "prompts/patch_reviewer.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# confidence_scorer +# --------------------------------------------------------------------------- + +class TestConfidenceScorer: + def test_returns_string(self): + from utilities.autopatcher.confidence_scorer import score_confidence + from utilities.autopatcher.llm_client import LLMClient + llm = LLMClient(api_key="") + result = score_confidence("vuln", "patch", "review", llm) + assert isinstance(result, str) + assert len(result) > 0 + + def test_response_mentions_score(self): + from utilities.autopatcher.confidence_scorer import score_confidence + from utilities.autopatcher.llm_client import LLMClient + llm = LLMClient(api_key="") + result = score_confidence("vuln", "patch", "review", llm) + assert "score" in result.lower() or "confidence" in result.lower() + + def test_prompt_file_exists(self): + prompt = PROMPTS_DIR / "confidence_scorer.md" + assert prompt.exists(), "prompts/confidence_scorer.md must exist" + assert prompt.stat().st_size > 0 + + +# --------------------------------------------------------------------------- +# pipeline +# --------------------------------------------------------------------------- + +class TestPipeline: + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + def test_run_produces_report(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert isinstance(report, str) + assert len(report) > 100 + + def test_report_contains_required_sections(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + required = [ + "Vulnerability summary", + "Vulnerability Sources", # renamed from "Primary Vulnerability References"; GHSA/CVE/Advisory URL only + "Proposed patch", + "Explanation", + "Affected areas", + "Reviewer Notes", # renamed from "Validation notes" (terminology cleanup) + "Recommendation", + "Trust Signals", # new Trust Package section + "Validation Actions", # new Trust Package section + "Review Results", # renamed from "Known Findings" (reviewer-experience terminology pass) + "Appendices", # new: consolidates diagnostics + legacy sections + ] + for section in required: + assert section in report, f"Report is missing section: '{section}'" + # "Validation Plan" was removed as a duplicate of Validation Actions + # (same underlying ≤3 items; Validation Actions now includes Reason + # too, so nothing unique was lost). + assert "## Validation Plan" not in report + # Superseded by the epistemic-category Review Results section. + assert "## Known Limitations" not in report + assert "## Known Security Gain" not in report + # Renamed to "Review Results" (reviewer-experience terminology pass). + assert "## Known Findings" not in report + # Reviewer-experience redesign: removed entirely as duplicated + # storytelling (restated Explanation/Known Findings/Reviewer Notes). + assert "## Patch Impact Summary" not in report + assert "## Impact Summary" not in report + assert "## Testing Notes" not in report + assert "### Testing Notes" not in report + # Upstream-reference policy: no fix-commit/remediation links in the + # reviewer report — those belong only in benchmark/evaluation artifacts. + assert "## Primary Vulnerability References" not in report + assert "Referenced Upstream Commit" not in report + # Presentation cleanup: legacy scoring sections, superseded by + # Trust Signals, are no longer rendered. + assert "Confidence Score" not in report + assert "Confidence Reasoning" not in report + assert "Confidence delta preview" not in report + # Presentation cleanup: the raw JSON dump duplicating the "Matching + # tests" list is no longer embedded in the human-facing report. + assert "```json" not in report + + def test_report_includes_validation_actions_format(self): + """Validation Actions is now the single, canonical checklist section + — it must include Reason as well as Next step (previously only the + now-removed "Validation Plan" section showed Reason).""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Validation Actions" in report + start = report.find("## Validation Actions") + assert start != -1 + # Patch Hygiene now precedes Validation Actions (promoted next to + # the diff) — Review Results is the next heading that follows it. + after = report.find("## Review Results", start) + block = report[start:after if after != -1 else start + 500] + assert "Reason:" in block and "Next step:" in block + actions = [ln for ln in block.splitlines() if re.match(r"^\d+\.\s+\*\*\[", ln.strip())] + assert len(actions) <= 3 + + def test_report_contains_recommendation_format(self): + """Trust Package V1: Recommendation uses the new decision vocabulary. + Trust Signals now precedes Recommendation (not after it), so this + just takes a fixed-width slice following the Recommendation heading.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Recommendation" in report + start = report.find("## Recommendation") + block = report[start:start + 400] + # New V1 format: bold decision on its own line, followed by reason text + bold_lines = [ln for ln in block.splitlines() if ln.strip().startswith("**") and ln.strip().endswith("**")] + assert bold_lines, "Recommendation must have a bold decision line" + decision = bold_lines[0].strip().strip("*").strip() + valid_decisions = ( + "Deploy After Validation", "Deploy With Caution", + "Manual Review Required", "Do Not Apply", + ) + assert decision in valid_decisions, f"Unexpected decision value: {decision!r}" + # Reason text must be a non-empty sentence after the decision line + non_empty = [ln.strip() for ln in block.splitlines() if ln.strip() and not ln.strip().startswith("#") and not ln.strip().startswith("**")] + assert non_empty, "Recommendation must contain reason text" + assert len(non_empty[0]) > 10 + + def test_recommendation_ordering(self): + """Reviewer-experience redesign: a reviewer first wants to know what + is broken and what patch is proposed — Vulnerability summary, + Vulnerability Sources, and Proposed patch now precede + Trust Signals and Recommendation entirely (reversing the previous + redesign's Trust-Signals-first placement). The legacy Confidence + Score/Reasoning sections that used to sit at the bottom, inside + Appendices, have been removed outright (superseded by Trust + Signals) rather than merely reordered. + + Patch Hygiene and Patch Applicability — the report's only two fully + deterministic checks — were promoted from Appendices to sit directly + after Proposed patch, before Trust Signals (a second reviewer- + experience pass): a reviewer who trusts deterministic evidence over + LLM narrative should not have to scroll past Explanation/Validation + Actions/Review Results to reach the actual git-apply result.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_vuln = report.find("## Vulnerability summary") + idx_refs = report.find("## Vulnerability Sources") + idx_patch = report.find("## Proposed patch") + idx_hygiene = report.find("## Patch Hygiene") + idx_applicability = report.find("## Patch Applicability") + idx_trust = report.find("## Trust Signals") + idx_rec = report.find("## Recommendation") + idx_val_actions = report.find("## Validation Actions") + idx_review_results = report.find("## Review Results") + for label, idx in [ + ("Vulnerability summary", idx_vuln), ("Vulnerability Sources", idx_refs), + ("Proposed patch", idx_patch), ("Patch Hygiene", idx_hygiene), + ("Patch Applicability", idx_applicability), ("Trust Signals", idx_trust), + ("Recommendation", idx_rec), ("Validation Actions", idx_val_actions), + ("Review Results", idx_review_results), + ]: + assert idx != -1, f"'{label}' section missing" + assert ( + idx_vuln < idx_refs < idx_patch < idx_hygiene < idx_applicability + < idx_trust < idx_rec < idx_val_actions < idx_review_results + ), ( + "Report order must be: Vulnerability summary, Primary Vulnerability " + "References, Proposed patch, Patch Hygiene, Patch Applicability, " + "Trust Signals, Recommendation, Validation Actions, then Review Results" + ) + + def test_report_omits_legacy_numeric_score(self): + """The legacy 'X / 1.0' confidence score was removed as a + presentation cleanup — it duplicated/contradicted the qualitative + Trust Signals verdicts. Trust Signals/Recommendation/decision logic + are unaffected; only this rendered artifact is gone.""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert not re.search(r"[0-9]+(?:\.[0-9]+)?\s*/\s*1\.0", report), ( + "Report should no longer contain the legacy numeric confidence score" + ) + + +# --------------------------------------------------------------------------- +# Report Structure v2, Phase 1 — Decision Card +# --------------------------------------------------------------------------- + +class TestDecisionCard: + """Unit tests for _render_decision_card as a Hero Banner: the decision + line itself is the heading (first visible line after the title), and + every field is a direct restatement of an already-computed value — + never a new derivation, never a separate "confidence" judgment.""" + + _EMOJI = { + "Deploy After Validation": "🟢", + "Deploy With Caution": "🟡", + "Manual Review Required": "🟠", + "Do Not Apply": "🔴", + } + + def _signals(self, patch_integrity="Clean"): + return {"patch_integrity": {"value": patch_integrity, "label": patch_integrity, "notes": ""}} + + def test_decision_line_is_first_line_and_uses_existing_emoji_mapping(self): + from utilities.autopatcher.pipeline import _render_decision_card + for decision, emoji in self._EMOJI.items(): + rec = {"decision": decision, "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + first_line = card.splitlines()[0] + assert first_line == f"## {emoji} {decision.upper()}" + + def test_patch_line_uses_existing_applicability_label(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Manual Review Required", "reason": "x"} + card = _render_decision_card(rec, self._signals("Critical Issues"), [], ["a.py"]) + assert "Patch has critical issues." in card + + def test_patch_line_clean(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals("Clean"), [], ["a.py"]) + assert "Patch applies cleanly." in card + + def test_no_separate_trust_or_confidence_field(self): + """Rule: never say 'High confidence' (or any confidence label) as a + separate Trust field — the banner has no independent confidence line.""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + assert "confidence" not in card.lower() + assert "Trust" not in card + + def test_validation_zero_actions(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], []) + assert "No additional validation actions identified." in card + + def test_validation_one_action_is_singular(self): + """Hero wording must not name a section — it must stay valid even if + section names or positions change (they already have, twice).""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [{}], []) + assert "Complete the recommended validation check before deployment." in card + assert "section" not in card.lower() + assert "1 checks" not in card + + def test_validation_multiple_actions_is_plural(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy With Caution", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [{}, {}, {}], []) + assert "Complete the recommended validation checks before deployment." in card + assert "section" not in card.lower() + + def test_files_changed_line(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Deploy After Validation", "reason": "x"} + card = _render_decision_card(rec, self._signals(), [], ["a.py", "b.py", "c.py"]) + assert "Files changed: 3" in card + + def test_do_not_apply_wording_does_not_imply_deployment(self): + """Reviewer-experience fix: 'before deployment' is contradictory when + the decision is Do Not Apply — there is no deployment to validate + toward. The same validation_actions still render; only the sentence + describing them changes.""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Do Not Apply", "reason": "x"} + card = _render_decision_card(rec, self._signals("Critical Issues"), [{}, {}], []) + assert "should not be deployed" in card + assert "before deployment" not in card + + def test_do_not_apply_wording_with_zero_actions(self): + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Do Not Apply", "reason": "x"} + card = _render_decision_card(rec, self._signals("Does Not Apply"), [], []) + assert "should not be deployed" in card + assert "before deployment" not in card + + def test_manual_review_required_wording_does_not_imply_deployment(self): + """Release-polish fix: Manual Review Required previously fell through + to the same "Complete the recommended validation check(s) before + deployment." wording as Deploy After Validation / Deploy With + Caution — differing only by the headline label above. It must + instead state plainly that deployment is not the next step and a + human reviewer must resolve the open questions first, regardless of + how many validation actions exist.""" + from utilities.autopatcher.pipeline import _render_decision_card + rec = {"decision": "Manual Review Required", "reason": "x"} + for actions in ([], [{}], [{}, {}, {}]): + card = _render_decision_card(rec, self._signals(), actions, []) + assert "before deployment" not in card + assert "section" not in card.lower() + assert "human reviewer" in card.lower() + assert "not a signal to deploy" in card.lower() + + +class TestTopAction: + """Unit tests for _select_top_action / _render_top_action_line + (release-polish change #7): a single "Top action" line rendered + immediately under Recommendation, echoing the highest-priority item + already in validation_actions — no new prioritization algorithm, no + duplication of Validation Actions' own reason/next_step text.""" + + def test_omits_when_no_actions(self): + from utilities.autopatcher.pipeline import _render_top_action_line + assert _render_top_action_line([]) == "" + assert _render_top_action_line(None) == "" + + def test_shows_only_the_title_not_reason_or_next_step(self): + from utilities.autopatcher.pipeline import _render_top_action_line + actions = [{ + "priority": "LOW", + "title": "Add targeted tests for X", + "reason": "This exact reason text must not repeat here", + "next_step": "This exact next step text must not repeat here", + }] + line = _render_top_action_line(actions) + assert "**Top action:** Add targeted tests for X" in line + assert "see Validation Actions below" in line + assert "This exact reason text must not repeat here" not in line + assert "This exact next step text must not repeat here" not in line + + def test_picks_highest_priority_even_when_not_first(self): + """Regression guard: build_validation_plan can unconditionally + prepend a MEDIUM-priority behavior-driven action ahead of an + existing HIGH-priority one (see its own "behavior" block, which + always does `final = [beh_action] + final`) — a naive + validation_actions[0] would under-represent the true priority.""" + from utilities.autopatcher.pipeline import _select_top_action + actions = [ + {"priority": "MEDIUM", "title": "Validate behavior", "reason": "r1", "next_step": "n1"}, + {"priority": "HIGH", "title": "Review authentication flow", "reason": "r2", "next_step": "n2"}, + {"priority": "LOW", "title": "Add targeted tests", "reason": "r3", "next_step": "n3"}, + ] + top = _select_top_action(actions) + assert top["title"] == "Review authentication flow" + + def test_ties_keep_first_list_order(self): + from utilities.autopatcher.pipeline import _select_top_action + actions = [ + {"priority": "HIGH", "title": "First high", "reason": "r1", "next_step": "n1"}, + {"priority": "HIGH", "title": "Second high", "reason": "r2", "next_step": "n2"}, + ] + top = _select_top_action(actions) + assert top["title"] == "First high" + + +class TestValidationActionsDecisionAware: + """Unit tests for _render_validation_actions_section's decision-aware + note (reviewer-experience fix): the actions themselves, their order, + priority, and count are untouched — only a leading note changes.""" + + _ACTION = {"priority": "MEDIUM", "title": "x", "reason": "y", "next_step": "z"} + + def test_do_not_apply_adds_note(self): + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION], "Do Not Apply") + assert "not recommended for deployment" in block + assert "1. **[MEDIUM]** x" in block + + def test_other_decisions_add_no_note(self): + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION], "Deploy After Validation") + assert "not recommended for deployment" not in block + assert "1. **[MEDIUM]** x" in block + + def test_default_decision_argument_adds_no_note(self): + """Backward-compatible default: omitting `decision` entirely must not + change existing callers' output.""" + from utilities.autopatcher.pipeline import _render_validation_actions_section + block = _render_validation_actions_section([self._ACTION]) + assert "not recommended for deployment" not in block + + +class TestDecisionCardInReport: + """End-to-end wiring: the Hero Banner sits first, Recommendation and + Trust Signals immediately follow it, and every other section keeps its + prior relative position.""" + + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + @staticmethod + def _find_hero_banner(report: str) -> int: + """Locate the Hero Banner heading (## ) — there is + no longer a literal '## Decision Card' label to search for.""" + import re + m = re.search(r"^## [🟢🟡🟠🔴] [A-Z ]+$", report, re.MULTILINE) + return m.start() if m else -1 + + def test_decision_card_appears_before_recommendation(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_card = self._find_hero_banner(report) + idx_rec = report.find("## Recommendation") + assert idx_card != -1, "Hero Banner heading not found" + assert idx_rec != -1 + assert idx_card < idx_rec + + def test_decision_card_is_first_section_after_title(self): + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + title_idx = report.find("# Auto Patcher MVP — Security Patch Report") + first_heading_after_title = report.find("##", title_idx) + assert first_heading_after_title == self._find_hero_banner(report) + + def test_vulnerability_summary_immediately_follows_hero_banner(self): + """Reviewer-experience redesign: a reviewer first wants to know what + is broken — Vulnerability summary now directly follows the Hero + Banner (reversing the previous redesign, which put Trust Signals + there instead).""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_card = self._find_hero_banner(report) + hero_line_end = report.find("\n", idx_card) + idx_vuln = report.find("## Vulnerability summary") + between = report[hero_line_end:idx_vuln] + assert "## " not in between + + def test_trust_signals_follows_proposed_patch(self): + """Trust Signals only becomes meaningful once a reviewer already + knows what is broken and what patch is proposed — it now follows + Vulnerability summary, Vulnerability Sources, and + Proposed patch, and precedes Recommendation.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_vuln = report.find("## Vulnerability summary") + idx_patch = report.find("## Proposed patch") + idx_trust = report.find("## Trust Signals") + idx_rec = report.find("## Recommendation") + assert idx_vuln < idx_patch < idx_trust < idx_rec + + def test_security_gain_appears_within_explanation(self): + """Known Security Gain was merged into Explanation rather than kept + as its own heading — it was always an extracted sentence from (or, + on the fallback path, a truncated paragraph of) the same explanation + text, so it's now a labeled lead-in inside that section instead of a + separate one repeating the same information. "Impact Summary" no + longer exists at all (removed as duplicated storytelling).""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx_patch = report.find("## Proposed patch") + idx_rec = report.find("## Recommendation") + idx_expl = report.find("## Explanation") + idx_val_actions = report.find("## Validation Actions") + assert idx_patch < idx_rec < idx_expl < idx_val_actions + assert "## Impact Summary" not in report + assert "## Known Security Gain" not in report + explanation_block = report[idx_expl:idx_val_actions] + assert "**Security gain:**" in explanation_block + + +class TestReportTerminologyCleanup: + """Terminology/navigation cleanup: one name per concept, and every + forward reference names a real, existing section heading.""" + + @staticmethod + def _vuln_text() -> str: + return (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + + def test_validation_plan_removed(self): + """Validation Plan repeated the same <=3 items already shown, with + Reason added, in Validation Actions — verified lossless to fold in + and remove rather than keep as a second name for the same checklist.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Validation Plan" not in report + assert "## Validation Actions" in report + + def test_reviewer_notes_replaces_validation_notes_heading(self): + """Renamed to stop sharing the word "Validation" with the actions + checklist, since this section is unrelated LLM reviewer prose, not + part of that concept.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Reviewer Notes" in report + assert "## Validation notes" not in report + + def test_testing_notes_removed_as_duplicated_storytelling(self): + """Testing Notes was a third restatement of validation_notes/challenger + findings, fully duplicating Reviewer Notes and Known Findings — + removed entirely rather than promoted.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert "## Testing Notes" not in report + assert "### Testing Notes" not in report + assert "### What should be tested" not in report + + def test_hero_banner_does_not_name_a_section(self): + """Hero wording must remain valid even if section names or positions + change elsewhere — it must not reference "Validation Actions" or any + other section name by name. + + This fixture's mock pipeline run deterministically resolves to + Manual Review Required (patch_integrity "Not Verified" — no + repo_root is passed here, so applicability is unavailable). Release- + polish pass: Manual Review Required no longer shares the generic + "Complete the recommended validation check(s) before deployment." + wording with Deploy After Validation / Deploy With Caution (see + TestDecisionCard.test_manual_review_required_wording_does_not_imply_deployment) + — it must still, per this test's own point, avoid naming a section.""" + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + idx = report.find("# Auto Patcher MVP") + banner = report[idx:report.find("## Vulnerability summary")] + assert "section" not in banner.lower() + assert "MANUAL REVIEW REQUIRED" in banner + assert "This is not a signal to deploy" in banner + + def test_no_dangling_see_analysis_below(self): + """Catches regression of the specific dangling phrase this cleanup + fixes in Trust Signals. Note: the Recommendation reason string + "Run the listed validation actions before deployment" (inside + _build_recommendation_v1, the Deploy After Validation branch) was + intentionally left untouched pending explicit confirmation that + editing that function's wording is in scope — not asserted here.""" + import re + from utilities.autopatcher.pipeline import run + report = run(vulnerability_text=self._vuln_text(), api_key="") + assert not re.search(r"see analysis below", report, re.IGNORECASE) +# --------------------------------------------------------------------------- +# Explanation rendering fix — dangling list-marker regression +# --------------------------------------------------------------------------- + +class TestExplanationDanglingListMarkerFix: + """Reviewer-experience fix: when the extracted security_gain sentence is + the entire body of a numbered/bulleted list item in the reviewer LLM's + explanation text, stripping it used to leave a bare marker behind (e.g. + a dangling "1." with nothing after it). This only touches how the + already-generated explanation text is rendered — _extract_security_gain + and the reviewer LLM's own output are untouched.""" + + @staticmethod + def _build(tmp_path, review): + from utilities.autopatcher.pipeline import _build_report, PipelineResult + result = PipelineResult( + vulnerability_text="# Test vulnerability\n\nSome description.", + patch=( + "--- a/mod.py\n+++ b/mod.py\n@@ -1,3 +1,3 @@\n" + " def foo():\n- return 1\n+ return 2\n" + ), + review=review, + score_text="**Confidence score:** 0.80\n\n**Reasons:**\n- ok", + challenger={"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""}, + impact={ + "impact_level": "low", "changed_files": [], "affected_files": [], + "impact_summary": "", "recommendations": [], "usage_matches": [], + }, + hygiene=[], + applicability={"applicable": True, "skipped": False, "skipped_reason": None, "error": None, "stderr": ""}, + repo_root=tmp_path, + detected_language="python", + ) + return _build_report(result) + + def test_no_dangling_list_marker_left_behind(self, tmp_path): + import re + review = ( + "**Explanation:**\n" + "1. This patch does not fix the vulnerability at all.\n\n" + "The underlying issue remains because the check was never added.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ) + report = self._build(tmp_path, review) + # The extracted sentence still appears once, as the Security gain callout. + assert "**Security gain:** This patch does not fix the vulnerability at all." in report + # No line consisting solely of a bare numbered/bulleted marker. + assert not re.search(r"^[ \t]*(?:\d+\.|[-*])[ \t]*$", report, re.MULTILINE) + # The rest of the explanation body is preserved. + assert "The underlying issue remains because the check was never added." in report + + def test_ordinary_explanation_unaffected(self, tmp_path): + """No list markers involved — behavior is unchanged from before this fix.""" + review = ( + "**Explanation:**\n" + "This patch fixes the vulnerability by validating input.\n\n" + "Additional context about the fix follows here.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ) + report = self._build(tmp_path, review) + assert "**Security gain:** This patch fixes the vulnerability by validating input." in report + assert "Additional context about the fix follows here." in report + + +# --------------------------------------------------------------------------- +# Slice 1 — Decision Consistency (report-level regression) +# --------------------------------------------------------------------------- + +class TestRecommendationConsistencyReport: + """Report-level regression coverage for Slice 1. + + Builds a PipelineResult directly and calls _build_report() — no LLM + calls, fully deterministic. Assertions check stable invariants (decision + label present, caveat text present/absent) rather than full-report + string equality, since equality would be brittle to unrelated formatting + and is not needed to prove this slice's behavior. + """ + + def _base_kwargs(self, tmp_path, challenger, applicability=None): + return dict( + vulnerability_text="# Test vulnerability\n\nSome description.", + patch=( + "--- a/mod.py\n" + "+++ b/mod.py\n" + "@@ -1,3 +1,3 @@\n" + " def foo():\n" + "- return 1\n" + "+ return 2\n" + ), + review=( + "**Explanation:**\n" + "The code was vulnerable because of X.\n\n" + "**Affected areas:**\n" + "- mod.py\n\n" + "**Validation notes:**\n" + "- Test with payload Y.\n" + ), + score_text="**Confidence score:** 0.80\n\n**Reasons:**\n- ok", + challenger=challenger, + impact={ + "impact_level": "low", "changed_files": [], "affected_files": [], + "impact_summary": "", "recommendations": [], "usage_matches": [], + }, + hygiene=[], + applicability=applicability or { + "applicable": True, "skipped": False, "skipped_reason": None, + "error": None, "stderr": "", + }, + repo_root=tmp_path, + detected_language="python", + ) + + def test_no_tests_found_gets_test_caveat_only(self, tmp_path): + """minimist-representative: no matching tests, no challenger findings. + Top-tier recommendation is unchanged; the test-coverage caveat is new.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Deploy After Validation**" in report + assert "test coverage" in report + assert "adversarial coverage" not in report + # Correction: never say "0 confirmed" / imply the patch is broken. + assert "0 confirmed" not in report.lower() + assert "issue(s) flagged" not in report + + def test_low_coverage_confidence_gets_coverage_caveat_only(self, tmp_path): + """pip-representative: matching tests exist, one unresolved plausible + finding. Top-tier recommendation is unchanged; the coverage-confidence + caveat is new.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + (tmp_path / "tests").mkdir() + (tmp_path / "tests" / "test_mod.py").write_text("def test_foo(): pass\n", encoding="utf-8") + challenger = { + "still_vulnerable": False, + "edge_cases": ["custom configurations may not benefit from this change"], + "potential_issues": [], + "summary": "", + } + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Deploy After Validation**" in report + assert "adversarial coverage" in report + assert "test coverage" not in report + # Correction: never say "0 confirmed" / imply the patch is broken. + assert "0 confirmed" not in report.lower() + assert "issue(s) flagged" not in report + + def test_manual_review_required_gets_no_caveat(self, tmp_path): + """A non-top-tier decision must not gain an "Evidence check" caveat + even when both underlying signals are weak — it already reads as + cautious. Distinct from the decision-relevant open-item scope note + added for Manual Review Required (release-polish change #6, see + test_manual_review_required_shows_open_item_scope_note below): that + note uses its own wording and is not gated by this "Evidence check" + mechanism — it simply doesn't fire here because this scenario's + empty edge_cases/potential_issues produce zero decision-relevant + findings to report.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": True, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + assert "**Manual Review Required**" in report + assert "This recommendation currently has no automated test coverage" not in report + assert "adversarial coverage is heuristic" not in report + assert "decision-relevant review item" not in report + + def test_manual_review_required_shows_open_item_scope_note(self, tmp_path): + """Release-polish change #6: Manual Review Required must surface the + same decision-relevant finding count Evidence-check caveats already + compute for the top two decisions — using distinct wording (never + the top-tier "Evidence check" / "adversarial coverage is heuristic" + phrasing, which stays reserved for Deploy After Validation / Deploy + With Caution per the test above).""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = { + "still_vulnerable": True, + "edge_cases": ["Cannot verify this without running the test suite"], + "potential_issues": [], + "summary": "", + } + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + rec_idx = report.find("## Recommendation") + explanation_idx = report.find("## Explanation") + rec_block = report[rec_idx:explanation_idx] + + assert "**Manual Review Required**" in rec_block + assert "1 decision-relevant review item remains open" in rec_block + assert "see Review Results below" in rec_block + assert "Evidence check" not in rec_block + assert "adversarial coverage is heuristic" not in rec_block + + def test_top_action_line_appears_below_recommendation(self, tmp_path): + """Release-polish change #7: a single "Top action" line must render + immediately under Recommendation, pointing to Validation Actions for + the full list, without repeating that action's reason/next_step.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = { + "still_vulnerable": False, + "edge_cases": ["custom configurations may not benefit from this change"], + "potential_issues": [], + "summary": "", + } + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + rec_idx = report.find("## Recommendation") + explanation_idx = report.find("## Explanation") + rec_block = report[rec_idx:explanation_idx] + + assert "**Top action:**" in rec_block + assert "see Validation Actions below" in rec_block + + def test_impact_surface_has_epistemic_disclaimer(self, tmp_path): + """Release-polish change #4: Impact Surface was the only major + section with no epistemic framing — must state this is static, + non-executing analysis.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + idx = report.find("## Impact Surface") + assert idx != -1 + block = report[idx:idx + 400] + assert "static" in block.lower() + assert "does not execute the code" in block + + def test_reviewer_notes_has_epistemic_disclaimer(self, tmp_path): + """Release-polish change #5: Reviewer Notes was the one un-hedged + LLM-prose section — must state it is reviewer-LLM guidance, not + independently verified evidence, and must not duplicate Explanation's + own disclaimer wording.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs(tmp_path, challenger)) + report = _build_report(result) + + idx = report.find("### Reviewer Notes") + assert idx != -1 + block = report[idx:idx + 300] + assert "not independently verified evidence" in block + assert "not independent execution or testing against the target repository" not in block + + def test_do_not_apply_gets_no_caveat(self, tmp_path): + """pygeoapi/curl-representative: applicability hard-block. Must remain + untouched by this slice.""" + from utilities.autopatcher.pipeline import _build_report, PipelineResult + challenger = {"still_vulnerable": False, "edge_cases": [], "potential_issues": [], "summary": ""} + result = PipelineResult(**self._base_kwargs( + tmp_path, challenger, + applicability={ + "applicable": False, "skipped": False, "skipped_reason": None, + "error": None, "stderr": "error: patch does not apply", + }, + )) + report = _build_report(result) + + assert "**Do Not Apply**" in report + assert "This recommendation currently has no automated test coverage" not in report + assert "adversarial coverage is heuristic" not in report + + +# --------------------------------------------------------------------------- +# pipeline helpers +# --------------------------------------------------------------------------- + +class TestPipelineHelpers: + def test_extract_score(self): + from utilities.autopatcher.pipeline import _extract_score + text = "**Confidence score:** 0.85\n**Reasons:**\n- reason 1" + assert _extract_score(text) == "0.85" + + def test_extract_score_missing(self): + from utilities.autopatcher.pipeline import _extract_score + assert _extract_score("no score here") == "N/A" + + def test_extract_summary(self): + from utilities.autopatcher.pipeline import _extract_summary + text = "# SQL Injection\n\nSome details" + assert _extract_summary(text) == "SQL Injection" + + def test_split_review_sections(self): + from utilities.autopatcher.pipeline import _split_review + review = textwrap.dedent("""\ + **Explanation:** + The code was vulnerable because of X. + + **Affected areas:** + - app/auth.py + + **Validation notes:** + - Test with payload Y. + """) + sections = _split_review(review) + assert "vulnerable" in sections["explanation"] + assert "auth.py" in sections["affected_areas"] + assert "payload" in sections["validation_notes"] + + def test_build_recommendation_helper(self): + from utilities.autopatcher.pipeline import build_recommendation + # still_vulnerable -> Do not deploy yet + rec = build_recommendation({"still_vulnerable": True}, 0.9, "Good", {"impact_level": "low"}) + assert rec["decision"] == "Do not deploy yet" + + # high impact and no tests -> Do not deploy yet (override) + rec = build_recommendation({}, 0.95, "None", {"impact_level": "high"}) + assert rec["decision"] == "Do not deploy yet" + + # safe case + rec = build_recommendation({}, 0.80, "Good", {"impact_level": "low"}) + assert rec["decision"] == "Safe to deploy" + + # reason constraints + rec = build_recommendation({"edge_cases": ["x"]}, 0.80, "Some", {"impact_level": "medium"}) + reason = rec.get("reason", "") + assert len(reason) > 0 and len(reason) <= 120 + assert "-" not in reason and "*" not in reason + + +# --------------------------------------------------------------------------- +# F-25 — a MEDIUM duplicate_assignment hygiene finding must land at +# "Minor Issues" / "Manual Review Required", never at "Critical Issues" / +# "Do Not Apply". Exercises _compute_trust_signals + _build_recommendation_v1 +# directly, isolating the hygiene-severity effect from the LLM-driven stages. +# --------------------------------------------------------------------------- + +class TestMediumHygieneRecommendation: + def test_medium_duplicate_assignment_yields_minor_issues_not_blocked(self): + from utilities.autopatcher.pipeline import _compute_trust_signals, _build_recommendation_v1 + + hygiene = [{ + "severity": "MEDIUM", + "check": "duplicate_assignment", + "detail": "`app/config.py`: `MAX_REQUEST_BYTES` is added, and an " + "unchanged assignment with the same name is also " + "visible in this diff — verify manually", + }] + applicability = {"applicable": True} + classified_challenger = { + "confirmed_defect_count": 0, + "plausible_risk_count": 0, + "validation_gap_count": 0, + "still_vulnerable": False, + } + + signals = _compute_trust_signals( + hygiene, applicability, classified_challenger, "Good", "low" + ) + assert signals["patch_integrity"]["value"] == "Minor Issues" + + recommendation = _build_recommendation_v1( + signals, still_vulnerable=False, defect_count=0 + ) + assert recommendation["decision"] == "Manual Review Required" + assert recommendation["decision"] != "Do Not Apply" + + +# --------------------------------------------------------------------------- +# Vulnerability Sources (GHSA/CVE/Advisory URL — no upstream remediation links) +# --------------------------------------------------------------------------- + +class TestPrimaryReferences: + """Direct unit coverage for _extract_primary_references/_render_primary_references + — previously only exercised indirectly through full report generation.""" + + def test_extracts_ghsa_and_cve_from_advisory_line(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "**Advisory:** GHSA-v845-jxx5-vc9f / CVE-2023-43804\n\nSome description.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] == "GHSA-v845-jxx5-vc9f" + assert refs["cve_id"] == "CVE-2023-43804" + assert refs["advisory_url"] == "https://github.com/advisories/GHSA-v845-jxx5-vc9f" + + def test_cve_only_advisory_line_uses_nvd_url(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "**Advisory:** CVE-2022-25883\n\nSome description.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] is None + assert refs["cve_id"] == "CVE-2022-25883" + assert refs["advisory_url"] == "https://nvd.nist.gov/vuln/detail/CVE-2022-25883" + + def test_file_mode_input_has_no_identifiers(self): + from utilities.autopatcher.pipeline import _extract_primary_references + text = "# Some vulnerability\n\nA hand-written description with no advisory line.\n" + refs = _extract_primary_references(text) + assert refs["ghsa_id"] is None + assert refs["cve_id"] is None + assert refs["advisory_url"] is None + + def test_extraction_never_returns_a_commit_reference(self): + """The extractor has no notion of an upstream commit at all anymore — + not computed, not hidden, removed entirely.""" + from utilities.autopatcher.pipeline import _extract_primary_references + text = ( + "**Advisory:** GHSA-v845-jxx5-vc9f\n\n" + "## References\n\n- https://github.com/urllib3/urllib3/commit/abc123\n" + ) + refs = _extract_primary_references(text) + assert "referenced_commit" not in refs + assert set(refs.keys()) == {"ghsa_id", "cve_id", "advisory_url"} + + +class TestRenderPrimaryReferences: + def test_heading_is_vulnerability_sources(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({"ghsa_id": "GHSA-xxxx", "cve_id": None, "advisory_url": "https://github.com/advisories/GHSA-xxxx"}) + assert "## Vulnerability Sources" in block + assert "## Primary Vulnerability References" not in block + + def test_ghsa_and_advisory_url_are_clickable_links(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-v845-jxx5-vc9f", "cve_id": "CVE-2023-43804", + "advisory_url": "https://github.com/advisories/GHSA-v845-jxx5-vc9f", + }) + assert "[GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f)" in block + assert "[https://github.com/advisories/GHSA-v845-jxx5-vc9f](https://github.com/advisories/GHSA-v845-jxx5-vc9f)" in block + + def test_cve_only_mode_links_to_nvd(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": None, "cve_id": "CVE-2022-25883", + "advisory_url": "https://nvd.nist.gov/vuln/detail/CVE-2022-25883", + }) + assert "[CVE-2022-25883](https://nvd.nist.gov/vuln/detail/CVE-2022-25883)" in block + + def test_file_mode_degrades_to_single_line(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({"ghsa_id": None, "cve_id": None, "advisory_url": None}) + assert "## Vulnerability Sources" in block + assert "User-provided vulnerability description" in block + assert "|" not in block # no table at all, not a table of placeholders + + def test_no_upstream_commit_or_remediation_link_ever_rendered(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-xxxx", "cve_id": "CVE-2023-1", "advisory_url": "https://github.com/advisories/GHSA-xxxx", + }) + assert "Referenced Upstream Commit" not in block + assert "commit" not in block.lower() + + def test_only_three_rows_in_table(self): + from utilities.autopatcher.pipeline import _render_primary_references + block = _render_primary_references({ + "ghsa_id": "GHSA-xxxx", "cve_id": "CVE-2023-1", "advisory_url": "https://github.com/advisories/GHSA-xxxx", + }) + row_lines = [l for l in block.splitlines() if l.startswith("| ") and "---" not in l and "Type" not in l] + assert len(row_lines) == 3 + + +# --------------------------------------------------------------------------- +# CLI (main.py) +# --------------------------------------------------------------------------- + +_FIXTURE_ADVISORY = { + "ghsa_id": "GHSA-1234-5678-9012", + "cve_id": "CVE-2021-12345", + "summary": "SQL injection in example library", + "description": "A SQL injection vulnerability exists in the authenticate() function.", + "severity": "critical", + "cvss": {"score": 9.8}, + "cwes": [{"cwe_id": "CWE-89", "name": "Improper Neutralization"}], + "vulnerabilities": [ + { + "package": {"name": "example-lib", "ecosystem": "pip"}, + "vulnerable_version_range": "< 1.2.3", + "first_patched_version": "1.2.3", + } + ], + "references": [{"url": "https://github.com/example/security/advisories/GHSA-1234"}], +} + + +# TestCLI (upstream): exercised main.py's argparse CLI, including GHSA mode. +# main.py is not ported -- its role is replaced by core/patch.py + cmd_patch +# in openant/cli.py, which have their own dedicated tests (see +# tests/patch/test_patch_wrapper_contract.py and openant-core's own CLI +# dispatch tests). GHSA mode (advisory_fetcher) is excluded entirely per the +# migration plan -- OpenAnt always supplies a rendered Finding, never a bare +# advisory ID. + + +# --------------------------------------------------------------------------- +# Pipeline code context integration +# --------------------------------------------------------------------------- + +class TestPipelineCodeContext: + def test_pipeline_passes_nonempty_code_context_when_repo_has_match(self, tmp_path): + """When repo_root contains a file matching the vulnerability, pipeline + should pass a non-empty code_context into generate_patch().""" + from utilities.autopatcher.pipeline import run + from unittest import mock as _mock + import utilities.autopatcher.patch_generator as _pg + + # Create a file that the example vulnerability.md references explicitly + auth_file = tmp_path / "app" / "auth.py" + auth_file.parent.mkdir(parents=True) + auth_file.write_text( + "def authenticate(username, password):\n" + " query = f\"SELECT * FROM users WHERE username='{username}'\"\n" + " return db.execute(query).fetchone()\n", + encoding="utf-8", + ) + + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + captured: list[str] = [] + _original = _pg.generate_patch + + def _capturing(vtext, llm, code_context=""): + captured.append(code_context) + return _original(vtext, llm) # use real (mock) implementation + + with _mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=_capturing): + run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + assert captured, "generate_patch was never called" + assert captured[0] != "", ( + "Expected non-empty code_context when repo contains app/auth.py" + ) + + def test_test_support_uses_target_repo_not_auto_patcher(self, tmp_path): + """Test Support must scan repo_root, not the Auto-Patcher's own test directory.""" + from utilities.autopatcher.pipeline import run + + # Target repo: a minimal Python project with one test file + target_test = tmp_path / "tests" / "test_auth.py" + target_test.parent.mkdir(parents=True) + target_test.write_text("def test_login(): pass\n", encoding="utf-8") + + vuln_text = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + report = run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + # The report must reference the target repo's test, not Auto-Patcher's tests + assert "test_auth.py" in report, ( + "Test Support should list the target repo's test_auth.py" + ) + # Auto-Patcher's own tests must not appear + assert "test_pipeline.py" not in report, ( + "Test Support must not list Auto-Patcher's own test files" + ) + assert "test_advisory_fetcher.py" not in report, ( + "Test Support must not list Auto-Patcher's own test files" + ) + + +# --------------------------------------------------------------------------- +# Repository Context section (Repository Grounding surfaced in the report) +# --------------------------------------------------------------------------- + +class TestRepositoryContextSection: + """Coverage for _render_repository_context_section() and its + _selected_reason_kind() adapter — the report-facing surface of + Repository Grounding (ground_repository()/RepositoryGroundingResult). + + Builds synthetic RepositoryGroundingResult objects directly rather than + running a real repo scan — these tests exercise the renderer's mapping + and presentation rules in isolation from repo_locator's discovery + algorithm, which is already covered by tests/test_repo_locator.py. + """ + + @staticmethod + def _evidence(pass_name, tier): + from utilities.autopatcher.repository_grounding_models import DiscoveryEvidence + return DiscoveryEvidence( + pass_name=pass_name, tier=tier, matched_tokens=None, + total_occurrences=None, hit_line=0, resolution_strategy=None, + ) + + @staticmethod + def _candidate(path, evidence, best_tier): + from utilities.autopatcher.repository_grounding_models import RepositoryCandidate + return RepositoryCandidate(path=path, evidence=evidence, best_tier=best_tier) + + @staticmethod + def _decision(path, outcome): + from utilities.autopatcher.repository_grounding_models import GroundingDecision + return GroundingDecision( + path=path, outcome=outcome, snippet_ranges=None, + bytes_contributed=0, truncated=False, + ) + + @staticmethod + def _grounding(candidates, decisions): + from utilities.autopatcher.repository_grounding_models import RepositoryGroundingResult + return RepositoryGroundingResult( + rendered_context="irrelevant to these tests", + candidates=candidates, decisions=decisions, + extraction_signals={}, budget=None, + ) + + # --- renderer behavior --------------------------------------------------- + + def test_renderer_basic_layout(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("retry.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decision = self._decision("retry.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert section.startswith("---\n\n## Repository Context\n\n") + assert ( + "The following repository locations were selected to provide " + "context for patch generation and review." in section + ) + assert "**`retry.py`**" in section + assert "Selected because\n- Explicitly referenced in the security advisory" in section + assert "Used for\n- Primary reference (full file)" in section + + def test_intro_clarifies_pre_patch_provenance(self): + """Release-polish change #8: Repository Context must state these + locations were selected before the patch was generated, and point + to Post-Patch Investigation for evidence gathered afterward — + without implying patch-touched evidence was selected beforehand.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("retry.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decision = self._decision("retry.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert "selected **before** the patch was generated" in section + assert "Post-Patch Investigation" in section + + def test_renderer_multi_location_separated_by_rule(self): + """§4.3 of the approved plan: a horizontal rule separates entries + when more than one location is selected.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + cand_a = self._candidate("first.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + cand_b = self._candidate("second.py", [self._evidence("symbol_search", tier=2)], best_tier=2) + decisions = [ + self._decision("first.py", "primary_full_file"), + self._decision("second.py", "secondary_snippet"), + ] + grounding = self._grounding([cand_a, cand_b], decisions) + + section = _render_repository_context_section(grounding) + + assert "**`first.py`**" in section + assert "**`second.py`**" in section + between = section[section.index("first.py"):section.index("second.py")] + assert "\n\n---\n\n" in between + + # --- semantic reason mapping ---------------------------------------------- + + @pytest.mark.parametrize("pass_name,expected_phrase", [ + ("explicit_path", "Explicitly referenced in the security advisory"), + ("symbol_definition", "Defines the exact symbol named in the advisory"), + ("symbol_search", "References a symbol named in the advisory"), + ("cwe_keywords", "Contains terminology associated with this vulnerability type"), + ]) + def test_semantic_reason_mapping(self, pass_name, expected_phrase): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence(pass_name, tier=1)], best_tier=1) + decision = self._decision("a_file.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert expected_phrase in section + + def test_selected_reason_kind_picks_evidence_matching_best_tier(self): + """A candidate discovered by more than one pass must report the + reason for whichever pass produced its best_tier — not just the + first evidence entry appended. This is the adapter that keeps the + renderer itself ignorant of DiscoveryEvidence/best_tier/tier + matching.""" + from utilities.autopatcher.pipeline import _selected_reason_kind + candidate = self._candidate( + "a_file.py", + [self._evidence("cwe_keywords", tier=1), self._evidence("symbol_search", tier=2)], + best_tier=2, + ) + assert _selected_reason_kind(candidate) == "symbol_search" + + def test_selected_reason_kind_falls_back_to_first_evidence_when_best_tier_none(self): + """Defensive fallback for a hypothetical best_tier=None candidate (no + current repo_locator.py pass produces one — every pass now assigns a + real tier, including the exact symbol-definition pass) — must not + crash, must use the one evidence entry present.""" + from utilities.autopatcher.pipeline import _selected_reason_kind + candidate = self._candidate( + "a_file.py", [self._evidence("some_future_tierless_pass", tier=None)], best_tier=None, + ) + assert _selected_reason_kind(candidate) == "some_future_tierless_pass" + + def test_selected_reason_kind_none_for_missing_or_empty_candidate(self): + from utilities.autopatcher.pipeline import _selected_reason_kind + assert _selected_reason_kind(None) is None + assert _selected_reason_kind(self._candidate("a_file.py", [], best_tier=None)) is None + + def test_unknown_reason_kind_falls_back_to_generic_phrase(self): + """An unrecognized pass_name (a future evidence kind not yet in the + lookup table) must degrade to a generic sentence, not KeyError.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("some_future_pass", tier=1)], best_tier=1) + decision = self._decision("a_file.py", "primary_full_file") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert "Identified during repository grounding" in section + + # --- usage-role mapping ----------------------------------------------- + + @pytest.mark.parametrize("outcome,expected_phrase", [ + ("primary_full_file", "Primary reference (full file)"), + ("primary_snippet", "Primary reference (excerpt)"), + ("secondary_snippet", "Supporting reference (excerpt)"), + ]) + def test_usage_role_mapping(self, outcome, expected_phrase): + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decision = self._decision("a_file.py", outcome) + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert expected_phrase in section + + # --- rejected locations omitted ----------------------------------------- + + def test_rejected_locations_omitted(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + kept = self._candidate("kept_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + dropped = self._candidate("dropped_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + decisions = [ + self._decision("kept_file.py", "primary_full_file"), + self._decision("dropped_file.py", "rejected"), + ] + grounding = self._grounding([kept, dropped], decisions) + + section = _render_repository_context_section(grounding) + + assert "kept_file.py" in section + assert "dropped_file.py" not in section + + def test_all_rejected_renders_zero_selection_sentence(self): + """Every discovered candidate rejected must read the same as no + candidates at all — not a silently empty location list.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + candidate = self._candidate("a_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + decision = self._decision("a_file.py", "rejected") + grounding = self._grounding([candidate], [decision]) + + section = _render_repository_context_section(grounding) + + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + # --- None / empty grounding handling ------------------------------------- + + def test_none_grounding_renders_zero_selection_sentence(self): + from utilities.autopatcher.pipeline import _render_repository_context_section + section = _render_repository_context_section(None) + assert "## Repository Context" in section + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + def test_empty_grounding_renders_zero_selection_sentence(self): + """A grounding result with no candidates/decisions at all (the real + empty/no-match exit path in ground_repository()) must render the + same zero-selection sentence as grounding=None.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + section = _render_repository_context_section(self._grounding([], [])) + assert ( + "No repository locations were identified to provide context " + "for this vulnerability." in section + ) + + # --- section ordering ---------------------------------------------------- + + def test_decision_order_preserved_not_resorted(self): + """The approved plan does not call for re-ordering by outcome — a + secondary location that comes before the primary in + grounding.decisions must still render in that order. No sort is + applied by the renderer.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + secondary = self._candidate("secondary_file.py", [self._evidence("cwe_keywords", tier=1)], best_tier=1) + primary = self._candidate("primary_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + decisions = [ + self._decision("secondary_file.py", "secondary_snippet"), + self._decision("primary_file.py", "primary_full_file"), + ] + grounding = self._grounding([secondary, primary], decisions) + + section = _render_repository_context_section(grounding) + + assert section.index("secondary_file.py") < section.index("primary_file.py") + + def test_repository_context_section_placed_before_impact_surface(self, tmp_path): + """§4.2 of the approved plan: Repository Context sits immediately + before Impact Surface, after Review Results — integration-level + check against a real ground_repository() result via run().""" + from utilities.autopatcher.pipeline import run + (tmp_path / "auth.py").write_text( + "def authenticate(u, p):\n return db.query(u, p)\n", encoding="utf-8" + ) + vuln_text = "SQL injection in authenticate() — see auth.py" + report = run(vulnerability_text=vuln_text, api_key="", repo_root=str(tmp_path)) + + idx_review = report.find("## Review Results") + idx_repo_context = report.find("## Repository Context") + idx_impact = report.find("## Impact Surface") + + assert idx_review != -1, "Review Results section missing" + assert idx_repo_context != -1, "Repository Context section missing" + assert idx_impact != -1, "Impact Surface section missing" + assert idx_review < idx_repo_context < idx_impact + + # --- path/reason/role pairing ----------------------------------------- + + def test_path_reason_role_pairing_not_mixed_up(self): + """Two locations with different reasons and different roles: each + path's own reason/role must appear paired with it, not swapped with + the other location's.""" + from utilities.autopatcher.pipeline import _render_repository_context_section + cand_a = self._candidate("alpha_file.py", [self._evidence("explicit_path", tier=3)], best_tier=3) + cand_b = self._candidate("beta_file.py", [self._evidence("symbol_search", tier=2)], best_tier=2) + decisions = [ + self._decision("alpha_file.py", "primary_full_file"), + self._decision("beta_file.py", "secondary_snippet"), + ] + grounding = self._grounding([cand_a, cand_b], decisions) + + section = _render_repository_context_section(grounding) + + block_a = section[section.index("alpha_file.py"):section.index("beta_file.py")] + block_b = section[section.index("beta_file.py"):] + + assert "Explicitly referenced in the security advisory" in block_a + assert "Primary reference (full file)" in block_a + assert "References a symbol named in the advisory" not in block_a + + assert "References a symbol named in the advisory" in block_b + assert "Supporting reference (excerpt)" in block_b diff --git a/libs/openant-core/tests/patch/test_pipeline_post_patch_investigation.py b/libs/openant-core/tests/patch/test_pipeline_post_patch_investigation.py new file mode 100644 index 00000000..3520a4ec --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline_post_patch_investigation.py @@ -0,0 +1,448 @@ +"""Tests for the Phase 4 Post-Patch Investigation pipeline wiring. + +Covers the orchestration boundary approved for this phase: derive +pre-patch Anchors right after Repository Understanding, run the +workspace-copy -> apply -> parse -> evaluate -> render chain exactly once +(right before the FIRST challenge_patch() call), thread its evidence into +that call plus calibrate_findings()/score_confidence() (guarded by a +patch-identity staleness check against the Challenger-driven repair +loop), and render a "Post-Patch Investigation" Trust Report section. + +Hermetic: LLM_PROVIDER=mock, no network. A real (tiny) on-disk git repo +under tmp_path gives Repository Understanding/Anchor derivation real, +deterministic work to do; the post-patch chain itself (workspace copy, +git apply, parser, evaluation) also runs for real against that repo -- +the mock patches deliberately don't apply cleanly to it, so the chain +naturally degrades to `evaluation_error` observations, which is itself a +real, useful thing to verify (see test_investigation_integration.py for +the identical fixture-repo convention this file reuses). +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path +from unittest import mock + +import utilities.autopatcher.post_patch_evaluation as _ppe_mod + +EXAMPLES_DIR = Path(__file__).parent / "fixtures" / "examples" +_VULN_TEXT = (EXAMPLES_DIR / "vulnerability.md").read_text(encoding="utf-8") + +_APPLICABILITY_CLEAN = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, +} + +_SOME_DIFF = """\ +```diff +--- a/app/auth.py ++++ b/app/auth.py +@@ -1,2 +1,3 @@ + def authenticate(): ++ pass + pass +```""" + +_REPAIR_DIFF = """\ +```diff +--- a/app/auth.py ++++ b/app/auth.py +@@ -1,2 +1,3 @@ + def authenticate(): ++ # repaired + pass +```""" + +_CHALLENGER_WITH_DEFECT = { + "still_vulnerable": False, + "edge_cases": ["An attacker can bypass this check via path traversal"], + "potential_issues": [], + "summary": "The patch has a confirmed bypass.", +} + +_CHALLENGER_CLEAN = { + "still_vulnerable": False, + "edge_cases": [], + "potential_issues": [], + "summary": "No issues found.", +} + + +def _write_auth_repo(root: Path) -> None: + """Same fixture as test_investigation_integration.py's _write_auth_repo + -- matches fixtures/examples/vulnerability.md's explicit `app/auth.py` + / `authenticate()` reference, giving grounding a real, strong-tier + candidate and the parser a real function to resolve.""" + auth = root / "app" / "auth.py" + auth.parent.mkdir(parents=True) + auth.write_text( + "import sqlite3\n\n" + "db = sqlite3.connect(\"users.db\")\n\n" + "def authenticate(username, password):\n" + " query = f\"SELECT * FROM users WHERE username='{username}'\"\n" + " return db.execute(query).fetchone() is not None\n", + encoding="utf-8", + ) + subprocess.run(["git", "init"], cwd=root, capture_output=True) + subprocess.run(["git", "config", "user.email", "t@t.com"], cwd=root, capture_output=True) + subprocess.run(["git", "config", "user.name", "T"], cwd=root, capture_output=True) + subprocess.run(["git", "add", "-A"], cwd=root, capture_output=True) + subprocess.run(["git", "commit", "-m", "init"], cwd=root, capture_output=True) + + +def _run_pipeline( + tmp_path, + *, + patches_gen, + patches_chall, + repo_root, + extra_patches=(), +): + import utilities.autopatcher.pipeline as _pipeline_mod + from contextlib import ExitStack + + captured_result = {} + orig_build = _pipeline_mod._build_report + + def _capture_build(r): + captured_result["result"] = r + return orig_build(r) + + calls = {"challenge": [], "calibrate": [], "score": []} + + def _challenge_side_effect(*a, **kw): + calls["challenge"].append(kw.get("code_context", "")) + idx = len(calls["challenge"]) - 1 + return patches_chall[min(idx, len(patches_chall) - 1)] + + def _calibrate_side_effect(*a, **kw): + calls["calibrate"].append(kw.get("code_context", "")) + return None + + def _score_side_effect(*a, **kw): + calls["score"].append(kw.get("code_context", "")) + return "Confidence score: 0.8" + + patchers = [ + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=patches_gen), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", return_value=_APPLICABILITY_CLEAN), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok review"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", side_effect=_challenge_side_effect), + mock.patch("utilities.autopatcher.pipeline.calibrate_findings", side_effect=_calibrate_side_effect), + mock.patch("utilities.autopatcher.pipeline.score_confidence", side_effect=_score_side_effect), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=_capture_build), + ] + patchers.extend(extra_patches) + + import tempfile + investigation_dir = Path(tempfile.mkdtemp(prefix="ppi-investigation-")) + + with ExitStack() as stack: + mocks = [stack.enter_context(p) for p in patchers] + from utilities.autopatcher.pipeline import run + report = run( + _VULN_TEXT, api_key="", repo_root=repo_root, + investigation_output_dir=(str(investigation_dir) if repo_root else None), + ) + + return captured_result["result"], report, calls, mocks + + +# --------------------------------------------------------------------------- +# Runs once, regardless of retry/repair +# --------------------------------------------------------------------------- + +class TestRunsExactlyOnce: + def test_evaluate_anchors_called_once_even_with_repair(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + with mock.patch.object( + _ppe_mod, "evaluate_anchors", side_effect=_ppe_mod.evaluate_anchors, + ) as m_evaluate: + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF, _REPAIR_DIFF], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert m_evaluate.call_count == 1 + assert result.repair_succeeded is True + + +# --------------------------------------------------------------------------- +# repo_root=None: never attempted +# --------------------------------------------------------------------------- + +class TestNoRepoRoot: + def test_section_states_not_evaluated_distinctly(self, tmp_path): + with mock.patch("utilities.autopatcher.patch_workspace.temporary_repo_copy") as m_copy: + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=None, + ) + + m_copy.assert_not_called() + assert result.post_patch_observations is None + assert "## Post-Patch Investigation" in report + assert "Not evaluated for this run" in report + # Must never collide with the exact F-01 string used elsewhere. + assert report.count("Not evaluated — no repository root was provided.") == 3 + + +# --------------------------------------------------------------------------- +# First challenge_patch() call receives the evidence +# --------------------------------------------------------------------------- + +class TestFirstChallengeCallEnriched: + def test_first_challenge_context_includes_post_patch_section(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert result.post_patch_observations is not None + assert len(calls["challenge"]) == 1 + assert "## Post-Patch Investigation" in calls["challenge"][0] + + def test_no_new_llm_calls_introduced(self, tmp_path): + """Only the already-existing generate_patch/challenge_patch/ + review_patch/score_confidence/calibrate_findings call sites should + fire -- the post-patch chain itself makes no LLM calls.""" + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + assert len(calls["challenge"]) == 1 + assert len(calls["score"]) == 1 + + +# --------------------------------------------------------------------------- +# Staleness guard: repair replacing the patch must not leak stale evidence +# --------------------------------------------------------------------------- + +class TestStalenessGuard: + def test_repair_succeeds_invalidates_evidence_for_later_consumers(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF, _REPAIR_DIFF], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert result.repair_succeeded is True + assert result.patch != result.post_patch_investigated_patch + + # The FIRST challenge_patch call (pre-repair) got the evidence... + assert "## Post-Patch Investigation" in calls["challenge"][0] + # ...but the repair loop's own re-challenge never does (by design, + # regardless of staleness -- extending fresh evidence to the + # repair path is explicitly deferred). + assert "## Post-Patch Investigation" not in calls["challenge"][1] + # ...and score_confidence (which runs after the repair loop, on the + # final patch) must not receive the now-stale evidence either. + assert "## Post-Patch Investigation" not in calls["score"][0] + + # The Trust Report must say so explicitly, not silently show stale data. + assert "revised after this evidence was computed" in report + + def test_no_repair_evidence_reaches_score_confidence(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert result.repair_attempted is False + assert result.patch == result.post_patch_investigated_patch + assert "## Post-Patch Investigation" in calls["score"][0] + + +# --------------------------------------------------------------------------- +# Failure isolation +# --------------------------------------------------------------------------- + +class TestFailureIsolation: + def test_exception_mid_chain_degrades_without_crashing(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + with mock.patch( + "utilities.autopatcher.patch_workspace.temporary_repo_copy", + side_effect=RuntimeError("boom"), + ): + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert result.post_patch_observations is None + assert "## Post-Patch Investigation" in report + assert "Not evaluated for this run" in report + # code_context passed to challenge_patch must fall back cleanly, + # never raise or contain a partial/garbled section. + assert "## Post-Patch Investigation" not in calls["challenge"][0] + + def test_trust_signals_unaffected_by_injected_failure(self, tmp_path): + """Recommendation/Trust Signals must be identical whether the + post-patch chain succeeds, degrades, or throws -- this feature is + provably inert with respect to that machinery (no new parameter + was added to _compute_trust_signals/_build_recommendation_v1).""" + repo_root = tmp_path / "repo" + _write_auth_repo(repo_root) + + result_ok, report_ok, _, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + with mock.patch( + "utilities.autopatcher.patch_workspace.temporary_repo_copy", + side_effect=RuntimeError("boom"), + ): + result_fail, report_fail, _, _ = _run_pipeline( + tmp_path, + patches_gen=[_SOME_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + assert result_ok.final_score == result_fail.final_score + assert result_ok.hygiene == result_fail.hygiene + + +# --------------------------------------------------------------------------- +# Candidate-selection-independent patch-touched Anchors (end-to-end regression) +# +# Reproduces, hermetically, the exact real-repo gap found while validating +# this feature against urllib3/CVE-2023-43804: Candidate Selection runs on +# the vulnerability TEXT before any patch exists, so it can never select a +# file the text doesn't textually resemble -- even when the eventual patch +# touches it. `app/rate_limit_config.py` below is deliberately unrelated, +# in vocabulary, to the SQL-injection vulnerability.md text (no "auth"/ +# "password"/"query"/"sql" overlap) precisely so real, unmodified grounding +# genuinely does not select it -- the same way real grounding for the +# urllib3 CVE never selected retry.py. No candidate is manually injected +# anywhere in this test. +# --------------------------------------------------------------------------- + +_RATE_LIMIT_CONFIG_DIFF = """\ +```diff +--- a/app/rate_limit_config.py ++++ b/app/rate_limit_config.py +@@ -1,5 +1,5 @@ + class RateLimitConfig: +- DEFAULT_ALLOWED_METHODS = frozenset(["GET"]) ++ DEFAULT_ALLOWED_METHODS = frozenset(["GET", "POST"]) + + def as_dict(self): + return {"methods": self.DEFAULT_ALLOWED_METHODS} +```""" + + +def _write_auth_and_rate_limit_repo(root: Path) -> None: + """_write_auth_repo's app/auth.py (matches vulnerability.md, so real + grounding selects it) plus a second, vocabulary-disjoint file holding + a class-level literal constant that the final patch below touches but + that no selected candidate ever surfaces. + + The class has a real method (not just the constant) deliberately -- + matching urllib3's actual `Retry` class, which has plenty of real + methods. A class with zero methods produces no RepositoryIndex entry + at all for its file (a separate, pre-existing parser limitation, + unrelated to this feature); this fixture avoids that degenerate shape + so the test exercises the real gap, not an incidental one. + """ + _write_auth_repo(root) # commits app/auth.py + git init + rate_limit = root / "app" / "rate_limit_config.py" + rate_limit.write_text( + "class RateLimitConfig:\n" + " DEFAULT_ALLOWED_METHODS = frozenset([\"GET\"])\n" + "\n" + " def as_dict(self):\n" + " return {\"methods\": self.DEFAULT_ALLOWED_METHODS}\n", + encoding="utf-8", + ) + subprocess.run(["git", "add", "-A"], cwd=root, capture_output=True) + subprocess.run(["git", "commit", "-m", "add rate_limit_config"], cwd=root, capture_output=True) + + +class TestPatchTouchedAnchorsIndependentOfCandidateSelection: + def test_constant_untouched_by_selection_still_detected_as_changed_and_covered(self, tmp_path): + repo_root = tmp_path / "repo" + _write_auth_and_rate_limit_repo(repo_root) + + result, report, calls, _ = _run_pipeline( + tmp_path, + patches_gen=[_RATE_LIMIT_CONFIG_DIFF], + patches_chall=[_CHALLENGER_CLEAN], + repo_root=str(repo_root), + ) + + # 1. The changed file is genuinely not in selection.selected -- + # real, unmodified Candidate Selection, no manual injection. + assert result.repository_understanding is not None + selected_paths = {c.path for c in result.repository_understanding.candidate_evidence} + assert "app/rate_limit_config.py" not in selected_paths + + # 2. No PRE-PATCH Anchor exists for the changed constant (proves + # the gap is real, not already closed by some other mechanism). + pre_patch_const_anchors = [ + o for o in result.post_patch_observations + if o.anchor_kind == "constant_value" and o.origin == "pre_patch" + and "rate_limit_config.py" in o.candidate_path + ] + assert pre_patch_const_anchors == [] + + # 3. A patch_touched constant_value Anchor WAS derived from the + # final diff, and (4) it reports Changed: GET -> {GET, POST}. + touched = [ + o for o in result.post_patch_observations + if o.anchor_kind == "constant_value" and o.origin == "patch_touched" + ] + assert len(touched) == 1 + obs = touched[0] + assert obs.status == "changed" + assert obs.before_value.value == frozenset({"GET"}) + assert obs.after_value.value == frozenset({"GET", "POST"}) + assert "app/rate_limit_config.py" in obs.candidate_path + + # 5. Coverage reports 1 of 1 covered, 0 uncovered for this element. + assert result.post_patch_coverage is not None + ref = obs.anchor_key.const_id + assert ref in result.post_patch_coverage.covered + assert ref not in result.post_patch_coverage.uncovered + + # Rendered report shows the fact, tagged as patch-discovered, and + # is reachable by the Challenger (evidence actually flows downstream). + assert "### Changed" in report + changed_section = report[report.index("### Changed"):report.index("### Disappeared")] + assert "discovered from patch diff" in changed_section + assert "1 of 1 element(s)" in calls["challenge"][0] or "1 of 1 element(s)" in report diff --git a/libs/openant-core/tests/patch/test_pipeline_repair.py b/libs/openant-core/tests/patch/test_pipeline_repair.py new file mode 100644 index 00000000..99e054ab --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline_repair.py @@ -0,0 +1,604 @@ +"""Tests for the Phase C challenger-driven repair loop in pipeline.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Shared test fixtures +# --------------------------------------------------------------------------- + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ pass + pass +```""" + +_REPAIR_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ # repaired + pass +```""" + +# A challenger result that contains a confirmed defect (bypasses _EXPLICIT_DEFECT_RE) +_CHALLENGER_WITH_DEFECT = { + "still_vulnerable": False, + "edge_cases": ["An attacker can bypass this check via path traversal"], + "potential_issues": [], + "summary": "The patch has a confirmed bypass.", +} + +# A challenger result with BOTH a confirmed defect and a plausible-risk finding +_CHALLENGER_MIXED = { + "still_vulnerable": False, + "edge_cases": ["An attacker can bypass this check via crafted input"], + "potential_issues": ["Performance may be impacted under high load"], + "summary": "Mixed findings.", +} + +# A clean challenger result (no defects) +_CHALLENGER_CLEAN = { + "still_vulnerable": False, + "edge_cases": [], + "potential_issues": [], + "summary": "No issues found.", +} + +# A challenger result whose finding is a plausible_risk only +_CHALLENGER_RISK_ONLY = { + "still_vulnerable": False, + "edge_cases": ["This might behave unexpectedly under concurrent access"], + "potential_issues": [], + "summary": "Unverified risk only.", +} + +_APPLICABILITY_CLEAN = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, +} +_APPLICABILITY_FAIL = { + "applicable": False, "skipped": False, "stderr": "error: patch failed", + "exit_code": 1, "skipped_reason": None, "error": None, +} +_APPLICABILITY_SKIPPED = { + "applicable": None, "skipped": True, "stderr": "", + "exit_code": None, "skipped_reason": "no repo_root", "error": None, +} + + +def _capture_result(tmp_path, *, patches_gen, patches_app, patches_chall, repo_root=None): + """Run pipeline.run() with controlled mocks and capture the PipelineResult.""" + captured = {} + import utilities.autopatcher.pipeline as _pipeline_mod + orig_build = _pipeline_mod._build_report + + def _capture(r): + captured["result"] = r + return orig_build(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=patches_gen) as _mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=patches_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok review"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", side_effect=patches_chall) as _mock_chall, + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=_capture), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=repo_root or str(tmp_path)) + + return captured["result"], _mock_gen, _mock_chall + + +# --------------------------------------------------------------------------- +# _build_repair_hint +# --------------------------------------------------------------------------- + +class TestBuildRepairHint: + def test_contains_defect_text(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["attacker can bypass the check via path traversal"]) + assert "bypass" in hint + + def test_operation_level_instruction_present(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["some defect"]) + assert "dangerous operation" in hint.lower() or "perimeter" in hint.lower() + + def test_multiple_defects_produce_multiple_bullets(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["defect one", "defect two"]) + assert "defect one" in hint + assert "defect two" in hint + assert hint.count("- defect") == 2 + + def test_helper_wiring_instruction_present(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint(["helper not called"]) + assert "helper" in hint.lower() and "every" in hint.lower() + + def test_empty_list_returns_string(self): + from utilities.autopatcher.pipeline import _build_repair_hint + hint = _build_repair_hint([]) + assert isinstance(hint, str) + assert len(hint) > 0 + + +# --------------------------------------------------------------------------- +# _render_repair_notice +# --------------------------------------------------------------------------- + +class TestRenderRepairNotice: + def _make_result(self, **kwargs): + from utilities.autopatcher.pipeline import PipelineResult + defaults = dict( + vulnerability_text="v", patch="p", review="r", + score_text="Confidence score: 0.8", challenger={}, + ) + defaults.update(kwargs) + return PipelineResult(**defaults) + + def test_no_notice_when_not_attempted(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result(repair_attempted=False) + assert _render_repair_notice(r) == "" + + def test_success_notice_contains_auto_repaired(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=True, + original_challenger_defect_count=2, + ) + notice = _render_repair_notice(r) + assert "auto-repaired" in notice.lower() + assert "2" in notice + + def test_failure_notice_contains_repair_attempted(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=True, + original_challenger_defect_count=1, repair_defect_count=1, + ) + notice = _render_repair_notice(r) + assert "repair attempted" in notice.lower() + assert "1" in notice + + def test_failure_notice_states_original_recommendation_stands(self): + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=True, + original_challenger_defect_count=2, repair_defect_count=1, + ) + notice = _render_repair_notice(r) + assert "original recommendation stands" in notice.lower() + + def test_never_rechallenged_notice_does_not_claim_a_defect_count(self): + """When the repair patch never reached re-challenge (applicability + failure or an internal error), the notice must say so explicitly and + must not present the untouched repair_defect_count default as if it + were an observed finding.""" + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=False, + original_challenger_defect_count=2, repair_defect_count=0, + ) + notice = _render_repair_notice(r) + assert "repair attempted" in notice.lower() + assert "did not reach re-challenge" in notice.lower() + assert "no repair defect count is available" in notice.lower() + assert "original recommendation stands" in notice.lower() + # Must not claim the repair patch "had 0 confirmed defect(s)" — that + # number was never observed. + assert "repair patch still had" not in notice.lower() + assert "0 confirmed defect(s)" not in notice.lower() + + def test_never_rechallenged_notice_still_reports_original_count(self): + """The original (pre-repair) defect count IS a real, observed value + and should still be reported.""" + from utilities.autopatcher.pipeline import _render_repair_notice + r = self._make_result( + repair_attempted=True, repair_succeeded=False, repair_rechallenged=False, + original_challenger_defect_count=3, repair_defect_count=0, + ) + notice = _render_repair_notice(r) + assert "3" in notice + + +# --------------------------------------------------------------------------- +# Repair NOT triggered +# --------------------------------------------------------------------------- + +class TestRepairNotTriggered: + """Repair must NOT fire when there are no confirmed defects or patch does not apply.""" + + def test_no_repair_when_no_confirmed_defects(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + assert mock_chall.call_count == 1 + + def test_no_repair_when_plausible_risk_only(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_RISK_ONLY], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_no_repair_when_applicable_false(self, tmp_path): + # Patch doesn't apply — repair must not fire (applicability retry handles this) + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_no_repair_when_applicable_none(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_SKIPPED], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_attempted is False + assert mock_gen.call_count == 1 + + def test_urllib3_analog_passes_through_unchanged(self, tmp_path): + # urllib3 pattern: applicable, no confirmed defects → single generate, single challenge + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert result.repair_succeeded is False + assert result.repair_patch is None + assert mock_gen.call_count == 1 + assert mock_chall.call_count == 1 + + +# --------------------------------------------------------------------------- +# Repair triggered +# --------------------------------------------------------------------------- + +class TestRepairTriggered: + """Repair fires when applicable=True and confirmed_defect_count > 0.""" + + def test_repair_calls_generate_patch_twice(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_attempted is True + assert mock_gen.call_count == 2 + + def test_repair_calls_challenge_patch_twice(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert mock_chall.call_count == 2 + + def test_repair_hint_passed_to_second_generate_call(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + _args, kwargs = mock_gen.call_args_list[1] + hint = kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) or "" + assert "bypass" in hint.lower() + + def test_repair_hint_contains_confirmed_defect_not_plausible_risk(self, tmp_path): + result, mock_gen, mock_chall = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_MIXED, _CHALLENGER_CLEAN], + ) + _args, kwargs = mock_gen.call_args_list[1] + hint = kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) or "" + # confirmed defect text is in the hint + assert "bypass" in hint.lower() + # plausible risk (performance) is NOT in the repair hint + assert "Performance" not in hint + + def test_repair_attempted_flag_set(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_attempted is True + + +# --------------------------------------------------------------------------- +# Repair outcomes +# --------------------------------------------------------------------------- + +class TestRepairOutcomes: + def test_repair_accepted_when_no_defects_remain(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_succeeded is True + # Final patch must be the repair diff content + assert "repaired" in result.patch + + def test_repair_rejected_when_defects_remain_after_repair(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + # repair challenger still has a confirmed defect + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + # patch must be the original, not the repair + assert "repaired" not in result.patch + + def test_repair_rejected_when_repair_patch_does_not_apply(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + # repair patch does not apply + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + assert result.repair_attempted is True + # Repair patch never reached re-challenge (applicability failed first), + # so repair_defect_count must not be presented as an observed value. + assert result.repair_rechallenged is False + + def test_review_runs_on_accepted_repair_patch(self, tmp_path): + """When repair is accepted, review_patch must be called with the repair diff.""" + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=[_CLEAN_DIFF, _REPAIR_DIFF]), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok") as mock_review, + mock.patch("utilities.autopatcher.pipeline.challenge_patch", + side_effect=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN]), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=str(tmp_path)) + # review_patch first arg is vuln text, second is the patch + _args, _ = mock_review.call_args + reviewed_patch = _args[1] + assert "repaired" in reviewed_patch + + def test_original_patch_kept_on_rejection(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_succeeded is False + assert "repaired" not in result.patch + + +# --------------------------------------------------------------------------- +# Repair metadata +# --------------------------------------------------------------------------- + +class TestRepairMetadata: + def test_repair_patch_stored_even_when_not_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_patch is not None + assert "repaired" in result.repair_patch + + def test_repair_patch_stored_even_when_rejected_on_defects(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_patch is not None + + def test_original_challenger_defect_count_stored(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.original_challenger_defect_count == 1 + + def test_repair_challenger_stored_when_repair_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_challenger is not None + + def test_repair_challenger_none_when_repair_not_applicable(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_challenger is None + + def test_repair_defect_count_zero_on_success(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert result.repair_defect_count == 0 + + def test_repair_defect_count_nonzero_on_rejection(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_defect_count > 0 + + def test_repair_rechallenged_true_when_repair_patch_applies(self, tmp_path): + """repair_defect_count is a real, observed value here — repair_rechallenged + must be True whether the re-challenge finds 0 or nonzero defects.""" + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert result.repair_rechallenged is True + + def test_repair_rechallenged_false_when_repair_patch_does_not_apply(self, tmp_path): + """repair_defect_count stays at its untouched default (0) here — this + must be distinguishable from an actual re-challenge finding 0 defects.""" + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert result.repair_defect_count == 0 + assert result.repair_rechallenged is False + + def test_defaults_when_repair_not_triggered(self, tmp_path): + result, _, _ = _capture_result( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert result.repair_attempted is False + assert result.repair_succeeded is False + assert result.repair_patch is None + assert result.repair_challenger is None + assert result.repair_defect_count == 0 + assert result.repair_rechallenged is False + assert result.original_challenger_defect_count == 0 + + +# --------------------------------------------------------------------------- +# Report content +# --------------------------------------------------------------------------- + +class TestRepairReport: + def _run_and_get_report(self, tmp_path, patches_gen, patches_app, patches_chall): + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=patches_gen), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=patches_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="**Explanation**\nok\n" + "**Affected areas**\nok\n**Validation notes**\nok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", side_effect=patches_chall), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="Confidence score: 0.8"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + return run("test vuln", api_key="", repo_root=str(tmp_path)) + + def test_report_contains_auto_repaired_on_success(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert "auto-repaired" in report.lower() + + def test_report_contains_repair_attempted_on_rejection(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_WITH_DEFECT], + ) + assert "repair attempted" in report.lower() + + def test_report_no_repair_notice_when_not_triggered(self, tmp_path): + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF], + patches_app=[_APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_CLEAN], + ) + assert "auto-repaired" not in report.lower() + assert "repair attempted" not in report.lower() + + def test_report_states_no_rechallenge_when_repair_patch_does_not_apply(self, tmp_path): + """End-to-end: when the repair patch fails applicability, the rendered + report must not claim a repair defect count that was never observed.""" + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_FAIL], + patches_chall=[_CHALLENGER_WITH_DEFECT], + ) + assert "repair attempted" in report.lower() + assert "did not reach re-challenge" in report.lower() + assert "no repair defect count is available" in report.lower() + assert "repair patch still had" not in report.lower() + assert "0 confirmed defect(s)" not in report.lower() + + def test_report_states_defect_count_before_repair(self, tmp_path): + # original had 1 confirmed defect; notice should say 1 + report = self._run_and_get_report( + tmp_path, + patches_gen=[_CLEAN_DIFF, _REPAIR_DIFF], + patches_app=[_APPLICABILITY_CLEAN, _APPLICABILITY_CLEAN], + patches_chall=[_CHALLENGER_WITH_DEFECT, _CHALLENGER_CLEAN], + ) + assert "1 confirmed defect" in report diff --git a/libs/openant-core/tests/patch/test_pipeline_retry.py b/libs/openant-core/tests/patch/test_pipeline_retry.py new file mode 100644 index 00000000..78511b7f --- /dev/null +++ b/libs/openant-core/tests/patch/test_pipeline_retry.py @@ -0,0 +1,795 @@ +"""Tests for applicability-aware retry logic in pipeline.py.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from unittest import mock + +import pytest + + +# --------------------------------------------------------------------------- +# Shared fixtures +# --------------------------------------------------------------------------- + +_PIP_STDERR = ( + "error: patch failed: src/pip/_internal/download.py:1\n" + "error: src/pip/_internal/download.py: patch does not apply\n" +) + +_CORRUPT_STDERR = "error: corrupt patch at line 7\n" + +_URLLIB3_MULTI_STDERR = ( + "error: patch failed: src/urllib3/util/retry.py:262\n" + "error: src/urllib3/util/retry.py: patch does not apply\n" + "error: patch failed: src/urllib3/poolmanager.py:8\n" + "error: src/urllib3/poolmanager.py: patch does not apply\n" +) + +_PIP_DIFF_ORIG = """\ +```diff +--- a/src/pip/_internal/download.py ++++ b/src/pip/_internal/download.py +@@ -1,3 +1,4 @@ + from __future__ import absolute_import ++# security fix + import os +```""" + +_PIP_DIFF_RETRY = """\ +```diff +--- a/src/pip/_internal/download.py ++++ b/src/pip/_internal/download.py +@@ -1,3 +1,4 @@ + import os ++# security fix + import sys +```""" + +_CLEAN_DIFF = """\ +```diff +--- a/src/urllib3/util/retry.py ++++ b/src/urllib3/util/retry.py +@@ -1,2 +1,3 @@ + def retry(): ++ # security fix + pass +```""" + + +# --------------------------------------------------------------------------- +# _extract_failed_file +# --------------------------------------------------------------------------- + +class TestExtractFailedFile: + def test_patch_failed_format(self): + from utilities.autopatcher.pipeline import _extract_failed_file + result = _extract_failed_file(_PIP_STDERR) + assert result == "src/pip/_internal/download.py" + + def test_does_not_apply_format(self): + from utilities.autopatcher.pipeline import _extract_failed_file + stderr = "error: some/path/file.py: patch does not apply\n" + assert _extract_failed_file(stderr) == "some/path/file.py" + + def test_corrupt_patch_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file(_CORRUPT_STDERR) is None + + def test_empty_stderr_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file("") is None + + def test_none_stderr_returns_none(self): + from utilities.autopatcher.pipeline import _extract_failed_file + assert _extract_failed_file(None) is None + + def test_patch_failed_takes_priority_over_does_not_apply(self): + from utilities.autopatcher.pipeline import _extract_failed_file + # Both formats present — patch failed is matched first + result = _extract_failed_file(_PIP_STDERR) + assert result == "src/pip/_internal/download.py" + + +# --------------------------------------------------------------------------- +# _extract_failed_files (multi-file) +# --------------------------------------------------------------------------- + +class TestExtractFailedFiles: + def test_single_file_stderr_returns_length_one_list(self): + from utilities.autopatcher.pipeline import _extract_failed_files + result = _extract_failed_files(_PIP_STDERR) + assert result == ["src/pip/_internal/download.py"] + + def test_two_file_stderr_returns_ordered_list(self): + from utilities.autopatcher.pipeline import _extract_failed_files + result = _extract_failed_files(_URLLIB3_MULTI_STDERR) + assert result == [ + "src/urllib3/util/retry.py", + "src/urllib3/poolmanager.py", + ] + + def test_file_matched_by_both_regexes_appears_once(self): + from utilities.autopatcher.pipeline import _extract_failed_files + # _PIP_STDERR names the same file via both the "patch failed" and + # "patch does not apply" lines. + result = _extract_failed_files(_PIP_STDERR) + assert result.count("src/pip/_internal/download.py") == 1 + + def test_empty_stderr_returns_empty_list(self): + from utilities.autopatcher.pipeline import _extract_failed_files + assert _extract_failed_files("") == [] + + def test_none_stderr_returns_empty_list(self): + from utilities.autopatcher.pipeline import _extract_failed_files + assert _extract_failed_files(None) == [] + + def test_three_file_stderr_returns_ordered_list(self): + from utilities.autopatcher.pipeline import _extract_failed_files + stderr = ( + "error: patch failed: src/urllib3/util/retry.py:1\n" + "error: src/urllib3/util/retry.py: patch does not apply\n" + "error: patch failed: src/urllib3/poolmanager.py:5\n" + "error: src/urllib3/poolmanager.py: patch does not apply\n" + "error: patch failed: src/urllib3/contrib/pyopenssl.py:9\n" + "error: src/urllib3/contrib/pyopenssl.py: patch does not apply\n" + ) + result = _extract_failed_files(stderr) + assert result == [ + "src/urllib3/util/retry.py", + "src/urllib3/poolmanager.py", + "src/urllib3/contrib/pyopenssl.py", + ] + + def test_mixed_dedup_with_other_single_match_files(self): + # fileA is named by BOTH regexes, fileB only by "patch failed", + # fileC only by "patch does not apply" -- dedup must not accidentally + # drop or reorder B/C just because A appears twice. + from utilities.autopatcher.pipeline import _extract_failed_files + stderr = ( + "error: patch failed: src/urllib3/util/retry.py:1\n" + "error: src/urllib3/util/retry.py: patch does not apply\n" + "error: patch failed: src/urllib3/poolmanager.py:5\n" + "error: src/urllib3/contrib/pyopenssl.py: patch does not apply\n" + ) + result = _extract_failed_files(stderr) + assert result == [ + "src/urllib3/util/retry.py", + "src/urllib3/poolmanager.py", + "src/urllib3/contrib/pyopenssl.py", + ] + + +# --------------------------------------------------------------------------- +# _extract_patch_target +# --------------------------------------------------------------------------- + +class TestExtractPatchTarget: + def test_fenced_diff_returns_target(self): + from utilities.autopatcher.pipeline import _extract_patch_target + result = _extract_patch_target(_PIP_DIFF_ORIG) + assert result == "src/pip/_internal/download.py" + + def test_unfenced_diff_returns_target(self): + from utilities.autopatcher.pipeline import _extract_patch_target + unfenced = ( + "--- a/src/foo.py\n" + "+++ b/src/foo.py\n" + "@@ -1,1 +1,1 @@\n" + "-old\n" + "+new\n" + ) + assert _extract_patch_target(unfenced) == "src/foo.py" + + def test_no_plus_plus_line_returns_none(self): + from utilities.autopatcher.pipeline import _extract_patch_target + assert _extract_patch_target("no diff here") is None + + def test_empty_patch_returns_none(self): + from utilities.autopatcher.pipeline import _extract_patch_target + assert _extract_patch_target("") is None + + +# --------------------------------------------------------------------------- +# _build_retry_hint +# --------------------------------------------------------------------------- + +class TestBuildRetryHint: + def test_contains_failed_file(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "src/pip/_internal/download.py") + assert "src/pip/_internal/download.py" in hint + + def test_contains_stderr_excerpt(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "some/file.py") + assert "error: patch failed" in hint + + def test_instructs_not_to_use_training_memory(self): + from utilities.autopatcher.pipeline import _build_retry_hint + hint = _build_retry_hint(_PIP_STDERR, "some/file.py") + assert "training" in hint.lower() or "ground truth" in hint.lower() + + def test_stderr_truncated_to_limit(self): + from utilities.autopatcher.pipeline import _build_retry_hint, _RETRY_STDERR_LINES + many_lines = "\n".join(f"line {i}" for i in range(50)) + hint = _build_retry_hint(many_lines, "f.py") + # Only first _RETRY_STDERR_LINES lines should appear + assert f"line {_RETRY_STDERR_LINES}" not in hint + assert "line 0" in hint + + +# --------------------------------------------------------------------------- +# Retry NOT triggered +# --------------------------------------------------------------------------- + +class TestRetryNotTriggered: + """Retry must NOT run when applicable=True, applicable=None, or no repo_root.""" + + def _run_with_mock(self, applicability_return, repo_root=None, vuln="test vuln"): + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", return_value=_CLEAN_DIFF) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", return_value=applicability_return), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok review"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run(vuln, api_key="", repo_root=repo_root) + return mock_gen.call_count + + def test_no_retry_when_applicable_true(self, tmp_path): + call_count = self._run_with_mock( + {"applicable": True, "skipped": False, "stderr": "", "exit_code": 0, + "skipped_reason": None, "error": None}, + repo_root=str(tmp_path), + ) + assert call_count == 1 + + def test_no_retry_when_applicable_none(self, tmp_path): + call_count = self._run_with_mock( + {"applicable": None, "skipped": True, "stderr": "", "exit_code": None, + "skipped_reason": "no repo_root", "error": None}, + repo_root=str(tmp_path), + ) + assert call_count == 1 + + def test_no_retry_when_no_repo_root(self): + call_count = self._run_with_mock( + {"applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None}, + repo_root=None, + ) + assert call_count == 1 + + +# --------------------------------------------------------------------------- +# Retry triggered — success and failure outcomes +# --------------------------------------------------------------------------- + +class TestRetryTriggered: + """applicable=False + repo_root → retry must be attempted.""" + + def _setup_mocks( + self, + tmp_path: Path, + retry_applicable: bool, + failed_file: str = "src/pip/_internal/download.py", + ): + """Return a context-manager tuple for the common retry scenario.""" + # Write the file the retry will read + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + return first_app, retry_app + + def test_retry_calls_generate_patch_twice(self, tmp_path): + first_app, retry_app = self._setup_mocks(tmp_path, retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient") as mock_llm_cls, + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + mock_llm_cls.return_value = mock.MagicMock() + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 2 + + def test_retry_call_includes_retry_hint(self, tmp_path): + first_app, retry_app = self._setup_mocks(tmp_path, retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + _args, kwargs = mock_gen.call_args_list[1] + assert kwargs.get("retry_hint") or (len(_args) > 3 and _args[3]) + + +# --------------------------------------------------------------------------- +# Retry triggered — multi-file stderr (regression test for the single-file +# recovery bug: a real multi-file "does not apply" failure must recover +# every named file, not just the first). +# --------------------------------------------------------------------------- + +class TestRetryMultiFile: + def _setup_mocks(self, retry_applicable: bool = True): + first_app = { + "applicable": False, "skipped": False, "stderr": _URLLIB3_MULTI_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + return first_app, retry_app + + def test_retry_code_context_includes_both_files(self, tmp_path): + retry_file = tmp_path / "src/urllib3/util/retry.py" + pool_file = tmp_path / "src/urllib3/poolmanager.py" + retry_file.parent.mkdir(parents=True, exist_ok=True) + pool_file.parent.mkdir(parents=True, exist_ok=True) + retry_file.write_text("RETRY_MARKER_CONTENT\ndef retry(): pass\n", encoding="utf-8") + pool_file.write_text("POOLMANAGER_MARKER_CONTENT\nclass PoolManager: pass\n", encoding="utf-8") + + first_app, retry_app = self._setup_mocks(retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_CLEAN_DIFF, _CLEAN_DIFF]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 2 + _args, kwargs = mock_gen.call_args_list[1] + retry_code_context = kwargs.get("code_context", "") + # Both files' real content must be present — not just the first + # file named in a multi-file "patch does not apply" failure. + assert "RETRY_MARKER_CONTENT" in retry_code_context + assert "POOLMANAGER_MARKER_CONTENT" in retry_code_context + assert "src/urllib3/util/retry.py" in retry_code_context + assert "src/urllib3/poolmanager.py" in retry_code_context + + def test_retry_proceeds_when_one_of_two_files_missing(self, tmp_path): + # Only the first failed file exists on disk; the second was + # deleted/renamed. The retry must still proceed using the readable + # file's real content, with the missing one noted (not silently + # dropped, not a crash). + retry_file = tmp_path / "src/urllib3/util/retry.py" + retry_file.parent.mkdir(parents=True, exist_ok=True) + retry_file.write_text("RETRY_MARKER_CONTENT\ndef retry(): pass\n", encoding="utf-8") + + first_app, retry_app = self._setup_mocks(retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_CLEAN_DIFF, _CLEAN_DIFF]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 2 + _args, kwargs = mock_gen.call_args_list[1] + retry_code_context = kwargs.get("code_context", "") + assert "RETRY_MARKER_CONTENT" in retry_code_context + assert "src/urllib3/poolmanager.py" in retry_code_context + assert "could not be read" in retry_code_context + + def test_retry_skipped_when_all_failed_files_missing(self, tmp_path): + # Neither failed file exists on disk. No real content can be + # included at all, so the retry must not be attempted a second time. + first_app, _retry_app = self._setup_mocks(retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + return_value=_CLEAN_DIFF) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + return_value=first_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + # generate_patch must be called exactly once — the initial + # generation — never a second time for a retry with no content. + assert mock_gen.call_count == 1 + + def test_aggregate_budget_omission_note(self, tmp_path): + # Each file is well under _RETRY_CONTENT_LIMIT alone, but together + # they exceed it -- the first must be included in full, the second + # omitted (not truncated) with an explicit note naming it. + from utilities.autopatcher.pipeline import _RETRY_CONTENT_LIMIT + + retry_file = tmp_path / "src/urllib3/util/retry.py" + pool_file = tmp_path / "src/urllib3/poolmanager.py" + retry_file.parent.mkdir(parents=True, exist_ok=True) + pool_file.parent.mkdir(parents=True, exist_ok=True) + chunk = int(_RETRY_CONTENT_LIMIT * 0.6) + retry_file.write_text("RETRY_MARKER_CONTENT\n" + ("a" * chunk), encoding="utf-8") + pool_file.write_text("POOLMANAGER_MARKER_CONTENT\n" + ("b" * chunk), encoding="utf-8") + + first_app, retry_app = self._setup_mocks(retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_CLEAN_DIFF, _CLEAN_DIFF]) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 2 + _args, kwargs = mock_gen.call_args_list[1] + retry_code_context = kwargs.get("code_context", "") + assert "RETRY_MARKER_CONTENT" in retry_code_context + assert "POOLMANAGER_MARKER_CONTENT" not in retry_code_context + assert "omitted to stay within" in retry_code_context + assert "src/urllib3/poolmanager.py" in retry_code_context + + def test_oversized_alone_plus_missing_file_skips_retry_entirely(self, tmp_path): + # File A is readable but exceeds the budget by itself (so it lands + # in omitted_files, contributing no real content); file B is simply + # missing (so it only ever contributes a "could not be read" note). + # No real content survives from either file, so the retry must not + # be attempted -- this is the regression test for the bug where + # "any read attempt succeeded" was wrongly treated as "there is + # real content to send." + from utilities.autopatcher.pipeline import _RETRY_CONTENT_LIMIT + + retry_file = tmp_path / "src/urllib3/util/retry.py" + retry_file.parent.mkdir(parents=True, exist_ok=True) + retry_file.write_text("a" * (_RETRY_CONTENT_LIMIT + 1_000), encoding="utf-8") + # src/urllib3/poolmanager.py deliberately not created on disk. + + first_app, _retry_app = self._setup_mocks(retry_applicable=True) + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + return_value=_CLEAN_DIFF) as mock_gen, + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + return_value=first_app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + assert mock_gen.call_count == 1 + + def test_retry_failed_file_is_first_of_multiple(self, tmp_path): + # retry_failed_file (the report-facing field) must stay a single + # string -- the first failed file -- even when multiple files failed. + retry_file = tmp_path / "src/urllib3/util/retry.py" + pool_file = tmp_path / "src/urllib3/poolmanager.py" + retry_file.parent.mkdir(parents=True, exist_ok=True) + pool_file.parent.mkdir(parents=True, exist_ok=True) + retry_file.write_text("RETRY_MARKER_CONTENT\n", encoding="utf-8") + pool_file.write_text("POOLMANAGER_MARKER_CONTENT\n", encoding="utf-8") + + first_app, retry_app = self._setup_mocks(retry_applicable=True) + captured_result = {} + import utilities.autopatcher.pipeline as _pipeline_mod + original_build_report = _pipeline_mod._build_report + + def capture_result(r): + captured_result["result"] = r + return original_build_report(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_CLEAN_DIFF, _CLEAN_DIFF]), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=capture_result), + ): + from utilities.autopatcher.pipeline import run + run("urllib3 vuln", api_key="", repo_root=str(tmp_path)) + + result = captured_result["result"] + assert result.retry_failed_file == "src/urllib3/util/retry.py" + assert isinstance(result.retry_failed_file, str) + + +# --------------------------------------------------------------------------- +# Retry outcomes — patch and metadata +# --------------------------------------------------------------------------- + +class TestRetryOutcomes: + def _run_retry_scenario(self, tmp_path, retry_applicable): + failed_file = "src/pip/_internal/download.py" + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + + captured_result = {} + + original_build_report = None + import utilities.autopatcher.pipeline as _pipeline_mod + original_build_report = _pipeline_mod._build_report + + def capture_result(r): + captured_result["result"] = r + return original_build_report(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", + side_effect=[_PIP_DIFF_ORIG, _PIP_DIFF_RETRY]), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", + side_effect=[first_app, retry_app]), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=capture_result), + ): + from utilities.autopatcher.pipeline import run + run("pip vuln", api_key="", repo_root=str(tmp_path)) + + return captured_result["result"] + + def test_retry_succeeded_uses_retry_patch(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=True) + assert result.retry_succeeded is True + # diff_hunk_repair rewrites @@ counts, so compare content not exact string + assert "import sys" in result.patch + assert result.patch != result.original_patch + + def test_retry_failed_keeps_original_patch(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert result.retry_succeeded is False + # patch must equal original_patch (both go through hunk repair, so compare relative identity) + assert result.patch == result.original_patch + + def test_retry_failed_keeps_original_applicability(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert result.applicability["applicable"] is False + + def test_retry_patch_stored_in_metadata_on_failure(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + # retry_patch is stored even on failure (for inspection) + assert result.retry_patch is not None + assert "import sys" in result.retry_patch + + def test_original_patch_preserved_in_metadata_on_success(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=True) + # original_patch must contain the original content (from __future__ not in retry) + assert "from __future__" in result.original_patch + + def test_original_patch_preserved_in_metadata_on_failure(self, tmp_path): + result = self._run_retry_scenario(tmp_path, retry_applicable=False) + assert "from __future__" in result.original_patch + + +# --------------------------------------------------------------------------- +# Retry metadata fields +# --------------------------------------------------------------------------- + +class TestRetryMetadata: + def _run_no_retry(self, tmp_path): + app = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, + } + captured_result = {} + import utilities.autopatcher.pipeline as _pipeline_mod + original_build_report = _pipeline_mod._build_report + + def capture(r): + captured_result["result"] = r + return original_build_report(r) + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", return_value=_CLEAN_DIFF), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", return_value=app), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + mock.patch("utilities.autopatcher.pipeline._build_report", side_effect=capture), + ): + from utilities.autopatcher.pipeline import run + run("test vuln", api_key="", repo_root=str(tmp_path)) + return captured_result["result"] + + def test_no_retry_metadata_defaults(self, tmp_path): + result = self._run_no_retry(tmp_path) + assert result.retry_attempted is False + assert result.retry_succeeded is False + assert result.retry_patch is None + assert result.retry_failed_file is None + assert result.retry_error_before is None + + def test_original_patch_matches_patch_when_no_retry(self, tmp_path): + result = self._run_no_retry(tmp_path) + assert result.original_patch == result.patch + + +# --------------------------------------------------------------------------- +# _render_retry_notice — isolated unit tests (mirrors +# tests/test_pipeline_repair.py::TestRenderRepairNotice) +# --------------------------------------------------------------------------- + +class TestRenderRetryNotice: + def _make_result(self, **kwargs): + from utilities.autopatcher.pipeline import PipelineResult + defaults = dict( + vulnerability_text="v", patch="p", review="r", + score_text="Confidence score: 0.8", challenger={}, + ) + defaults.update(kwargs) + return PipelineResult(**defaults) + + def test_no_notice_when_not_attempted(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=False) + assert _render_retry_notice(r) == "" + + def test_success_notice_content(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=True, retry_succeeded=True) + notice = _render_retry_notice(r) + assert "Applicability-aware retry" in notice + assert "Initial patch did not apply." in notice + assert "Applicability-aware retry was attempted." in notice + assert "Retry succeeded" in notice + assert "Retry failed" not in notice + + def test_failure_notice_content(self): + from utilities.autopatcher.pipeline import _render_retry_notice + r = self._make_result(retry_attempted=True, retry_succeeded=False) + notice = _render_retry_notice(r) + assert "Applicability-aware retry" in notice + assert "Initial patch did not apply." in notice + assert "Applicability-aware retry was attempted." in notice + assert "Retry failed to produce an applicable patch." in notice + assert "Retry succeeded" not in notice + + +# --------------------------------------------------------------------------- +# Retry notice — end-to-end wiring into the rendered report +# --------------------------------------------------------------------------- + +class TestRetryNoticeInReport: + """Proves the notice is actually wired into _build_report's output, in + the right place, and has zero footprint when no retry occurred.""" + + def _run_and_capture_report(self, tmp_path, *, trigger_retry: bool, retry_applicable=None): + if trigger_retry: + failed_file = "src/pip/_internal/download.py" + target = tmp_path / failed_file + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("import os\nimport sys\n", encoding="utf-8") + + first_app = { + "applicable": False, "skipped": False, "stderr": _PIP_STDERR, + "exit_code": 1, "skipped_reason": None, "error": None, + } + retry_app = { + "applicable": retry_applicable, "skipped": False, "stderr": "", + "exit_code": 0 if retry_applicable else 1, + "skipped_reason": None, "error": None, + } + gen_side_effect = [_PIP_DIFF_ORIG, _PIP_DIFF_RETRY] + app_side_effect = [first_app, retry_app] + else: + clean_app = { + "applicable": True, "skipped": False, "stderr": "", + "exit_code": 0, "skipped_reason": None, "error": None, + } + gen_side_effect = [_CLEAN_DIFF] + app_side_effect = [clean_app] + + with ( + mock.patch("utilities.autopatcher.pipeline.LLMClient"), + mock.patch("utilities.autopatcher.pipeline.generate_patch", side_effect=gen_side_effect), + mock.patch("utilities.autopatcher.patch_applicability.check_applicability", side_effect=app_side_effect), + mock.patch("utilities.autopatcher.pipeline.review_patch", return_value="ok"), + mock.patch("utilities.autopatcher.pipeline.challenge_patch", return_value={}), + mock.patch("utilities.autopatcher.pipeline.score_confidence", return_value="score: 7"), + mock.patch("utilities.autopatcher.pipeline.LightweightImpactAnalyzer"), + mock.patch("utilities.autopatcher.patch_hygiene.check_patch", return_value=[]), + ): + from utilities.autopatcher.pipeline import run + report = run("pip vuln", api_key="", repo_root=str(tmp_path)) + return report + + def test_no_notice_in_report_when_no_retry_occurred(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=False) + assert "Applicability-aware retry" not in report + + def test_notice_appears_in_report_on_retry_success(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=True) + assert "Applicability-aware retry" in report + assert "Retry succeeded" in report + + def test_notice_appears_in_report_on_retry_failure(self, tmp_path): + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=False) + assert "Applicability-aware retry" in report + assert "Retry failed to produce an applicable patch." in report + + def test_notice_positioned_after_patch_applicability(self, tmp_path): + """Patch Applicability and the retry notice were promoted out of + Appendices (second reviewer-experience pass) to sit directly after + Proposed patch, before Trust Signals — Appendices now comes well + after both.""" + report = self._run_and_capture_report(tmp_path, trigger_retry=True, retry_applicable=False) + applicability_idx = report.index("## Patch Applicability") + notice_idx = report.index("Applicability-aware retry") + trust_idx = report.index("## Trust Signals") + appendices_idx = report.index("## Appendices") + assert applicability_idx < notice_idx < trust_idx < appendices_idx diff --git a/libs/openant-core/tests/patch/test_post_patch_evaluation.py b/libs/openant-core/tests/patch/test_post_patch_evaluation.py new file mode 100644 index 00000000..1498e947 --- /dev/null +++ b/libs/openant-core/tests/patch/test_post_patch_evaluation.py @@ -0,0 +1,1172 @@ +"""Tests for post_patch_evaluation.evaluate_anchors (Phase 3: Post-Patch +Anchor Evaluation). + +evaluate_anchors() is pure over two already-computed inputs (a list of +Anchor and an InvestigationContext | None), so every fixture here is built +purely in memory -- RepositoryIndex and ReachabilityAnalyzer both accept +plain dicts directly, with no file I/O -- exactly like Phase 2's tests. +""" + +from __future__ import annotations + +import copy +import inspect +from pathlib import Path +from unittest import mock + +from utilities.agentic_enhancer.reachability_analyzer import ReachabilityAnalyzer +from utilities.agentic_enhancer.repository_index import RepositoryIndex +from utilities.autopatcher.candidate_enrichment import InvestigationContext +from utilities.autopatcher.post_patch_investigation import ( + Anchor, + CallEdgeKey, + ReachabilityKey, + ReachabilityValue, + ResolvedFunctionKey, + ResolvedFunctionValue, + SinkMatchKey, + SinkMatchValue, +) + + +# --------------------------------------------------------------------------- +# Fixture helpers +# --------------------------------------------------------------------------- + +def _function(func_id, start_line=1, end_line=5, name=None, unit_type="function", class_name=None): + return { + "id": func_id, + "name": name or func_id.rsplit(":", 1)[-1], + "startLine": start_line, + "endLine": end_line, + "unitType": unit_type, + "className": class_name, + } + + +def _context(functions, call_graph=None, reverse_call_graph=None, entry_points=None) -> InvestigationContext: + index = RepositoryIndex({"functions": functions}) + reverse_call_graph = reverse_call_graph or {} + entry_points = entry_points or set() + reachability = ReachabilityAnalyzer(functions, reverse_call_graph, entry_points) + return InvestigationContext( + index=index, + call_graph=call_graph or {}, + reverse_call_graph=reverse_call_graph, + reachability=reachability, + ) + + +def _resolved_function_anchor(func_id, start_line=1, end_line=5, candidate_path="a.py", origin="pre_patch"): + return Anchor( + kind="resolved_function", + candidate_path=candidate_path, + key=ResolvedFunctionKey(func_id=func_id, name=func_id.rsplit(":", 1)[-1], class_name=None, unit_type="function"), + before_value=ResolvedFunctionValue(start_line=start_line, end_line=end_line), + source="candidate_enrichment.resolved_function", + origin=origin, + ) + + +def _call_edge_anchor(caller_id, callee_id, candidate_path="a.py"): + return Anchor( + kind="call_edge", + candidate_path=candidate_path, + key=CallEdgeKey(caller_func_id=caller_id, callee_func_id=callee_id), + before_value=True, + source="candidate_enrichment.callees", + ) + + +def _reachability_anchor(func_id, reachable, entry_point_path=None, candidate_path="a.py"): + return Anchor( + kind="reachability", + candidate_path=candidate_path, + key=ReachabilityKey(func_id=func_id), + before_value=ReachabilityValue( + reachable=reachable, + entry_point_path=tuple(entry_point_path) if entry_point_path else None, + ), + source="candidate_enrichment.is_reachable_from_entry_point", + ) + + +def _sink_match_anchor(candidate_path="a.py", method="run", line=10, snippet="os.system(x)"): + return Anchor( + kind="sink_match", + candidate_path=candidate_path, + key=SinkMatchKey(candidate_path=candidate_path, method=method), + before_value=SinkMatchValue(line=line, snippet=snippet), + source="candidate_enrichment.sink_matches", + ) + + +# --------------------------------------------------------------------------- +# resolved_function +# --------------------------------------------------------------------------- + +class TestResolvedFunctionEvaluation: + def test_unchanged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchor = _resolved_function_anchor(func_id, start_line=1, end_line=5) + context = _context({func_id: _function(func_id, start_line=1, end_line=5)}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unchanged" + assert obs.after_value == ResolvedFunctionValue(start_line=1, end_line=5) + assert obs.evaluated_via == "agentic_enhancer.repository_index.RepositoryIndex.get_function" + + def test_changed(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchor = _resolved_function_anchor(func_id, start_line=1, end_line=5) + context = _context({func_id: _function(func_id, start_line=20, end_line=30)}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "changed" + assert obs.after_value == ResolvedFunctionValue(start_line=20, end_line=30) + assert obs.before_value == ResolvedFunctionValue(start_line=1, end_line=5) + + def test_disappeared(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _resolved_function_anchor("a.py:foo") + context = _context({}) # func_id no longer present + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "disappeared" + assert obs.after_value is None + assert "no longer present" in obs.details + + def test_evaluation_error_on_unexpected_exception(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _resolved_function_anchor("a.py:foo") + broken_context = mock.MagicMock() + broken_context.index.get_function.side_effect = RuntimeError("index corrupted") + + obs = evaluate_anchors([anchor], broken_context)[0] + assert obs.status == "evaluation_error" + assert obs.after_value is None + assert "index corrupted" in obs.details + + +# --------------------------------------------------------------------------- +# call_edge +# --------------------------------------------------------------------------- + +class TestCallEdgeEvaluation: + def test_unchanged_when_edge_still_present(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + caller, callee = "a.py:foo", "b.py:bar" + anchor = _call_edge_anchor(caller, callee) + context = _context( + {caller: _function(caller), callee: _function(callee)}, + call_graph={caller: [callee]}, + ) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unchanged" + assert obs.after_value is True + + def test_disappeared_when_caller_no_longer_calls_callee(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + caller, callee = "a.py:foo", "b.py:bar" + anchor = _call_edge_anchor(caller, callee) + context = _context( + {caller: _function(caller), callee: _function(callee)}, + call_graph={caller: []}, # caller exists but no longer calls callee + ) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "disappeared" + assert obs.after_value is False + + def test_unresolved_when_caller_no_longer_resolves(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _call_edge_anchor("a.py:foo", "b.py:bar") + context = _context({}) # caller itself is gone + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unresolved" + assert obs.after_value is None + + +# --------------------------------------------------------------------------- +# reachability +# --------------------------------------------------------------------------- + +class TestReachabilityEvaluation: + def test_unchanged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id, entry_id = "a.py:foo", "entry.py:main" + anchor = _reachability_anchor(func_id, reachable=True, entry_point_path=[entry_id, func_id]) + context = _context( + {func_id: _function(func_id), entry_id: _function(entry_id)}, + reverse_call_graph={func_id: [entry_id]}, + entry_points={entry_id}, + ) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unchanged" + assert obs.after_value.reachable is True + + def test_changed_true_to_false(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchor = _reachability_anchor(func_id, reachable=True, entry_point_path=["entry.py:main", func_id]) + context = _context({func_id: _function(func_id)}) # no reverse edges, no entry points -> unreachable now + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "changed" + assert obs.after_value.reachable is False + assert obs.after_value.entry_point_path is None + + def test_changed_when_previously_unresolved(self): + """before_value.reachable is None (a rare pre-patch + enrichment-exception state) transitioning to a definite value is a + known/unknown transition, not a presence/absence one -- reported + as `changed`, the same as any other value difference, not a + dedicated status.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id, entry_id = "a.py:foo", "entry.py:main" + anchor = _reachability_anchor(func_id, reachable=None, entry_point_path=None) + context = _context( + {func_id: _function(func_id), entry_id: _function(entry_id)}, + reverse_call_graph={func_id: [entry_id]}, + entry_points={entry_id}, + ) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "changed" + assert obs.after_value.reachable is True + + def test_unresolved_when_function_gone(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _reachability_anchor("a.py:foo", reachable=True, entry_point_path=["entry.py:main", "a.py:foo"]) + context = _context({}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unresolved" + assert obs.after_value is None + + def test_evaluation_error_on_unexpected_exception(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchor = _reachability_anchor(func_id, reachable=True) + broken_context = mock.MagicMock() + broken_context.index.get_function.return_value = _function(func_id) + broken_context.reachability.is_reachable_from_entry_point.side_effect = RuntimeError("bfs exploded") + + obs = evaluate_anchors([anchor], broken_context)[0] + assert obs.status == "evaluation_error" + assert "bfs exploded" in obs.details + + +# --------------------------------------------------------------------------- +# constant_value +# --------------------------------------------------------------------------- + +def _constant_context(functions=None, constants=None) -> InvestigationContext: + functions = functions or {} + return InvestigationContext( + index=RepositoryIndex({"functions": functions}), + call_graph={}, + reverse_call_graph={}, + reachability=ReachabilityAnalyzer(functions, {}, set()), + constants=constants or {}, + ) + + +def _constant_value_anchor(candidate_path, qualified_name, class_name, kind, value, origin="pre_patch"): + from utilities.autopatcher.post_patch_investigation import ConstantValueKey, ConstantValueValue + + return Anchor( + kind="constant_value", + candidate_path=candidate_path, + key=ConstantValueKey( + const_id=f"{candidate_path}:{qualified_name}", qualified_name=qualified_name, class_name=class_name, + ), + before_value=ConstantValueValue(ast_literal_kind=kind, value=value), + source="candidate_enrichment.scope_constants", + origin=origin, + ) + + +def _constant_entry(qualified_name, class_name, name, outcome="literal", kind="frozenset_call", value=None, line=1, end_line=1): + return { + "qualified_name": qualified_name, "class_name": class_name, "name": name, + "outcome": outcome, "ast_literal_kind": kind, "value": value, "line": line, "end_line": end_line, + } + + +class TestConstantValueEvaluation: + def test_unchanged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor("retry.py", "Retry.X", "Retry", "frozenset_call", frozenset({"Authorization"})) + context = _constant_context(constants={ + "retry.py": {"Retry.X": _constant_entry("Retry.X", "Retry", "X", value=frozenset({"Authorization"}))} + }) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unchanged" + assert obs.after_value.value == frozenset({"Authorization"}) + assert obs.evaluated_via == "candidate_enrichment.InvestigationContext.constants" + + def test_changed_detects_the_actual_cve_2023_43804_fix(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor( + "retry.py", "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", + "frozenset_call", frozenset({"Authorization"}), + ) + context = _constant_context(constants={ + "retry.py": {"Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": _constant_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + value=frozenset({"Authorization", "Cookie"}), + )} + }) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "changed" + assert obs.before_value.value == frozenset({"Authorization"}) + assert obs.after_value.value == frozenset({"Authorization", "Cookie"}) + + def test_disappeared_when_target_no_longer_found(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor("a.py", "X", None, "Constant", 30) + context = _constant_context(constants={}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "disappeared" + assert obs.after_value is None + + def test_unresolved_when_now_non_literal(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor("a.py", "BACKEND", None, "Call", "default_backend") + context = _constant_context(constants={ + "a.py": {"BACKEND": _constant_entry("BACKEND", None, "BACKEND", outcome="non_literal", kind=None, value=None)} + }) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unresolved" + assert "non_literal" in obs.details + + def test_evaluation_error_becomes_evaluation_error_status_when_no_context(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor("a.py", "X", None, "Constant", 1) + observations = evaluate_anchors([anchor], None) + assert observations[0].status == "evaluation_error" + + +# --------------------------------------------------------------------------- +# origin: pre_patch (default) vs patch_touched -- mechanical propagation only, +# never a branch in evaluation logic itself. +# --------------------------------------------------------------------------- + +class TestAnchorOriginPropagation: + def test_default_origin_is_pre_patch(self): + anchor = _resolved_function_anchor("a.py:foo") + assert anchor.origin == "pre_patch" + + def test_observation_echoes_the_anchors_origin(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + pre_patch = _resolved_function_anchor(func_id, origin="pre_patch") + patch_touched = _resolved_function_anchor("b.py:bar", origin="patch_touched") + context = _context({func_id: _function(func_id), "b.py:bar": _function("b.py:bar")}) + + observations = evaluate_anchors([pre_patch, patch_touched], context) + by_kind_key = {(o.anchor_key.func_id): o for o in observations} + assert by_kind_key[func_id].origin == "pre_patch" + assert by_kind_key["b.py:bar"].origin == "patch_touched" + + def test_origin_never_changes_evaluated_status_or_after_value(self): + """Two anchors identical except for origin must evaluate + identically in every other respect -- proves evaluate_anchors() + does not branch on origin, it only carries it through.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + pre_patch = _resolved_function_anchor(func_id, start_line=1, end_line=5, origin="pre_patch") + patch_touched = _resolved_function_anchor(func_id, start_line=1, end_line=5, origin="patch_touched") + context = _context({func_id: _function(func_id, start_line=20, end_line=30)}) + + obs_pre, obs_patch = evaluate_anchors([pre_patch, patch_touched], context) + assert obs_pre.status == obs_patch.status == "changed" + assert obs_pre.after_value == obs_patch.after_value + assert obs_pre.evaluated_via == obs_patch.evaluated_via + assert obs_pre.origin == "pre_patch" + assert obs_patch.origin == "patch_touched" + + def test_origin_defaults_on_constant_value_and_evaluation_error_paths_too(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _constant_value_anchor("a.py", "X", None, "Constant", 1, origin="patch_touched") + observations = evaluate_anchors([anchor], None) # no context -> evaluation_error + assert observations[0].status == "evaluation_error" + assert observations[0].origin == "patch_touched" + + +# --------------------------------------------------------------------------- +# sink_match -- deferred, always unresolved +# --------------------------------------------------------------------------- + +class TestSinkMatchDeferred: + def test_always_unresolved_regardless_of_context(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _sink_match_anchor() + context = _context({"a.py:run": _function("a.py:run")}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.status == "unresolved" + assert "deferred" in obs.details + assert obs.evaluated_via == "deferred" + + def test_unresolved_even_with_no_context(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchor = _sink_match_anchor() + obs = evaluate_anchors([anchor], None)[0] + assert obs.status == "unresolved" + + +# --------------------------------------------------------------------------- +# Missing context +# --------------------------------------------------------------------------- + +class TestMissingContext: + def test_context_dependent_kinds_become_evaluation_error(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + anchors = [ + _resolved_function_anchor("a.py:foo"), + _call_edge_anchor("a.py:foo", "b.py:bar"), + _reachability_anchor("a.py:foo", reachable=True), + _constant_value_anchor("a.py", "X", None, "Constant", 1), + ] + observations = evaluate_anchors(anchors, None) + assert all(o.status == "evaluation_error" for o in observations) + assert all(o.after_value is None for o in observations) + + +# --------------------------------------------------------------------------- +# Ordering, provenance, determinism, purity +# --------------------------------------------------------------------------- + +class TestOrderingAndProvenance: + def test_ordering_preserved_across_mixed_kinds(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchors = [ + _sink_match_anchor(), + _resolved_function_anchor(func_id), + _call_edge_anchor(func_id, "b.py:bar"), + _reachability_anchor(func_id, reachable=True), + ] + context = _context({func_id: _function(func_id)}) + + observations = evaluate_anchors(anchors, context) + assert [o.anchor_kind for o in observations] == [a.kind for a in anchors] + assert [o.anchor_key for o in observations] == [a.key for a in anchors] + + def test_provenance_preserved(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchor = _resolved_function_anchor(func_id) + context = _context({func_id: _function(func_id)}) + + obs = evaluate_anchors([anchor], context)[0] + assert obs.anchor_kind == anchor.kind + assert obs.anchor_key == anchor.key + assert obs.candidate_path == anchor.candidate_path + assert obs.source == anchor.source + assert obs.evaluated_via == "agentic_enhancer.repository_index.RepositoryIndex.get_function" + + +class TestDeterminismAndPurity: + def test_repeated_evaluation_is_deterministic(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchors = [_resolved_function_anchor(func_id), _reachability_anchor(func_id, reachable=True)] + context = _context({func_id: _function(func_id)}) + + result1 = evaluate_anchors(anchors, context) + result2 = evaluate_anchors(anchors, context) + assert result1 == result2 + + def test_no_mutation_of_anchors_or_context(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchors = [_resolved_function_anchor(func_id, start_line=1, end_line=5)] + context = _context({func_id: _function(func_id, start_line=1, end_line=5)}) + anchors_snapshot = copy.deepcopy(anchors) + + evaluate_anchors(anchors, context) + + assert anchors == anchors_snapshot + assert context.index.get_function(func_id) == _function(func_id, start_line=1, end_line=5) + + def test_empty_anchors_returns_empty_list(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + assert evaluate_anchors([], None) == [] + assert evaluate_anchors([], _context({})) == [] + + def test_no_disallowed_imports(self): + import utilities.autopatcher.post_patch_evaluation as mod + + source = inspect.getsource(mod) + disallowed = [ + "import subprocess", "import socket", "import requests", + "import tempfile", "import shutil", + "anthropic", "openai", "urllib", + ] + for token in disallowed: + assert token not in source, f"unexpected token found: {token}" + + def test_no_recommendation_or_verdict_vocabulary(self): + """#16: statuses and details must never read as a recommendation -- + only comparison-neutral facts.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors + + func_id = "a.py:foo" + anchors = [ + _resolved_function_anchor(func_id), + _call_edge_anchor(func_id, "b.py:bar"), + _reachability_anchor(func_id, reachable=True), + _sink_match_anchor(), + ] + context = _context({}) # force every branch to produce a details string + + observations = evaluate_anchors(anchors, context) + blocklist = ["fixed", "correct", "success", "vulnerable", "safe", "remediat"] + for obs in observations: + text = (obs.details or "").lower() + for word in blocklist: + assert word not in text, f"{word!r} found in details: {text!r}" + + +# --------------------------------------------------------------------------- +# render_post_patch_investigation +# --------------------------------------------------------------------------- + +def _diff(file_path, before_lines, after_lines, start=1): + """Minimal unified-diff builder covering exactly what diff_parsing.parse_diff + understands: '--- a/', '+++ b/', an '@@ ... +start,count @@' header, and + ' '/'+'/'-' body lines. Produces one hunk with every before-line removed + and every after-line added (a full-file replace) -- simple and sufficient + for these tests, which only care about symbol attribution, not minimal diffs.""" + count = max(len(before_lines), len(after_lines)) + lines = [f"--- a/{file_path}", f"+++ b/{file_path}", f"@@ -{start},{len(before_lines)} +{start},{count} @@"] + lines += [f"-{l}" for l in before_lines] + lines += [f"+{l}" for l in after_lines] + return "\n".join(lines) + "\n" + + +class TestComputeCoverage: + def test_returns_none_without_a_context(self): + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + assert compute_coverage("--- a/x.py\n+++ b/x.py\n", [], Path("/nonexistent"), None) is None + + def test_treats_pre_patch_and_patch_touched_origins_equally(self, tmp_path): + """Explicit requirement: coverage accounting must not care which + phase produced the covering Anchor.""" + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + (tmp_path / "a.py").write_text("def foo():\n return 1\n", encoding="utf-8") + diff = _diff("a.py", [" return 1"], [" return 2"], start=2) + func_id = "a.py:foo" + context = _constant_context(functions={func_id: _function(func_id, start_line=1, end_line=2)}) + + pre_patch_result = compute_coverage( + diff, [_resolved_function_anchor(func_id, start_line=1, end_line=2, origin="pre_patch")], tmp_path, context, + ) + patch_touched_result = compute_coverage( + diff, [_resolved_function_anchor(func_id, start_line=1, end_line=2, origin="patch_touched")], tmp_path, context, + ) + assert pre_patch_result.covered == patch_touched_result.covered == (func_id,) + assert pre_patch_result.uncovered == patch_touched_result.uncovered == () + + def test_covered_element_matches_a_resolved_function_anchor(self, tmp_path): + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + (tmp_path / "a.py").write_text("def foo():\n return 1\n", encoding="utf-8") + diff = _diff("a.py", [" return 1"], [" return 2"], start=2) + func_id = "a.py:foo" + anchors = [_resolved_function_anchor(func_id, start_line=1, end_line=2, candidate_path="a.py")] + context = _constant_context(functions={func_id: _function(func_id, start_line=1, end_line=2)}) + + result = compute_coverage(diff, anchors, tmp_path, context) + assert result.total == 1 + assert result.covered == (func_id,) + assert result.uncovered == () + + def test_uncovered_element_when_no_anchor_names_it(self, tmp_path): + """The CVE-2023-43804 shape before constant_value anchors exist: + the diff touches a class constant, but only resolved_function + anchors exist -- must report uncovered, never fabricate coverage.""" + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + (tmp_path / "retry.py").write_text( + "class Retry:\n DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Authorization\"])\n", + encoding="utf-8", + ) + diff = _diff( + "retry.py", + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])'], + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])'], + start=2, + ) + # The element is resolvable (it's a known constant, on line 2 of + # the file) -- but no Anchor names it, since anchors=[] below. + context = _constant_context(constants={ + "retry.py": {"Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": _constant_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + value=frozenset({"Authorization"}), + line=2, end_line=2, + )} + }) + result = compute_coverage(diff, anchors=[], repo_root=tmp_path, context=context) + assert result.total == 1 + assert result.covered == () + assert len(result.uncovered) == 1 + assert "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result.uncovered[0] + + def test_covered_once_constant_value_anchor_exists(self, tmp_path): + """Same diff as above, but now a constant_value anchor exists for + the touched constant -- must report covered.""" + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + (tmp_path / "retry.py").write_text( + "class Retry:\n DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Authorization\"])\n", + encoding="utf-8", + ) + diff = _diff( + "retry.py", + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])'], + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])'], + start=2, + ) + anchors = [_constant_value_anchor( + "retry.py", "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "frozenset_call", frozenset({"Authorization"}), + )] + context = _constant_context( + functions={}, + constants={"retry.py": {"Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": _constant_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + value=frozenset({"Authorization"}), + line=2, end_line=2, + )}}, + ) + result = compute_coverage(diff, anchors, tmp_path, context) + assert result.total == 1 + assert len(result.covered) == 1 + assert "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in result.covered[0] + assert result.uncovered == () + + def test_unreadable_file_counts_as_unattributed_not_dropped(self, tmp_path): + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + diff = _diff("missing.py", ["x = 1"], ["x = 2"]) + context = _constant_context() + result = compute_coverage(diff, [], tmp_path, context) + assert result.total == 0 + assert result.unattributed == 1 + + def test_call_edge_anchors_never_credited_as_coverage(self, tmp_path): + """call_edge's key names a relationship between two OTHER + locations, not an observable property of the touched location's + own content -- must never manufacture false coverage.""" + from utilities.autopatcher.post_patch_evaluation import compute_coverage + + (tmp_path / "a.py").write_text("def foo():\n return 1\n", encoding="utf-8") + diff = _diff("a.py", [" return 1"], [" return 2"], start=2) + anchors = [_call_edge_anchor("a.py:foo", "b.py:bar", candidate_path="a.py")] + context = _constant_context(functions={"a.py:foo": _function("a.py:foo", start_line=1, end_line=2)}) + + result = compute_coverage(diff, anchors, tmp_path, context) + assert result.covered == () + assert result.uncovered == ("a.py:foo",) + + +class TestDerivePatchTouchedAnchors: + """derive_patch_touched_anchors -- the candidate-selection-independent + fix: resolves the final patch's own diff directly against the + pre-patch InvestigationContext, never against selected candidates.""" + + def test_returns_empty_list_without_a_context(self): + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + assert derive_patch_touched_anchors("--- a/x.py\n+++ b/x.py\n", Path("/nonexistent"), None, []) == [] + + def test_derives_a_new_constant_value_anchor_for_the_cve_2023_43804_shape(self, tmp_path): + """The exact motivating scenario: retry.py was never a selected + candidate (existing_anchors has nothing for it), but the final + patch touches Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT -- must + derive a fresh, origin="patch_touched" Anchor for it.""" + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "retry.py").write_text( + "class Retry:\n DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset([\"Authorization\"])\n", + encoding="utf-8", + ) + diff = _diff( + "retry.py", + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"])'], + [' DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"])'], + start=2, + ) + context = _constant_context(constants={ + "retry.py": {"Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": _constant_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + value=frozenset({"Authorization"}), line=2, end_line=2, + )} + }) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=[]) + assert len(derived) == 1 + anchor = derived[0] + assert anchor.kind == "constant_value" + assert anchor.origin == "patch_touched" + assert anchor.key.qualified_name == "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" + assert anchor.before_value.value == frozenset({"Authorization"}) + + def test_skips_a_ref_already_covered_by_an_existing_anchor(self, tmp_path): + """No duplicate anchor when the file WAS a selected candidate and + already has an anchor for this exact element.""" + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "a.py").write_text("def foo():\n return 1\n", encoding="utf-8") + diff = _diff("a.py", [" return 1"], [" return 2"], start=2) + existing = [_resolved_function_anchor("a.py:foo", start_line=1, end_line=2, candidate_path="a.py")] + context = _constant_context(functions={"a.py:foo": _function("a.py:foo", start_line=1, end_line=2)}) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=existing) + assert derived == [] + + def test_skips_module_level_fallback_matches(self, tmp_path): + """A hunk resolving only to the whole-file module_level catch-all + unit carries no meaningful signal -- must never fabricate a + near-content-free resolved_function anchor for it (it should + render as uncovered via Coverage Analysis instead).""" + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "a.py").write_text("x = 1\ny = 2\n", encoding="utf-8") + diff = _diff("a.py", ["x = 1"], ["x = 100"], start=1) + context = _constant_context(functions={ + "a.py:__module__": _function("a.py:__module__", start_line=1, end_line=2, unit_type="module_level"), + }) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=[]) + assert derived == [] + + def test_skips_non_literal_constants(self, tmp_path): + """No fabricated value for a constant whose RHS isn't a + supported literal shape.""" + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "a.py").write_text("BACKEND = default_backend()\n", encoding="utf-8") + diff = _diff("a.py", ["BACKEND = default_backend()"], ["BACKEND = other_backend()"], start=1) + context = _constant_context(constants={ + "a.py": {"BACKEND": _constant_entry("BACKEND", None, "BACKEND", outcome="non_literal", kind=None, value=None)} + }) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=[]) + assert derived == [] + + def test_deduplicates_multiple_hunks_resolving_to_the_same_element(self, tmp_path): + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "a.py").write_text("def foo():\n x = 1\n return x\n", encoding="utf-8") + diff = ( + "--- a/a.py\n+++ b/a.py\n@@ -1,3 +1,3 @@\n" + " def foo():\n- x = 1\n+ x = 2\n- return x\n+ return x + 1\n" + ) + context = _constant_context(functions={"a.py:foo": _function("a.py:foo", start_line=1, end_line=3)}) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=[]) + assert len(derived) == 1 + + def test_new_anchors_only_never_returns_existing_anchors_object(self, tmp_path): + """Contract: existing_anchors is read for identity comparison only, + never mutated, never echoed back.""" + from utilities.autopatcher.post_patch_evaluation import derive_patch_touched_anchors + + (tmp_path / "retry.py").write_text( + "class Retry:\n X = frozenset([\"A\"])\n", encoding="utf-8", + ) + diff = _diff("retry.py", [' X = frozenset(["A"])'], [' X = frozenset(["A", "B"])'], start=2) + context = _constant_context(constants={ + "retry.py": {"Retry.X": _constant_entry("Retry.X", "Retry", "X", value=frozenset({"A"}), line=2, end_line=2)} + }) + existing = [_resolved_function_anchor("unrelated.py:bar")] + snapshot = list(existing) + + derived = derive_patch_touched_anchors(diff, tmp_path, context, existing_anchors=existing) + assert existing == snapshot + assert all(a not in existing for a in derived) + + +class TestRenderPostPatchInvestigation: + def test_empty_observations(self): + from utilities.autopatcher.post_patch_evaluation import render_post_patch_investigation + + rendered = render_post_patch_investigation([]) + assert rendered.startswith("## Post-Patch Investigation") + assert "No anchors were available to re-evaluate." in rendered + + def test_grouped_by_status(self): + from utilities.autopatcher.post_patch_evaluation import ( + AnchorObservation, render_post_patch_investigation, + ) + + def _obs(status, kind="resolved_function", details=None): + return AnchorObservation( + anchor_kind=kind, + anchor_key=ResolvedFunctionKey(func_id="a.py:foo", name="foo", class_name=None, unit_type="function"), + candidate_path="a.py", + status=status, + before_value=ResolvedFunctionValue(start_line=1, end_line=5), + after_value=ResolvedFunctionValue(start_line=1, end_line=9) if status == "changed" else None, + details=details, + source="candidate_enrichment.resolved_function", + evaluated_via="agentic_enhancer.repository_index.RepositoryIndex.get_function", + ) + + observations = [ + _obs("changed"), + _obs("disappeared", details="function id no longer present in the patched copy"), + _obs("unchanged"), + _obs("unresolved", details="function id no longer resolves in the patched copy"), + _obs("evaluation_error", details="lookup failed: RuntimeError: boom"), + ] + rendered = render_post_patch_investigation(observations) + + assert "### Changed" in rendered + assert "### Disappeared" in rendered + assert "### Unchanged" in rendered + assert "### Remaining Unknowns" in rendered + # Changed/Disappeared content appears under their own headings. + changed_idx = rendered.index("### Changed") + disappeared_idx = rendered.index("### Disappeared") + unchanged_idx = rendered.index("### Unchanged") + unknowns_idx = rendered.index("### Remaining Unknowns") + assert changed_idx < disappeared_idx < unchanged_idx < unknowns_idx + assert "resolved_function:a.py:foo" in rendered[changed_idx:disappeared_idx] + assert "no longer present" in rendered[disappeared_idx:unchanged_idx] + assert "1 anchor(s) confirmed unchanged" in rendered[unchanged_idx:unknowns_idx] + # Both unresolved and evaluation_error land in Remaining Unknowns, never elsewhere. + unknowns_section = rendered[unknowns_idx:] + assert "unresolved" in unknowns_section + assert "evaluation_error" in unknowns_section + + def test_sink_match_always_lands_in_remaining_unknowns(self): + """sink_match is always status='unresolved' today (Phase 3 defers + it) -- confirms it's never miscategorized into a determinate group.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchors = [_sink_match_anchor()] + observations = evaluate_anchors(anchors, None) + rendered = render_post_patch_investigation(observations) + + assert "### Remaining Unknowns" in rendered + unknowns_idx = rendered.index("### Remaining Unknowns") + assert "sink_match" in rendered[unknowns_idx:] + # Never appears in the determinate-looking sections above. + assert "sink_match" not in rendered[:rendered.index("### Changed")] + + def test_never_mutates_input(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + func_id = "a.py:foo" + anchors = [_resolved_function_anchor(func_id)] + context = _context({func_id: _function(func_id)}) + observations = evaluate_anchors(anchors, context) + snapshot = list(observations) + + render_post_patch_investigation(observations) + + assert observations == snapshot + + def test_never_exceeds_max_chars(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchors = [ + _resolved_function_anchor(f"a{i}.py:foo", candidate_path=f"a{i}.py") + for i in range(50) + ] + context = _context({}) # everything -> disappeared, exercises the per-group item cap + observations = evaluate_anchors(anchors, context) + + rendered = render_post_patch_investigation(observations, max_chars=500) + assert len(rendered) <= 500 + assert "(+" in rendered or "truncated" in rendered + + def test_no_recommendation_or_verdict_vocabulary_in_data_sections(self): + """The blocklist applies to the data-driven sections (Changed/ + Disappeared/Unchanged/Remaining Unknowns), not the fixed preamble + -- which legitimately says "not a verdict that the patch is + correct... or a successful fix" to explicitly disclaim exactly + those words, the same way evidence_fusion.py's own preamble does.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + func_id = "a.py:foo" + anchors = [ + _resolved_function_anchor(func_id), + _call_edge_anchor(func_id, "b.py:bar"), + _reachability_anchor(func_id, reachable=True), + _sink_match_anchor(), + ] + context = _context({}) + rendered = render_post_patch_investigation(evaluate_anchors(anchors, context)) + + data_sections = rendered[rendered.index("### Changed"):] + blocklist = ["fixed", "correct", "success", "vulnerable", "safe", "remediat"] + lowered = data_sections.lower() + for word in blocklist: + assert word not in lowered, f"{word!r} found in rendered data sections" + + def test_coverage_omitted_entirely_when_not_provided(self): + """No fabricated coverage section when the caller didn't compute + one (coverage=None is the default).""" + from utilities.autopatcher.post_patch_evaluation import render_post_patch_investigation + + rendered = render_post_patch_investigation([]) + assert "Anchor Coverage" not in rendered + + def test_coverage_section_appears_before_changed_and_survives_truncation(self): + from utilities.autopatcher.post_patch_evaluation import CoverageResult, render_post_patch_investigation + + coverage = CoverageResult(total=8, covered=("a.py:foo",), uncovered=tuple(f"b{i}.py:x" for i in range(7)), unattributed=0) + rendered = render_post_patch_investigation([], coverage) + + assert "### Anchor Coverage" in rendered + assert "1 of 8 element(s)" in rendered + assert "7 are not" in rendered + coverage_idx = rendered.index("### Anchor Coverage") + assert coverage_idx < rendered.index("No anchors were available") + + # Placed early enough to survive a tight character budget, unlike + # a section placed last (which _hard_clamp would drop first). The + # budget below is tight enough to truncate well before any of the + # Changed/Disappeared/Unchanged/Remaining-Unknowns sections could + # render, yet still fits header + preamble + Anchor Coverage. + tight = render_post_patch_investigation([], coverage, max_chars=760) + assert "Anchor Coverage" in tight + assert "*(truncated to fit the character budget)*" in tight + + def test_uncovered_list_capped_with_plus_n_more(self): + from utilities.autopatcher.post_patch_evaluation import CoverageResult, render_post_patch_investigation + + coverage = CoverageResult(total=7, covered=(), uncovered=tuple(f"f{i}.py:x" for i in range(7)), unattributed=0) + rendered = render_post_patch_investigation([], coverage) + assert "(+2 more)" in rendered + + def test_unchanged_caveat_appears_only_when_something_is_uncovered(self): + from utilities.autopatcher.post_patch_evaluation import ( + CoverageResult, evaluate_anchors, render_post_patch_investigation, + ) + + func_id = "a.py:foo" + anchors = [_resolved_function_anchor(func_id)] + context = _context({func_id: _function(func_id)}) + observations = evaluate_anchors(anchors, context) + + fully_covered = CoverageResult(total=1, covered=(func_id,), uncovered=(), unattributed=0) + rendered_full = render_post_patch_investigation(observations, fully_covered) + unchanged_full = rendered_full[rendered_full.index("### Unchanged"):rendered_full.index("### Remaining Unknowns")] + assert "see Anchor Coverage above" not in unchanged_full + + partially_covered = CoverageResult(total=2, covered=(func_id,), uncovered=("b.py:bar",), unattributed=0) + rendered_partial = render_post_patch_investigation(observations, partially_covered) + unchanged_partial = rendered_partial[rendered_partial.index("### Unchanged"):rendered_partial.index("### Remaining Unknowns")] + assert "see Anchor Coverage above" in unchanged_partial + + def test_reproduces_the_cve_2023_43804_report_shape(self): + """End-to-end render check for the motivating case: a + constant_value anchor shows the actual fix under Changed, and + Anchor Coverage names what else in the file has no anchor at + all -- instead of today's misleading "0 changed, 7 unchanged".""" + from utilities.autopatcher.post_patch_evaluation import ( + CoverageResult, evaluate_anchors, render_post_patch_investigation, + ) + + anchor = _constant_value_anchor( + "retry.py", "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", + "frozenset_call", frozenset({"Authorization"}), + ) + context = _constant_context(constants={ + "retry.py": {"Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT": _constant_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "Retry", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + value=frozenset({"Authorization", "Cookie"}), + )} + }) + observations = evaluate_anchors([anchor], context) + coverage = CoverageResult(total=1, covered=("retry.py:Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT",), uncovered=(), unattributed=0) + + rendered = render_post_patch_investigation(observations, coverage) + changed_idx = rendered.index("### Changed") + disappeared_idx = rendered.index("### Disappeared") + assert "constant_value:retry.py:Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in rendered[changed_idx:disappeared_idx] + assert "frozenset({'Authorization'})" in rendered[changed_idx:disappeared_idx] + assert "1 of 1 element(s)" in rendered + + def test_patch_touched_changed_item_is_tagged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchor = _constant_value_anchor( + "retry.py", "Retry.X", "Retry", "frozenset_call", frozenset({"Authorization"}), origin="patch_touched", + ) + context = _constant_context(constants={ + "retry.py": {"Retry.X": _constant_entry("Retry.X", "Retry", "X", value=frozenset({"Authorization", "Cookie"}))} + }) + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + changed_section = rendered[rendered.index("### Changed"):rendered.index("### Disappeared")] + assert "(discovered from patch diff)" in changed_section + + def test_pre_patch_changed_item_is_not_tagged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchor = _constant_value_anchor( + "retry.py", "Retry.X", "Retry", "frozenset_call", frozenset({"Authorization"}), origin="pre_patch", + ) + context = _constant_context(constants={ + "retry.py": {"Retry.X": _constant_entry("Retry.X", "Retry", "X", value=frozenset({"Authorization", "Cookie"}))} + }) + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + changed_section = rendered[rendered.index("### Changed"):rendered.index("### Disappeared")] + assert "(discovered from patch diff)" not in changed_section + + def test_patch_touched_disappeared_item_is_tagged(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchor = _resolved_function_anchor("a.py:foo", origin="patch_touched") + context = _context({}) # func_id no longer present -> disappeared + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + disappeared_section = rendered[rendered.index("### Disappeared"):rendered.index("### Unchanged")] + assert "(discovered from patch diff)" in disappeared_section + + def test_unchanged_summary_has_no_second_origin_breakdown(self): + """Explicit requirement: Unchanged stays a single breakdown axis + (by anchor_kind), never a second one by origin.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + func_id = "a.py:foo" + anchors = [ + _resolved_function_anchor(func_id, origin="pre_patch"), + _resolved_function_anchor("b.py:bar", origin="patch_touched"), + ] + context = _context({func_id: _function(func_id), "b.py:bar": _function("b.py:bar")}) + rendered = render_post_patch_investigation(evaluate_anchors(anchors, context)) + unchanged_section = rendered[rendered.index("### Unchanged"):rendered.index("### Remaining Unknowns")] + assert "2 anchor(s) confirmed unchanged (resolved_function: 2)" in unchanged_section + assert "patch_touched" not in unchanged_section + assert "pre_patch" not in unchanged_section + assert "discovered from patch diff" not in unchanged_section + + +# --------------------------------------------------------------------------- +# Release-polish change #2: human-readable Anchor values in Changed +# --------------------------------------------------------------------------- + +class TestHumanReadableAnchorValues: + """The Changed section must render concise, maintainer-readable + before/after values -- never a raw Python NamedTuple repr (e.g. + "ResolvedFunctionValue(start_line=1, end_line=5)" or + "ConstantValueValue(ast_literal_kind=...)"). Presentation only -- + evaluate_anchors()'s own status/comparison logic is untouched.""" + + def test_resolved_function_shows_line_range_not_namedtuple_repr(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchor = _resolved_function_anchor("a.py:foo", start_line=1, end_line=5) + context = _context({"a.py:foo": _function("a.py:foo", start_line=20, end_line=30)}) + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + changed_section = rendered[rendered.index("### Changed"):rendered.index("### Disappeared")] + + assert "ResolvedFunctionValue" not in changed_section + assert "lines 1-5" in changed_section + assert "lines 20-30" in changed_section + + def test_reachability_shows_plain_words_not_namedtuple_repr(self): + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + func_id = "a.py:foo" + anchor = _reachability_anchor(func_id, reachable=True, entry_point_path=["entry.py:main", func_id]) + context = _context({func_id: _function(func_id)}) # no reverse edges/entry points -> unreachable now + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + changed_section = rendered[rendered.index("### Changed"):rendered.index("### Disappeared")] + + assert "ReachabilityValue" not in changed_section + assert "reachable" in changed_section + assert "not reachable" in changed_section + + def test_constant_value_shows_bare_literal_not_wrapper_repr(self): + """Regression guard alongside test_reproduces_the_cve_2023_43804_report_shape: + the wrapper's own class name / ast_literal_kind field must not + appear -- only the literal's own value.""" + from utilities.autopatcher.post_patch_evaluation import evaluate_anchors, render_post_patch_investigation + + anchor = _constant_value_anchor( + "retry.py", "Retry.X", "Retry", "frozenset_call", frozenset({"Authorization"}), + ) + context = _constant_context(constants={ + "retry.py": {"Retry.X": _constant_entry("Retry.X", "Retry", "X", value=frozenset({"Authorization", "Cookie"}))} + }) + rendered = render_post_patch_investigation(evaluate_anchors([anchor], context)) + changed_section = rendered[rendered.index("### Changed"):rendered.index("### Disappeared")] + + assert "ConstantValueValue" not in changed_section + assert "ast_literal_kind" not in changed_section + assert "frozenset({'Authorization'})" in changed_section + + def test_unknown_value_shape_falls_back_safely(self): + """Defensive fallback: a value that doesn't match its kind's + expected NamedTuple shape must never raise.""" + from utilities.autopatcher.post_patch_evaluation import _format_anchor_value + + assert _format_anchor_value("resolved_function", "unexpected-shape") == "unexpected-shape" + assert _format_anchor_value("resolved_function", None) == "unknown" + + +# --------------------------------------------------------------------------- +# Release-polish change #8: Post-Patch Investigation preamble provenance +# --------------------------------------------------------------------------- + +class TestPreamblePatchTouchedProvenance: + def test_preamble_clarifies_patch_touched_provenance(self): + """The preamble must distinguish this evidence (gathered from the + final patch diff, after generation) from Repository Context + (locations selected before the patch existed) -- worded without a + directional "above"/"below" claim, since this function also renders + correctly when exercised standalone (as this test does).""" + from utilities.autopatcher.post_patch_evaluation import render_post_patch_investigation + + rendered = render_post_patch_investigation([]) + + assert "after the patch was generated" in rendered + assert "Repository Context" in rendered + assert "before the patch existed" in rendered diff --git a/libs/openant-core/tests/patch/test_post_patch_investigation.py b/libs/openant-core/tests/patch/test_post_patch_investigation.py new file mode 100644 index 00000000..d2ee042f --- /dev/null +++ b/libs/openant-core/tests/patch/test_post_patch_investigation.py @@ -0,0 +1,570 @@ +"""Tests for post_patch_investigation.derive_pre_patch_anchors (Phase 2: +Pre-Patch Anchor Derivation). + +Builds fixtures from the real dataclasses (RepositoryCandidate, +CandidateEnrichment, DiscoveryEvidence, RepositoryUnderstanding) rather than +mocking -- this module is pure data transformation, so real objects are +both simpler and more honest than mocks. +""" + +from __future__ import annotations + +import copy +import dataclasses +import inspect + +import pytest + +from utilities.autopatcher.evidence_fusion import RepositoryUnderstanding +from utilities.autopatcher.repository_grounding_models import ( + CandidateEnrichment, + DiscoveryEvidence, + RepositoryCandidate, +) + + +# --------------------------------------------------------------------------- +# Fixture helpers +# --------------------------------------------------------------------------- + +def _evidence(tier=3, hit_line=10, pass_name="explicit_path"): + return DiscoveryEvidence( + pass_name=pass_name, + tier=tier, + matched_tokens=None, + total_occurrences=None, + hit_line=hit_line, + resolution_strategy=None, + ) + + +def _candidate(path: str, enrichment=None, best_tier=3) -> RepositoryCandidate: + return RepositoryCandidate( + path=path, + evidence=[_evidence(tier=best_tier)], + best_tier=best_tier, + enrichment=enrichment, + ) + + +def _enrichment(**overrides) -> CandidateEnrichment: + defaults = dict( + functions_in_file=[], + resolved_function=None, + resolution_note=None, + callees=[], + callers_by_call_graph=[], + callers_by_text_search=[], + is_reachable_from_entry_point=None, + entry_point_path=None, + related_tests=[], + test_support_rating=None, + sink_matches=None, + enrichment_errors=[], + ) + defaults.update(overrides) + return CandidateEnrichment(**defaults) + + +def _resolved(func_id, name="authenticate", start_line=1, end_line=5, unit_type="function", class_name=None): + return { + "id": func_id, + "name": name, + "startLine": start_line, + "endLine": end_line, + "unitType": unit_type, + "className": class_name, + } + + +def _understanding(candidates, relationships=None, notes=None, context_available=True) -> RepositoryUnderstanding: + return RepositoryUnderstanding( + candidate_evidence=candidates, + relationships=relationships or [], + fusion_notes=notes or [], + investigation_context_available=context_available, + ) + + +# --------------------------------------------------------------------------- +# 1. resolved_function anchor +# --------------------------------------------------------------------------- + +class TestResolvedFunctionAnchor: + def test_derived_with_stable_deterministic_id(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + func_id = "auth.py:authenticate" + candidate = _candidate("auth.py", _enrichment(resolved_function=_resolved(func_id))) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + + matches = [a for a in anchors if a.kind == "resolved_function"] + assert len(matches) == 1 + anchor = matches[0] + assert anchor.display_id == f"resolved_function:{func_id}" + assert anchor.candidate_path == "auth.py" + assert anchor.key.func_id == func_id + assert anchor.key.name == "authenticate" + assert anchor.key.class_name is None + assert anchor.key.unit_type == "function" + assert anchor.before_value.start_line == 1 + assert anchor.before_value.end_line == 5 + assert anchor.source == "candidate_enrichment.resolved_function" + + def test_line_number_changes_alone_do_not_change_identity(self): + """#2/#3: identical qualified function, different line range -> + same (kind, key) identity and same display_id, different + before_value.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + func_id = "auth.py:authenticate" + candidate_v1 = _candidate("auth.py", _enrichment( + resolved_function=_resolved(func_id, start_line=1, end_line=5) + )) + candidate_v2 = _candidate("auth.py", _enrichment( + resolved_function=_resolved(func_id, start_line=20, end_line=30) + )) + + anchors_v1 = derive_pre_patch_anchors(_understanding([candidate_v1])) + anchors_v2 = derive_pre_patch_anchors(_understanding([candidate_v2])) + + a1 = next(a for a in anchors_v1 if a.kind == "resolved_function") + a2 = next(a for a in anchors_v2 if a.kind == "resolved_function") + + assert (a1.kind, a1.key) == (a2.kind, a2.key) + assert a1.key == a2.key + assert a1.display_id == a2.display_id + assert a1.before_value != a2.before_value + assert a1.before_value.start_line == 1 + assert a2.before_value.start_line == 20 + + +# --------------------------------------------------------------------------- +# 4/5/14. call_edge anchors +# --------------------------------------------------------------------------- + +class TestCallEdgeAnchor: + def test_derived_once_and_deduplicated_across_both_endpoints(self): + """A calls B: A's callees names B's func_id, and B's + callers_by_call_graph names A's func_id -- same real edge, must + collapse to exactly one anchor.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + func_a = "a.py:funcA" + func_b = "b.py:funcB" + candidate_a = _candidate("a.py", _enrichment( + resolved_function=_resolved(func_a, name="funcA"), + callees=[func_b], + )) + candidate_b = _candidate("b.py", _enrichment( + resolved_function=_resolved(func_b, name="funcB"), + callers_by_call_graph=[func_a], + )) + + anchors = derive_pre_patch_anchors(_understanding([candidate_a, candidate_b])) + + edges = [a for a in anchors if a.kind == "call_edge"] + assert len(edges) == 1 + edge = edges[0] + assert edge.display_id == f"call_edge:{func_a}->{func_b}" + assert edge.key.caller_func_id == func_a + assert edge.key.callee_func_id == func_b + assert edge.before_value is True + assert edge.candidate_path == "a.py" # caller's file, per docstring + + def test_duplicate_entries_within_one_field_also_deduplicated(self): + """#14 variant: the same callee listed twice in one candidate's own + callees list must still produce one anchor.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + func_a = "a.py:funcA" + func_b = "b.py:funcB" + candidate = _candidate("a.py", _enrichment( + resolved_function=_resolved(func_a, name="funcA"), + callees=[func_b, func_b], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + edges = [a for a in anchors if a.kind == "call_edge"] + assert len(edges) == 1 + + def test_callers_by_text_search_never_creates_call_edge(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + func_id = "auth.py:authenticate" + candidate = _candidate("auth.py", _enrichment( + resolved_function=_resolved(func_id, name="authenticate"), + callers_by_text_search=[ + {"id": "other.py:caller", "name": "caller", "file": "other.py", "matches": []} + ], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert not [a for a in anchors if a.kind == "call_edge"] + + +# --------------------------------------------------------------------------- +# 6. reachability anchor +# --------------------------------------------------------------------------- + +class TestReachabilityAnchor: + def test_true_false_and_none_are_distinguishable(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + reachable_candidate = _candidate("a.py", _enrichment( + resolved_function=_resolved("a.py:funcA", name="funcA"), + is_reachable_from_entry_point=True, + entry_point_path=["entry.py:main", "a.py:funcA"], + )) + unreachable_candidate = _candidate("b.py", _enrichment( + resolved_function=_resolved("b.py:funcB", name="funcB"), + is_reachable_from_entry_point=False, + )) + # resolved but reachability computation itself failed (real path in + # candidate_enrichment._enrich_one's try/except): resolved_function + # is set, is_reachable_from_entry_point stays None. + unresolved_candidate = _candidate("c.py", _enrichment( + resolved_function=_resolved("c.py:funcC", name="funcC"), + is_reachable_from_entry_point=None, + )) + + anchors = derive_pre_patch_anchors(_understanding( + [reachable_candidate, unreachable_candidate, unresolved_candidate] + )) + reach = {a.candidate_path: a for a in anchors if a.kind == "reachability"} + + assert reach["a.py"].before_value.reachable is True + assert reach["a.py"].before_value.entry_point_path == ("entry.py:main", "a.py:funcA") + assert reach["b.py"].before_value.reachable is False + assert reach["b.py"].before_value.entry_point_path is None + assert reach["c.py"].before_value.reachable is None + assert reach["c.py"].before_value.entry_point_path is None + + +# --------------------------------------------------------------------------- +# 7. sink_match anchor +# --------------------------------------------------------------------------- + +class TestSinkMatchAnchor: + def test_preserves_provenance_and_stable_key(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment( + sink_matches=[{"file": "auth.py", "line": 10, "method": "authenticate", "snippet": "os.system(cmd)"}], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + matches = [a for a in anchors if a.kind == "sink_match"] + assert len(matches) == 1 + anchor = matches[0] + assert anchor.display_id == "sink_match:auth.py:authenticate" + assert anchor.key.candidate_path == "auth.py" + assert anchor.key.method == "authenticate" + assert anchor.before_value.line == 10 + assert anchor.before_value.snippet == "os.system(cmd)" + assert anchor.source == "candidate_enrichment.sink_matches" + + def test_module_level_sink_uses_module_label(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment( + sink_matches=[{"file": "auth.py", "line": 3, "method": None, "snippet": "os.system(x)"}], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + anchor = next(a for a in anchors if a.kind == "sink_match") + assert anchor.display_id == "sink_match:auth.py:" + assert anchor.key.method is None + + def test_none_vs_empty_sink_matches_both_produce_no_anchors(self): + """None = not attempted, [] = attempted and found nothing -- both + are honest "no anchor" states, never fabricated.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + none_candidate = _candidate("a.py", _enrichment(sink_matches=None)) + empty_candidate = _candidate("b.py", _enrichment(sink_matches=[])) + anchors = derive_pre_patch_anchors(_understanding([none_candidate, empty_candidate])) + assert not [a for a in anchors if a.kind == "sink_match"] + + def test_sink_match_derived_even_when_resolved_function_is_none(self): + """sink_matches is computed independently of function resolution + in candidate_enrichment._enrich_one -- must not be silently + skipped just because no function was resolved.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment( + resolved_function=None, + sink_matches=[{"file": "auth.py", "line": 10, "method": "run", "snippet": "os.system(x)"}], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert len(anchors) == 1 + assert anchors[0].kind == "sink_match" + + +def _literal_entry(qualified_name, name, class_name=None, kind="frozenset_call", value=None, line=1, end_line=1): + return { + "qualified_name": qualified_name, "class_name": class_name, "name": name, + "outcome": "literal", "ast_literal_kind": kind, "value": value, + "line": line, "end_line": end_line, + } + + +class TestConstantValueAnchor: + def test_derived_for_the_cve_2023_43804_shape(self): + """The exact motivating case: a class-level frozenset constant.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + entry = _literal_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + class_name="Retry", value=frozenset({"Authorization"}), + ) + candidate = _candidate("retry.py", _enrichment( + resolved_function=_resolved("retry.py:Retry.increment", class_name="Retry"), + scope_constants=[entry], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + matches = [a for a in anchors if a.kind == "constant_value"] + assert len(matches) == 1 + anchor = matches[0] + assert anchor.display_id == "constant_value:retry.py:Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" + assert anchor.key.qualified_name == "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT" + assert anchor.key.class_name == "Retry" + assert anchor.before_value.value == frozenset({"Authorization"}) + assert anchor.before_value.ast_literal_kind == "frozenset_call" + assert anchor.source == "candidate_enrichment.scope_constants" + + def test_non_literal_outcome_produces_no_anchor(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + entry = _literal_entry("BACKEND", "BACKEND", kind=None, value=None) + entry["outcome"] = "non_literal" + candidate = _candidate("a.py", _enrichment(scope_constants=[entry])) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert not [a for a in anchors if a.kind == "constant_value"] + + def test_derived_even_when_resolved_function_is_none(self): + """Mirrors sink_match's independence from resolved_function -- + candidate_enrichment already decided scoping; derivation must not + re-gate on it.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + entry = _literal_entry("TIMEOUT", "TIMEOUT", value=30, kind="Constant") + candidate = _candidate("a.py", _enrichment(resolved_function=None, scope_constants=[entry])) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert len(anchors) == 1 + assert anchors[0].kind == "constant_value" + + def test_deduplicated_across_two_candidates_in_the_same_class(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + entry = _literal_entry( + "Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT", "DEFAULT_REMOVE_HEADERS_ON_REDIRECT", + class_name="Retry", value=frozenset({"Authorization"}), + ) + candidate_a = _candidate("retry.py", _enrichment( + resolved_function=_resolved("retry.py:Retry.increment", name="increment", class_name="Retry"), + scope_constants=[entry], + )) + candidate_b = _candidate("retry.py", _enrichment( + resolved_function=_resolved("retry.py:Retry.__repr__", name="__repr__", class_name="Retry"), + scope_constants=[dict(entry)], + )) + anchors = derive_pre_patch_anchors(_understanding([candidate_a, candidate_b])) + assert len([a for a in anchors if a.kind == "constant_value"]) == 1 + + def test_none_vs_empty_scope_constants_both_produce_no_anchors(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + empty_candidate = _candidate("a.py", _enrichment(scope_constants=[])) + anchors = derive_pre_patch_anchors(_understanding([empty_candidate])) + assert not [a for a in anchors if a.kind == "constant_value"] + + +# --------------------------------------------------------------------------- +# 8. related_test is deferred -- never produced by this phase +# --------------------------------------------------------------------------- + +class TestRelatedTestDeferred: + def test_related_tests_data_produces_no_anchors(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment( + related_tests=[{"path": "/abs/tests/test_auth.py", "proximity": "same-file", "reason": "x"}], + test_support_rating=("Good", 0.05, {}), + )) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert anchors == [] + assert not any(a.kind == "related_test" for a in anchors) + + +# --------------------------------------------------------------------------- +# 9/10. no fabrication / empty input +# --------------------------------------------------------------------------- + +class TestNoFabrication: + def test_candidate_without_enrichment_produces_no_anchors(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", enrichment=None) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert anchors == [] + + def test_empty_understanding_returns_empty_list(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + anchors = derive_pre_patch_anchors(_understanding([])) + assert anchors == [] + + +# --------------------------------------------------------------------------- +# 11/12. determinism and purity +# --------------------------------------------------------------------------- + +class TestDeterminismAndPurity: + def test_repeated_derivation_produces_equal_anchors_in_equal_order(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment( + resolved_function=_resolved("auth.py:authenticate"), + callees=["auth.py:helper"], + is_reachable_from_entry_point=True, + entry_point_path=["entry.py:main", "auth.py:authenticate"], + sink_matches=[{"file": "auth.py", "line": 10, "method": "authenticate", "snippet": "x"}], + )) + understanding = _understanding([candidate]) + + result1 = derive_pre_patch_anchors(understanding) + result2 = derive_pre_patch_anchors(understanding) + assert result1 == result2 + assert [(a.kind, a.key) for a in result1] == [(a.kind, a.key) for a in result2] + assert [a.display_id for a in result1] == [a.display_id for a in result2] + + def test_input_understanding_and_candidates_not_mutated(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + enrichment = _enrichment( + resolved_function=_resolved("auth.py:authenticate"), + callees=["auth.py:helper"], + sink_matches=[{"file": "auth.py", "line": 10, "method": "authenticate", "snippet": "x"}], + ) + candidate = _candidate("auth.py", enrichment) + understanding = _understanding([candidate]) + snapshot = copy.deepcopy(understanding) + + derive_pre_patch_anchors(understanding) + + assert understanding == snapshot + assert candidate.enrichment is enrichment # same object, never replaced + + def test_no_disallowed_imports_and_no_repo_root_parameter(self): + """#13: no I/O, parsing, environment, network, subprocess, or LLM + imports; confirms the pure "RepositoryUnderstanding -> Anchors" + signature has no repo_root (or any other) parameter beyond + `understanding`.""" + import utilities.autopatcher.post_patch_investigation as mod + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + source = inspect.getsource(mod) + disallowed = [ + "import subprocess", "import socket", "import requests", + "import os", "import shutil", "import tempfile", + "anthropic", "openai", "urllib", + ] + for token in disallowed: + assert token not in source, f"unexpected token found: {token}" + + params = list(inspect.signature(derive_pre_patch_anchors).parameters) + assert params == ["understanding"] + + +# --------------------------------------------------------------------------- +# 15. global identity uniqueness +# --------------------------------------------------------------------------- + +class TestIdentityUniqueness: + def test_identities_unique_across_candidates_and_kinds(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate_a = _candidate("a.py", _enrichment( + resolved_function=_resolved("a.py:funcA", name="funcA"), + callees=["b.py:funcB"], + is_reachable_from_entry_point=True, + entry_point_path=["entry.py:main", "a.py:funcA"], + sink_matches=[{"file": "a.py", "line": 5, "method": "funcA", "snippet": "x"}], + )) + candidate_b = _candidate("b.py", _enrichment( + resolved_function=_resolved("b.py:funcB", name="funcB"), + callers_by_call_graph=["a.py:funcA"], + is_reachable_from_entry_point=False, + sink_matches=[{"file": "b.py", "line": 8, "method": "funcB", "snippet": "y"}], + )) + + anchors = derive_pre_patch_anchors(_understanding([candidate_a, candidate_b])) + + # (kind, key) is the authoritative semantic identity. + identities = [(a.kind, a.key) for a in anchors] + assert len(identities) == len(set(identities)) + + # display_id happens to also be unique for this fixture, but that's + # incidental to the rendering format, not a guarantee -- (kind, key) + # is what's actually relied on for uniqueness. + display_ids = [a.display_id for a in anchors] + assert len(display_ids) == len(set(display_ids)) + + assert len(anchors) >= 5 # 2 resolved_function + 1 call_edge + 2 reachability + 2 sink_match + + def test_anchor_instances_are_hashable(self): + """Bonus of the typed key/value design: unlike a dict-based Anchor, + every field here is hashable, so Anchor itself can be hashed -- + confirms the typed representation is a real improvement, not just + a relabeling.""" + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment(resolved_function=_resolved("auth.py:authenticate"))) + anchors = derive_pre_patch_anchors(_understanding([candidate])) + assert {a for a in anchors} # no TypeError: unhashable type + + +# --------------------------------------------------------------------------- +# Identity semantics: (kind, key), not key alone, not a stored id field +# --------------------------------------------------------------------------- + +class TestIdentitySemantics: + def test_kind_disambiguates_structurally_identical_keys(self): + """Regression guard for the exact reasoning behind using (kind, key) + instead of key alone: NamedTuple equality/hash ignore the declared + subclass (they inherit tuple.__eq__/__hash__, which compare + positionally) -- so a CallEdgeKey and a SinkMatchKey with the same + field values compare equal as plain tuples. Including `kind` in the + identity is what tells them apart.""" + from utilities.autopatcher.post_patch_investigation import CallEdgeKey, SinkMatchKey + + call_edge_key = CallEdgeKey(caller_func_id="auth.py", callee_func_id="authenticate") + sink_match_key = SinkMatchKey(candidate_path="auth.py", method="authenticate") + + assert call_edge_key == sink_match_key # NamedTuple equality ignores subclass + assert hash(call_edge_key) == hash(sink_match_key) + assert ("call_edge", call_edge_key) != ("sink_match", sink_match_key) + + def test_anchor_has_no_stored_id_field(self): + from utilities.autopatcher.post_patch_investigation import Anchor + + field_names = {f.name for f in dataclasses.fields(Anchor)} + assert "id" not in field_names + assert field_names == {"kind", "candidate_path", "key", "before_value", "source", "origin"} + + def test_display_id_is_a_computed_property_not_a_field(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment(resolved_function=_resolved("auth.py:authenticate"))) + anchor = derive_pre_patch_anchors(_understanding([candidate]))[0] + + assert isinstance(type(anchor).display_id, property) + assert anchor.display_id == "resolved_function:auth.py:authenticate" + + def test_anchor_remains_frozen_and_hashable_without_id_field(self): + from utilities.autopatcher.post_patch_investigation import derive_pre_patch_anchors + + candidate = _candidate("auth.py", _enrichment(resolved_function=_resolved("auth.py:authenticate"))) + anchor = derive_pre_patch_anchors(_understanding([candidate]))[0] + + assert dataclasses.fields(anchor) # sanity: still a dataclass + with pytest.raises(dataclasses.FrozenInstanceError): + anchor.candidate_path = "changed.py" + hash(anchor) # must not raise diff --git a/libs/openant-core/tests/patch/test_repo_locator.py b/libs/openant-core/tests/patch/test_repo_locator.py new file mode 100644 index 00000000..e99aad00 --- /dev/null +++ b/libs/openant-core/tests/patch/test_repo_locator.py @@ -0,0 +1,2124 @@ +"""Unit tests for repo_locator. All tests use tmp_path repos — no real repos needed.""" + +from __future__ import annotations + +import json +import os +import sys +import tempfile +from pathlib import Path + +import pytest + + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +class TestExplicitFilePath: + def test_finds_file_mentioned_by_path(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "app" / "auth.py", "def authenticate(u, p):\n pass\n") + vuln = "Vulnerability in app/auth.py — SQL injection in authenticate()" + result = find_code_context(vuln, tmp_path) + assert "authenticate" in result + + def test_explicit_path_preferred_over_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context, _extract_file_paths + write(tmp_path / "app" / "auth.py", "def authenticate(u, p):\n pass\n") + write(tmp_path / "other" / "stuff.py", "def authenticate(x):\n pass\n") + vuln = "Vulnerability in app/auth.py — authenticate() is exploitable" + paths = _extract_file_paths(vuln) + assert "app/auth.py" in paths + result = find_code_context(vuln, tmp_path) + assert result != "" + # auth.py should appear (highest score) + assert "auth" in result + + +class TestRepositoryPathResolver: + """Unit tests for RepositoryPathResolver: exact match + unique suffix + match only. Basename-only matching is explicitly out of scope for this + slice — see test_bare_filename_does_not_fall_back_to_basename_match.""" + + def test_exact_match_resolves(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("app/auth.py") + assert result.strategy == "exact" + assert result.path == (tmp_path / "app" / "auth.py").resolve() + + def test_suffix_match_resolves_src_layout(self, tmp_path): + """Mirrors pip's shape: advisory names `_internal/download.py`, + the real file lives under a src-layout prefix the advisory text + never mentions.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "suffix" + assert result.path == ( + tmp_path / "src" / "pip" / "_internal" / "download.py" + ).resolve() + + def test_exact_match_preferred_over_suffix(self, tmp_path): + """When both an exact-join file and a deeper suffix-matching file + exist, the exact match wins — the fallback must never override a + working resolution.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "_internal" / "download.py", "# shallow\n") + write(tmp_path / "src" / "pip" / "_internal" / "download.py", "# deep\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "exact" + assert result.path == (tmp_path / "_internal" / "download.py").resolve() + + def test_ambiguous_suffix_match_is_not_resolved(self, tmp_path): + """Two files sharing the same suffix under different top-level + directories must not be silently guessed — this is the safety + property the whole fallback depends on.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "ambiguous" + assert result.path is None + + def test_bare_filename_does_not_fall_back_to_basename_match(self, tmp_path): + """A single-segment path (no directory component) must not trigger + suffix matching — that would be basename matching, which is + explicitly out of scope for this slice.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "deeply" / "nested" / "auth.py", "def authenticate(): pass\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("auth.py") + assert result.strategy == "unresolved" + assert result.path is None + + def test_no_match_returns_unresolved(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "other.py", "print('hello')\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("app/auth.py") + assert result.strategy == "unresolved" + assert result.path is None + + def test_suffix_match_respects_path_segment_boundaries(self, tmp_path): + """A directory named `my_internal` must not satisfy a suffix + search for `_internal/download.py` — matching is segment-aware, + not a raw string-suffix comparison.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + write(tmp_path / "my_internal" / "download.py", "# decoy\n") + resolver = RepositoryPathResolver(tmp_path) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "unresolved" + assert result.path is None + + +class TestExplicitPathSrcLayoutIntegration: + """find_code_context() integration: the src-layout fallback must surface + the real file end-to-end, through Pass 1, not just at the resolver + unit-test level.""" + + def test_src_layout_file_included_via_suffix_fallback(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(link, location):\n pass\n", + ) + vuln = "Vulnerability in _internal/download.py — unpack_url() follows unsafe redirects" + result = find_code_context(vuln, tmp_path) + assert "unpack_url" in result + assert "src/pip/_internal/download.py" in result + + def test_ambiguous_src_layout_path_does_not_crash_or_leak_wrong_file(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "a" / "_internal" / "download.py", "def a_impl(): pass\n") + write(tmp_path / "b" / "_internal" / "download.py", "def b_impl(): pass\n") + vuln = "Vulnerability in _internal/download.py" + result = find_code_context(vuln, tmp_path) + assert isinstance(result, str) # no crash + # Neither ambiguous candidate should be silently chosen via Pass 1 + assert "a_impl" not in result + assert "b_impl" not in result + + +class TestSymbolMatch: + def test_finds_function_by_backtick_name(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "src" / "db.py", "def execute_query(sql):\n cursor.execute(sql)\n") + vuln = "The `execute_query` function does not use parameterized queries" + result = find_code_context(vuln, tmp_path) + assert "execute_query" in result + + def test_finds_snake_case_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "lib" / "auth.py", "def validate_token(tok):\n return True\n") + vuln = "validate_token does not check expiry — security bypass possible" + result = find_code_context(vuln, tmp_path) + assert "validate_token" in result + + def test_finds_pascal_case_symbol(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "models" / "user.py", "class UserManager:\n pass\n") + vuln = "UserManager lacks access control checks" + result = find_code_context(vuln, tmp_path) + assert "UserManager" in result + + +class TestCweKeywordFallback: + def test_sql_injection_cwe_finds_cursor(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "db" / "queries.py", + "def run(sql):\n cursor.execute(sql)\n return cursor.fetchone()\n", + ) + vuln = "SQL injection (CWE-89) in authentication module" + result = find_code_context(vuln, tmp_path) + assert result != "" + assert "execute" in result or "cursor" in result + + +# --------------------------------------------------------------------------- +# Stage 2: CWE-name token expansion (cwe_name_tokens) +# +# Per reports/implementation_plan_stage2_token_expansion_2026-07-14.md: +# union with _CWE_KEYWORDS (not replacement), lowercase tokens, minimal +# grammatical stopword filter only (no _GENERIC_TOKENS reuse — §0.2), +# parenthetical abbreviations kept (§0.3), no ranking/scoring/policy change. +# --------------------------------------------------------------------------- + +class TestCweNameTokens: + """Unit tests for cwe_name_tokens() — pure text function, no repo needed.""" + + def test_all_three_type_line_shapes_parse_without_bare_cwe_token(self): + """The three real **Type:** line shapes observed in this project's + corpus (CWE-id-first with parens, name-first with parens, name-first + with an em-dash and a nested parenthetical abbreviation) must all + parse without a bare CWE-NNN token surviving into the output.""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + curl_text = "**Type:** Insufficiently Protected Credentials (CWE-522)" + node_semver_text = ( + "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + ) + ghsa_text = "**Type:** CWE-798 (Use of Hard-coded Credentials)" + for text in (curl_text, node_semver_text, ghsa_text): + tokens = cwe_name_tokens(text) + assert not any("cwe" in t.lower() for t in tokens), tokens + + def test_curl_type_line_yields_expected_tokens(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Insufficiently Protected Credentials (CWE-522)" + tokens = cwe_name_tokens(text) + assert "credentials" in tokens + assert "protected" in tokens + assert "insufficiently" in tokens + + def test_node_semver_type_line_keeps_nested_abbreviation(self): + """The nested (ReDoS) abbreviation must survive as `redos` — approved + Stage 2 design decision (§0.3): kept, not discarded. `service` must + also survive (§0.2: _GENERIC_TOKENS reuse was rejected).""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + tokens = cwe_name_tokens(text) + assert "redos" in tokens + assert "regular" in tokens + assert "expression" in tokens + assert "service" in tokens + + def test_ghsa_style_cwe_id_first_shape_parses(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** CWE-798 (Use of Hard-coded Credentials)" + tokens = cwe_name_tokens(text) + assert "credentials" in tokens + assert "hard-coded" in tokens + + def test_generic_cwe_name_does_not_crash_or_special_case(self): + """CWE-200's real name is broad; must not crash, and no special-casing + for 'generic' CWE ids — the same minimal filter as every other case.""" + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = ( + "**Type:** CWE-200 " + "(Exposure of Sensitive Information to an Unauthorized Actor)" + ) + tokens = cwe_name_tokens(text) + assert isinstance(tokens, list) + assert all(isinstance(t, str) for t in tokens) + assert not any("cwe" in t.lower() for t in tokens) + + def test_no_type_line_returns_empty_list(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + assert cwe_name_tokens("No Type line present in this text at all.") == [] + + def test_tokens_are_lowercase(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + tokens = cwe_name_tokens(text) + assert tokens == [t.lower() for t in tokens] + assert "Regular" not in tokens + assert "Service" not in tokens + + def test_proven_valuable_words_survive_stopword_filter(self): + """expression/regular/prototype/credentials must never be filtered — + encodes the prior investigation's own false-negative lesson.""" + from utilities.autopatcher.repo_locator import _CWE_NAME_STOPWORDS + for word in ("expression", "regular", "prototype", "credentials"): + assert word not in _CWE_NAME_STOPWORDS + + def test_deterministic_ordering(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = "**Type:** Regular Expression Denial of Service (ReDoS) — CWE-1333" + assert cwe_name_tokens(text) == cwe_name_tokens(text) + + def test_bare_cwe_id_never_survives_prototype_pollution_name(self): + from utilities.autopatcher.repo_locator import cwe_name_tokens + text = ( + "**Type:** CWE-1321 (Improperly Controlled Modification of " + "Object Prototype Attributes ('Prototype Pollution'))" + ) + tokens = cwe_name_tokens(text) + assert not any("1321" in t for t in tokens) + assert "prototype" in tokens + assert "pollution" in tokens + + +class TestCweKeywordDictUnionUnaffected: + """Union, not replacement: CWE-22's existing _CWE_KEYWORDS coverage must + be unaffected by the addition of cwe_name_tokens().""" + + def test_cwe22_dict_token_still_reachable_via_find_code_context(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "fs.py", + "def resolve(p):\n return os.path.abspath(p)\n", + ) + vuln = "**Type:** CWE-22 (Path Traversal)\n\nA path traversal vulnerability." + ctx = find_code_context(vuln, tmp_path) + assert "abspath" in ctx + + +class TestCweNameTokensPass3Integration: + """find_code_context() integration: a CWE with no _CWE_KEYWORDS dict + entry (CWE-522) must still surface a real file via cwe_name_tokens, + with no Pass 1/2 signal available to find it any other way.""" + + def test_undicted_cwe_finds_file_via_derived_token(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write( + tmp_path / "lib" / "vauth" / "digest.c", + "/* stores user credentials for digest authentication */\n" + "struct auth_state { char *credentials; };\n", + ) + vuln = ( + "**Type:** Insufficiently Protected Credentials (CWE-522)\n\n" + "curl leaks credentials to a different host on redirect." + ) + ctx = find_code_context(vuln, tmp_path) + assert "credentials" in ctx + assert "digest.c" in ctx + + +class TestNoMatch: + def test_returns_empty_string_when_nothing_found(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "unrelated.py", "print('hello')\n") + vuln = "Vulnerability in authenticate_user() — SQL injection (CWE-89)" + # authenticate_user is specific; unrelated.py has no such content + result = find_code_context(vuln, tmp_path) + # May or may not be empty depending on CWE fallback hits — key thing: no crash + assert isinstance(result, str) + + def test_empty_repo_returns_empty_string(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + result = find_code_context("SQL injection in authenticate()", tmp_path) + assert result == "" + + +class TestCharCap: + def test_snippet_mode_respects_char_cap(self, tmp_path): + """Files > 20 000 chars use snippet mode; result must stay near the cap.""" + from utilities.autopatcher.repo_locator import find_code_context + # File must exceed _FULL_FILE_THRESHOLD_CHARS (20 000) to trigger snippet mode + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000, "file must be large enough to bypass full-file mode" + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in app/auth.py — authenticate()" + result = find_code_context(vuln, tmp_path) + assert len(result) <= 4_500 # snippet budget + header overhead + + def test_small_file_fully_included(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + content = "def authenticate(u, p):\n pass\n" + write(tmp_path / "app" / "auth.py", content) + vuln = "SQL injection in app/auth.py" + result = find_code_context(vuln, tmp_path) + assert "def authenticate" in result + + +class TestGenericTokensIgnored: + def test_generic_tokens_produce_no_symbol_matches(self, tmp_path): + from utilities.autopatcher.repo_locator import _extract_symbols + vuln = "Error in handler: request response data output config settings" + symbols = _extract_symbols(vuln) + assert symbols == [] + + def test_specific_tokens_extracted(self, tmp_path): + from utilities.autopatcher.repo_locator import _extract_symbols + vuln = "SQL injection in `authenticate_user` and `validate_session`" + symbols = _extract_symbols(vuln) + assert "authenticate_user" in symbols + assert "validate_session" in symbols + + +# --------------------------------------------------------------------------- +# Change 1: ranking by total occurrence count +# --------------------------------------------------------------------------- + +class TestRankingByTotalOccurrences: + def test_higher_occurrence_count_ranks_first(self, tmp_path): + """File with more token hits should rank before file with fewer hits.""" + from utilities.autopatcher.repo_locator import _grep_repo + + # a.py: 4 occurrences of 'authenticate' + write(tmp_path / "a.py", + "authenticate()\nauthenticate()\nauthenticate()\nauthenticate()\n") + # b.py: 1 occurrence — sorts before a.py alphabetically but should rank lower + write(tmp_path / "b.py", "def authenticate(): pass\n") + + results = _grep_repo(tmp_path, ["authenticate"]) + assert results, "expected at least one result" + assert results[0][0].name == "a.py", ( + f"expected a.py (4 hits) first, got {results[0][0].name}" + ) + + def test_tie_broken_consistently(self, tmp_path): + """Two files with equal hit counts should both be returned.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "x.py", "frozenset(['Authorization'])\n") + write(tmp_path / "y.py", "frozenset(['Authorization'])\n") + results = _grep_repo(tmp_path, ["Authorization"]) + names = {r[0].name for r in results} + assert "x.py" in names and "y.py" in names + + +# --------------------------------------------------------------------------- +# Change 2: test-file exclusion +# --------------------------------------------------------------------------- + +class TestTestFileExclusion: + def test_test_prefix_file_excluded(self, tmp_path): + """test_*.py files must not appear in _grep_repo results.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "tests" / "test_auth.py", + "authenticate() " * 50) # many hits + write(tmp_path / "src" / "auth.py", + "def authenticate(): pass\n") # 1 hit + + results = _grep_repo(tmp_path, ["authenticate"]) + names = [r[0].name for r in results] + assert "test_auth.py" not in names, "test files must be excluded" + assert "auth.py" in names + + def test_tests_directory_excluded(self, tmp_path): + """Files inside a tests/ directory must not appear.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "tests" / "integration.py", + "Authorization " * 20) + write(tmp_path / "lib" / "http.py", + "Authorization = 'Bearer'\n") + + results = _grep_repo(tmp_path, ["Authorization"]) + names = [r[0].name for r in results] + assert "integration.py" not in names + assert "http.py" in names + + def test_spec_directory_excluded(self, tmp_path): + """Files inside a spec/ directory must not appear.""" + from utilities.autopatcher.repo_locator import _grep_repo + write(tmp_path / "spec" / "retry_spec.py", + "Authorization " * 20) + write(tmp_path / "retry.py", + "DEFAULT_REMOVE = frozenset(['Authorization'])\n") + + results = _grep_repo(tmp_path, ["Authorization"]) + names = [r[0].name for r in results] + assert "retry_spec.py" not in names + assert "retry.py" in names + + +# --------------------------------------------------------------------------- +# Change 3: backtick term extraction +# --------------------------------------------------------------------------- + +class TestBacktickTermExtraction: + def test_cookie_extracted_despite_generic_filter(self): + """'cookie' is in _GENERIC_TOKENS but `Cookie` in backticks must be extracted.""" + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "urllib3 doesn't strip the `Cookie` header on redirects." + terms = _extract_backtick_terms(text) + assert "Cookie" in terms + + def test_authorization_extracted(self): + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "The `Authorization` header is leaked to the redirect target." + terms = _extract_backtick_terms(text) + assert "Authorization" in terms + + def test_multiple_backtick_terms_extracted(self): + from utilities.autopatcher.repo_locator import _extract_backtick_terms + text = "Use `Cookie` and `Authorization` with `remove_headers_on_redirect`." + terms = _extract_backtick_terms(text) + assert "Cookie" in terms + assert "Authorization" in terms + assert "remove_headers_on_redirect" in terms + + def test_generic_tokens_not_filtered_from_backtick_terms(self): + """Backtick extraction bypasses _GENERIC_TOKENS.""" + from utilities.autopatcher.repo_locator import _extract_backtick_terms, _GENERIC_TOKENS + # Build text where every token is in GENERIC_TOKENS + generic = " ".join(f"`{t}`" for t in list(_GENERIC_TOKENS)[:5]) + terms = _extract_backtick_terms(generic) + # All of them should be present (no filtering) + for t in list(_GENERIC_TOKENS)[:5]: + if len(t) >= 2: + assert t in terms, f"'{t}' should not be filtered from backtick terms" + + def test_backtick_term_used_as_search_signal(self, tmp_path): + """Advisory with backtick term finds file containing that token.""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "retry.py", + "DEFAULT_REMOVE = frozenset(['Authorization'])\n") + # Advisory mentions Authorization in backticks — should find retry.py + vuln = "The `Authorization` header is not stripped on cross-origin redirects." + ctx = find_code_context(vuln, tmp_path) + assert "Authorization" in ctx + assert "retry.py" in ctx + + +# --------------------------------------------------------------------------- +# Context grounding: _is_docstring_line, _find_code_block_after, _extract_snippet +# --------------------------------------------------------------------------- + +def _make_large_file( + tmp_path: "Path", name: str, docstring_lines: int = 60, _force_large: bool = False +) -> "Path": + """Create a >150-line Python file with a docstring section then constants. + + When _force_large=True, pad with trailing comments until the file exceeds + 20 000 chars (full-file threshold), keeping the constant close enough for + the code-anchor scan to find it. + """ + lines = ["class Retry:"] + lines.append(' """') + for i in range(docstring_lines): + lines.append(f" :param int p{i}: Param {i} description text here.") + lines.append(' """') + lines.append("") + lines.append(" DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])") + lines.append(" DEFAULT_BACKOFF_MAX = 120") + lines.append("") + lines.append(" def __init__(self, total=10, redirect=None):") + lines.append(" self.total = total") + lines.append(" self.redirect = redirect") + # Pad to exceed _SMALL_FILE_THRESHOLD (150 lines) + while len(lines) < 160: + lines.append("") + # Optionally pad with trailing comments to exceed the full-file threshold + # (20 000 chars), keeping the constant reachable within the anchor scan. + content = "\n".join(lines) + while len(content) < 22_000 if _force_large else False: + lines.append("# padding " + "x" * 70) + content = "\n".join(lines) + p = tmp_path / name + p.write_text(content, encoding="utf-8") + return p + + +class TestIsDocstringLine: + def test_rst_directive_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(":param int redirect: How many redirects") is True + + def test_indented_rst_directive_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" :param int redirect: How many redirects") is True + + def test_prose_without_operators_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" How many redirects to perform.") is True + + def test_triple_quote_is_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(' """') is True + + def test_constant_assignment_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line( + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])" + ) is False + + def test_def_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line(" def __init__(self, total=10):") is False + + def test_class_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("class Retry:") is False + + def test_empty_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("") is False + + def test_import_line_is_not_docstring(self): + from utilities.autopatcher.repo_locator import _is_docstring_line + assert _is_docstring_line("import re") is False + + +class TestFindCodeBlockAfter: + def test_finds_constant_after_docstring(self, tmp_path): + from utilities.autopatcher.repo_locator import _find_code_block_after + _make_large_file(tmp_path, "retry.py") + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + dq_close = next( + i for i, l in enumerate(lines) if l.strip() == '"""' and i > 0 + ) + result_tuple = _find_code_block_after(lines, dq_close + 1) + assert result_tuple is not None + text, start_0 = result_tuple + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert isinstance(start_0, int) + + def test_returns_none_when_no_code_found(self): + from utilities.autopatcher.repo_locator import _find_code_block_after + lines = [" prose line only" for _ in range(210)] + assert _find_code_block_after(lines, 0) is None + + def test_respects_max_lines_when_anchor_is_def(self): + """max_lines fallback applies when the first anchor is a def/class line.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # Class with only methods, no constants — anchor is a def. + lines = [ + " def method_a(self):", + " return 1", + " def method_b(self):", + " return 2", + ] * 20 # many lines; max_lines cap must kick in + result_tuple = _find_code_block_after(lines, 0, max_lines=3) + assert result_tuple is not None + text, _ = result_tuple + assert len(text.splitlines()) <= 3 + + def test_structural_collection_ignores_max_lines_for_constants(self): + """When anchor is a constant, all constants are collected even if > max_lines.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # 20 constants followed by a def — no max_lines truncation for constants. + lines = [f" CONST_{i} = {i}" for i in range(20)] + [ + " def __init__(self):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0, max_lines=5) + assert result_tuple is not None + text, _ = result_tuple + # All 20 constants must be present despite max_lines=5 + for i in range(20): + assert f"CONST_{i}" in text + # Method must not be included + assert "def __init__" not in text + + def test_stops_at_first_method_definition(self): + """Structural extraction stops exactly at the first def line.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + lines = [ + " DEFAULT_HEADERS = frozenset(['Authorization'])", + " DEFAULT_TIMEOUT = 30", + "", + " def __init__(self):", + " self.x = 1", + " def increment(self):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + assert "DEFAULT_HEADERS" in text + assert "DEFAULT_TIMEOUT" in text + assert "def __init__" not in text + assert "def increment" not in text + + def test_critical_constant_visible_when_not_first_in_block(self): + """A constant that is not the first anchor must still appear in the result.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + # Mirrors urllib3 retry.py: DEFAULT_ALLOWED_METHODS is the first anchor, + # DEFAULT_REMOVE_HEADERS_ON_REDIRECT comes 9 lines later. + lines = [ + " #: Default methods", + " DEFAULT_ALLOWED_METHODS = frozenset(['GET', 'POST', 'HEAD'])", + "", + " #: Status codes", + " RETRY_AFTER_STATUS_CODES = frozenset([429, 503])", + "", + " #: Headers to strip on redirect", + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " #: Backoff", + " DEFAULT_BACKOFF_MAX = 120", + "", + " def __init__(self, total=10):", + " pass", + ] + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + # The critical constant must be visible even though it is not the first anchor + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert "frozenset(['Authorization'])" in text + # Method must not bleed into the result + assert "def __init__" not in text + + +class TestExtractSnippetGrounding: + def test_docstring_hit_appends_code_anchor(self, tmp_path): + """When hit_line is inside a docstring, snippet must include the constant.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + text, ranges = _extract_snippet(content, hit_line, 4000) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Code anchor must appear in snippet when hit is inside a docstring" + ) + assert len(ranges) == 2, "Non-contiguous snippet must return two ranges" + + def test_code_hit_does_not_add_anchor(self, tmp_path): + """When hit_line is already a code line, only the window range is returned.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next( + i for i, l in enumerate(lines) + if "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in l + ) + text, ranges = _extract_snippet(content, hit_line, 4000) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + assert len(ranges) == 1, "Code hit should produce a single range" + + def test_anchor_preserved_when_budget_tight(self, tmp_path): + """When budget is tight, code anchor survives even if window is cut.""" + from utilities.autopatcher.repo_locator import _extract_snippet + _make_large_file(tmp_path, "retry.py", docstring_lines=60) + content = (tmp_path / "retry.py").read_text() + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + text, ranges = _extract_snippet(content, hit_line, 500) + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Code anchor must be preserved over docstring window when budget is tight" + ) + + def test_small_file_returns_one_range(self, tmp_path): + """Files at or below _SMALL_FILE_THRESHOLD return a single range.""" + from utilities.autopatcher.repo_locator import _extract_snippet + small = "def foo():\n pass\n" * 5 # << 150 lines + text, ranges = _extract_snippet(small, 0, 4000) + assert "def foo" in text + assert len(ranges) == 1 + + +# --------------------------------------------------------------------------- +# Integration: urllib3 context now includes the vulnerable constant +# --------------------------------------------------------------------------- + +_URLLIB3_EVAL = Path("/tmp/urllib3-eval") +_URLLIB3_RETRY_PY = _URLLIB3_EVAL / "src" / "urllib3" / "util" / "retry.py" + +_run_live = ( + os.environ.get("RUN_LIVE_REPO_TESTS") == "1" + and _URLLIB3_RETRY_PY.exists() +) + + +@pytest.mark.skipif( + not _run_live, + reason=( + "Live repo tests opt-in only — " + "set RUN_LIVE_REPO_TESTS=1 and populate /tmp/urllib3-eval " + "with a urllib3 checkout containing src/urllib3/util/retry.py" + ), +) +class TestUrllib3ContextGrounding: + def test_default_remove_headers_in_context(self): + """After grounding improvement, the urllib3 context must include + DEFAULT_REMOVE_HEADERS_ON_REDIRECT so the LLM can produce the real fix.""" + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + from advisory_fetcher import fetch_ghsa + from advisory_converter import ghsa_to_vuln_text + from utilities.autopatcher.repo_locator import find_code_context + + adv = fetch_ghsa("GHSA-v845-jxx5-vc9f") + vuln_text = ghsa_to_vuln_text(adv) + ctx = find_code_context(vuln_text, _URLLIB3_EVAL) + + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in ctx, ( + "Context must include the vulnerable constant so the LLM can generate the fix" + ) + + def test_context_header_uses_full_relative_path(self): + """Context header must show src/urllib3/util/retry.py, not just retry.py.""" + sys.path.insert(0, str(Path(__file__).parent.parent / "src")) + from advisory_fetcher import fetch_ghsa + from advisory_converter import ghsa_to_vuln_text + from utilities.autopatcher.repo_locator import find_code_context + + adv = fetch_ghsa("GHSA-v845-jxx5-vc9f") + vuln_text = ghsa_to_vuln_text(adv) + ctx = find_code_context(vuln_text, _URLLIB3_EVAL) + + assert "# src/urllib3/util/retry.py" in ctx, ( + "Context header must contain the full repo-relative path" + ) + assert "# retry.py\n" not in ctx, ( + "Basename-only header must not appear" + ) + + +# --------------------------------------------------------------------------- +# Context header path grounding (no live repo required) +# --------------------------------------------------------------------------- + +class TestContextHeaderLineRanges: + def test_header_includes_line_range(self, tmp_path): + """Files > 20 000 chars use snippet mode and must show a (lines N-M) annotation.""" + from utilities.autopatcher.repo_locator import find_code_context + nested = tmp_path / "app" / "auth.py" + nested.parent.mkdir(parents=True) + # Build a large file (> 20 000 chars) so snippet mode is used + content = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(content) > 20_000 + nested.write_text(content, encoding="utf-8") + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + assert "(lines " in ctx, f"Expected line range in header, got:\n{ctx[:200]}" + + def test_non_contiguous_snippet_shows_two_ranges(self, tmp_path): + """When window + code anchor are non-contiguous, header shows both ranges. + Uses _force_large=True so the file exceeds 20 000 chars (snippet mode) + while keeping the constant close enough for the anchor scan. + """ + from utilities.autopatcher.repo_locator import find_code_context + _make_large_file(tmp_path, "retry.py", docstring_lines=60, _force_large=True) + vuln = "The `p0` parameter is not bounded correctly." + ctx = find_code_context(vuln, tmp_path) + import re as _re + header_match = _re.search(r'\(lines ([0-9]+-[0-9]+), ([0-9]+-[0-9]+)\)', ctx) + assert header_match is not None, ( + f"Expected two ranges in header for non-contiguous snippet, got:\n{ctx[:300]}" + ) + + def test_range_numbers_are_sensible(self, tmp_path): + """Line range start must be >= 1 and end >= start.""" + from utilities.autopatcher.repo_locator import find_code_context + import re as _re + _make_large_file(tmp_path, "retry.py", docstring_lines=60, _force_large=True) + vuln = "The `p0` parameter is not bounded correctly." + ctx = find_code_context(vuln, tmp_path) + for m in _re.finditer(r'(\d+)-(\d+)', ctx.split("\n")[0]): + start, end = int(m.group(1)), int(m.group(2)) + assert start >= 1 + assert end >= start + + +class TestContextHeaderPaths: + def test_nested_file_uses_full_relative_path(self, tmp_path): + """Header must be app/auth.py, not auth.py.""" + from utilities.autopatcher.repo_locator import find_code_context + + nested = tmp_path / "app" / "auth.py" + nested.parent.mkdir(parents=True) + nested.write_text("def authenticate(u, p):\n pass\n", encoding="utf-8") + + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + + assert "# app/auth.py" in ctx, f"Expected '# app/auth.py' in context, got:\n{ctx[:300]}" + assert "# auth.py\n" not in ctx + + def test_deeply_nested_file_uses_full_relative_path(self, tmp_path): + """Header must be src/urllib3/util/retry.py for deeply nested files.""" + from utilities.autopatcher.repo_locator import find_code_context + + deep = tmp_path / "src" / "urllib3" / "util" / "retry.py" + deep.parent.mkdir(parents=True) + deep.write_text( + "class Retry:\n" + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])\n", + encoding="utf-8", + ) + + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + + assert "# src/urllib3/util/retry.py" in ctx, ( + f"Expected full relative path in header, got:\n{ctx[:300]}" + ) + + +# --------------------------------------------------------------------------- +# Full-file mode +# --------------------------------------------------------------------------- + +class TestFullFileMode: + def test_small_file_uses_full_file_context(self, tmp_path): + """Files <= 20 000 chars must be sent in full, not as a snippet.""" + from utilities.autopatcher.repo_locator import find_code_context + content = ( + "class Retry:\n" + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])\n" + " def __init__(self): pass\n" + ) + assert len(content) < 20_000 + write(tmp_path / "retry.py", content) + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + # Full file content must be present + assert "class Retry:" in ctx + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in ctx + assert "def __init__" in ctx + + def test_full_file_header_format(self, tmp_path): + """Header must say '(full file, N lines)' in full-file mode.""" + from utilities.autopatcher.repo_locator import find_code_context + import re as _re + content = "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n" + write(tmp_path / "src" / "retry.py", content) + vuln = "The `Authorization` header is not stripped." + ctx = find_code_context(vuln, tmp_path) + assert "(full file," in ctx, f"Header must say full file, got:\n{ctx[:200]}" + n_lines = len(content.splitlines()) + assert f"{n_lines} lines)" in ctx + + def test_full_file_mode_includes_only_one_file(self, tmp_path): + """When only one candidate matches, full-file mode returns exactly that file.""" + from utilities.autopatcher.repo_locator import find_code_context + # Only retry.py contains Authorization — no backtick on redirects so + # connectionpool.py produces zero signal hits and is not a candidate. + write(tmp_path / "retry.py", + "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n") + write(tmp_path / "connectionpool.py", + "class ConnectionPool:\n redirects = True\n") + vuln = "The `Authorization` header is not stripped on redirects." + ctx = find_code_context(vuln, tmp_path) + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, ( + f"With one matching candidate, output must have exactly one header, got: {headers}" + ) + + def test_large_file_falls_back_to_snippet_range_mode(self, tmp_path): + """Files > 20 000 chars must use snippet + range mode, not full-file.""" + from utilities.autopatcher.repo_locator import find_code_context + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000 + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in `authenticate` function (app/auth.py)" + ctx = find_code_context(vuln, tmp_path) + assert "(full file," not in ctx, "Large file must not use full-file mode" + assert "(lines " in ctx, "Large file must show line ranges" + + +# --------------------------------------------------------------------------- +# Structural constant extraction regression +# --------------------------------------------------------------------------- + +class TestStructuralConstantExtraction: + """Regression: structural extraction captures the complete constant block. + + Previously _find_code_block_after used a fixed 40-line window from the + first anchor. When many constants precede the target constant, it fell + outside the window. The structural approach (scan until first def/class) + is invariant to constant count and ordering. + """ + + def _build_many_constants_content(self, num_before: int = 50) -> str: + """Return file content with num_before constants before the critical one. + + The docstring is intentionally long (60 param lines) so that the ±30-line + window around the docstring hit contains only docstring — never the + constants section or method definition below it. This makes the anchor + block the sole source of constants in the snippet. + """ + lines = ["class Retry:", ' """'] + # 60 docstring param lines; hit_line will be somewhere in here (> 30 + # lines from the constants section, so the window never reaches them). + for i in range(60): + lines.append(f" :param int p{i}: Param {i} description.") + lines.append(' """') + lines.append("") + for i in range(num_before): + lines.append(f" PRECEDING_CONST_{i:02d} = {i}") + lines += [ + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " def __init__(self, redirect=None):", + " self.redirect = redirect", + ] + content = "\n".join(lines) + # Pad well past both thresholds (>150 lines, >20 000 chars) to force + # snippet mode and trigger the docstring-anchor path. + content += "\n" + (" # padding " + "x" * 70 + "\n") * 320 + return content + + def test_critical_constant_visible_with_many_preceding_constants(self): + """Structural extraction exposes the target even when > 40 constants precede it.""" + from utilities.autopatcher.repo_locator import _extract_snippet + + content = self._build_many_constants_content(num_before=50) + lines = content.splitlines() + + # Sanity: file is large enough to be in snippet mode + assert len(content) > 20_000 + + # Sanity: the constant is more than 40 lines from the start of the + # constants section (which is what old fixed-window would have given). + first_const_line = next( + i for i, l in enumerate(lines) if "PRECEDING_CONST_00" in l + ) + target_line = next( + i for i, l in enumerate(lines) + if "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in l + ) + assert (target_line - first_const_line) > 40, ( + "Test precondition: target must be > 40 lines from first constant " + "to prove the fixed-window approach would have missed it" + ) + + # Hit line is the docstring — triggers anchor mode + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + + text, ranges = _extract_snippet(content, hit_line, 4000) + + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text, ( + "Critical constant must appear in snippet even when preceded by " + "more than 40 constants" + ) + + def test_method_body_does_not_bleed_into_constant_block(self): + """Structural extraction stops before the first def — no method body lines.""" + from utilities.autopatcher.repo_locator import _extract_snippet + + content = self._build_many_constants_content(num_before=5) + lines = content.splitlines() + hit_line = next(i for i, l in enumerate(lines) if ":param int p0:" in l) + + text, _ = _extract_snippet(content, hit_line, 4000) + + assert "def __init__" not in text, ( + "def __init__ must not appear in the constant block snippet" + ) + assert "self.redirect = redirect" not in text, ( + "Method body must not bleed into the constant block" + ) + + def test_snippet_size_bounded_by_constant_block_not_fixed_window(self): + """With few constants the result is smaller than the old 40-line window.""" + from utilities.autopatcher.repo_locator import _find_code_block_after + + # 4 constants, then __init__ — same shape as urllib3 retry.py + lines = [ + " #: Default allowed methods", + " DEFAULT_ALLOWED_METHODS = frozenset(['GET', 'POST'])", + "", + " #: Status codes", + " RETRY_AFTER_STATUS_CODES = frozenset([429, 503])", + "", + " #: Headers to strip", + " DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(['Authorization'])", + "", + " #: Max backoff", + " DEFAULT_BACKOFF_MAX = 120", + "", + " def __init__(self, total=10, redirect=None):", + " self.total = total", + ] + [" # body line\n"] * 50 # would inflate a fixed-40 result + + result_tuple = _find_code_block_after(lines, 0) + assert result_tuple is not None + text, _ = result_tuple + + # Structural result is tighter than the old 40-line block + n_lines = len(text.splitlines()) + assert n_lines < 40, ( + f"Structural extraction ({n_lines} lines) should be smaller than " + "old fixed 40-line window for a typical small constant block" + ) + # Critical constant is present + assert "DEFAULT_REMOVE_HEADERS_ON_REDIRECT" in text + + +# --------------------------------------------------------------------------- +# Hybrid context budget: secondary snippets appended after full-file primary +# --------------------------------------------------------------------------- + +class TestHybridContextBudget: + """Full-file mode now injects snippets from ranked[1] and ranked[2] within + a bounded secondary budget so implementation files below the primary are + visible to the model.""" + + def _setup_two_candidates(self, tmp_path, primary_hits: int = 5, secondary_hits: int = 2): + """Create two files that both match on FileSystemProvider. + + primary.py has more hits and is always below _FULL_FILE_THRESHOLD_CHARS, + so it wins rank #1 and triggers full-file mode. + secondary.py has fewer hits and becomes the secondary candidate. + """ + primary_content = "class FileSystemProvider:\n pass\n" * primary_hits + " # pad\n" * 20 + secondary_content = ( + "class FileSystemProvider:\n def items(self, dirpath):\n" + " data_path = self.data + dirpath\n" + ) * secondary_hits + assert len(primary_content) < 20_000 + write(tmp_path / "api.py", primary_content) + write(tmp_path / "provider.py", secondary_content) + return "FileSystemProvider path traversal" + + def test_full_file_mode_includes_secondary_candidates(self, tmp_path): + """Primary is returned in full; secondary candidate snippet is appended.""" + from utilities.autopatcher.repo_locator import find_code_context + vuln = self._setup_two_candidates(tmp_path) + ctx = find_code_context(vuln, tmp_path) + + assert "api.py" in ctx, "primary file must appear in context" + assert "(full file," in ctx, "primary must use full-file header" + assert "provider.py" in ctx, "secondary candidate must be appended" + + def test_single_candidate_behavior_unchanged(self, tmp_path): + """When only one file matches, output is identical to pre-hybrid behavior.""" + from utilities.autopatcher.repo_locator import find_code_context + content = "class FileSystemProvider:\n pass\n" * 5 + assert len(content) < 20_000 + write(tmp_path / "only.py", content) + # unrelated file — no matching signal + write(tmp_path / "other.py", "def unrelated(): pass\n") + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + assert "only.py" in ctx + assert "(full file," in ctx + assert "other.py" not in ctx, "non-matching file must not appear" + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, f"single candidate must produce one header, got: {headers}" + + def test_secondary_budget_respected(self, tmp_path): + """Total secondary snippet size stays within _SECONDARY_CONTEXT_BUDGET.""" + from utilities.autopatcher.repo_locator import find_code_context, _SECONDARY_CONTEXT_BUDGET + # Primary: small, triggers full-file mode + primary_content = "class FileSystemProvider:\n pass\n" * 5 + assert len(primary_content) < 20_000 + write(tmp_path / "api.py", primary_content) + # Secondary: much larger than the secondary budget + big_secondary = "class FileSystemProvider:\n" + " # line\n" * 2000 + assert len(big_secondary) > _SECONDARY_CONTEXT_BUDGET + write(tmp_path / "provider.py", big_secondary) + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + # Locate the secondary section and measure its size + sep = "\n\n# provider.py" + assert sep in ctx, "secondary section must be present" + secondary_portion = ctx[ctx.index(sep) + 2:] # from "# provider.py" onward + assert len(secondary_portion) <= _SECONDARY_CONTEXT_BUDGET + 200, ( + f"secondary portion ({len(secondary_portion)} chars) exceeds budget " + f"{_SECONDARY_CONTEXT_BUDGET} + 200 header overhead" + ) + + def test_secondary_snippet_has_header(self, tmp_path): + """Secondary snippet header shows the file path and a line range.""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "api.py", + "class FileSystemProvider:\n pass\n" * 5) + write(tmp_path / "provider.py", + "class FileSystemProvider:\n" + " def items(self, dirpath):\n" + " data_path = self.data + dirpath\n") + + vuln = "FileSystemProvider path traversal" + ctx = find_code_context(vuln, tmp_path) + + # Secondary header must include path and line range annotation + assert "# provider.py (lines " in ctx, ( + f"secondary header must show path + line range; got context start:\n{ctx[:400]}" + ) + + +# --------------------------------------------------------------------------- +# _find_symbol_definitions unit tests +# --------------------------------------------------------------------------- + +class TestFindSymbolDefinitions: + """Unit tests for the exact symbol-definition lookup (Pass 2): finds + files that define a class, function, or method named in the advisory.""" + + def test_finds_defining_file(self, tmp_path): + """Returns the file that contains 'class FileSystemProvider'.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n def get_data_path(self): pass\n") + results = _find_symbol_definitions("FileSystemProvider path traversal", tmp_path) + assert len(results) == 1 + p, _, hit_line = results[0] + assert p.name == "filesystem.py" + assert hit_line == 0 + + def test_hit_line_points_to_class_not_file_top(self, tmp_path): + """hit_line is the 0-indexed line of the class definition, not always 0.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + preamble = "import os\n" * 20 # 20 lines before the class + write(tmp_path / "fs.py", preamble + "class FileSystemProvider:\n pass\n") + results = _find_symbol_definitions("FileSystemProvider path traversal", tmp_path) + assert results, "should find fs.py" + _, _, hit_line = results[0] + assert hit_line == 20, f"expected hit_line=20, got {hit_line}" + + def test_ignores_names_below_min_length(self, tmp_path): + """Names shorter than _MIN_CLASS_NAME_LENGTH (5) must not trigger a scan.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "foo.py", "class Foo:\n pass\n") # len 3 + write(tmp_path / "abcd.py", "class Abcd:\n pass\n") # len 4 + results = _find_symbol_definitions("Foo and Abcd have vulnerabilities", tmp_path) + assert results == [], f"short names must be ignored, got {results}" + + def test_excludes_test_files(self, tmp_path): + """Files inside tests/ or named test_*.py must not appear.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "tests" / "test_provider.py", + "class FileSystemProvider:\n pass\n") + write(tmp_path / "provider.py", + "class FileSystemProvider:\n pass\n") + results = _find_symbol_definitions("FileSystemProvider path traversal", tmp_path) + assert len(results) == 1 + assert results[0][0].name == "provider.py", "test file must be excluded" + + def test_finds_defining_file_via_def_when_no_pascal_case_in_advisory(self, tmp_path): + """An advisory with only a snake_case/backtick symbol (no PascalCase + class name) must still find the file that *defines* it, via the + "def" branch — this is the generalization from class-only lookup to + symbol lookup (functions/methods), the core of F-30's fix.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "auth.py", "def authenticate_user(): pass\n") + results = _find_symbol_definitions( + "SQL injection in `authenticate_user` (CWE-89)", tmp_path + ) + assert len(results) == 1 + assert results[0][0].name == "auth.py" + + def test_multiple_class_names_matched(self, tmp_path): + """Multiple PascalCase class names in the advisory each trigger discovery.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "stac_handler.py", + "class StacHandler:\n pass\n") + write(tmp_path / "filesystem.py", + "class FileSystemProvider:\n pass\n") + results = _find_symbol_definitions( + "FileSystemProvider and StacHandler both have path traversal", tmp_path + ) + names = {r[0].name for r in results} + assert "stac_handler.py" in names + assert "filesystem.py" in names + + def test_sorted_by_definition_count_descending(self, tmp_path): + """File with more class definitions appears first.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + # two.py defines FileSystemProvider twice (e.g. re-export + real class) + write(tmp_path / "two.py", + "class FileSystemProvider:\n pass\nclass FileSystemProvider:\n pass\n") + write(tmp_path / "one.py", + "class FileSystemProvider:\n pass\n") + results = _find_symbol_definitions("FileSystemProvider path traversal", tmp_path) + assert results[0][0].name == "two.py", "higher definition count must rank first" + + +# --------------------------------------------------------------------------- +# Exact symbol-definition pass integration tests (find_code_context) +# --------------------------------------------------------------------------- + +class TestSymbolDefinitionGrounding: + """Exact symbol-definition pass (F-30): a file that *defines* an + advisory-named class/function outranks ordinary grep hits, so it cannot + be displaced by occurrence-count ranking — regardless of how many times + an unrelated file repeats some other, more generic token.""" + + def _write_high_hit_primary(self, tmp_path: Path, signal: str, n: int = 25) -> Path: + """Write api.py with many occurrences of signal but no class definition.""" + content = ( + f"# {signal} routing module\n" + + f"def handle_{signal}(req): pass # {signal}\n" * n + ) + assert len(content) < 20_000 + return write(tmp_path / "api.py", content) + + def test_class_def_file_injected_despite_low_occurrence_rank(self, tmp_path): + """filesystem.py defines the advisory-named class but has the lowest + raw \\bstac\\b occurrence count of the four files below. + + Hit counts verified against \\bstac\\b (underscore is a word char so + "handle_stac" does NOT contribute a standalone "stac" hit; only the + trailing "# stac" comment in _write_high_hit_primary does): + api.py 26 hits (1 header + 25 comments) + urls.py 10 hits (1 header + 9 /stac/ paths) + flask_app 8 hits (1 header + 7 /stac/ paths) + filesystem 5 hits (4 standalone + 1 FileSystemProvider) -- lowest count + Under pure occurrence-count ranking filesystem.py would rank 4th and + be excluded by _grep_repo[:3] entirely. The exact symbol-definition + pass (F-30) instead ranks it above all three occurrence-based hits, + since it *defines* FileSystemProvider — so it becomes the full-file + primary, not merely a low-priority secondary. + """ + from utilities.autopatcher.repo_locator import find_code_context + # api.py: 26 stac hits (1 header + 25 trailing comments) + self._write_high_hit_primary(tmp_path, "stac", 25) + # urls.py: 10 stac hits (1 header + 9 "/stac/" route paths) + write(tmp_path / "urls.py", + "# stac route\n" + "path('/stac/')\n" * 9) + # flask_app.py: 8 stac hits (1 header + 7 "/stac/" route paths) + write(tmp_path / "flask_app.py", + "# stac routes\n" + "@app.route('/stac/')\n" * 7) + # filesystem.py: 5 hits (4 standalone stac + 1 FileSystemProvider) -- lowest count + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n") + ctx = find_code_context( + "FileSystemProvider path traversal in `stac` collection", tmp_path + ) + assert "# provider/filesystem.py (full file," in ctx, ( + "the file defining FileSystemProvider must become the full-file primary " + "despite having the lowest raw occurrence count" + ) + + def test_symbol_definition_file_becomes_primary_despite_lower_occurrence_count( + self, tmp_path + ): + """F-30 core regression: a generic token repeated many times in an + unrelated file (api.py, 26 raw \\bstac\\b hits) must NOT evict or + outrank the file that actually *defines* the advisory-named symbol + (filesystem.py, only 5 raw hits) — the defining file must become the + full-file primary candidate. + """ + from utilities.autopatcher.repo_locator import find_code_context + # api.py: 26 stac hits -- highest raw occurrence count, no definition + self._write_high_hit_primary(tmp_path, "stac", 25) + write(tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n") + ctx = find_code_context( + "FileSystemProvider path traversal in `stac` collection", tmp_path + ) + assert "# provider/filesystem.py (full file," in ctx, ( + "the symbol-defining file must be the primary, not the high-occurrence file" + ) + assert "# api.py (full file," not in ctx, ( + "a generic token's raw occurrence count must not win primary status " + "over the file that defines the named symbol" + ) + + def test_class_def_file_not_duplicated_when_already_primary(self, tmp_path): + """If the class-defining file IS the primary, it must not appear twice.""" + from utilities.autopatcher.repo_locator import find_code_context + content = ( + "class FileSystemProvider:\n" + " def get_data_path(self, p): return p\n" + ) * 5 + assert len(content) < 20_000 + write(tmp_path / "filesystem.py", content) + ctx = find_code_context("FileSystemProvider path traversal", tmp_path) + import re as _re + headers = _re.findall(r"^# \S+", ctx, _re.MULTILINE) + assert len(headers) == 1, ( + f"class-def file as primary must not be duplicated; headers: {headers}" + ) + + def test_function_found_via_def_branch_when_no_class_name_in_advisory(self, tmp_path): + """Advisory without a PascalCase class name still finds the file that + *defines* the named function, via the generalized def-matching + branch — not just supported "by coincidence" through ordinary grep.""" + from utilities.autopatcher.repo_locator import find_code_context + write(tmp_path / "auth.py", + "def authenticate_user(u, p):\n return u == 'admin'\n") + ctx = find_code_context("SQL injection in `authenticate_user` (CWE-89)", tmp_path) + assert "authenticate_user" in ctx + + def test_short_names_do_not_trigger_symbol_definition_scan(self, tmp_path): + """PascalCase names < _MIN_CLASS_NAME_LENGTH must not trigger a scan.""" + from utilities.autopatcher.repo_locator import _find_symbol_definitions + write(tmp_path / "foo.py", "class Foo:\n pass\n") + write(tmp_path / "base.py", "class Base:\n pass\n") + assert _find_symbol_definitions("Foo and Base vulnerability", tmp_path) == [] + + def test_class_def_snippet_includes_class_body(self, tmp_path): + """The symbol-definition file's content must include the class + definition itself, not just the top-of-file preamble (hit_line + points to the class line) — here as the full-file primary, since + the defining file outranks the high-occurrence file.""" + from utilities.autopatcher.repo_locator import find_code_context + self._write_high_hit_primary(tmp_path, "stac", 25) + preamble = "import os\nimport sys\n\n" # 3 lines before class + class_body = ( + "class FileSystemProvider:\n" + " def get_data_path(self, path):\n" + " return self.root + path\n" + ) + write(tmp_path / "provider.py", preamble + class_body) + ctx = find_code_context("FileSystemProvider path traversal in stac", tmp_path) + assert "# provider.py (full file," in ctx, ( + "the symbol-defining file must be the full-file primary" + ) + assert "class FileSystemProvider" in ctx + + def test_symbol_definition_file_is_primary_in_snippet_mode_when_large(self, tmp_path): + """F-30 regression (snippet mode): when the file that defines the + named symbol is itself large enough to exceed the full-file + threshold, it must still become the FIRST candidate considered in + snippet mode (ranked[0]) — not be displaced by a smaller, unrelated + file with a higher raw occurrence count of some generic token.""" + from utilities.autopatcher.repo_locator import ( + find_code_context, _FULL_FILE_THRESHOLD_CHARS, _MAX_CONTEXT_CHARS, + ) + # api.py: 26 stac hits, tiny file, no definition of anything named. + self._write_high_hit_primary(tmp_path, "stac", 25) + # provider.py: defines FileSystemProvider, but is large enough to + # exceed the full-file threshold and force snippet mode. + big_provider = ( + "class FileSystemProvider:\n" + + " # implementation line\n" * 2000 + ) + assert len(big_provider) > _FULL_FILE_THRESHOLD_CHARS + write(tmp_path / "provider.py", big_provider) + + ctx = find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + + assert "class FileSystemProvider" in ctx, ( + "the large defining file must still be included, anchored at its " + "class definition, despite exceeding the full-file threshold" + ) + assert "provider.py" in ctx and "api.py" in ctx + # provider.py (ranked[0]) must be rendered before api.py (ranked[1]) — + # confirms it was not displaced from the first slot. + assert ctx.index("provider.py") < ctx.index("api.py"), ( + "the symbol-defining file must occupy the first (primary) slot " + "in snippet mode, not be pushed behind the high-occurrence file" + ) + assert len(ctx) <= _MAX_CONTEXT_CHARS + 500, ( + f"snippet-mode output ({len(ctx)} chars) exceeds the expected budget allowance" + ) + + +# --------------------------------------------------------------------------- +# F-30 regression: a generic token's raw occurrence count must not evict or +# outrank the file that defines the advisory's actual named symbol. Mirrors +# the finding's own example ("path" — a plain _GENERIC_TOKENS word, not a +# repo-specific one like "stac") and its exact failure-mode narrative: three +# unrelated files each repeating a generic word many times, competing +# against one real file that defines the named vulnerable symbol. +# --------------------------------------------------------------------------- + +class TestF30GenericTokenCannotEvictDefiningFile: + def test_defining_file_survives_full_eviction_scenario(self, tmp_path): + """Before the fix, three unrelated files each repeating a generic + backtick term ("path") enough times would fill _grep_repo's own + top-3 cut, silently evicting the file that defines the named + symbol from the candidate list entirely — see the F-30 root-cause + investigation. The exact symbol-definition pass finds the defining + file independently of _grep_repo's occurrence-based cut, so it must + survive regardless of how much generic-token noise exists.""" + from utilities.autopatcher.repo_locator import find_code_context + + # The real vulnerable file: defines the named symbol, barely + # mentions the generic term. + write(tmp_path / "auth.py", + "def authenticate_user(username, password, path):\n" + " return db.check(username, password)\n") + + # Three unrelated files, each with many occurrences of the generic + # term and zero mentions of the actual vulnerable symbol -- enough + # to have filled _grep_repo's own top-3 cutoff before the fix. + for n, fname in enumerate(["routes.py", "storage.py", "uploads.py"], start=1): + write(tmp_path / fname, + "\n".join(f"def helper_{n}_{i}(path):\n return path" for i in range(20))) + + vuln_text = ( + "**Type:** CWE-287 Improper Authentication\n\n" + "The `authenticate_user` function does not validate the redirect `path` " + "parameter, allowing an attacker-controlled `path` to bypass auth checks." + ) + + ctx = find_code_context(vuln_text, tmp_path) + + assert "def authenticate_user" in ctx, ( + "the file defining the named vulnerable symbol must not be evicted " + "by unrelated files repeating a generic token" + ) + assert "# auth.py (full file," in ctx, ( + "the defining file must be the primary candidate, not merely present" + ) + + def test_generic_token_does_not_outrank_defining_file(self, tmp_path): + """Two-file version: even without enough noise files to trigger full + eviction from _grep_repo's own cut, a single unrelated file with a + higher raw occurrence count of a generic token must not become the + primary ahead of the file that defines the named symbol.""" + from utilities.autopatcher.repo_locator import find_code_context + + write(tmp_path / "auth.py", + "def authenticate_user(username, password):\n" + " # path traversal via unsanitized redirect path\n" + " return db.check(username, password)\n") + write(tmp_path / "unrelated_paths.py", + "\n".join(f"def helper_{i}(path):\n return path + str({i})" for i in range(30))) + + vuln_text = ( + "**Type:** CWE-287 Improper Authentication\n\n" + "The `authenticate_user` function does not validate the redirect `path` " + "parameter, allowing an attacker-controlled `path` to bypass auth checks." + ) + + ctx = find_code_context(vuln_text, tmp_path) + + assert "# auth.py (full file," in ctx, ( + "the symbol-defining file must be the primary candidate" + ) + assert "# unrelated_paths.py (full file," not in ctx, ( + "a generic token's raw occurrence count must not win primary status" + ) + + +# --------------------------------------------------------------------------- +# Live pygeoapi integration (opt-in) +# --------------------------------------------------------------------------- + +_PYGEOAPI_EVAL = Path(tempfile.gettempdir()) / "pygeoapi-eval" +_PYGEOAPI_STAC_PY = _PYGEOAPI_EVAL / "pygeoapi" / "api" / "stac.py" +_PYGEOAPI_FS_PY = _PYGEOAPI_EVAL / "pygeoapi" / "provider" / "filesystem.py" + +_run_pygeoapi_live = ( + os.environ.get("RUN_LIVE_REPO_TESTS") == "1" + and _PYGEOAPI_STAC_PY.exists() + and _PYGEOAPI_FS_PY.exists() +) + + +@pytest.mark.skipif( + not _run_pygeoapi_live, + reason=( + "Live pygeoapi tests opt-in only — " + "set RUN_LIVE_REPO_TESTS=1 and ensure " + f"{_PYGEOAPI_EVAL} contains both " + "pygeoapi/api/stac.py and pygeoapi/provider/filesystem.py" + ), +) +class TestPygeoAPIClassDefinitionGrounding: + _VULN_TEXT = ( + "# pygeoapi 0.23.x: Path Traversal in STAC FileSystemProvider\n\n" + "**Type:** CWE-22\n\n" + "A raw string path concatenation vulnerability in pygeoapi's STAC " + "FileSystemProvider plugin allows path traversal via `stac-collection` " + "resources." + ) + + def test_filesystem_py_appears_in_context(self): + """After class-def supplement, filesystem.py must appear in injected context.""" + from utilities.autopatcher.repo_locator import find_code_context + ctx = find_code_context(self._VULN_TEXT, _PYGEOAPI_EVAL) + assert "filesystem.py" in ctx, ( + "filesystem.py must be injected via class-def supplement" + ) + + def test_stac_py_remains_primary(self): + """stac.py (highest occurrence count) must remain the full-file primary.""" + from utilities.autopatcher.repo_locator import find_code_context + ctx = find_code_context(self._VULN_TEXT, _PYGEOAPI_EVAL) + assert "# pygeoapi/api/stac.py (full file," in ctx, ( + "stac.py must remain the full-file primary" + ) + + +# --------------------------------------------------------------------------- +# Stage 6A: runtime observability — resolution_strategy propagation, +# explicit_path_resolutions, selected_pass, selected. Additive only: the +# pre-existing explicit_paths/explicit_paths_unresolved/explicit_paths_ambiguous +# fields must be unaffected by any of this. +# --------------------------------------------------------------------------- + +def _latest_debug_artifact(tmp_path: Path) -> dict: + debug_dir = tmp_path / "reports" / "debug" + files = sorted(debug_dir.glob("context_selection_*.json")) + assert files, f"expected a debug artifact under {debug_dir}" + return json.loads(files[-1].read_text(encoding="utf-8")) + + +class TestExplicitPathResolutionsAdditive: + """explicit_path_resolutions is added alongside the three existing + fields, which must remain exactly as they were.""" + + def test_existing_three_fields_unchanged_on_exact_match(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + assert signals["explicit_paths"] == ["app/auth.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == [] + + def test_existing_three_fields_unchanged_on_ambiguous_match(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + assert signals["explicit_paths"] == ["_internal/download.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == ["_internal/download.py"] + + def test_explicit_path_resolutions_present_alongside_old_fields(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + signals = record["extraction_signals"] + + # Old fields still present and correct + assert signals["explicit_paths"] == ["_internal/download.py"] + assert signals["explicit_paths_unresolved"] == [] + assert signals["explicit_paths_ambiguous"] == [] + # New field, additive + assert signals["explicit_path_resolutions"] == [ + { + "raw_path": "_internal/download.py", + "strategy": "suffix", + "resolved_file": "src/pip/_internal/download.py", + } + ] + + def test_one_entry_per_extracted_path_including_unresolved_and_ambiguous( + self, tmp_path, monkeypatch + ): + """A single call whose advisory names three paths — one exact, one + ambiguous, one unresolved — must produce exactly three + explicit_path_resolutions entries, one per path, each with the + correct strategy.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + write(tmp_path / "a" / "_internal" / "download.py", "# a\n") + write(tmp_path / "b" / "_internal" / "download.py", "# b\n") + + vuln = ( + "Vulnerability in app/auth.py and _internal/download.py " + "and does/not/exist.py" + ) + find_code_context(vuln, tmp_path) + record = _latest_debug_artifact(tmp_path) + resolutions = record["extraction_signals"]["explicit_path_resolutions"] + + by_path = {r["raw_path"]: r for r in resolutions} + assert len(resolutions) == 3 + assert by_path["app/auth.py"]["strategy"] == "exact" + assert by_path["app/auth.py"]["resolved_file"] == "app/auth.py" + assert by_path["_internal/download.py"]["strategy"] == "ambiguous" + assert by_path["_internal/download.py"]["resolved_file"] is None + assert by_path["does/not/exist.py"]["strategy"] == "unresolved" + assert by_path["does/not/exist.py"]["resolved_file"] is None + + +class TestResolutionStrategyPropagation: + """The Stage 6A bugfix: resolution_strategy must survive into the + per-candidate `passes` entry written to disk.""" + + def test_resolution_strategy_exact_in_written_artifact(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + explicit_pass = next(p for p in candidate["passes"] if p["pass"] == "explicit_path") + + assert explicit_pass["resolution_strategy"] == "exact" + + def test_resolution_strategy_suffix_in_written_artifact(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write( + tmp_path / "src" / "pip" / "_internal" / "download.py", + "def unpack_url(): pass\n", + ) + + find_code_context("Vulnerability in _internal/download.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next( + c for c in record["candidates"] + if c["file"] == "src/pip/_internal/download.py" + ) + explicit_pass = next(p for p in candidate["passes"] if p["pass"] == "explicit_path") + + assert explicit_pass["resolution_strategy"] == "suffix" + + +class TestSelectedPassAndSelected: + """selected_pass and selected are derived, single-source-of-truth + fields — never independently maintained state.""" + + def test_selected_pass_matches_final_score_owning_pass(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + + assert candidate["selected_pass"] == "explicit_path" + + def test_selected_true_for_primary_file(self, tmp_path, monkeypatch): + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + write(tmp_path / "app" / "auth.py", "def authenticate(): pass\n") + + find_code_context("Vulnerability in app/auth.py", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next(c for c in record["candidates"] if c["file"] == "app/auth.py") + + assert candidate["selected"] is True + assert candidate["selection_outcome"] != "rejected" + + # Fixture shared by the next four tests: mirrors + # TestSymbolDefinitionGrounding's api.py/urls.py/flask_app.py/filesystem.py + # setup elsewhere in this file. filesystem.py defines FileSystemProvider, + # so the exact symbol-definition pass (tier 3) ranks it above all three + # ordinary grep hits (tier 2) regardless of occurrence count — it becomes + # the full-file primary (ranked[0]). api.py(26 `stac` hits) and + # urls.py(10) are the two highest-occurrence tier-2 hits and fill the + # 2-slot secondary queue (ranked[1], ranked[2]); flask_app.py(8), the + # third and lowest tier-2 hit, is pushed out by the 2-slot cap and ends + # up "rejected" — giving a real, non-null final_score paired with a + # rejected outcome. + def _write_rejection_fixture(self, tmp_path: Path) -> None: + write( + tmp_path / "api.py", + "# stac routing module\n" + "def handle_stac(req): pass # stac\n" * 25, + ) + write(tmp_path / "urls.py", "# stac route\n" + "path('/stac/')\n" * 9) + write(tmp_path / "flask_app.py", "# stac routes\n" + "@app.route('/stac/')\n" * 7) + write( + tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n", + ) + + def test_selected_false_for_rejected_candidate(self, tmp_path, monkeypatch): + """A candidate outside the final secondary-slot cut must have + selected=False, matching its selection_outcome of 'rejected'.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + rejected = [c for c in record["candidates"] if c["selection_outcome"] == "rejected"] + + assert rejected, "expected at least one rejected candidate in this setup" + for c in rejected: + assert c["selected"] is False + + def test_selected_matches_selection_outcome_for_every_candidate(self, tmp_path, monkeypatch): + """selected is always exactly (selection_outcome != 'rejected') — + single source of truth, never independently wrong.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + + for c in record["candidates"]: + assert c["selected"] == (c["selection_outcome"] != "rejected") + + def test_selected_pass_never_null_for_any_candidate(self, tmp_path, monkeypatch): + """Every entry that made it into the candidates list arrived via at + least one pass, so selected_pass must always resolve to a name.""" + from utilities.autopatcher.repo_locator import find_code_context + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + + for c in record["candidates"]: + assert c["selected_pass"] is not None, f"selected_pass unexpectedly null for {c['file']}" + + def test_selected_pass_symbol_definition_for_promoted_low_occurrence_file( + self, tmp_path, monkeypatch + ): + """filesystem.py has the lowest raw occurrence count of the four + files in this fixture, but the exact symbol-definition pass (F-30) + ranks it above every ordinary grep hit because it *defines* + FileSystemProvider. final_score must be the real symbol-definition + tier (not None — there is no more supplement-only, tier-less path), + and selected_pass must resolve to 'symbol_definition'.""" + from utilities.autopatcher.repo_locator import ( + find_code_context, _TIER_SYMBOL_DEFINITION, + ) + monkeypatch.setenv("AUTOPATCHER_DEBUG", "1") + monkeypatch.chdir(tmp_path) + self._write_rejection_fixture(tmp_path) + + find_code_context("FileSystemProvider path traversal in `stac` collection", tmp_path) + record = _latest_debug_artifact(tmp_path) + candidate = next( + c for c in record["candidates"] if c["file"] == "provider/filesystem.py" + ) + + assert candidate["final_score"] == _TIER_SYMBOL_DEFINITION + assert candidate["selected_pass"] == "symbol_definition" + assert candidate["selection_outcome"] == "primary_full_file", ( + "the defining file must be the primary, not merely a rejected-adjacent entry" + ) + + +class TestGroundRepository: + """ground_repository() must return the exact same rendered_context as + find_code_context(), plus one RepositoryCandidate/GroundingDecision per + discovered file with today's exact outcome values, across the four + exit paths find_code_context() has: empty/no-match, primary full-file, + primary snippet, and multi-candidate secondary-context.""" + + def test_empty_no_match(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + vuln = "SQL injection in authenticate()" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected == "" + assert result.candidates == [] + assert result.decisions == [] + assert result.budget is None + + def test_primary_full_file(self, tmp_path): + """Single-candidate case: only retry.py matches (connectionpool.py + produces zero signal hits), so it must be the sole candidate and + be selected via the full-file path.""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + write(tmp_path / "retry.py", + "class Retry:\n DEFAULT_REMOVE = frozenset(['Authorization'])\n") + write(tmp_path / "connectionpool.py", + "class ConnectionPool:\n redirects = True\n") + vuln = "The `Authorization` header is not stripped on redirects." + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + assert [c.path for c in result.candidates] == ["retry.py"] + assert [d.path for d in result.decisions] == ["retry.py"] + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + assert result.decisions[0].outcome == "primary_full_file" + + def test_primary_snippet(self, tmp_path): + """Single-candidate case forced into snippet mode: the file exceeds + _FULL_FILE_THRESHOLD_CHARS, and only Pass 1 (explicit path) fires — + 'authenticate' has no underscore/PascalCase so Pass 2 stays empty.""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + big = "def authenticate(u, p):\n" + " # padding line here\n" * 1500 + assert len(big) > 20_000 + write(tmp_path / "app" / "auth.py", big) + vuln = "SQL injection in app/auth.py — authenticate()" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + assert [c.path for c in result.candidates] == ["app/auth.py"] + assert [d.path for d in result.decisions] == ["app/auth.py"] + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + assert result.decisions[0].outcome == "primary_snippet" + + def test_secondary_context_selected_and_rejected(self, tmp_path): + """Reuses TestSelectedPassAndSelected's rejection fixture: 4 + candidates, 3 selected (1 primary full-file — the symbol-definition + file, ranked above all ordinary grep hits — plus 2 secondary + snippets), 1 rejected by the 2-slot secondary cap despite having + real, non-null evidence (final_score=2).""" + from utilities.autopatcher.repo_locator import find_code_context, ground_repository + write(tmp_path / "api.py", + "# stac routing module\n" + "def handle_stac(req): pass # stac\n" * 25) + write(tmp_path / "urls.py", "# stac route\n" + "path('/stac/')\n" * 9) + write(tmp_path / "flask_app.py", "# stac routes\n" + "@app.route('/stac/')\n" * 7) + write( + tmp_path / "provider" / "filesystem.py", + "class FileSystemProvider:\n" + " def get_data_path(self, path): return path\n" + " # stac stac stac stac\n", + ) + vuln = "FileSystemProvider path traversal in `stac` collection" + + expected = find_code_context(vuln, tmp_path) + result = ground_repository(vuln, tmp_path) + + assert result.rendered_context == expected + + expected_paths = {"api.py", "urls.py", "flask_app.py", "provider/filesystem.py"} + assert {c.path for c in result.candidates} == expected_paths + assert len(result.decisions) == len(result.candidates) + for cand, dec in zip(result.candidates, result.decisions): + assert cand.path == dec.path + + outcomes = {d.path: d.outcome for d in result.decisions} + assert outcomes["provider/filesystem.py"] == "primary_full_file" + assert outcomes["api.py"] == "secondary_snippet" + assert outcomes["urls.py"] == "secondary_snippet" + assert outcomes["flask_app.py"] == "rejected" + + selected = {p for p, o in outcomes.items() if o != "rejected"} + rejected = {p for p, o in outcomes.items() if o == "rejected"} + assert selected == {"provider/filesystem.py", "api.py", "urls.py"} + assert rejected == {"flask_app.py"} + + +# --------------------------------------------------------------------------- +# F-18: symlink / path-containment regression tests +# +# _safe_under() is a security boundary, not a convenience filter: a symlink +# that *lives* inside the repo but whose target *resolves* outside it must +# never be enumerated, grepped, class-def-matched, or suffix-resolved. These +# tests exercise the resolve-and-compare check directly through the three +# call sites it was added to (_grep_repo, _find_symbol_definitions, +# RepositoryPathResolver._iter_files), plus find_code_context end-to-end. +# --------------------------------------------------------------------------- + +class TestSymlinkContainment: + def test_external_absolute_symlink_rejected_by_grep(self, tmp_path): + from utilities.autopatcher.repo_locator import _grep_repo + repo = tmp_path / "repo" + repo.mkdir() + write(tmp_path / "secret.py", "HOST_SECRET = 'hunter2'\n") + os.symlink(tmp_path / "secret.py", repo / "evil.py") + + results = _grep_repo(repo, ["HOST_SECRET"]) + assert results == [] + + def test_external_relative_symlink_rejected_by_grep(self, tmp_path): + from utilities.autopatcher.repo_locator import _grep_repo + repo = tmp_path / "repo" + repo.mkdir() + write(tmp_path / "secret.py", "HOST_SECRET = 'hunter2'\n") + # Target given as a path relative to the symlink's own directory + # (repo/), so it escapes without ever mentioning an absolute path. + os.symlink(os.path.join("..", "secret.py"), repo / "evil.py") + + results = _grep_repo(repo, ["HOST_SECRET"]) + assert results == [] + + def test_external_symlink_rejected_by_class_definitions(self, tmp_path): + from utilities.autopatcher.repo_locator import _find_symbol_definitions + repo = tmp_path / "repo" + repo.mkdir() + write(tmp_path / "secret.py", "class LeakyProvider:\n pass\n") + os.symlink(tmp_path / "secret.py", repo / "evil.py") + + results = _find_symbol_definitions("LeakyProvider vulnerability", repo) + assert results == [] + + def test_suffix_match_cannot_return_escaping_symlink(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + repo = tmp_path / "repo" + (repo / "_internal").mkdir(parents=True) + write(tmp_path / "secret.py", "HOST_SECRET = 1\n") + os.symlink(tmp_path / "secret.py", repo / "_internal" / "download.py") + + resolver = RepositoryPathResolver(repo) + result = resolver.resolve("_internal/download.py") + assert result.path is None + assert result.strategy == "unresolved" + + def test_find_code_context_never_surfaces_external_content(self, tmp_path): + from utilities.autopatcher.repo_locator import find_code_context + repo = tmp_path / "repo" + repo.mkdir() + write(tmp_path / "secret.py", "HOST_SECRET_TOKEN = 'do-not-leak'\n") + os.symlink(tmp_path / "secret.py", repo / "evil.py") + write(repo / "app.py", "def authenticate(): pass\n") + + vuln = "authenticate() is exploitable — see `HOST_SECRET_TOKEN`" + result = find_code_context(vuln, repo) + assert "do-not-leak" not in result + assert "HOST_SECRET_TOKEN" not in result + + def test_in_repo_symlink_still_supported_by_grep(self, tmp_path): + """A symlink whose target resolves inside the repo is not an escape + and must keep working exactly as before (no regression).""" + from utilities.autopatcher.repo_locator import _grep_repo + repo = tmp_path / "repo" + write(repo / "impl" / "real.py", "def authenticate(): pass\n") + os.symlink(repo / "impl" / "real.py", repo / "alias.py") + + results = _grep_repo(repo, ["authenticate"]) + names = {p.name for p, _content, _hit in results} + assert "alias.py" in names + + def test_in_repo_symlink_still_supported_by_suffix_match(self, tmp_path): + """Mirrors test_suffix_match_resolves_src_layout: the advisory names + `_internal/download.py`, but the real file lives under a src-layout + prefix reached only via a symlinked alias — no `_internal/` directory + exists at the repo root, so this can only resolve via suffix match.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + repo = tmp_path / "repo" + write(repo / "real" / "download.py", "def unpack_url(): pass\n") + (repo / "src" / "pip" / "_internal").mkdir(parents=True) + os.symlink(repo / "real" / "download.py", repo / "src" / "pip" / "_internal" / "download.py") + + resolver = RepositoryPathResolver(repo) + result = resolver.resolve("_internal/download.py") + assert result.strategy == "suffix" + assert result.path is not None + assert result.path.resolve() == (repo / "real" / "download.py").resolve() + + def test_broken_symlink_does_not_crash_grep(self, tmp_path): + from utilities.autopatcher.repo_locator import _grep_repo + repo = tmp_path / "repo" + repo.mkdir() + os.symlink(tmp_path / "does_not_exist.py", repo / "broken.py") + write(repo / "app.py", "def authenticate(): pass\n") + + results = _grep_repo(repo, ["authenticate"]) + names = {p.name for p, _content, _hit in results} + assert "broken.py" not in names + assert "app.py" in names + + def test_broken_symlink_does_not_crash_class_definitions(self, tmp_path): + from utilities.autopatcher.repo_locator import _find_symbol_definitions + repo = tmp_path / "repo" + repo.mkdir() + os.symlink(tmp_path / "does_not_exist.py", repo / "broken.py") + + results = _find_symbol_definitions("Anything at all", repo) + assert results == [] + + def test_broken_symlink_does_not_crash_suffix_match(self, tmp_path): + from utilities.autopatcher.repo_locator import RepositoryPathResolver + repo = tmp_path / "repo" + (repo / "_internal").mkdir(parents=True) + os.symlink(tmp_path / "does_not_exist.py", repo / "_internal" / "download.py") + + resolver = RepositoryPathResolver(repo) + result = resolver.resolve("_internal/download.py") + assert result.path is None + + def test_symlink_loop_does_not_hang_or_crash(self, tmp_path): + """A mutual symlink loop must be rejected quickly, not hang the scan + or propagate the OS-level 'too many levels of symbolic links' error.""" + from utilities.autopatcher.repo_locator import _grep_repo + repo = tmp_path / "repo" + repo.mkdir() + os.symlink(repo / "loop_b.py", repo / "loop_a.py") + os.symlink(repo / "loop_a.py", repo / "loop_b.py") + write(repo / "app.py", "def authenticate(): pass\n") + + results = _grep_repo(repo, ["authenticate"]) + names = {p.name for p, _content, _hit in results} + assert "loop_a.py" not in names + assert "loop_b.py" not in names + assert "app.py" in names + + def test_symlink_loop_does_not_crash_class_definitions(self, tmp_path): + from utilities.autopatcher.repo_locator import _find_symbol_definitions + repo = tmp_path / "repo" + repo.mkdir() + os.symlink(repo / "loop_a.py", repo / "loop_a.py") + + results = _find_symbol_definitions("Anything at all", repo) + assert results == [] + + def test_exact_match_symlink_loop_fails_closed(self, tmp_path): + """A symlink loop at the exact path an advisory names must not + crash resolution — `.resolve()` raises RuntimeError for a loop, and + _resolve_exact must catch that and fail closed (unresolved), not + propagate it. A single path segment guarantees this exercises only + the exact-match branch, never the suffix fallback.""" + from utilities.autopatcher.repo_locator import RepositoryPathResolver + repo = tmp_path / "repo" + repo.mkdir() + os.symlink(repo / "loop.py", repo / "loop.py") + + resolver = RepositoryPathResolver(repo) + result = resolver.resolve("loop.py") + assert result.path is None + assert result.strategy == "unresolved" + + def test_repo_root_beneath_symlinked_ancestor_still_works(self, tmp_path): + """The repo root itself sitting under a symlinked ancestor directory + (e.g. macOS's /tmp -> /private/tmp) must not break containment — + both sides of the comparison need to resolve to the same canonical + location.""" + from utilities.autopatcher.repo_locator import ( + RepositoryPathResolver, + _find_symbol_definitions, + _grep_repo, + ) + real_root = tmp_path / "real_root" + write(real_root / "repo" / "app.py", "def authenticate(): pass\n") + write(real_root / "repo" / "provider.py", "class FileSystemProvider:\n pass\n") + link_root = tmp_path / "link_root" + os.symlink(real_root, link_root) + repo_via_link = link_root / "repo" + + grep_results = _grep_repo(repo_via_link, ["authenticate"]) + assert {p.name for p, _c, _h in grep_results} == {"app.py"} + + class_results = _find_symbol_definitions("FileSystemProvider vuln", repo_via_link) + assert {p.name for p, _c, _h in class_results} == {"provider.py"} + + resolver = RepositoryPathResolver(repo_via_link) + result = resolver.resolve("app.py") + assert result.strategy == "exact" + assert result.path is not None diff --git a/libs/openant-core/tests/patch/test_run_metadata.py b/libs/openant-core/tests/patch/test_run_metadata.py new file mode 100644 index 00000000..62106fa8 --- /dev/null +++ b/libs/openant-core/tests/patch/test_run_metadata.py @@ -0,0 +1,286 @@ +import os +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +import pytest + + +from utilities.autopatcher.run_metadata import RunMetadata, auto_output_path, collect_git_info, render_metadata_section + +_TS = datetime(2026, 6, 16, 18, 35, 42, tzinfo=timezone.utc) + + +def _meta(**overrides) -> RunMetadata: + defaults = dict( + timestamp="2026-06-16 18:35:42 UTC", + input_source="GHSA-v845-jxx5-vc9f", + repo_root="/tmp/urllib3-eval", + repo_commit="d9f85a74", + llm_provider="anthropic", + llm_model="claude-sonnet-4-6", + llm_mode="LIVE", + output_path="reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md", + patcher_commit="e0f8737", + ) + defaults.update(overrides) + return RunMetadata(**defaults) + + +# --------------------------------------------------------------------------- +# auto_output_path +# --------------------------------------------------------------------------- + +class TestAutoOutputPath: + def test_ghsa_mode_full_format(self): + path = auto_output_path(_TS, "GHSA-v845-jxx5-vc9f", None, "/tmp/urllib3-eval") + assert path == "reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" + + def test_file_mode_no_repo(self): + path = auto_output_path(_TS, None, "examples/vulnerability.md", None) + assert path == "reports/20260616-183542-vulnerability.md" + + def test_file_mode_with_repo(self): + path = auto_output_path(_TS, None, "examples/vulnerability.md", "/tmp/myrepo") + assert path == "reports/20260616-183542-myrepo-vulnerability.md" + + def test_output_always_under_reports_dir(self): + path = auto_output_path(_TS, "GHSA-x-y-z", None, "/tmp/proj") + assert path.startswith("reports/") + assert path.endswith(".md") + + def test_ghsa_id_is_lowercased(self): + path = auto_output_path(_TS, "GHSA-V845-JXX5-VC9F", None, "/tmp/urllib3-eval") + assert "ghsa-v845-jxx5-vc9f" in path + assert "GHSA" not in path + + def test_repo_slug_is_lowercased(self): + path = auto_output_path(_TS, "GHSA-x-y-z", None, "/tmp/MyProject") + assert "myproject" in path + assert "MyProject" not in path + + def test_seconds_included_in_timestamp(self): + ts = datetime(2026, 6, 16, 9, 5, 3, tzinfo=timezone.utc) + path = auto_output_path(ts, "GHSA-a-b-c", None, "/tmp/repo") + assert "20260616-090503" in path + + def test_no_ghsa_no_file_fallback(self): + path = auto_output_path(_TS, None, None, None) + assert path == "reports/20260616-183542-report.md" + + +# --------------------------------------------------------------------------- +# collect_git_info +# --------------------------------------------------------------------------- + +class TestCollectGitInfo: + def test_non_repo_returns_unknown(self, tmp_path): + result = collect_git_info(tmp_path) + assert result == "unknown" + + def test_nonexistent_path_returns_unknown(self): + result = collect_git_info(Path("/nonexistent/path/xyz")) + assert result == "unknown" + + def test_never_raises(self, tmp_path): + result = collect_git_info(tmp_path / "does_not_exist") + assert isinstance(result, str) + + def test_real_repo_returns_short_sha(self): + project_root = Path(__file__).parent.parent + result = collect_git_info(project_root) + assert result != "unknown" + assert len(result) >= 7 + assert all(c in "0123456789abcdef" for c in result), f"Not a hex SHA: {result!r}" + + def test_real_repo_sha_matches_git_log(self): + project_root = Path(__file__).parent.parent + expected = subprocess.run( + ["git", "log", "-1", "--format=%h"], + cwd=str(project_root), + capture_output=True, + text=True, + ).stdout.strip() + assert collect_git_info(project_root) == expected + + +# --------------------------------------------------------------------------- +# render_metadata_section +# --------------------------------------------------------------------------- + +class TestRenderMetadataSection: + def test_all_required_fields_present(self): + md = render_metadata_section(_meta()) + for field in [ + "Generated", + "Input", + "Repository", + "Repo commit", + "LLM provider", + "LLM model", + "LLM mode", + "Output", + "Auto-patcher", + ]: + assert field in md, f"Required field missing from metadata: {field!r}" + + def test_values_rendered_in_table(self): + md = render_metadata_section(_meta()) + assert "GHSA-v845-jxx5-vc9f" in md + assert "d9f85a74" in md + assert "anthropic" in md + assert "claude-sonnet-4-6" in md + assert "LIVE" in md + assert "e0f8737" in md + + def test_mock_warning_present_in_mock_mode(self): + md = render_metadata_section(_meta(llm_mode="MOCK")) + assert "MOCK MODE" in md + assert "must not be used as benchmark evidence" in md + + def test_mock_warning_absent_in_live_mode(self): + md = render_metadata_section(_meta(llm_mode="LIVE")) + assert "MOCK MODE" not in md + + def test_empty_repo_root_renders_dash(self): + md = render_metadata_section(_meta(repo_root="")) + assert "| Repository | — |" in md + + def test_output_path_in_table(self): + md = render_metadata_section(_meta( + output_path="reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" + )) + assert "reports/20260616-183542-urllib3-eval-ghsa-v845-jxx5-vc9f.md" in md + + +# --------------------------------------------------------------------------- +# render_metadata_section — max_tokens_configured / stage_stop_reasons +# --------------------------------------------------------------------------- + +class TestTokenBudgetAndStopReasons: + def test_max_tokens_row_rendered_when_present(self): + md = render_metadata_section(_meta(max_tokens_configured=4096)) + assert "| Max output tokens | 4096 |" in md + + def test_max_tokens_row_renders_dash_when_absent(self): + md = render_metadata_section(_meta()) + assert "| Max output tokens | — |" in md + + def test_stage_stop_reason_table_rendered_when_present(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "end_turn", + "patch_review": "end_turn", + "challenger": "end_turn", + "confidence_scorer": "end_turn", + })) + assert "### Stage Stop Reasons" in md + assert "| Patch generation | end_turn |" in md + assert "| Patch review | end_turn |" in md + assert "| Challenger | end_turn |" in md + assert "| Confidence scorer | end_turn |" in md + + def test_stage_stop_reason_table_absent_when_no_stages_recorded(self): + md = render_metadata_section(_meta()) + assert "Stage Stop Reasons" not in md + + def test_truncated_stage_gets_warning_icon_in_table(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "max_tokens", + "patch_review": "end_turn", + })) + assert "| Patch generation | ⚠️ max_tokens |" in md + assert "| Patch review | end_turn |" in md + + def test_openai_length_stop_reason_also_flagged(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "length", + })) + assert "| Patch generation | ⚠️ length |" in md + + def test_truncation_banner_present_when_any_stage_truncated(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "max_tokens", + "patch_review": "end_turn", + })) + assert "TRUNCATED OUTPUT DETECTED" in md + + def test_truncation_banner_absent_when_no_stage_truncated(self): + md = render_metadata_section(_meta(stage_stop_reasons={ + "patch_generation": "end_turn", + "patch_review": "stop", + })) + assert "TRUNCATED OUTPUT DETECTED" not in md + + def test_truncation_banner_absent_when_no_stages_recorded(self): + md = render_metadata_section(_meta()) + assert "TRUNCATED OUTPUT DETECTED" not in md + + def test_truncation_banner_coexists_with_mock_warning(self): + md = render_metadata_section(_meta( + llm_mode="MOCK", + stage_stop_reasons={"patch_generation": "max_tokens"}, + )) + assert "MOCK MODE" in md + assert "TRUNCATED OUTPUT DETECTED" in md + + +# --------------------------------------------------------------------------- +# render_metadata_section — CVE input-source disclosure (additive; must not +# change output for the default "finding" input_type) +# --------------------------------------------------------------------------- + +class TestCveInputSourceDisclosure: + def test_default_input_type_renders_no_disclosure_or_row(self): + md = render_metadata_section(_meta()) + assert "Input Source: CVE" not in md + assert "Input type" not in md + + def test_finding_input_type_renders_no_disclosure_or_row(self): + md = render_metadata_section(_meta(input_type="finding")) + assert "Input Source: CVE" not in md + assert "Input type" not in md + + def test_cve_input_type_renders_disclosure(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + )) + assert "Input Source: CVE (CVE-2022-25883)" in md + assert "NVD" in md + + def test_cve_disclosure_states_not_repository_verified(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + )) + assert "not been verified against this repository" in md + + def test_cve_disclosure_states_recommendation_based_on_collected_evidence(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + )) + assert "based only on the evidence this pipeline run actually collected" in md + + def test_cve_input_type_renders_table_row(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + )) + assert "| Input type | CVE (CVE-2022-25883, NVD) |" in md + + def test_cve_disclosure_handles_missing_advisory_fields_without_crash(self): + md = render_metadata_section(_meta(input_type="cve")) + assert "Input Source: CVE (unknown)" in md + + def test_cve_disclosure_coexists_with_mock_warning(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + llm_mode="MOCK", + )) + assert "MOCK MODE" in md + assert "Input Source: CVE" in md + + def test_rest_of_table_unaffected_by_cve_disclosure(self): + md = render_metadata_section(_meta( + input_type="cve", advisory_id="CVE-2022-25883", advisory_source="NVD", + )) + for field in ["Generated", "Repository", "Repo commit", "LLM provider", "Output", "Auto-patcher"]: + assert field in md diff --git a/libs/openant-core/tests/patch/test_run_patch_cve.py b/libs/openant-core/tests/patch/test_run_patch_cve.py new file mode 100644 index 00000000..3c982980 --- /dev/null +++ b/libs/openant-core/tests/patch/test_run_patch_cve.py @@ -0,0 +1,245 @@ +"""Integration tests for run_patch_cve: CVE id -> fetch -> InvestigationCase -> +ContextProjection -> the existing (unmodified) Auto Patcher pipeline -> +artifacts on disk. + +Hermetic: LLM_PROVIDER=mock, no network (fetch_cve is mocked at the +utilities.autopatcher.cve_fetcher module boundary), no real repo beyond a +tmp_path directory, no Docker. Mirrors test_patch_wrapper_contract.py's style +for the equivalent run_patch() (Finding-mode) tests. +""" + +from __future__ import annotations + +import os +from unittest import mock + +import pytest + +from core.patch import PatchStepResult, run_patch_cve +from utilities.autopatcher.cve_fetcher import CVEFetchError, CVENotFoundError + +FIXTURE_CVE = { + "id": "CVE-2021-12345", + "descriptions": [ + {"lang": "en", "value": "A SQL injection vulnerability exists in the authenticate() function."} + ], + "metrics": { + "cvssMetricV31": [ + {"cvssData": {"baseScore": 9.8, "baseSeverity": "CRITICAL"}}, + ] + }, + "weaknesses": [ + {"description": [{"lang": "en", "value": "CWE-89"}]} + ], +} + + +# core.patch imports fetch_cve locally inside run_patch_cve's body (from +# utilities.autopatcher.cve_fetcher import fetch_cve), so the patch target +# must be the function's home module, not core.patch's namespace. +def _mock_fetch_cve_at_source(cve=FIXTURE_CVE, side_effect=None): + if side_effect is not None: + return mock.patch("utilities.autopatcher.cve_fetcher.fetch_cve", side_effect=side_effect) + return mock.patch("utilities.autopatcher.cve_fetcher.fetch_cve", return_value=cve) + + +class TestRunPatchCveHappyPath: + def test_writes_artifacts_named_after_cve_id(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(): + result = run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + assert isinstance(result, PatchStepResult) + assert result.finding_id == "CVE-2021-12345" + assert result.input_type == "cve" + assert result.input_id == "CVE-2021-12345" + assert result.vulnerability_path == str(tmp_path / "patch" / "CVE-2021-12345-vulnerability.md") + assert result.trust_report_path == str(tmp_path / "patch" / "CVE-2021-12345-trust-report.md") + assert os.path.exists(result.vulnerability_path) + assert os.path.exists(result.trust_report_path) + + def test_trust_report_discloses_cve_input_source(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(): + result = run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + report_text = open(result.trust_report_path, encoding="utf-8").read() + assert "Input Source: CVE (CVE-2021-12345)" in report_text + assert "not been verified against this repository" in report_text + assert "| Input type | CVE (CVE-2021-12345, NVD) |" in report_text + + def test_vulnerability_artifact_matches_cve_to_vuln_text(self, tmp_path, monkeypatch): + from utilities.autopatcher.cve_converter import cve_to_vuln_text + + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(): + result = run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + vuln_text = open(result.vulnerability_path, encoding="utf-8").read() + assert vuln_text == cve_to_vuln_text(FIXTURE_CVE) + + def test_trust_report_mock_mode_is_self_disclosing(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(): + result = run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + report_text = open(result.trust_report_path, encoding="utf-8").read() + assert "MOCK MODE" in report_text + assert "LLM mode | MOCK" in report_text + + def test_never_leaves_stale_trust_report_after_failed_rerun(self, tmp_path, monkeypatch): + """F-39 guarantee, ported to the CVE entry point.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(): + first = run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + assert os.path.exists(first.trust_report_path) + + import utilities.autopatcher.pipeline as _pipeline_module + + def _boom(**kwargs): + raise RuntimeError("simulated pipeline failure") + + monkeypatch.setattr(_pipeline_module, "run", _boom) + + with _mock_fetch_cve_at_source(): + with pytest.raises(RuntimeError, match="simulated pipeline failure"): + run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + assert not os.path.exists(first.trust_report_path) + assert os.path.exists(first.vulnerability_path) + + +class TestRunPatchCveRepoRootValidation: + def test_missing_repo_root_raises_before_any_fetch(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + nonexistent = str(tmp_path / "does-not-exist") + + with _mock_fetch_cve_at_source() as mocked_fetch: + with pytest.raises(ValueError, match="does-not-exist"): + run_patch_cve("CVE-2021-12345", nonexistent, str(tmp_path)) + mocked_fetch.assert_not_called() + + def test_empty_repo_root_raises(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + with pytest.raises(ValueError): + run_patch_cve("CVE-2021-12345", "", str(tmp_path)) + + def test_repo_root_pointing_at_a_file_not_a_directory_raises(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + a_file = tmp_path / "not-a-dir" + a_file.write_text("x") + with pytest.raises(ValueError): + run_patch_cve("CVE-2021-12345", str(a_file), str(tmp_path)) + + +class TestRunPatchCveFetchFailures: + def test_cve_not_found_propagates(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(side_effect=CVENotFoundError("no such CVE")): + with pytest.raises(CVENotFoundError): + run_patch_cve("CVE-9999-99999", str(repo_root), str(tmp_path)) + + def test_network_failure_propagates(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(side_effect=CVEFetchError("network error")): + with pytest.raises(CVEFetchError): + run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + + def test_no_artifacts_written_when_fetch_fails(self, tmp_path, monkeypatch): + """Fetch failures happen before any artifact is written -- mirrors + run_patch()'s existing behavior for an unknown/ineligible finding.""" + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source(side_effect=CVENotFoundError("no such CVE")): + with pytest.raises(CVENotFoundError): + run_patch_cve("CVE-9999-99999", str(repo_root), str(tmp_path)) + + assert not os.path.exists(tmp_path / "patch") + + +class TestRunPatchCveRequiresLlmProvider: + def test_requires_llm_provider(self, tmp_path, monkeypatch): + monkeypatch.delenv("LLM_PROVIDER", raising=False) + repo_root = tmp_path / "repo" + repo_root.mkdir() + + with _mock_fetch_cve_at_source() as mocked_fetch: + with pytest.raises(RuntimeError, match="LLM_PROVIDER"): + run_patch_cve("CVE-2021-12345", str(repo_root), str(tmp_path)) + # repo_root check and fetch both happen before the LLM_PROVIDER + # check inside the shared helper -- fetch_cve is still called here, + # unlike the repo_root-invalid case above. + mocked_fetch.assert_called_once() + + +class TestRunPatchCveRepoRootNormalization: + """Repository Understanding integration: repo_root must be normalized + (resolved) once, at this entry point, before InvestigationCase / + ground_repository / parsing ever see it -- an unresolved path (e.g. a + macOS /var/... symlink to /private/var/...) can otherwise degrade + repository-grounding candidate paths to bare filenames.""" + + def test_repo_root_is_resolved_before_reaching_pipeline_run(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + real_repo = tmp_path / "real_repo" + real_repo.mkdir() + link_repo = tmp_path / "link_repo" + link_repo.symlink_to(real_repo) + + import utilities.autopatcher.pipeline as _pipeline_module + + captured = {} + original_run = _pipeline_module.run + + def _capturing_run(*, vulnerability_text, api_key, repo_root=None, investigation_output_dir=None): + captured["repo_root"] = repo_root + return original_run( + vulnerability_text=vulnerability_text, api_key=api_key, + repo_root=repo_root, investigation_output_dir=investigation_output_dir, + ) + + with _mock_fetch_cve_at_source(), mock.patch.object(_pipeline_module, "run", side_effect=_capturing_run): + run_patch_cve("CVE-2021-12345", str(link_repo), str(tmp_path)) + + assert captured["repo_root"] == str(real_repo.resolve()) + assert captured["repo_root"] != str(link_repo) + + +class TestRunPatchCveInvestigationDirectory: + """Run-scoped Repository Understanding parser-artifact directory.""" + + def test_created_outside_repo_and_under_output_dir(self, tmp_path, monkeypatch): + monkeypatch.setenv("LLM_PROVIDER", "mock") + repo_root = tmp_path / "repo" + repo_root.mkdir() + output_dir = tmp_path / "out" + + with _mock_fetch_cve_at_source(): + run_patch_cve("CVE-2021-12345", str(repo_root), str(output_dir)) + + expected = output_dir / "patch" / "CVE-2021-12345-investigation" + assert expected.is_dir() + assert not str(expected).startswith(str(repo_root)) diff --git a/libs/openant-core/tests/patch/test_sink_extractor.py b/libs/openant-core/tests/patch/test_sink_extractor.py new file mode 100644 index 00000000..ee0beb86 --- /dev/null +++ b/libs/openant-core/tests/patch/test_sink_extractor.py @@ -0,0 +1,372 @@ +"""Tests for Phase D repo-wide sink extraction. + +Tests: extract_repo_sinks, _find_enclosing_def, _build_sink_table, and +the integration path through build_vulnerability_pattern_context. + +All file-system tests use pytest's tmp_path fixture — no real repos needed. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.vulnerability_patterns import ( + _build_sink_table, + _find_enclosing_def, + build_vulnerability_pattern_context, + extract_repo_sinks, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def write(path: Path, content: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- +# _find_enclosing_def +# --------------------------------------------------------------------------- + + +class TestFindEnclosingDef: + def test_finds_simple_def(self): + lines = [ + "def my_function(self, x):", + " os.system(x)", + ] + assert _find_enclosing_def(lines, 1) == "my_function" + + def test_finds_async_def(self): + lines = [ + "async def async_func(self):", + " await do_thing()", + " os.system('x')", + ] + assert _find_enclosing_def(lines, 2) == "async_func" + + def test_returns_none_when_no_def_above(self): + lines = [ + "import os", + "os.system('cmd')", + ] + assert _find_enclosing_def(lines, 1) is None + + def test_finds_nearest_not_outer(self): + # Should return inner, not outer + lines = [ + "def outer(self):", + " x = 1", + " def inner(y):", + " os.system(y)", + ] + assert _find_enclosing_def(lines, 3) == "inner" + + def test_sink_on_def_line_itself(self): + lines = [ + "def risky(self, shell=True):", + ] + assert _find_enclosing_def(lines, 0) == "risky" + + def test_indented_method_inside_class(self): + lines = [ + "class Foo:", + " def method(self):", + " os.system('x')", + ] + assert _find_enclosing_def(lines, 2) == "method" + + def test_empty_lines_list(self): + assert _find_enclosing_def([], 0) is None + + def test_walks_past_blank_lines(self): + lines = [ + "def handler(self):", + "", + " x = 1", + " os.system(x)", + ] + assert _find_enclosing_def(lines, 3) == "handler" + + +# --------------------------------------------------------------------------- +# extract_repo_sinks +# --------------------------------------------------------------------------- + + +class TestExtractRepoSinks: + def test_multi_file_coverage(self, tmp_path): + write(tmp_path / "a.py", "def func_a(cmd):\n os.system(cmd)\n") + write(tmp_path / "b.py", "def func_b(cmd):\n x = cmd; os.system(x)\n") + write(tmp_path / "c.py", "def func_c(cmd):\n subprocess.call(cmd, shell=True)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + methods = {s["method"] for s in sinks} + assert "func_a" in methods + assert "func_b" in methods + assert "func_c" in methods + + def test_enclosing_method_detected(self, tmp_path): + write(tmp_path / "app.py", ( + "def safe_func(x):\n" + " return x\n" + "\n" + "def danger_func(path):\n" + " full = os.path.normpath(path)\n" + " return full\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "danger_func" + + def test_deduplication_keeps_first_occurrence(self, tmp_path): + write(tmp_path / "app.py", ( + "def multi_sink(a, b):\n" + " x = os.path.normpath(a)\n" + " y = os.path.normpath(b)\n" + " return x, y\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "multi_sink" + assert sinks[0]["line"] == 2 # first occurrence, 1-indexed + + def test_no_sinks_returns_empty(self, tmp_path): + write(tmp_path / "clean.py", ( + "from pathlib import Path\n" + "\n" + "def safe(path, base):\n" + " return Path(path).resolve().is_relative_to(Path(base).resolve())\n" + )) + assert extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") == [] + + def test_comment_lines_skipped(self, tmp_path): + write(tmp_path / "commented.py", ( + "def func(x):\n" + " # old: os.path.normpath(x)\n" + " return x\n" + )) + assert extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") == [] + + def test_test_files_by_name_skipped(self, tmp_path): + write(tmp_path / "test_app.py", "def test_f(cmd):\n os.system(cmd)\n") + write(tmp_path / "app.py", "def safe(x):\n return x\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert all(s["file"] != "test_app.py" for s in sinks) + + def test_tests_subdir_skipped(self, tmp_path): + write(tmp_path / "tests" / "test_foo.py", "def test_x():\n os.system('x')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks == [] + + def test_module_level_sink_has_none_method(self, tmp_path): + write(tmp_path / "script.py", "os.system('cmd')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks) == 1 + assert sinks[0]["method"] is None + + def test_results_sorted_by_file_then_line(self, tmp_path): + write(tmp_path / "z_file.py", "def zzz(x):\n os.system(x)\n") + write(tmp_path / "a_file.py", "def aaa(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + files = [s["file"] for s in sinks] + assert files == sorted(files) + + def test_unknown_vuln_class_returns_empty(self, tmp_path): + write(tmp_path / "app.py", "def f(x):\n os.system(x)\n") + assert extract_repo_sinks(tmp_path, "SQL_INJECTION") == [] + + def test_none_repo_root_returns_empty(self): + assert extract_repo_sinks(None, "COMMAND_INJECTION") == [] + + def test_capped_at_max_rows(self, tmp_path): + for i in range(25): + write(tmp_path / f"mod{i:02d}.py", f"def func_{i}(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks) <= 20 + + def test_line_number_is_1_indexed(self, tmp_path): + write(tmp_path / "app.py", ( + "def func(x):\n" + " safe()\n" + " os.system(x)\n" + )) + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks[0]["line"] == 3 + + def test_snippet_stripped(self, tmp_path): + write(tmp_path / "app.py", "def func(x):\n os.system(x)\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert sinks[0]["snippet"] == "os.system(x)" + + def test_path_traversal_normpath_detected(self, tmp_path): + write(tmp_path / "server.py", ( + "def serve(self, path):\n" + " full = os.path.normpath(self.root + path)\n" + " return open(full).read()\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert len(sinks) == 1 + assert sinks[0]["method"] == "serve" + + def test_path_traversal_string_concat_detected(self, tmp_path): + write(tmp_path / "fs.py", ( + "def get_file(self, name):\n" + " path = self.root + name\n" + " return open(path).read()\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert any(s["method"] == "get_file" for s in sinks) + + def test_path_traversal_os_path_join_not_detected(self, tmp_path): + # os.path.join is excluded from repo_sink_patterns to avoid noise + write(tmp_path / "utils.py", ( + "def build_path(base, name):\n" + " return os.path.join(base, name)\n" + )) + sinks = extract_repo_sinks(tmp_path, "PATH_TRAVERSAL") + assert sinks == [] + + def test_ignored_dirs_skipped(self, tmp_path): + write(tmp_path / ".venv" / "lib" / "bad.py", "os.system('x')\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert all(".venv" not in s["file"] for s in sinks) + + def test_snippet_truncated_at_80_chars(self, tmp_path): + long_line = " os.system(" + "x" * 100 + ")" + write(tmp_path / "app.py", f"def f(x):\n{long_line}\n") + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + assert len(sinks[0]["snippet"]) <= 80 + + def test_multiple_methods_in_one_file(self, tmp_path): + write(tmp_path / "multi.py", ( + "def method_a(self, cmd):\n" + " os.system(cmd)\n" + "\n" + "def method_b(self, cmd):\n" + " os.system(cmd + ' extra')\n" + )) + sinks = extract_repo_sinks(tmp_path, "COMMAND_INJECTION") + methods = {s["method"] for s in sinks} + assert "method_a" in methods + assert "method_b" in methods + + +# --------------------------------------------------------------------------- +# _build_sink_table +# --------------------------------------------------------------------------- + + +class TestBuildSinkTable: + def test_empty_input_has_headers_only(self): + table = _build_sink_table([]) + assert "| File |" in table + assert "| Line |" in table + assert "| Method |" in table + assert "| Dangerous call |" in table + assert len(table.splitlines()) == 2 # header + separator + + def test_row_content_present(self): + sinks = [{"file": "app/cmd.py", "line": 42, "method": "execute", "snippet": "os.system(cmd)"}] + table = _build_sink_table(sinks) + assert "app/cmd.py" in table + assert "42" in table + assert "execute" in table + assert "os.system(cmd)" in table + + def test_none_method_shown_as_module(self): + sinks = [{"file": "script.py", "line": 1, "method": None, "snippet": "os.system('x')"}] + table = _build_sink_table(sinks) + assert "" in table + + def test_pipe_in_snippet_escaped(self): + sinks = [{"file": "f.py", "line": 1, "method": "fn", "snippet": "a | b"}] + table = _build_sink_table(sinks) + assert "a \\| b" in table + + def test_multiple_rows(self): + sinks = [ + {"file": "a.py", "line": 10, "method": "foo", "snippet": "os.system(x)"}, + {"file": "b.py", "line": 20, "method": "bar", "snippet": "os.system(y)"}, + ] + table = _build_sink_table(sinks) + lines = table.splitlines() + # header + separator + 2 rows = 4 lines + assert len(lines) == 4 + assert "foo" in table + assert "bar" in table + + +# --------------------------------------------------------------------------- +# Integration: build_vulnerability_pattern_context +# --------------------------------------------------------------------------- + + +class TestBuildContextWithRepoSinks: + def test_table_included_when_sinks_found(self, tmp_path): + write(tmp_path / "app.py", "def handler(cmd):\n os.system(cmd)\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" in result + assert "handler" in result + + def test_table_absent_when_no_sinks(self, tmp_path): + write(tmp_path / "app.py", "def safe(x):\n return x\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" not in result + + def test_table_absent_when_no_repo_root(self): + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=None + ) + assert "Dangerous operation locations" not in result + + def test_line_number_in_output(self, tmp_path): + write(tmp_path / "app.py", ( + "def func(x):\n" + " safe()\n" + " os.system(x)\n" + )) + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", "", repo_root=tmp_path + ) + assert "| 3 |" in result # 1-indexed line 3 + + def test_both_context_checklist_and_repo_table_present(self, tmp_path): + # code_context method needs >= _MIN_METHOD_BODY_LINES (3) lines to + # appear in the context-level sink checklist. + code_ctx = ( + "def context_method(x):\n" + " y = x\n" + " os.system(y)\n" + ) + write(tmp_path / "app.py", "def repo_method(x):\n os.system(x)\n") + result = build_vulnerability_pattern_context( + "OS command injection CWE-78", code_ctx, repo_root=tmp_path + ) + assert "context_method" in result # from context-level scan + assert "Dangerous operation locations" in result # repo table + assert "repo_method" in result # from repo scan + + def test_path_traversal_repo_table_present(self, tmp_path): + write(tmp_path / "server.py", ( + "def serve(self, path):\n" + " full = os.path.normpath(self.root + path)\n" + " return open(full).read()\n" + )) + result = build_vulnerability_pattern_context( + "path traversal CWE-22", "", repo_root=tmp_path + ) + assert "Dangerous operation locations" in result + assert "serve" in result diff --git a/libs/openant-core/tests/patch/test_test_suggester.py b/libs/openant-core/tests/patch/test_test_suggester.py new file mode 100644 index 00000000..40078ca8 --- /dev/null +++ b/libs/openant-core/tests/patch/test_test_suggester.py @@ -0,0 +1,77 @@ +def load_module(): + import utilities.autopatcher.test_suggester as mod + return mod + + +def test_extract_findings_from_bullets_and_sections(): + mod = load_module() + text = """ +Edge cases: +- Database drivers that use `%s` placeholders (driver mismatch) +- Unicode and binary username encodings + +Potential issues: +- Missing unit tests for edge-case payloads +- Assumes `db.execute` accepts parameterized args as shown +""" + findings = mod.extract_findings(text) + # headings/status labels should not be included + assert not any(f.lower().startswith("still vulnerable") for f in findings) + assert not any(f.lower().startswith("edge cases") for f in findings) + assert not any(f.lower().startswith("potential issues") for f in findings) + + # actionable findings should be present + assert any("unicode and binary" in f.lower() for f in findings) + assert any("database drivers" in f.lower() or "%s" in f for f in findings) + assert len(findings) <= 6 + # ensure deterministic ordering: first finding should be the database driver mismatch + assert findings[0] == "Database drivers that use `%s` placeholders (driver mismatch)" + + +def test_suggest_tests_creates_valid_skeletons(): + mod = load_module() + findings = [ + "Unicode and binary username encodings", + "Database drivers that use %s placeholders", + ] + suggestions = mod.suggest_tests(findings) + assert isinstance(suggestions, list) + assert len(suggestions) == 2 + for s, f in zip(suggestions, findings): + assert s["reason"] == f + assert s["name"].startswith("test_") + assert "pytest.mark.skip" in s["code"] + + +def test_limit_findings_to_six(): + mod = load_module() + # create many bullet lines + text = "\n".join(f"- finding {i}" for i in range(10)) + findings = mod.extract_findings(text) + assert len(findings) == 6 + + +def test_suggest_tests_with_behavior_generates_skeletons_and_dedupe(): + mod = load_module() + # Behavior with function name and two primary behaviors + behavior = { + "function": "authenticate", + "file": "app/auth.py", + "summary": "Does authentication", + "primary_behaviors": ["valid login", "invalid login"], + } + + # No initial findings -> behavior-derived tests should appear + suggestions = mod.suggest_tests([], behavior=behavior) + names = [s["name"] for s in suggestions] + assert "test_authenticate_valid_login" in names + assert "test_authenticate_invalid_login" in names + + # Dedupe: if findings already produce same name, behaviour test not duplicated + findings = ["valid login"] + suggestions2 = mod.suggest_tests(findings, behavior=behavior) + names2 = [s["name"] for s in suggestions2] + # only one test for valid_login should exist + assert names2.count("test_valid_login") + names2.count("test_authenticate_valid_login") >= 1 + # ensure behavior marker in code for at least one suggestion + assert any("# Behavior-focused validation" in s["code"] for s in suggestions2) diff --git a/libs/openant-core/tests/patch/test_trust_package.py b/libs/openant-core/tests/patch/test_trust_package.py new file mode 100644 index 00000000..4829dbaa --- /dev/null +++ b/libs/openant-core/tests/patch/test_trust_package.py @@ -0,0 +1,1710 @@ +"""Unit tests for Trust Package V1 helpers. + +Tests cover the four building blocks: + _classify_finding — categorises single challenger finding strings + _classify_challenger — classifies all findings + produces summary counts + _compute_trust_signals — derives the six trust signals deterministically + _build_recommendation_v1 — produces a V1 deployment decision from signals + _extract_security_gain — extracts the benefit sentence from reviewer text + _build_known_findings — groups classified findings into the Known + Findings section's epistemic categories + +All tests use synthetic inputs; no LLM calls are made. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.pipeline import ( + _classify_finding, + _classify_challenger, + _compute_trust_signals, + _build_recommendation_v1, + _extract_security_gain, + _build_known_findings, + _render_known_findings, + _check_recommendation_consistency, + _resolve_impact_level, +) + + +# --------------------------------------------------------------------------- +# _classify_finding +# --------------------------------------------------------------------------- + +class TestClassifyFinding: + def test_confirmed_defect_still_vulnerable(self): + assert _classify_finding("The patch is still vulnerable to the original attack") == "confirmed_defect" + + def test_confirmed_defect_does_not_fix(self): + assert _classify_finding("This does not fix the underlying issue") == "confirmed_defect" + + def test_confirmed_defect_bypass(self): + assert _classify_finding("An attacker can bypass this check via the login endpoint") == "confirmed_defect" + + def test_confirmed_defect_attack_remains(self): + assert _classify_finding("The attack vector remains exploitable through direct API access") == "confirmed_defect" + + def test_validation_gap_cannot_verify(self): + assert _classify_finding("Cannot verify this without running the test suite") == "validation_gap" + + def test_validation_gap_without_testing(self): + assert _classify_finding("The fix cannot be confirmed without testing the redirect scenario") == "validation_gap" + + def test_validation_gap_needs_test(self): + assert _classify_finding("This requires testing to validate the fix") == "validation_gap" + + def test_validation_gap_untested(self): + assert _classify_finding("This path is untested and should be verified") == "validation_gap" + + def test_generic_tests_should(self): + assert _classify_finding("Tests should be added to cover this scenario") == "generic" + + def test_generic_consider_adding(self): + assert _classify_finding("Consider adding logging for audit purposes") == "generic" + + def test_plausible_risk_default(self): + assert _classify_finding("Case-insensitive matching may not handle all header variants") == "plausible_risk" + + def test_plausible_risk_specific_path(self): + assert _classify_finding("Custom Retry configurations that override DEFAULT may not benefit") == "plausible_risk" + + def test_empty_string_returns_generic(self): + assert _classify_finding("") == "generic" + + def test_case_insensitive_matching(self): + assert _classify_finding("STILL VULNERABLE to injection") == "confirmed_defect" + + +# --------------------------------------------------------------------------- +# Scope-marker exclusion for "does not fix/address/prevent/close" +# --------------------------------------------------------------------------- + +class TestFindingClassificationScopeMarker: + """Regression: scope-marker exclusion prevents false Confirmed Defects. + + Findings that say "does not fix/address/prevent/close" alongside a version + reference or scope qualifier should be Plausible Risk (scope limitation). + Those without any scope qualifier remain Confirmed Defect (primary failure). + Explicit exploitability language overrides scope markers in both directions. + """ + + # --- Scope-limited findings → Plausible Risk --- + + def test_does_not_address_v1x_is_plausible_risk(self): + assert _classify_finding( + "does not address the urllib3 v1.x branch" + ) == "plausible_risk" + + def test_does_not_prevent_users_who_override_is_plausible_risk(self): + assert _classify_finding( + "does not prevent users who override defaults" + ) == "plausible_risk" + + def test_does_not_close_for_users_running_version_is_plausible_risk(self): + assert _classify_finding( + "does not close the attack vector for users running 1.26.x" + ) == "plausible_risk" + + def test_does_not_fix_older_versions_is_plausible_risk(self): + assert _classify_finding( + "does not fix the issue for older versions of the library" + ) == "plausible_risk" + + def test_does_not_prevent_when_configured_is_plausible_risk(self): + assert _classify_finding( + "does not prevent the attack when configured with custom retry settings" + ) == "plausible_risk" + + def test_does_not_address_unless_is_plausible_risk(self): + assert _classify_finding( + "does not address this unless users explicitly opt in to the new behaviour" + ) == "plausible_risk" + + def test_does_not_fix_separate_fix_needed_is_plausible_risk(self): + assert _classify_finding( + "does not fix the v1 branch; a separate fix would be required" + ) == "plausible_risk" + + def test_does_not_close_legacy_branch_is_plausible_risk(self): + assert _classify_finding( + "does not close the vulnerability on the legacy branch" + ) == "plausible_risk" + + def test_does_not_prevent_only_when_is_plausible_risk(self): + assert _classify_finding( + "does not prevent exploitation only when the application runs in debug mode" + ) == "plausible_risk" + + # --- Primary-failure findings → Confirmed Defect --- + + def test_does_not_fix_sql_injection_is_confirmed_defect(self): + assert _classify_finding( + "does not fix the SQL injection vulnerability in execute_query()" + ) == "confirmed_defect" + + def test_does_not_prevent_exploitation_is_confirmed_defect(self): + assert _classify_finding( + "does not prevent exploitation through the original endpoint" + ) == "confirmed_defect" + + def test_does_not_address_attack_at_line_is_confirmed_defect(self): + assert _classify_finding( + "does not address the attack at line 47 in the vulnerable function" + ) == "confirmed_defect" + + def test_does_not_close_path_traversal_is_confirmed_defect(self): + assert _classify_finding( + "does not close the path traversal vulnerability" + ) == "confirmed_defect" + + def test_does_not_fix_underlying_issue_is_confirmed_defect(self): + assert _classify_finding( + "does not fix the underlying race condition" + ) == "confirmed_defect" + + # --- Explicit exploitability overrides scope markers --- + + def test_still_vulnerable_with_version_still_confirmed_defect(self): + assert _classify_finding( + "still vulnerable in v1.x and v2.x contexts" + ) == "confirmed_defect" + + def test_still_vulnerable_plain_confirmed_defect(self): + assert _classify_finding("still vulnerable to the original attack") == "confirmed_defect" + + def test_validation_gap_unaffected_by_scope_markers(self): + assert _classify_finding("cannot verify without testing") == "validation_gap" + + +# --------------------------------------------------------------------------- +# Narrowed bypass pattern: active exploit vs architectural observation +# --------------------------------------------------------------------------- + +class TestBypassClassification: + """Regression tests for the narrowed bypass pattern. + + Bare 'bypass' (architectural observation, speculation) → plausible_risk. + 'can bypass' / 'can be bypassed' / 'allows bypass' (active exploit) → confirmed_defect. + """ + + # Previously false positives — should now be Plausible Risk + + def test_architectural_bypass_is_plausible_risk(self): + assert _classify_finding( + "Cookies set via other mechanisms (e.g. adapters or poolmanagers) bypass this entirely" + ) == "plausible_risk" + + def test_may_bypass_conditional_is_plausible_risk(self): + assert _classify_finding( + "headers like `cookie`, `COOKIE`, or `CookIe` may bypass the frozenset check " + "if header comparison isn't case-insensitive" + ) == "plausible_risk" + + def test_might_bypass_is_plausible_risk(self): + assert _classify_finding( + "URL-encoded variants might bypass the header comparison in some configurations" + ) == "plausible_risk" + + def test_plain_bypass_verb_is_plausible_risk(self): + assert _classify_finding( + "Requests using connection-level cookies bypass this stripping logic" + ) == "plausible_risk" + + # Active exploit framing — should remain Confirmed Defect + + def test_can_bypass_is_confirmed_defect(self): + assert _classify_finding( + "An attacker can bypass this check via the login endpoint" + ) == "confirmed_defect" + + def test_can_be_bypassed_is_confirmed_defect(self): + assert _classify_finding( + "The vulnerability can be bypassed by sending a crafted request" + ) == "confirmed_defect" + + def test_could_bypass_is_confirmed_defect(self): + assert _classify_finding( + "An attacker could bypass the authentication with a malformed token" + ) == "confirmed_defect" + + def test_allows_bypass_is_confirmed_defect(self): + assert _classify_finding( + "This flaw allows bypass of the session validation" + ) == "confirmed_defect" + + def test_allows_bypassing_is_confirmed_defect(self): + assert _classify_finding( + "The missing check allows bypassing the access control" + ) == "confirmed_defect" + + # Other explicit defect patterns unaffected by this change + + def test_still_vulnerable_unaffected(self): + assert _classify_finding( + "The patch is still vulnerable to the original attack" + ) == "confirmed_defect" + + def test_attack_vector_remains_unaffected(self): + assert _classify_finding( + "The attack vector remains open via the original endpoint" + ) == "confirmed_defect" + + def test_remains_exploitable_unaffected(self): + assert _classify_finding( + "The injection point remains exploitable after this change" + ) == "confirmed_defect" + + +# --------------------------------------------------------------------------- +# _classify_challenger +# --------------------------------------------------------------------------- + +class TestClassifyChallenger: + def _make(self, still_vulnerable=False, edge_cases=None, potential_issues=None): + return { + "still_vulnerable": still_vulnerable, + "edge_cases": edge_cases or [], + "potential_issues": potential_issues or [], + "summary": "Test summary", + } + + def test_empty_challenger_zero_counts(self): + result = _classify_challenger(self._make()) + assert result["confirmed_defect_count"] == 0 + assert result["plausible_risk_count"] == 0 + assert result["validation_gap_count"] == 0 + + def test_confirmed_defect_increments_count(self): + result = _classify_challenger(self._make( + edge_cases=["attack vector remains exploitable"] + )) + assert result["confirmed_defect_count"] == 1 + assert result["plausible_risk_count"] == 0 + + def test_validation_gap_increments_count(self): + result = _classify_challenger(self._make( + potential_issues=["cannot verify without running tests"] + )) + assert result["validation_gap_count"] == 1 + assert result["confirmed_defect_count"] == 0 + + def test_mixed_findings_counted_independently(self): + result = _classify_challenger(self._make( + edge_cases=["still vulnerable via header", "cannot verify without testing"], + potential_issues=["case mismatch may occur"], + )) + assert result["confirmed_defect_count"] == 1 + assert result["validation_gap_count"] == 1 + assert result["plausible_risk_count"] == 1 + + def test_original_fields_preserved(self): + challenger = self._make(still_vulnerable=True) + result = _classify_challenger(challenger) + assert result["still_vulnerable"] is True + assert result["summary"] == "Test summary" + + def test_classified_lists_have_text_and_category(self): + result = _classify_challenger(self._make( + edge_cases=["cannot verify without running tests"] + )) + items = result["classified_edge_cases"] + assert len(items) == 1 + assert "text" in items[0] and "category" in items[0] + assert items[0]["category"] == "validation_gap" + + def test_none_challenger_returns_empty_counts(self): + result = _classify_challenger(None) + assert result.get("confirmed_defect_count", 0) == 0 + + +# --------------------------------------------------------------------------- +# _compute_trust_signals +# --------------------------------------------------------------------------- + +def _clean_applicability(): + return {"applicable": True, "skipped": False, "skipped_reason": None, "error": None, "stderr": ""} + +def _skip_applicability(): + return {"applicable": None, "skipped": True, "skipped_reason": "no .git directory", "error": None, "stderr": ""} + +def _fail_applicability(stderr="error: patch does not apply"): + return {"applicable": False, "skipped": False, "skipped_reason": None, "error": None, "stderr": stderr} + +def _error_applicability(error="git apply --check timed out after 10s"): + # F-23: timeout and unexpected-exception outcomes share this exact shape + # (applicable=None, skipped=False, error=) — distinct from the + # pre-flight skip fixture above, but policy-equivalent per I1. + return {"applicable": None, "skipped": False, "skipped_reason": None, "error": error, "stderr": ""} + +def _classified(still_vulnerable=False, defects=0, risks=0, gaps=0): + c = { + "still_vulnerable": still_vulnerable, + "confirmed_defect_count": defects, + "plausible_risk_count": risks, + "validation_gap_count": gaps, + "classified_edge_cases": [], + "classified_potential_issues": [], + } + # Populate classified lists to match counts + for _ in range(defects): + c["classified_edge_cases"].append({"text": "still vulnerable", "category": "confirmed_defect"}) + for _ in range(risks): + c["classified_edge_cases"].append({"text": "plausible edge case", "category": "plausible_risk"}) + for _ in range(gaps): + c["classified_potential_issues"].append({"text": "cannot verify without tests", "category": "validation_gap"}) + return c + + +class TestComputeTrustSignals: + + # Patch Integrity + def test_integrity_clean_when_no_issues(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Clean" + + def test_integrity_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Does Not Apply" + + def test_integrity_not_verified_when_skipped(self): + s = _compute_trust_signals([], _skip_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Not Verified" + + def test_integrity_not_verified_when_applicability_errors_or_times_out(self): + """F-23: an applicability timeout/exception (applicable=None, + skipped=False) must read as Not Verified, never fall through to + Clean — it is unavailable evidence, not evidence of a clean apply.""" + s = _compute_trust_signals([], _error_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Not Verified" + assert s["patch_integrity"]["value"] != "Clean" + + def test_integrity_critical_when_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Critical Issues" + + def test_integrity_minor_when_medium_hygiene_only(self): + hygiene = [{"severity": "MEDIUM", "check": "unused_import", "detail": "Unused import"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["patch_integrity"]["value"] == "Minor Issues" + + # Security Improvement + def test_improvement_none_when_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "None" + + def test_improvement_unknown_when_skipped(self): + s = _compute_trust_signals([], _skip_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "Unknown" + + def test_improvement_unknown_when_applicability_errors_or_times_out(self): + """F-23: same timeout/exception state must read as Unknown security + improvement, never High/Medium — an unresolved check is not evidence + the patch improved security, no matter what the challenger found. + (still_vulnerable defaults to False in _classified(), which is + exactly the case that previously fell through to "High" — the + strongest false-positive of F-23.)""" + s = _compute_trust_signals([], _error_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "Unknown" + assert s["security_improvement"]["value"] not in ("High", "Medium", "Low") + + def test_improvement_low_when_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["security_improvement"]["value"] == "Low" + + def test_improvement_low_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, defects=1), "None", "low") + assert s["security_improvement"]["value"] == "Low" + + def test_improvement_high_when_not_still_vulnerable(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=False), "None", "low") + assert s["security_improvement"]["value"] == "High" + + def test_improvement_high_when_only_validation_gaps(self): + # still_vulnerable=True but only gaps (no confirmed defects, no plausible risks) + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, gaps=2), "None", "low") + assert s["security_improvement"]["value"] == "High" + + def test_improvement_medium_when_plausible_risks_present(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, risks=1), "None", "low") + assert s["security_improvement"]["value"] == "Medium" + + # Remediation Alignment + def test_alignment_misaligned_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, defects=1), "None", "low") + assert s["remediation_alignment"]["value"] == "Misaligned" + + def test_alignment_aligned_when_not_still_vulnerable(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=False), "None", "low") + assert s["remediation_alignment"]["value"] == "Aligned" + + def test_alignment_likely_aligned_when_only_gaps(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, gaps=2), "None", "low") + assert s["remediation_alignment"]["value"] == "Likely Aligned" + + def test_alignment_partial_when_plausible_risks(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(still_vulnerable=True, risks=1), "None", "low") + assert s["remediation_alignment"]["value"] == "Partial" + + # Coverage Confidence + def test_coverage_high_when_no_findings(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["coverage_confidence"]["value"] == "High" + + def test_coverage_medium_when_gaps(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(gaps=2), "None", "low") + assert s["coverage_confidence"]["value"] == "Medium" + + def test_coverage_medium_when_plausible_risks(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(risks=1), "None", "low") + assert s["coverage_confidence"]["value"] == "Medium" + + def test_coverage_low_when_confirmed_defect(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(defects=1), "None", "low") + assert s["coverage_confidence"]["value"] == "Low" + + # Test Availability + def test_test_availability_tests_available_for_good(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Good", "low") + assert s["test_availability"]["value"] == "Tests Available" + + def test_test_availability_tests_available_for_some(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Some", "low") + assert s["test_availability"]["value"] == "Tests Available" + + def test_test_availability_no_tests_for_none(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["test_availability"]["value"] == "No Tests Found" + + # Deployment Safety + def test_safety_low_risk_for_low_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert s["deployment_safety"]["value"] == "Low Risk" + + def test_safety_medium_risk_for_medium_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "medium") + assert s["deployment_safety"]["value"] == "Medium Risk" + + def test_safety_high_risk_for_high_impact(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "high") + assert s["deployment_safety"]["value"] == "High Risk" + + def test_safety_high_risk_for_high_hygiene(self): + hygiene = [{"severity": "HIGH", "check": "empty_hunk", "detail": "Empty hunk"}] + s = _compute_trust_signals(hygiene, _clean_applicability(), _classified(), "None", "low") + assert s["deployment_safety"]["value"] == "High Risk" + + # Composite: urllib3-representative inputs + def test_urllib3_representative_signals(self): + """All six signals for a well-grounded correct urllib3-style patch.""" + classified = _classified(still_vulnerable=True, risks=1, gaps=1) + s = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + assert s["patch_integrity"]["value"] == "Clean" + assert s["security_improvement"]["value"] == "Medium" + assert s["remediation_alignment"]["value"] == "Partial" + assert s["coverage_confidence"]["value"] == "Medium" + assert s["test_availability"]["value"] == "No Tests Found" + assert s["deployment_safety"]["value"] == "Low Risk" + + # Labels contain icons + def test_labels_contain_icon_for_clean(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "low") + assert "✓" in s["patch_integrity"]["label"] + + def test_labels_contain_icon_for_does_not_apply(self): + s = _compute_trust_signals([], _fail_applicability(), _classified(), "None", "low") + assert "✗" in s["patch_integrity"]["label"] + + # ---- Language guardrail: "Not Applicable" / "not_applicable" must not + # collapse into the same reassuring labels as a genuine clean finding. ---- + + def test_test_availability_not_applicable_is_not_verified_not_no_tests_found(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Not Applicable", "low") + assert s["test_availability"]["value"] == "Not Verified" + assert s["test_availability"]["value"] != "No Tests Found" + assert s["test_availability"]["value"] != "Tests Available" + + def test_deployment_safety_not_applicable_is_not_verified_not_low_risk(self): + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "not_applicable") + assert s["deployment_safety"]["value"] == "Not Verified" + assert s["deployment_safety"]["value"] != "Low Risk" + + def test_deployment_safety_not_verified_when_impact_unavailable(self): + """F-24: impact_level="unavailable" (impact analysis raised and was + swallowed — see _resolve_impact_level) must never read as Low Risk. + A crashed analysis is not evidence of low regression risk.""" + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "unavailable") + assert s["deployment_safety"]["value"] == "Not Verified" + assert s["deployment_safety"]["value"] != "Low Risk" + + def test_deployment_safety_not_verified_for_unrecognized_impact_level(self): + """Whitelist, not blacklist: any impact_level string that isn't one + of the known-good {low, medium, high} must fall to Not Verified, + not to the old permissive default.""" + s = _compute_trust_signals([], _clean_applicability(), _classified(), "None", "totally-malformed") + assert s["deployment_safety"]["value"] == "Not Verified" + + def test_non_python_repo_signals_never_read_as_clean(self): + # Simulates the actual values a non-Python (e.g. curl/C) run produces: + # testing_rating="Not Applicable" from score_test_support, and + # impact_level="not_applicable" from LightweightImpactAnalyzer. + s = _compute_trust_signals([], _clean_applicability(), _classified(), "Not Applicable", "not_applicable") + assert s["test_availability"]["value"] not in ("Tests Available", "No Tests Found") + assert s["deployment_safety"]["value"] not in ("Low Risk", "Medium Risk", "High Risk") + assert s["test_availability"]["value"] == "Not Verified" + assert s["deployment_safety"]["value"] == "Not Verified" + + +# --------------------------------------------------------------------------- +# _resolve_impact_level +# --------------------------------------------------------------------------- + +class TestResolveImpactLevel: + def test_none_impact_is_unavailable(self): + """F-24: result.impact is None only when impact analysis raised and + was swallowed by pipeline.run()'s bare except — must resolve to the + distinct "unavailable" sentinel, never silently to "low".""" + assert _resolve_impact_level(None) == "unavailable" + + def test_present_dict_reads_impact_level(self): + assert _resolve_impact_level({"impact_level": "high"}) == "high" + + def test_present_dict_missing_key_defaults_to_low(self): + # A real analyzer result always includes impact_level (see + # ImpactReport.to_dict()); this only guards a malformed dict, which + # is not the same failure mode as impact=None. + assert _resolve_impact_level({}) == "low" + + +# --------------------------------------------------------------------------- +# _build_recommendation_v1 +# --------------------------------------------------------------------------- + +def _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Low Risk"): + return { + "patch_integrity": {"value": integrity, "label": integrity, "notes": ""}, + "security_improvement": {"value": improvement, "label": improvement, "notes": ""}, + "remediation_alignment": {"value": alignment, "label": alignment, "notes": ""}, + "coverage_confidence": {"value": "High", "label": "High", "notes": ""}, + "test_availability": {"value": "No Tests Found", "label": "No Tests Found", "notes": ""}, + "deployment_safety": {"value": safety, "label": safety, "notes": ""}, + } + + +# --------------------------------------------------------------------------- +# Trust Signals v2 — question-style rendering (display only) +# --------------------------------------------------------------------------- + +def _signals_full(**overrides): + """All six keys with a neutral 'good' baseline, so tests only need to + override the one signal they're checking.""" + base = { + "patch_integrity": {"value": "Clean", "label": "Clean", "notes": "Applies cleanly · no hygiene issues"}, + "security_improvement": {"value": "High", "label": "High", "notes": "Adversarial review found no remaining exploit path"}, + "remediation_alignment": {"value": "Aligned", "label": "Aligned", "notes": "Adversarial review confirms fix approach"}, + "coverage_confidence": {"value": "High", "label": "High", "notes": "No gaps identified by adversarial analysis"}, + "test_availability": {"value": "Tests Available", "label": "Tests Available", "notes": "Good — test files cover this module"}, + "deployment_safety": {"value": "Low Risk", "label": "Low Risk", "notes": "Localized change · low regression risk"}, + } + base.update(overrides) + return base + + +class TestTrustSignalsV2Table: + """Rendering-only tests for the redesigned Trust Signals table. None of + these touch _compute_trust_signals or _build_recommendation_v1.""" + + def test_heading_unchanged(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "## Trust Signals\n" in table + + def test_question_rows_present(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + for question in [ + "Does the patch apply?", + "Does it address the vulnerability?", + "Are there unresolved concerns?", + "Do relevant tests already exist?", + "Is deployment risk low?", + ]: + assert question in table + + def test_security_improvement_not_displayed_as_a_row(self): + """Requirement 3: overlapping aggregate signals must not appear as + peer rows. security_improvement must not surface anywhere in the + rendered table, even though it's still computed and still read by + _build_recommendation_v1 elsewhere.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "Security Improvement" not in table + assert "security_improvement" not in table + + def test_old_icon_vocabulary_does_not_leak_through(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + for old_icon in ["✓", "✗", "◑", "○"]: + assert old_icon not in table + + def test_patch_integrity_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Clean": "✅ Good", + "Minor Issues": "⚠️ Needs review", + "Critical Issues": "❌ Blocked", + "Does Not Apply": "❌ Blocked", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + patch_integrity={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Does the patch apply?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_remediation_alignment_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Aligned": "✅ Good", + "Likely Aligned": "⚠️ Needs review", + "Partial": "⚠️ Needs review", + "Misaligned": "❌ Blocked", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + remediation_alignment={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Does it address")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_coverage_confidence_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = {"High": "✅ Good", "Medium": "⚠️ Needs review", "Low": "❌ Blocked"} + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + coverage_confidence={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Are there unresolved")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_test_availability_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Tests Available": "✅ Good", + "No Tests Found": "⚠️ Needs review", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + test_availability={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Do relevant tests already exist?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_deployment_safety_status_mapping(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + expected = { + "Low Risk": "✅ Good", + "Medium Risk": "⚠️ Needs review", + "High Risk": "❌ Blocked", + "Not Verified": "? Not verified", + } + for value, status in expected.items(): + table = _render_trust_signals_table(_signals_full( + deployment_safety={"value": value, "label": value, "notes": ""} + )) + row = [l for l in table.splitlines() if l.startswith("| Is deployment risk low?")][0] + assert status in row, f"{value} -> expected {status!r} in row: {row!r}" + + def test_coverage_medium_note_points_to_review_results_section(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + coverage_confidence={ + "value": "Medium", "label": "Medium", + "notes": "12 review finding(s) · no deterministic blocker identified", + } + )) + assert "see Review Results section below" in table + assert "see analysis below" not in table + + def test_notes_text_otherwise_preserved(self): + """Requirement 5: the underlying computed notes text is not altered + beyond the one dangling-reference substitution.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + assert "Applies cleanly · no hygiene issues" in table + assert "Adversarial review confirms fix approach" in table + assert "No gaps identified by adversarial analysis" in table + assert "Good — test files cover this module" in table + assert "Localized change · low regression risk" in table + + def test_good_rows_get_no_forward_pointer(self): + """A row that's already ✅ Good has nothing to send the reader to — + except the testing row, which always carries its Existing-Test- + Coverage-vs-Missing-Behavioral-Validation bridge regardless of + status (see test_testing_row_bridge_present_regardless_of_status).""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + rows = [l for l in table.splitlines() if l.startswith("| ") and "Do relevant tests" not in l] + for row in rows: + assert "see" not in row.lower(), f"Unexpected forward pointer in a good row: {row!r}" + + def test_testing_row_bridge_present_regardless_of_status(self): + """The Existing Test Coverage vs Missing Behavioral Validation bridge + must appear whether or not test coverage is good — a "✅ Good" status + here must never look like it also answers whether the new behavior + itself is validated.""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full()) + row = [l for l in table.splitlines() if l.startswith("| Do relevant tests already exist?")][0] + assert "Review Results" in row + # Points at Review Results as a whole, not a specific subsection — + # verified against a real challenger run that "no test validates X" + # findings don't reliably classify as validation_gap (they can land + # in Behavior Notes instead), so naming one subsection here would + # risk pointing at an empty section. + assert "Validation Gaps" not in row + + def test_non_good_patch_integrity_points_to_patch_applicability(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + patch_integrity={"value": "Does Not Apply", "label": "Does Not Apply", "notes": "rejected by git apply"} + )) + assert "see Patch Applicability section below" in table + + def test_non_good_test_availability_points_to_test_support(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + test_availability={"value": "No Tests Found", "label": "No Tests Found", "notes": "No test files cover this module"} + )) + assert "see Test Support section below" in table + + def test_non_good_deployment_safety_points_to_impact_surface(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table(_signals_full( + deployment_safety={"value": "High Risk", "label": "High Risk", "notes": "HIGH impact surface"} + )) + assert "see Impact Surface section below" in table + + def test_non_good_remediation_alignment_points_to_review_results_when_rendered(self): + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table( + _signals_full(remediation_alignment={"value": "Partial", "label": "Partial", "notes": "still_vulnerable flag set"}), + known_findings_rendered=True, + ) + assert "see Review Results section below" in table + + def test_remediation_alignment_pointer_suppressed_when_review_results_empty(self): + """Edge case: remediation_alignment can be non-good ("Likely Aligned") + while no relevant Review Results category is populated. The pointer + must not be shown in that case — never reference a section that + won't exist. (The testing row's own Review Results bridge is + independent of this gate and is checked separately.)""" + from utilities.autopatcher.pipeline import _render_trust_signals_table + table = _render_trust_signals_table( + _signals_full(remediation_alignment={ + "value": "Likely Aligned", "label": "Likely Aligned", + "notes": "Correct mechanism · runtime verification pending", + }), + known_findings_rendered=False, + ) + row = [l for l in table.splitlines() if l.startswith("| Does it address the vulnerability?")][0] + assert "see Review Results" not in row + + +class TestRecommendationV1: + def test_do_not_apply_for_does_not_apply_integrity(self): + rec = _build_recommendation_v1(_signals_for(integrity="Does Not Apply")) + assert rec["decision"] == "Do Not Apply" + + def test_do_not_apply_for_critical_issues(self): + rec = _build_recommendation_v1(_signals_for(integrity="Critical Issues")) + assert rec["decision"] == "Do Not Apply" + + def test_misaligned_alone_is_manual_review_not_do_not_apply(self): + """Recommendation Policy v2: alignment=Misaligned is heuristic-only evidence + (challenger-derived confirmed_defect_count) and must not hard-block by itself.""" + rec = _build_recommendation_v1(_signals_for(alignment="Misaligned")) + assert rec["decision"] == "Manual Review Required" + # Reason must not state the exploit as a confirmed fact. + assert "confirmed exploit path" not in rec["reason"].lower() + + def test_deploy_after_validation_high_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="High", safety="Low Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_after_validation_medium_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Medium", safety="Low Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_after_validation_medium_improvement_medium_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Medium", safety="Medium Risk")) + assert rec["decision"] == "Deploy After Validation" + + def test_deploy_after_validation_never_fires_with_not_verified_safety(self): + """F-37: the gate is a whitelist (I3) — "Not Verified" safety must + never satisfy it, regardless of improvement. Prior to the fix the + gate was `safety != "High Risk"`, which admitted this exact case.""" + rec = _build_recommendation_v1(_signals_for(improvement="High", safety="Not Verified")) + assert rec["decision"] != "Deploy After Validation" + assert rec["decision"] == "Manual Review Required" + + def test_unknown_improvement_never_satisfies_deploy_after_validation(self): + """I3 invariant preservation (NOT an F-37 regression case — this + already passed before the F-37 fix, since `improvement in ("High", + "Medium")` was already whitelist-shaped pre-fix; only the `safety` + side was ever blacklist-shaped). Kept as documentation that + "Unknown" improvement must never satisfy the gate, even paired with + an otherwise-positive safety value.""" + rec = _build_recommendation_v1(_signals_for(improvement="Unknown", safety="Low Risk")) + assert rec["decision"] != "Deploy After Validation" + assert rec["decision"] == "Manual Review Required" + + def test_deploy_after_validation_never_fires_with_not_verified_integrity(self): + """Boundary found in final review: _build_recommendation_v1 only + ever reads `integrity` in the Do Not Apply gate above — nothing + re-checked it here before this fix. A signals dict with + integrity="Not Verified" (applicability unavailable) but otherwise + fully positive improvement/safety must still be blocked; in the + live pipeline this combination can't occur (both are derived from + the same _unavailable_reason in _compute_trust_signals), but + _build_recommendation_v1 takes an arbitrary signals dict and must + not rely on that upstream coupling to stay safe.""" + rec = _build_recommendation_v1(_signals_for(integrity="Not Verified", improvement="High", safety="Low Risk")) + assert rec["decision"] != "Deploy After Validation" + assert rec["decision"] == "Manual Review Required" + + def test_deploy_after_validation_never_fires_with_minor_issues_integrity(self): + """"Minor Issues" reports an actual observed hygiene defect + (currently only "unused_import" — see patch_hygiene.py), not a + verified-clean patch — it must not satisfy the positive-integrity + allowlist even though it doesn't hard-block via _BLOCKING_INTEGRITY. + Not being blocked is not the same claim as being positive evidence.""" + rec = _build_recommendation_v1(_signals_for(integrity="Minor Issues", improvement="High", safety="Low Risk")) + assert rec["decision"] != "Deploy After Validation" + assert rec["decision"] == "Manual Review Required" + + def test_deploy_with_caution_low_improvement_low_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Low", safety="Low Risk")) + assert rec["decision"] == "Deploy With Caution" + + def test_manual_review_required_high_risk(self): + rec = _build_recommendation_v1(_signals_for(improvement="Low", safety="High Risk")) + assert rec["decision"] == "Manual Review Required" + + def test_recommendation_has_reason(self): + rec = _build_recommendation_v1(_signals_for()) + assert len(rec["reason"]) > 20 + + def test_urllib3_representative_deploy_after_validation(self): + """A well-grounded correct patch with medium coverage → Deploy After Validation.""" + classified = _classified(still_vulnerable=True, risks=1, gaps=1) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1(signals) + assert rec["decision"] == "Deploy After Validation" + + +# --------------------------------------------------------------------------- +# Recommendation Policy boundary table — every row is traceable to one of +# I1-I6 (see the Recommendation Policy Invariants block in pipeline.py, +# directly above _compute_trust_signals). still_vulnerable/defect_count are +# passed straight through to _build_recommendation_v1, independent of the +# signals dict, matching that function's own signature/contract. +# --------------------------------------------------------------------------- + +_POLICY_TABLE = [ + # (integrity, alignment, still_vulnerable, defect_count, improvement, safety) -> decision + ("Clean", "Aligned", False, 0, "High", "Low Risk", "Deploy After Validation"), + ("Clean", "Aligned", False, 0, "Medium", "Medium Risk", "Deploy After Validation"), + ("Clean", "Aligned", False, 0, "Low", "Low Risk", "Deploy With Caution"), + ("Does Not Apply", "Aligned", False, 0, "None", "Low Risk", "Do Not Apply"), + ("Critical Issues", "Aligned", False, 0, "Low", "Low Risk", "Do Not Apply"), + ("Clean", "Misaligned", False, 0, "High", "Low Risk", "Manual Review Required"), + ("Clean", "Aligned", True, 0, "High", "Low Risk", "Manual Review Required"), + ("Clean", "Aligned", False, 0, "High", "High Risk", "Manual Review Required"), + # F-23 boundary: applicability UNAVAILABLE (skip or error/timeout) always + # yields integrity=Not Verified / improvement=Unknown; neither may reach + # Deploy After Validation regardless of safety. + ("Not Verified", "Aligned", False, 0, "Unknown", "Low Risk", "Manual Review Required"), + ("Not Verified", "Aligned", False, 0, "Unknown", "Medium Risk", "Manual Review Required"), + # F-24 boundary: impact analysis UNAVAILABLE always yields + # deployment_safety=Not Verified; must never reach Deploy After + # Validation even with strong improvement evidence. + ("Clean", "Aligned", False, 0, "High", "Not Verified", "Manual Review Required"), + ("Clean", "Aligned", False, 0, "Medium", "Not Verified", "Manual Review Required"), + # F-37 boundary at the gate itself: Not Verified/Unknown on either axis + # never satisfies the whitelist, independent of how they were produced. + ("Clean", "Aligned", False, 0, "Unknown", "Not Verified", "Manual Review Required"), + # Integrity boundary found in final review: a decoupled signals dict + # (integrity=Not Verified paired directly with a positive improvement/ + # safety, bypassing the coupling _compute_trust_signals normally + # guarantees) must still be blocked — integrity is a mandatory gate too. + ("Not Verified", "Aligned", False, 0, "High", "Low Risk", "Manual Review Required"), + # "Minor Issues" reports a real hygiene defect, not verified-clean — + # excluded from the positive-integrity allowlist even though it clears + # the Do Not Apply gate. + ("Minor Issues", "Aligned", False, 0, "High", "Low Risk", "Manual Review Required"), +] + + +class TestRecommendationPolicyBoundaries: + @pytest.mark.parametrize( + "integrity,alignment,still_vulnerable,defect_count,improvement,safety,expected", + _POLICY_TABLE, + ) + def test_policy_boundary( + self, integrity, alignment, still_vulnerable, defect_count, improvement, safety, expected + ): + signals = _signals_for( + integrity=integrity, improvement=improvement, alignment=alignment, safety=safety + ) + rec = _build_recommendation_v1( + signals, still_vulnerable=still_vulnerable, defect_count=defect_count + ) + assert rec["decision"] == expected + + +# --------------------------------------------------------------------------- +# Evaluation case regressions — still_vulnerable guard +# --------------------------------------------------------------------------- + +class TestRecommendationV1EvaluationCases: + """Phase A regression suite for the still_vulnerable hard-block guard. + + Each test represents the definitive signal state observed in the evaluation + run for that case and asserts the expected recommendation before and after + the guard was introduced. Tests named *_unchanged confirm no regression. + """ + + # --- GitPython: the bug case --- + + def test_still_vulnerable_no_defects_produces_manual_review(self): + """still_vulnerable=True with no confirmed defects → Manual Review Required. + + Signals: integrity=Clean, improvement=Medium, alignment=Partial, + safety=Low Risk, still_vulnerable=True, defect_count=0. + The LLM binary verdict alone is insufficient for a hard-block; + the escalation tier asks a human to review the plausible risks. + """ + signals = _signals_for( + integrity="Clean", + improvement="Medium", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + def test_gitpython_reason_names_challenger(self): + """Escalation reason must mention the challenger, not applicability.""" + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert "challenger" in rec["reason"].lower() or "review" in rec["reason"].lower() + + def test_gitpython_before_guard_would_have_passed(self): + """Confirm the pre-fix path: same signals without still_vulnerable → Deploy After Validation.""" + signals = _signals_for( + integrity="Clean", + improvement="Medium", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- urllib3: must not regress --- + + def test_urllib3_unchanged(self): + """urllib3: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- pip: must not regress --- + + def test_pip_unchanged(self): + """pip: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Medium Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- minimist: must not regress --- + + def test_minimist_unchanged(self): + """minimist v3: still_vulnerable=False, improvement=High → Deploy After Validation unchanged.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Deploy After Validation" + + # --- pygeoapi: integrity check takes priority over still_vulnerable --- + + def test_pygeoapi_integrity_fires_before_still_vulnerable(self): + """pygeoapi: Does Not Apply integrity fires before still_vulnerable guard.""" + signals = _signals_for( + integrity="Does Not Apply", + improvement="None", + alignment="Partial", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True) + assert rec["decision"] == "Do Not Apply" + assert "does not apply" in rec["reason"].lower() or "critical" in rec["reason"].lower() + + def test_pygeoapi_do_not_apply_without_still_vulnerable(self): + """pygeoapi: Do Not Apply fires on integrity alone when still_vulnerable=False.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=False) + assert rec["decision"] == "Do Not Apply" + + # --- Guard ordering --- + + def test_still_vulnerable_does_not_fire_when_integrity_fails(self): + """Reason text should reflect integrity failure, not still_vulnerable, when both are true.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Partial") + rec_integrity = _build_recommendation_v1(signals, still_vulnerable=False) + rec_both = _build_recommendation_v1(signals, still_vulnerable=True) + assert rec_integrity["decision"] == rec_both["decision"] == "Do Not Apply" + assert rec_integrity["reason"] == rec_both["reason"] + + def test_still_vulnerable_no_defects_escalates_regardless_of_improvement(self): + """still_vulnerable=True with defect_count=0 escalates even when other signals are clean.""" + signals = _signals_for( + integrity="Clean", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + +# --------------------------------------------------------------------------- +# _extract_security_gain +# --------------------------------------------------------------------------- + +class TestExtractSecurityGain: + def test_extracts_sentence_with_fix_verb(self): + explanation = "The vulnerability exists because the header is not stripped. The patch fixes this by adding Cookie to the removal list." + gain = _extract_security_gain(explanation) + assert "patch" in gain.lower() or "fix" in gain.lower() + assert len(gain) >= 40 + + def test_extracts_sentence_with_prevent_verb(self): + explanation = "Sensitive tokens were logged. This patch prevents token values from appearing in log output." + gain = _extract_security_gain(explanation) + assert "prevent" in gain.lower() + + def test_extracts_sentence_with_add_verb(self): + explanation = "Cookie was not stripped. The fix adds Cookie to DEFAULT_REMOVE_HEADERS_ON_REDIRECT alongside Authorization." + gain = _extract_security_gain(explanation) + assert "add" in gain.lower() or "Cookie" in gain + + def test_skips_short_sentences(self): + # A sentence with a verb but too short should be skipped + explanation = "Bug found. The patch fixes it. Here is why the fix adds significant protection against cross-origin leakage of credentials." + gain = _extract_security_gain(explanation) + assert len(gain) >= 40 + + def test_fallback_to_first_paragraph(self): + explanation = "This change modifies the retry behavior." + gain = _extract_security_gain(explanation) + assert len(gain) > 0 + + def test_empty_explanation_returns_empty(self): + assert _extract_security_gain("") == "" + + +# --------------------------------------------------------------------------- +# _build_known_findings / _render_known_findings +# --------------------------------------------------------------------------- + +class TestBuildKnownFindings: + """_build_known_findings regroups the same four classifier categories + into epistemic-state labels — no new classification, no change to the + counts _compute_trust_signals/_build_recommendation_v1 read. + + plausible_risk/generic findings are further split three ways by the + (optional) finding_calibration stage output — evidence-quality pass — + with a conservative fallback (plausible_risk -> Validation Hypotheses, + generic -> Future Hardening Ideas) when no calibration entry exists for + a given finding, so a calibration failure degrades gracefully instead of + losing information.""" + + def _classified_with(self, defects=(), risks=(), gaps=(), generic=()): + classified_edge = [] + classified_issues = [] + for d in defects: + classified_edge.append({"text": d, "category": "confirmed_defect"}) + for r in risks: + classified_edge.append({"text": r, "category": "plausible_risk"}) + for g in gaps: + classified_issues.append({"text": g, "category": "validation_gap"}) + for gen in generic: + classified_issues.append({"text": gen, "category": "generic"}) + return { + "classified_edge_cases": classified_edge, + "classified_potential_issues": classified_issues, + "confirmed_defect_count": len(defects), + "plausible_risk_count": len(risks), + "validation_gap_count": len(gaps), + } + + def test_confirmed_defects_map_to_potential_remaining_risks(self): + cc = self._classified_with(defects=["confirmed issue"]) + findings = _build_known_findings(cc) + assert findings["potential_remaining_risks"] == ["confirmed issue"] + + def test_validation_gaps_map_to_validation_gaps(self): + cc = self._classified_with(gaps=["gap1"]) + findings = _build_known_findings(cc) + assert findings["validation_gaps"] == ["gap1"] + + def test_plausible_risk_without_calibration_falls_back_to_hypotheses(self): + cc = self._classified_with(risks=["plausible risk"]) + findings = _build_known_findings(cc, finding_calibration=None) + assert findings["validation_hypotheses"] == ["plausible risk"] + assert findings["observed_implementation_notes"] == [] + assert findings["future_hardening_ideas"] == [] + + def test_generic_without_calibration_falls_back_to_hardening(self): + cc = self._classified_with(generic=["tests should be added"]) + findings = _build_known_findings(cc, finding_calibration=None) + assert findings["future_hardening_ideas"] == ["tests should be added"] + + def test_calibration_routes_finding_to_its_assigned_group(self): + cc = self._classified_with(risks=["case normalization detail"]) + calibration = [{ + "original": "case normalization detail", + "group": "observed", + "reworded": "The constructor normalizes header casing via h.lower().", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["observed_implementation_notes"] == [ + "The constructor normalizes header casing via h.lower()." + ] + assert findings["validation_hypotheses"] == [] + + def test_calibration_can_move_a_finding_out_of_its_classifier_bucket(self): + """A plausible_risk-classified finding can still land in Future + Hardening Ideas if calibration judges it unrelated to the advisory — + the classifier bucket only decides which findings are eligible for + calibration, not the final presentation group.""" + cc = self._classified_with(risks=["Proxy-Authorization header not stripped"]) + calibration = [{ + "original": "Proxy-Authorization header not stripped", + "group": "hardening", + "reworded": "Proxy-Authorization is not covered by this advisory; a separate hardening improvement.", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["future_hardening_ideas"] == [ + "Proxy-Authorization is not covered by this advisory; a separate hardening improvement." + ] + assert findings["validation_hypotheses"] == [] + + def test_missing_calibration_entry_for_one_finding_still_falls_back(self): + """Calibration covering only some findings must not drop the rest — + each uncovered finding still gets its conservative default.""" + cc = self._classified_with(risks=["covered finding", "uncovered finding"]) + calibration = [{ + "original": "covered finding", "group": "observed", "reworded": "Covered, reworded.", + }] + findings = _build_known_findings(cc, finding_calibration=calibration) + assert findings["observed_implementation_notes"] == ["Covered, reworded."] + assert findings["validation_hypotheses"] == ["uncovered finding"] + + def test_validation_gaps_capped_at_three(self): + cc = self._classified_with(gaps=["g1", "g2", "g3", "g4", "g5"]) + findings = _build_known_findings(cc) + assert len(findings["validation_gaps"]) <= 3 + + def test_empty_classified_challenger_returns_all_empty(self): + cc = self._classified_with() + findings = _build_known_findings(cc) + assert all(v == [] for v in findings.values()) + + def test_signature_takes_optional_calibration(self): + """Unlike the old _build_known_limitations(cc, coverage_value), this + function takes classified_challenger plus an optional + finding_calibration — gating on coverage is the renderer/caller's + job, and calibration is optional so a calibration failure doesn't + change this function's contract.""" + import inspect + params = inspect.signature(_build_known_findings).parameters + assert list(params) == ["classified_challenger", "finding_calibration"] + assert params["finding_calibration"].default is None + + +class TestRenderKnownFindings: + def _findings(self, risks=(), gaps=(), observed=(), hypotheses=(), hardening=()): + return { + "potential_remaining_risks": list(risks), + "validation_gaps": list(gaps), + "observed_implementation_notes": list(observed), + "validation_hypotheses": list(hypotheses), + "future_hardening_ideas": list(hardening), + } + + def test_empty_findings_render_nothing(self): + assert _render_known_findings(self._findings()) == "" + + def test_bullet_count_disclaimer_present_when_rendered(self): + """Reviewer-experience fix: the number of bullets below this heading + must not read as a count of confirmed defects — a standing + disclaimer states this once, directly under the heading.""" + block = _render_known_findings(self._findings(risks=["r1"])) + assert "not a count of confirmed defects" in block.lower() + + def test_bullet_count_disclaimer_absent_when_empty(self): + assert "confirmed defects" not in _render_known_findings(self._findings()).lower() + + def test_heading_and_subheadings_present(self): + block = _render_known_findings(self._findings( + risks=["r1"], gaps=["g1"], observed=["o1"], hypotheses=["h1"], hardening=["f1"], + )) + assert "## Review Results" in block + assert "### Potential Remaining Risks" in block + assert "### Validation Gaps" in block + assert "### Confirmed Observations" in block + assert "### Validation Questions" in block + assert "### Future Improvements" in block + assert "r1" in block and "g1" in block and "o1" in block and "h1" in block and "f1" in block + + def test_potential_remaining_risks_labeled_as_heuristic_not_confirmed(self): + """Correction: confirmed_defect findings must not be presented as + confirmed facts — the challenger is heuristic LLM analysis.""" + block = _render_known_findings(self._findings(risks=["r1"])) + assert "heuristic" in block.lower() + assert "**Confirmed" not in block + assert "confirmed gap" not in block.lower() + + def test_observed_implementation_notes_labeled_as_repository_backed(self): + block = _render_known_findings(self._findings(observed=["h.lower() normalizes casing"])) + assert "### Confirmed Observations" in block + assert "repository evidence" in block.lower() + + def test_validation_hypotheses_labeled_as_unconfirmed(self): + """Correction: hypotheses must read as conditional reasoning, not + observed behavior.""" + block = _render_known_findings(self._findings(hypotheses=["same-origin redirects may also strip Cookie"])) + assert "### Validation Questions" in block + assert "not directly observed" in block.lower() + assert "not confirmed outcomes" in block.lower() + + def test_hardening_ideas_do_not_reduce_confidence(self): + block = _render_known_findings(self._findings(hardening=["Proxy-Authorization header"])) + assert "### Future Improvements" in block + assert "do not reduce confidence" in block.lower() + + def test_validation_gaps_bridge_to_test_coverage(self): + """Correction: Validation Gaps must clarify it's independent of + whether the repo already has pre-existing tests for this module.""" + block = _render_known_findings(self._findings(gaps=["no test validates the new behavior"])) + assert "Trust Signals" in block + + def test_only_populated_subsections_render(self): + block = _render_known_findings(self._findings(gaps=["g1"])) + assert "### Validation Gaps" in block + assert "### Potential Remaining Risks" not in block + assert "### Confirmed Observations" not in block + assert "### Validation Questions" not in block + assert "### Future Improvements" not in block + + def test_future_hardening_ideas_render_alone(self): + """The section must render even when only hardening ideas are + present — this is the previously-fully-discarded generic bucket.""" + block = _render_known_findings(self._findings(hardening=["consider adding a comment"])) + assert "## Review Results" in block + assert "### Future Improvements" in block + assert "consider adding a comment" in block + + +# --------------------------------------------------------------------------- +# _check_recommendation_consistency (Slice 1 — Decision Consistency) +# --------------------------------------------------------------------------- + +class TestRecommendationConsistency: + """Unit tests for the Slice 1 consistency check. + + Goal: a top-tier recommendation must acknowledge, in its own text, any + already-displayed evidence (test availability, decision-relevant open + findings) that runs against it. This function never changes the decision + itself — these tests only assert on the caveat list it returns. + + The coverage-related caveat is deliberately NOT driven by + coverage_confidence's own value/notes: Coverage Confidence answers "how + thoroughly did we explore?" (Future Hardening Ideas count there, + unchanged) while this caveat answers "should this recommendation be + discounted?" (Future Hardening Ideas must not count here). It computes + its own decision-relevant count from `known_findings` instead. + """ + + def _signals(self, test_availability="Tests Available"): + return { + "test_availability": { + "value": test_availability, "label": test_availability, + "notes": "No test files cover this module" if test_availability == "No Tests Found" else "some notes", + }, + # coverage_confidence is realistic fixture data only — this + # function no longer reads it at all. + "coverage_confidence": { + "value": "Medium", "label": "Medium", + "notes": "11 review finding(s) · no deterministic blocker identified", + }, + } + + def _known_findings(self, risks=0, gaps=0, observed=0, hypotheses=0, hardening=0): + return { + "potential_remaining_risks": [f"risk {i}" for i in range(risks)], + "validation_gaps": [f"gap {i}" for i in range(gaps)], + "observed_implementation_notes": [f"observed {i}" for i in range(observed)], + "validation_hypotheses": [f"hypothesis {i}" for i in range(hypotheses)], + "future_hardening_ideas": [f"hardening {i}" for i in range(hardening)], + } + + # --- Decision gating: only top-tier decisions are checked at all --- + + def test_no_caveat_for_manual_review_required(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=2) + assert _check_recommendation_consistency(signals, "Manual Review Required", findings) == [] + + def test_no_caveat_for_do_not_apply(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=2) + assert _check_recommendation_consistency(signals, "Do Not Apply", findings) == [] + + # --- Test Availability condition --- + + def test_caveat_for_no_tests_found_at_top_tier(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "test coverage" in caveats[0].lower() + + def test_no_caveat_when_tests_available_at_top_tier(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_not_verified_is_not_treated_as_no_tests_found(self): + """Language-guardrail 'Not Verified' means the check didn't run, not + that tests are confirmed absent — must not trip the same caveat.""" + signals = self._signals(test_availability="Not Verified") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + # --- Decision-relevant findings condition --- + + def test_caveat_when_decision_relevant_findings_open(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hypotheses=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "adversarial coverage" in caveats[0].lower() + + def test_no_caveat_when_no_findings_at_all(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings() + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_hardening_only_findings_produce_no_caveat(self): + """The core fix: Future Hardening Ideas are real findings (they still + count toward Coverage Confidence and appear in Known Findings) but + must not, on their own, make this caveat fire — they are not reasons + to distrust this deployment recommendation.""" + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hardening=5) + assert _check_recommendation_consistency(signals, "Deploy After Validation", findings) == [] + + def test_hardening_findings_excluded_from_caveat_count(self): + """Mixed case: decision-relevant findings trigger the caveat, but its + wording counts only those — hardening findings present alongside + must not inflate the number shown.""" + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(hypotheses=2, hardening=7) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 1 + assert "2 decision-relevant finding(s)" in caveats[0] + assert "7" not in caveats[0] + + def test_confirmed_risks_and_gaps_also_count_as_decision_relevant(self): + signals = self._signals(test_availability="Tests Available") + findings = self._known_findings(risks=1, gaps=1, observed=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert "3 decision-relevant finding(s)" in caveats[0] + + # --- Both conditions at once --- + + def test_both_caveats_when_both_weak_at_top_tier(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings(hypotheses=1) + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert len(caveats) == 2 + + # --- Deploy With Caution is also top-tier --- + + def test_deploy_with_caution_is_also_checked(self): + """Deploy With Caution is reachable through _build_recommendation_v1's + API even though today's _compute_trust_signals output cannot produce + it in practice (see benchmark notes) — the check must still cover it + defensively rather than assume it will never be seen.""" + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy With Caution", findings) + assert len(caveats) == 1 + + # --- Wording reuses already-displayed evidence, not new judgment --- + + def test_caveat_reuses_displayed_notes_text(self): + signals = self._signals(test_availability="No Tests Found") + findings = self._known_findings() + caveats = _check_recommendation_consistency(signals, "Deploy After Validation", findings) + assert "No test files cover this module" in caveats[0] + + # --- Defensive: missing keys never raise --- + + def test_missing_signal_keys_do_not_raise(self): + assert _check_recommendation_consistency({}, "Deploy After Validation", {}) == [] + + +class TestDecisionRelevantFindingCount: + """Unit tests for the small pure helper the consistency check now uses + instead of reusing coverage_confidence's value.""" + + def test_excludes_future_hardening_ideas(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + findings = { + "potential_remaining_risks": [], "validation_gaps": [], + "observed_implementation_notes": [], "validation_hypotheses": [], + "future_hardening_ideas": ["a", "b", "c"], + } + assert _decision_relevant_finding_count(findings) == 0 + + def test_counts_all_other_categories(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + findings = { + "potential_remaining_risks": ["r"], "validation_gaps": ["g"], + "observed_implementation_notes": ["o"], "validation_hypotheses": ["h1", "h2"], + "future_hardening_ideas": ["ignored"], + } + assert _decision_relevant_finding_count(findings) == 5 + + def test_missing_keys_default_to_empty(self): + from utilities.autopatcher.pipeline import _decision_relevant_finding_count + assert _decision_relevant_finding_count({}) == 0 + + +# --------------------------------------------------------------------------- +# Hybrid blocking policy +# --------------------------------------------------------------------------- + +class TestHybridBlockingPolicy: + """Regression suite for the hybrid blocking policy (Recommendation Policy v2). + + Hard-block → integrity failure only (deterministic evidence) + Escalation → still_vulnerable=True, confirmed_defect_count == 0 + OR confirmed_defect_count > 0 (alignment=Misaligned) — both heuristic-only, + both land at Manual Review Required, never Do Not Apply + Forward → still_vulnerable=False, confirmed_defect_count == 0 + + Benchmark expectations + ---------------------- + urllib3 : still_vulnerable=True, defect_count=0, risks=12 → Manual Review Required + minimist : still_vulnerable=False, defect_count=0 → Deploy After Validation + pip : still_vulnerable=False, defect_count=0 → Deploy After Validation + pygeoapi : integrity=Does Not Apply → Do Not Apply + GitPython : still_vulnerable=True, defect_count=1 → Manual Review Required (via alignment) + """ + + # --- urllib3: still_vulnerable=True, defect_count=0 → escalation --- + + def test_urllib3_still_vulnerable_no_defects_is_manual_review(self): + """urllib3 regression: plausible risks with no confirmed defect → Manual Review Required.""" + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial", safety="Low Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert rec["decision"] == "Manual Review Required" + + def test_urllib3_full_signals_manual_review(self): + """End-to-end urllib3 signal path via _compute_trust_signals.""" + classified = _classified(still_vulnerable=True, risks=12) + signals = _compute_trust_signals([], _clean_applicability(), classified, "Good", "high") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + def test_urllib3_escalation_reason_mentions_challenger(self): + signals = _signals_for(integrity="Clean", improvement="Medium", alignment="Partial") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=0) + assert "challenger" in rec["reason"].lower() + + # --- GitPython v2: defect_count=1 → alignment=Misaligned → hard-block --- + + def test_gitpython_confirmed_defect_is_manual_review_required(self): + """GitPython v2: one confirmed defect drives alignment=Misaligned → Manual Review + Required. Per Recommendation Policy v2, heuristic evidence (challenger-derived + confirmed_defect_count) escalates to human review, it does not hard-block.""" + classified = _classified(still_vulnerable=True, defects=1, risks=13) + signals = _compute_trust_signals([], _clean_applicability(), classified, "Some", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + assert signals["remediation_alignment"]["value"] == "Misaligned" + + def test_gitpython_escalation_survives_still_vulnerable_false(self): + """Contradictory case: LLM says No but a finding classifies as confirmed_defect. + defect_count > 0 drives alignment=Misaligned and escalates to Manual Review + Required regardless of still_vulnerable — still never Do Not Apply on its own.""" + classified = _classified(still_vulnerable=False, defects=1) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + # --- pygeoapi: integrity failure fires before any challenger signal --- + + def test_pygeoapi_integrity_hard_block(self): + """pygeoapi: Does Not Apply integrity blocks regardless of challenger state.""" + signals = _signals_for(integrity="Does Not Apply", improvement="None", alignment="Misaligned") + rec = _build_recommendation_v1(signals, still_vulnerable=True, defect_count=1) + assert rec["decision"] == "Do Not Apply" + assert "does not apply" in rec["reason"].lower() or "critical" in rec["reason"].lower() + + # --- minimist / pip: clean path --- + + def test_minimist_clean_path_deploy_after_validation(self): + """minimist: still_vulnerable=False, defect_count=0 → Deploy After Validation.""" + signals = _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Low Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Deploy After Validation" + + def test_pip_clean_path_deploy_after_validation(self): + """pip: still_vulnerable=False, defect_count=0, medium risk → Deploy After Validation.""" + signals = _signals_for(integrity="Clean", improvement="High", alignment="Aligned", safety="Medium Risk") + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Deploy After Validation" + + # --- Boundary: still_vulnerable=True with defect_count>0 also hard-blocks --- + + def test_both_still_vulnerable_and_defects_is_manual_review_required(self): + """When still_vulnerable=True AND defect_count>0, alignment=Misaligned fires → + Manual Review Required (heuristic evidence escalates, it does not hard-block).""" + classified = _classified(still_vulnerable=True, defects=2, risks=5) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + # --- Boundary: still_vulnerable=True with only gaps (risk_count=0) --- + + def test_still_vulnerable_gaps_only_is_manual_review(self): + """still_vulnerable=True with only validation gaps (no plausible risks, no defects) + also hits the escalation branch.""" + classified = _classified(still_vulnerable=True, gaps=3) + signals = _compute_trust_signals([], _clean_applicability(), classified, "None", "low") + rec = _build_recommendation_v1( + signals, + still_vulnerable=classified["still_vulnerable"], + defect_count=classified["confirmed_defect_count"], + ) + assert rec["decision"] == "Manual Review Required" + + +# --------------------------------------------------------------------------- +# Recommendation Policy v2 — explicit policy statement as a regression test +# --------------------------------------------------------------------------- + +class TestPureHeuristicEvidenceNeverBlocks: + """Recommendation Policy v2: pure heuristic evidence must never produce Do Not Apply. + + Do Not Apply may only be produced by deterministic evidence — currently that + means `patch_integrity` (git-apply / static hygiene). Every other input to + `_build_recommendation_v1` (`security_improvement`, `remediation_alignment`, + `still_vulnerable`, `defect_count`) is derived entirely from the adversarial + challenger's free-text output and is therefore heuristic, not deterministic + (see docs/recommendation-policy-v2.md). This test holds `patch_integrity` at + its cleanest, non-blocking value and sweeps every value the heuristic + signals can take — including the worst case on every axis at once — and + asserts none of them, alone or combined, ever reach Do Not Apply. + + This expresses the policy itself, not one input combination: if a future + change reintroduces a path from heuristic evidence to Do Not Apply, this + test fails regardless of which heuristic signal caused it. + """ + + _ALIGNMENT_VALUES = ["Misaligned", "Partial", "Likely Aligned", "Aligned"] + _IMPROVEMENT_VALUES = ["None", "Low", "Medium", "High"] + _SAFETY_VALUES = ["Low Risk", "Medium Risk", "High Risk"] + _DEFECT_COUNTS = (0, 1, 5) + + def test_pure_heuristic_evidence_never_produces_do_not_apply(self): + for alignment in self._ALIGNMENT_VALUES: + for improvement in self._IMPROVEMENT_VALUES: + for safety in self._SAFETY_VALUES: + for still_vulnerable in (True, False): + for defect_count in self._DEFECT_COUNTS: + signals = _signals_for( + integrity="Clean", # the one deterministic control variable + improvement=improvement, + alignment=alignment, + safety=safety, + ) + rec = _build_recommendation_v1( + signals, + still_vulnerable=still_vulnerable, + defect_count=defect_count, + ) + assert rec["decision"] != "Do Not Apply", ( + "Pure heuristic evidence produced Do Not Apply with " + f"integrity=Clean, alignment={alignment!r}, " + f"improvement={improvement!r}, safety={safety!r}, " + f"still_vulnerable={still_vulnerable}, " + f"defect_count={defect_count}" + ) + + def test_integrity_is_still_the_only_path_to_do_not_apply(self): + """Sanity check the sweep isn't vacuous: deterministic integrity failure + must still produce Do Not Apply even with every heuristic signal clean.""" + signals = _signals_for( + integrity="Does Not Apply", + improvement="High", + alignment="Aligned", + safety="Low Risk", + ) + rec = _build_recommendation_v1(signals, still_vulnerable=False, defect_count=0) + assert rec["decision"] == "Do Not Apply" diff --git a/libs/openant-core/tests/patch/test_vulnerability_patterns.py b/libs/openant-core/tests/patch/test_vulnerability_patterns.py new file mode 100644 index 00000000..cd3c9a70 --- /dev/null +++ b/libs/openant-core/tests/patch/test_vulnerability_patterns.py @@ -0,0 +1,252 @@ +"""Tests for src/vulnerability_patterns.py + +Validates classification, sink scanning, and context building. +No references to specific project names (pygeoapi, gitpython, etc.). +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + + +from utilities.autopatcher.vulnerability_patterns import ( + build_vulnerability_pattern_context, + classify_vuln_class, + find_sink_methods, +) + + +# --------------------------------------------------------------------------- +# classify_vuln_class +# --------------------------------------------------------------------------- + + +class TestClassifyVulnClass: + def test_cwe22_returns_path_traversal(self): + assert classify_vuln_class("This advisory tracks CWE-22") == "PATH_TRAVERSAL" + + def test_cwe23_returns_path_traversal(self): + assert classify_vuln_class("Classified as CWE-23 path traversal") == "PATH_TRAVERSAL" + + def test_path_traversal_keyword(self): + assert classify_vuln_class( + "A path traversal vulnerability in the file serve endpoint." + ) == "PATH_TRAVERSAL" + + def test_directory_traversal_keyword(self): + assert classify_vuln_class( + "directory traversal allows reading /etc/passwd" + ) == "PATH_TRAVERSAL" + + def test_cwe78_returns_command_injection(self): + assert classify_vuln_class( + "CWE-78 OS command injection via user input" + ) == "COMMAND_INJECTION" + + def test_cwe88_returns_command_injection(self): + assert classify_vuln_class( + "Argument injection (CWE-88) in the build runner." + ) == "COMMAND_INJECTION" + + def test_command_injection_keyword(self): + assert classify_vuln_class( + "An OS command injection flaw in the exec handler." + ) == "COMMAND_INJECTION" + + def test_shell_injection_keyword(self): + assert classify_vuln_class( + "Shell injection vulnerability in the build step." + ) == "COMMAND_INJECTION" + + def test_unrelated_cwe_returns_none(self): + assert classify_vuln_class( + "Prototype pollution in merge utility (CWE-1321)" + ) is None + + def test_empty_string_returns_none(self): + assert classify_vuln_class("") is None + + def test_unrecognized_text_returns_none(self): + assert classify_vuln_class("Generic advisory with no class indicators.") is None + + def test_cwe_takes_priority_over_keyword(self): + # CWE-78 is explicit; "path traversal" keyword is also present — CWE wins. + text = "CWE-78 injection triggered via a crafted path traversal string" + assert classify_vuln_class(text) == "COMMAND_INJECTION" + + +# --------------------------------------------------------------------------- +# find_sink_methods +# --------------------------------------------------------------------------- + +_PATH_CONTEXT = ( + "def safe_method(self, path):\n" + " return open(path).read()\n" + "\n" + "def vulnerable_serve(self, dirpath):\n" + " data_path = self.data + dirpath\n" + " return open(data_path).read()\n" + "\n" + "def also_vulnerable(self, filename):\n" + " full = os.path.normpath(self.base + filename)\n" + " return full\n" +) + + +class TestFindSinkMethods: + def test_finds_both_vulnerable_methods(self): + hits = find_sink_methods(_PATH_CONTEXT, "PATH_TRAVERSAL") + assert "vulnerable_serve" in hits + assert "also_vulnerable" in hits + + def test_does_not_include_safe_method(self): + hits = find_sink_methods(_PATH_CONTEXT, "PATH_TRAVERSAL") + assert "safe_method" not in hits + + def test_none_class_returns_empty(self): + assert find_sink_methods(_PATH_CONTEXT, None) == [] + + def test_empty_context_returns_empty(self): + assert find_sink_methods("", "PATH_TRAVERSAL") == [] + + def test_no_sinks_returns_empty(self): + context = ( + "def clean_method(self, x):\n" + " return str(x)\n" + "\n" + "def another_clean(self, y):\n" + " return y.strip()\n" + "\n" + ) + assert find_sink_methods(context, "PATH_TRAVERSAL") == [] + + def test_sink_in_comment_not_counted(self): + context = ( + "def documented_method(self, path):\n" + " # old: os.path.normpath(self.base + path) was used here\n" + " return os.path.realpath(path)\n" + "\n" + "def unrelated(self):\n" + " return 42\n" + ) + hits = find_sink_methods(context, "PATH_TRAVERSAL") + assert "documented_method" not in hits + + def test_command_injection_shell_true(self): + context = ( + "def run_build(self, cmd):\n" + " subprocess.call(cmd, shell=True)\n" + "\n" + "def clean_runner(self, args):\n" + " subprocess.run(args)\n" + "\n" + ) + hits = find_sink_methods(context, "COMMAND_INJECTION") + assert "run_build" in hits + assert "clean_runner" not in hits + + def test_unknown_class_returns_empty(self): + assert find_sink_methods(_PATH_CONTEXT, "SQL_INJECTION") == [] + + +# --------------------------------------------------------------------------- +# build_vulnerability_pattern_context +# --------------------------------------------------------------------------- + + +class TestBuildVulnerabilityPatternContext: + def test_path_traversal_returns_nonempty(self): + result = build_vulnerability_pattern_context( + "Path traversal vulnerability in file server (CWE-22)", "" + ) + assert result != "" + + def test_command_injection_returns_nonempty(self): + result = build_vulnerability_pattern_context( + "OS command injection via user-controlled input (CWE-78)", "" + ) + assert result != "" + + def test_unknown_class_returns_empty(self): + result = build_vulnerability_pattern_context( + "Prototype pollution in lodash merge (CWE-1321)", "" + ) + assert result == "" + + def test_path_traversal_contains_realpath(self): + result = build_vulnerability_pattern_context("path traversal issue", "") + assert "realpath" in result.lower() + + def test_path_traversal_names_normpath_antipattern(self): + result = build_vulnerability_pattern_context("path traversal issue", "") + assert "normpath" in result.lower() + assert "avoid" in result.lower() + + def test_command_injection_contains_shell_true_warning(self): + result = build_vulnerability_pattern_context( + "command injection via shell=True", "" + ) + assert "shell=True" in result or "shell = True" in result + + def test_command_injection_contains_avoid_section(self): + result = build_vulnerability_pattern_context("OS command injection CWE-78", "") + assert "avoid" in result.lower() + + def test_sink_methods_appear_in_output(self): + context = ( + "def serve_file(self, dirpath):\n" + " path = self.base + dirpath\n" + " return open(path).read()\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "serve_file" in result + + def test_sink_checklist_instruction_present(self): + context = ( + "def file_handler(self, path):\n" + " p = self.root + path\n" + " return p\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + lower = result.lower() + assert "checklist" in lower or "every listed" in lower or "must address" in lower + + def test_no_sinks_no_sink_section(self): + context = ( + "def clean_method(self, x):\n" + " return str(x)\n" + "\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "Methods in the provided code" not in result + + def test_repo_root_none_does_not_raise(self): + result = build_vulnerability_pattern_context( + "path traversal CWE-22", "", repo_root=None + ) + assert isinstance(result, str) + + def test_bad_vuln_text_returns_empty(self): + result = build_vulnerability_pattern_context(None, "") + assert result == "" + + def test_output_starts_with_guidance_header(self): + result = build_vulnerability_pattern_context("path traversal CWE-22", "") + assert result.startswith("## Vulnerability class guidance") + + def test_multiple_sink_methods_all_listed(self): + context = ( + "def handler_a(self, dirpath):\n" + " p = self.base + dirpath\n" + " return p\n" + "\n" + "def handler_b(self, name):\n" + " q = os.path.normpath(self.root + name)\n" + " return q\n" + ) + result = build_vulnerability_pattern_context("path traversal CWE-22", context) + assert "handler_a" in result + assert "handler_b" in result diff --git a/libs/openant-core/tests/test_patch_eligible_verdicts.py b/libs/openant-core/tests/test_patch_eligible_verdicts.py new file mode 100644 index 00000000..f685d71a --- /dev/null +++ b/libs/openant-core/tests/test_patch_eligible_verdicts.py @@ -0,0 +1,33 @@ +"""Pins core.verdict_taxonomy.PATCH_ELIGIBLE, the eligibility filter used by +core/patch.py to decide whether a finding may be sent to the patch-trust +pipeline. + +Deliberately a distinct set from DISCLOSURE_ELIGIBLE and DYNAMIC_TESTABLE -- +see the constant's docstring in core/verdict_taxonomy.py for why. +""" + +import sys +from pathlib import Path + +_CORE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(_CORE_ROOT)) + +from core.verdict_taxonomy import PATCH_ELIGIBLE, DISCLOSURE_ELIGIBLE, DYNAMIC_TESTABLE + + +def test_patch_eligible_exact_membership(): + assert PATCH_ELIGIBLE == frozenset({"confirmed", "agreed", "vulnerable", "bypassable"}) + + +def test_patch_eligible_excludes_non_vulnerability_verdicts(): + for verdict in ("unverified", "error", "rejected", "safe", "protected", "inconclusive"): + assert verdict not in PATCH_ELIGIBLE + + +def test_patch_eligible_is_its_own_set_not_an_alias(): + """PATCH_ELIGIBLE must differ from both DISCLOSURE_ELIGIBLE (broader -- + includes unverified/error) and DYNAMIC_TESTABLE (narrower -- excludes + bypassable). Guards against someone "simplifying" it into an alias.""" + assert PATCH_ELIGIBLE != DISCLOSURE_ELIGIBLE + assert PATCH_ELIGIBLE != DYNAMIC_TESTABLE + assert "bypassable" in PATCH_ELIGIBLE and "bypassable" not in DYNAMIC_TESTABLE diff --git a/libs/openant-core/utilities/autopatcher/__init__.py b/libs/openant-core/utilities/autopatcher/__init__.py new file mode 100644 index 00000000..6ed1eeab --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/__init__.py @@ -0,0 +1,7 @@ +"""Auto Patcher's patch-generation and trust-scoring engine, merged into OpenAnt. + +Ported from the standalone auto-patcher-mvp project so end users don't need a +separate checkout, interpreter, or virtualenv. Entry point: :func:`pipeline.run`. +See ``core/patch.py`` for the thin OpenAnt-side wrapper (finding lookup, +eligibility, artifact paths) that calls into this package. +""" diff --git a/libs/openant-core/utilities/autopatcher/behavior_summary.py b/libs/openant-core/utilities/autopatcher/behavior_summary.py new file mode 100644 index 00000000..9cd57162 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/behavior_summary.py @@ -0,0 +1,80 @@ +"""Minimal deterministic behavior summary analyzer. + +Produces a tiny BehaviorReport dict with a single function (or file fallback), +one-line summary, and 2-4 primary behaviors to validate. +""" +from __future__ import annotations + +import re +from typing import List + + +def _pick_purpose_and_behaviors(name_tokens: List[str], path_tokens: List[str]): + tokens = set(t.lower() for t in name_tokens + path_tokens) + + # Simple deterministic mapping + if any(t in tokens for t in ("auth", "authenticate", "login", "logout")): + purpose = "authentication" + behaviors = ["valid login", "invalid login", "malformed/injection-style input"] + elif any(t in tokens for t in ("validate", "sanitize", "clean", "normalize")): + purpose = "input validation" + behaviors = ["valid input acceptance", "invalid/malformed input rejection"] + elif any(t in tokens for t in ("db", "query", "execute", "insert", "update", "delete", "cursor")): + purpose = "database operations" + behaviors = ["query correctness", "parameterization vs raw queries"] + elif any(t in tokens for t in ("api", "request", "handler", "route", "view")): + purpose = "request/handler logic" + behaviors = ["happy-path response", "error handling paths"] + else: + purpose = "application logic" + behaviors = ["normal flow", "edge-case handling"] + + # Limit to 4 + return purpose, behaviors[:4] + + +class BehaviorAnalyzer: + """Very small deterministic analyzer for Behavior Summary. + + analyze(patch_diff, repo_context=None) -> dict + """ + + # Accept diff markers like '+' at line start (e.g. '+def foo(') + FUNC_RE = re.compile(r"^[\+\-\s]*def\s+([A-Za-z0-9_]+)\s*\(") + + def analyze(self, patch_diff: str, repo_context=None) -> dict: + # 1) find first changed file + file_path = "unknown" + for line in patch_diff.splitlines(): + if line.startswith("+++ b/"): + file_path = line[6:].strip() + break + + # 2) find first function definition in diff hunks + func_name = "" + for line in patch_diff.splitlines(): + m = self.FUNC_RE.match(line) + if m: + func_name = m.group(1) + break + + # derive tokens + name_tokens = func_name.replace("_", " ").split() if func_name else [] + path_tokens = [p for p in re.split(r"[/_.\\]+", file_path) if p] + + purpose, behaviors = _pick_purpose_and_behaviors(name_tokens, path_tokens) + + if func_name: + summary = f"This patch likely affects {purpose} in {file_path}." + else: + summary = f"This patch likely affects {purpose} in {file_path}." + + # limit behaviors to 2-4 deterministic choices + primary_behaviors = behaviors[:4] + + return { + "function": func_name, + "file": file_path, + "summary": summary, + "primary_behaviors": primary_behaviors, + } diff --git a/libs/openant-core/utilities/autopatcher/candidate_enrichment.py b/libs/openant-core/utilities/autopatcher/candidate_enrichment.py new file mode 100644 index 00000000..2ac31ba3 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/candidate_enrichment.py @@ -0,0 +1,489 @@ +"""Deterministic candidate enrichment. + +Attaches ``CandidateEnrichment`` metadata (see +``repository_grounding_models.py``) to selected ``RepositoryCandidate`` +objects, using only existing OpenAnt parser, call-graph, reachability, and +test-discovery capabilities. No LLM calls anywhere in this module, and no +vulnerability judgement is made -- this is deterministic repository fact +gathering only, meant to run *before* any model is asked to reason about +the vulnerability. + +``build_investigation_context()`` performs the one repo-wide, real-parse +step (``core.parser_adapter.parse_repository``) and assembles the shared, +read-only artifacts every candidate's enrichment reuses -- the same +read-call_graph.json / EntryPointDetector / ReachabilityAnalyzer sequence +``core.parser_adapter.apply_reachability_filter`` already performs +internally, reused here for enrichment instead of dataset filtering. + +``enrich_candidates()`` is pure orchestration over an already-built +``InvestigationContext`` (or ``None``): it never parses anything itself, +which keeps it trivially testable with hand-built fixtures. + +Enrichment is intentionally depth-1 only: callers/callees are recorded as +names, never recursively re-enriched into their own ``CandidateEnrichment`` +objects. Expanding further is the same "generic full-repository scan" +failure mode this whole design exists to avoid, relocated to the graph +level instead of eliminated. +""" + +from __future__ import annotations + +import ast +import os +from dataclasses import dataclass, field +from pathlib import Path + +from core.parser_adapter import parse_repository +from utilities.agentic_enhancer.entry_point_detector import EntryPointDetector +from utilities.agentic_enhancer.reachability_analyzer import ReachabilityAnalyzer +from utilities.agentic_enhancer.repository_index import RepositoryIndex, load_index_from_file +from utilities.autopatcher import testing_support, vulnerability_patterns +from utilities.autopatcher.candidate_selection import CandidateSelection +from utilities.autopatcher.repository_grounding_models import ( + CandidateEnrichment, + RepositoryCandidate, +) +from utilities.file_io import read_json + + +@dataclass +class InvestigationContext: + """Shared, repo-wide artifacts built once per patch run from a single + ``parse_repository()`` call. Nothing here is candidate-specific -- + every candidate's enrichment reads from this read-only.""" + + index: RepositoryIndex + call_graph: dict + reverse_call_graph: dict + reachability: ReachabilityAnalyzer + # file_path -> {qualified_name -> literal-assignment record}. Populated + # once here (see _collect_repo_constants below), covering every Python + # file RepositoryIndex already knows about (i.e. every file containing + # at least one function/class) -- a file with zero functions/classes is + # never scanned, an accepted gap for a "smallest extension" (see + # build_investigation_context's docstring). Default {} keeps existing + # hand-built test fixtures (which construct this dataclass without the + # field) working unchanged. + constants: dict = field(default_factory=dict) + + +_LITERAL_WRAPPER_CTORS = {"frozenset": frozenset, "set": set, "list": list, "tuple": tuple, "dict": dict} +_BASE_LITERAL_NODE_TYPES = (ast.Set, ast.List, ast.Tuple, ast.Dict, ast.Constant, ast.UnaryOp) + + +def _canonicalize_literal(value: object) -> object: + """Recursively convert an ast.literal_eval() result into a hashable + form, so it can live inside a frozen, hashable Anchor/AnchorValue + (mutable set/list/dict are not hashable). Shape-preserving: a set-like + input always canonicalizes to frozenset, list-like to tuple, dict to a + sorted tuple of (key, value) pairs -- recursively, so nested literal + containers are handled too.""" + if isinstance(value, dict): + return tuple(sorted( + (_canonicalize_literal(k), _canonicalize_literal(v)) for k, v in value.items() + )) + if isinstance(value, (set, frozenset)): + return frozenset(_canonicalize_literal(v) for v in value) + if isinstance(value, (list, tuple)): + return tuple(_canonicalize_literal(v) for v in value) + return value # str/int/float/bool/bytes/None/complex -- already hashable + + +def _evaluate_literal_rhs(node: "ast.AST") -> "tuple[str, str | None, object]": + """Evaluate one assignment's RHS deterministically. Returns + ``(outcome, ast_literal_kind, value)``. ``outcome`` is ``"literal"`` or + ``"non_literal"`` -- never guessed, never partially evaluated (e.g. a + dict literal with one non-literal value fails whole, never yields a + partial dict). + + Supports bare literal expressions (Set/List/Tuple/Dict/Constant, or + UnaryOp of one -- e.g. ``-1``) directly via ``ast.literal_eval``, plus + exactly one call-shaped exception: a zero-or-one-argument call to + ``frozenset``/``set``/``list``/``tuple``/``dict`` wrapping a bare + literal (e.g. ``frozenset(["Authorization"])``) -- the exact shape + urllib3's own ``DEFAULT_REMOVE_HEADERS_ON_REDIRECT`` declaration uses. + Any other call, attribute access, name reference, or expression is + ``"non_literal"``. + """ + ctor_name: "str | None" = None + literal_node = node + + if isinstance(node, ast.Call): + func = node.func + if not ( + isinstance(func, ast.Name) + and func.id in _LITERAL_WRAPPER_CTORS + and not node.keywords + and len(node.args) <= 1 + ): + return "non_literal", None, None + ctor_name = func.id + if not node.args: + try: + return "literal", f"{ctor_name}_call", _canonicalize_literal(_LITERAL_WRAPPER_CTORS[ctor_name]()) + except Exception: + return "non_literal", None, None + literal_node = node.args[0] + + if not isinstance(literal_node, _BASE_LITERAL_NODE_TYPES): + return "non_literal", None, None + + try: + raw = ast.literal_eval(literal_node) + except Exception: + return "non_literal", None, None + + if ctor_name is not None: + try: + raw = _LITERAL_WRAPPER_CTORS[ctor_name](raw) + except Exception: + return "non_literal", None, None + return "literal", f"{ctor_name}_call", _canonicalize_literal(raw) + + return "literal", type(literal_node).__name__, _canonicalize_literal(raw) + + +def _extract_literal_constants(file_text: str) -> "dict[str, dict]": + """Walk module-level and class-level simple assignments in ``file_text`` + and return ``{qualified_name: record}`` -- ``qualified_name`` is the + bare name at module scope, ``"ClassName.name"`` at class scope, matching + ``RepositoryIndex``'s own func_id convention. + + Only ``Assign``/``AnnAssign`` nodes with a single ``ast.Name`` target, + at module level or directly inside a class body, are considered -- + mirroring ``impact_surface.py``'s own module/class-scope restriction. + Multi-target (``a = b = {...}``) and destructuring/attribute targets + are excluded here entirely (not "non_literal" -- they're not a + single-name assignment at all). Augmented assignment (``X |= {...}``) + and annotation-only (``x: int``, no RHS) are recorded with their own + explicit outcome, never a guessed/fabricated value. + + Returns ``{}`` when ``file_text`` does not parse as Python -- callers + must treat that as "no constants available", never guess. + """ + try: + tree = ast.parse(file_text) + except (SyntaxError, ValueError): + return {} + + result: "dict[str, dict]" = {} + + def _record(name: str, class_name: "str | None", line: int, end_line: int, + outcome: str, kind: "str | None", value: object) -> None: + qualified = f"{class_name}.{name}" if class_name else name + result[qualified] = { + "qualified_name": qualified, + "class_name": class_name, + "name": name, + "outcome": outcome, + "ast_literal_kind": kind, + "value": value, + "line": line, + "end_line": end_line, + } + + def _handle(node: "ast.Assign | ast.AnnAssign | ast.AugAssign", class_name: "str | None") -> None: + end_line = getattr(node, "end_lineno", node.lineno) or node.lineno + if isinstance(node, ast.AugAssign): + if isinstance(node.target, ast.Name): + _record(node.target.id, class_name, node.lineno, end_line, "augmented_assign", None, None) + return + targets = node.targets if isinstance(node, ast.Assign) else [node.target] + if len(targets) != 1 or not isinstance(targets[0], ast.Name): + return # multi-target / destructuring / attribute target -- out of scope, not a guess + name = targets[0].id + if isinstance(node, ast.AnnAssign) and node.value is None: + _record(name, class_name, node.lineno, end_line, "annotation_only", None, None) + return + outcome, kind, value = _evaluate_literal_rhs(node.value) + _record(name, class_name, node.lineno, end_line, outcome, kind, value) + + for node in ast.iter_child_nodes(tree): + if isinstance(node, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + _handle(node, None) + elif isinstance(node, ast.ClassDef): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.Assign, ast.AnnAssign, ast.AugAssign)): + _handle(child, node.name) + + return result + + +def _collect_repo_constants(repo_root: Path, index: RepositoryIndex) -> dict: + """Build the ``InvestigationContext.constants`` table: for every Python + file ``index`` already knows about (i.e. every file containing at least + one function/class -- ``index.by_file``), read and AST-walk it once for + module/class-level literal assignments. A single file's read/parse + failure is isolated to that file (skipped, never aborts the whole + table); there is no dedicated error channel here because this mirrors + ``list_functions_in_file``'s own best-effort, per-file posture.""" + constants: dict = {} + for file_path in index.by_file.keys(): + if not file_path.endswith((".py", ".pyi")): + continue + try: + file_text = (repo_root / file_path).read_text(encoding="utf-8") + except Exception: + continue + per_file = _extract_literal_constants(file_text) + if per_file: + constants[file_path] = per_file + return constants + + +def build_investigation_context(repo_root: Path, output_dir: Path) -> "InvestigationContext | None": + """Parse the repository once and assemble the shared artifacts every + candidate's enrichment reuses. + + Uses ``processing_level="all"`` deliberately: candidate enrichment + needs the *complete* function/call-graph picture for a candidate's + file so ``list_functions_in_file`` isn't missing anything a narrower + processing level would have already filtered out before reachability + is queried per-candidate, below. + + Returns ``None`` (never raises) if parsing fails or produces no usable + ``analyzer_output.json``/``call_graph.json`` (e.g. an unsupported + language). Callers must treat ``None`` as "enrichment degrades to + file/test/sink-only facts", never as a reason to fail the patch run. + + ``repo_root`` is resolved internally (``Path.resolve()``) before + parsing. This matters: ``repo_locator.py``'s own ``_rel()`` helper + silently falls back to a bare filename (instead of a proper + repo-relative path) when a discovered file's resolved absolute path + doesn't share a common root with an *unresolved* ``repo_root`` -- a + real, observed failure mode on macOS, where a raw temp directory + (``/var/folders/...``) is a symlink to its resolved form + (``/private/var/folders/...``). Resolving here keeps this module's own + parse/index/reachability construction internally consistent regardless + of what the caller passed -- but it cannot retroactively fix a + ``RepositoryCandidate.path`` that an *earlier*, differently-resolved + call to ``ground_repository()`` already computed as a bare filename. + Whatever calls ``ground_repository()`` to build the + ``RepositoryGroundingResult`` this whole chain starts from must also + pass an already-``.resolve()``d ``repo_root``, or candidate paths may + degrade to bare filenames and enrichment will correctly, but + unhelpfully, fail to resolve a containing function for them. + """ + repo_root = Path(repo_root).resolve() + output_dir_str = str(output_dir) + + try: + parse_result = parse_repository(str(repo_root), output_dir_str, processing_level="all") + except Exception: + return None + + if not parse_result.analyzer_output_path or not os.path.exists(parse_result.analyzer_output_path): + return None + + call_graph_path = os.path.join(output_dir_str, "call_graph.json") + if not os.path.exists(call_graph_path): + return None + + try: + index = load_index_from_file(parse_result.analyzer_output_path, str(repo_root)) + call_graph_data = read_json(call_graph_path) + functions = call_graph_data.get("functions", {}) + call_graph = call_graph_data.get("call_graph", {}) + reverse_call_graph = call_graph_data.get("reverse_call_graph", {}) + entry_points = EntryPointDetector(functions, call_graph).detect_entry_points() + reachability = ReachabilityAnalyzer(functions, reverse_call_graph, entry_points) + except Exception: + return None + + try: + constants = _collect_repo_constants(repo_root, index) + except Exception: + constants = {} + + return InvestigationContext( + index=index, + call_graph=call_graph, + reverse_call_graph=reverse_call_graph, + reachability=reachability, + constants=constants, + ) + + +def enrich_candidates( + selection: CandidateSelection, + repo_root: Path, + vulnerability_text: str, + context: "InvestigationContext | None", +) -> list[RepositoryCandidate]: + """Attach deterministic ``CandidateEnrichment`` metadata to every + selected candidate, in place. + + Returns the same ``RepositoryCandidate`` objects passed in via + ``selection.selected`` (same identity, same order) -- never a second + candidate model. ``path``/``evidence``/``best_tier`` are never + modified; only ``.enrichment`` is set. + + A failure enriching one candidate is isolated to that candidate's + ``enrichment.enrichment_errors`` and never prevents enriching the + others. No LLM calls anywhere in this function or anything it calls. + + ``repo_root`` is resolved internally for the same reason + ``build_investigation_context()`` resolves it -- see that function's + docstring. This cannot fix a candidate path already computed as a bare + filename by an unresolved-root call to ``ground_repository()``. + """ + repo_root = Path(repo_root).resolve() + vuln_class = vulnerability_patterns.classify_vuln_class(vulnerability_text) + for candidate in selection.selected: + _enrich_one(candidate, repo_root, vuln_class, context) + return selection.selected + + +def _enrich_one( + candidate: RepositoryCandidate, + repo_root: Path, + vuln_class: "str | None", + context: "InvestigationContext | None", +) -> None: + errors: list[str] = [] + functions_in_file: list[dict] = [] + resolved_function: "dict | None" = None + resolution_note: "str | None" = None + callees: list[str] = [] + callers_by_call_graph: list[str] = [] + callers_by_text_search: list[dict] = [] + is_reachable: "bool | None" = None + entry_point_path: "list[str] | None" = None + + if context is not None: + try: + functions_in_file = context.index.list_functions_in_file(candidate.path) + resolved_function, resolution_note = _resolve_containing_function( + functions_in_file, candidate + ) + if resolved_function is not None: + func_id = resolved_function["id"] + callees = list(context.call_graph.get(func_id, [])) + callers_by_call_graph = list(context.reverse_call_graph.get(func_id, [])) + name = resolved_function.get("name") + if name: + callers_by_text_search = context.index.search_usages(name) + is_reachable = context.reachability.is_reachable_from_entry_point(func_id) + if is_reachable: + entry_point_path = context.reachability.get_entry_point_path(func_id) + except Exception as exc: # noqa: BLE001 -- isolate to this candidate, never propagate + errors.append(f"graph enrichment failed: {type(exc).__name__}: {exc}") + else: + resolution_note = ( + "no investigation context available " + "(parse produced no usable analyzer_output.json/call_graph.json)" + ) + + scope_constants: list[dict] = [] + if context is not None: + try: + file_constants = context.constants.get(candidate.path, {}) + # unitType == "module_level" is this parser's one-per-file + # whole-module catch-all unit (spans the entire file), used by + # _resolve_containing_function whenever no real function's + # line range contains the hit_line -- exactly what happens for + # a class-body constant sitting between methods (observed + # directly against a real repo: a hit_line on + # Retry.DEFAULT_REMOVE_HEADERS_ON_REDIRECT resolved to this + # catch-all, with className=None, even though the constant + # itself is class-scoped). That className=None carries no real + # narrowing signal -- it must be treated the same as + # resolved_function being None entirely, not as "the real + # scope is module-only" (which would incorrectly exclude every + # class-level constant in the file). A genuine top-level + # function/method resolution (unitType != "module_level") + # still narrows normally. + have_narrowing_signal = ( + resolved_function is not None + and resolved_function.get("unitType") != "module_level" + ) + resolved_class_name = resolved_function.get("className") if have_narrowing_signal else None + for entry in file_constants.values(): + entry_class = entry["class_name"] + # Module-level constants (entry_class is None) are always + # in scope. Class-level constants are scoped to the + # resolved function's own class when there is a real + # narrowing signal; otherwise every class-level constant + # in the file is included rather than silently missed + # (mirrors sink_matches' independence from + # resolved_function, below, applied to the analogous "no + # signal to narrow by" case here). + if entry_class is not None and have_narrowing_signal and entry_class != resolved_class_name: + continue + scope_constants.append(entry) + except Exception as exc: # noqa: BLE001 + errors.append(f"constant scope resolution failed: {type(exc).__name__}: {exc}") + + try: + target_file = repo_root / candidate.path + related_tests = testing_support.tests_for_file(repo_root, target_file) + test_support_rating = testing_support.score_test_support(related_tests) + except Exception as exc: # noqa: BLE001 + related_tests = [] + test_support_rating = None + errors.append(f"test discovery failed: {type(exc).__name__}: {exc}") + + sink_matches: "list[dict] | None" = None + if vuln_class: + try: + all_sinks = vulnerability_patterns.extract_repo_sinks(repo_root, vuln_class) + sink_matches = [s for s in all_sinks if s.get("file") == candidate.path] + except Exception as exc: # noqa: BLE001 + errors.append(f"sink extraction failed: {type(exc).__name__}: {exc}") + + candidate.enrichment = CandidateEnrichment( + functions_in_file=functions_in_file, + resolved_function=resolved_function, + resolution_note=resolution_note, + callees=callees, + callers_by_call_graph=callers_by_call_graph, + callers_by_text_search=callers_by_text_search, + is_reachable_from_entry_point=is_reachable, + entry_point_path=entry_point_path, + related_tests=related_tests, + test_support_rating=test_support_rating, + sink_matches=sink_matches, + scope_constants=scope_constants, + enrichment_errors=errors, + ) + + +def _resolve_containing_function( + functions_in_file: list[dict], + candidate: RepositoryCandidate, +) -> "tuple[dict | None, str | None]": + """Resolve which function in ``functions_in_file`` contains -- or is + nearest to -- the candidate's strongest evidence's ``hit_line``. + + Never fabricates a match: an explicit note accompanies any fallback, + and ``None`` with a note is returned rather than guessing when nothing + can be resolved. + """ + if not functions_in_file: + return None, "file has no parsed functions (module-level code, or unsupported/unparsed file)" + + evidence_with_tier = [e for e in candidate.evidence if e.tier is not None] + if not evidence_with_tier: + return None, "no evidence carries a tier" + strongest = max(evidence_with_tier, key=lambda e: e.tier) + + hit_line = strongest.hit_line + if hit_line is None: + return None, "strongest evidence carries no hit_line" + + containing = [ + f + for f in functions_in_file + if f.get("startLine") is not None + and f.get("endLine") is not None + and f["startLine"] <= hit_line <= f["endLine"] + ] + if containing: + return containing[0], None + + with_start_line = [f for f in functions_in_file if f.get("startLine") is not None] + if not with_start_line: + return None, "no function has line-range metadata to match against" + + nearest = min(with_start_line, key=lambda f: (abs(f["startLine"] - hit_line), f["startLine"])) + return nearest, f"no function contains hit_line {hit_line}; used nearest function by start line" diff --git a/libs/openant-core/utilities/autopatcher/candidate_selection.py b/libs/openant-core/utilities/autopatcher/candidate_selection.py new file mode 100644 index 00000000..3a2fb141 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/candidate_selection.py @@ -0,0 +1,134 @@ +"""Bounded candidate selection over existing repository-grounding output. + +Phase 1 of the multi-candidate investigation design: this module decides +*which* of the candidates ``ground_repository()`` already found are worth +investigating later, and *how many* -- nothing else. It adds no discovery, +ranking, or scoring logic of its own: ordering reuses ``RepositoryCandidate. +best_tier`` verbatim (the existing repository-grounding tier, not a new +confidence/investigation/fusion score -- those are separate concepts for a +later phase, if introduced at all). + +Selection proves nothing about a candidate. A ``selected`` candidate is a +repository location worth investigating later, not a confirmed vulnerable +location. The cap exists solely to bound the LLM cost, latency, and Trust +Report noise a later investigation phase would otherwise incur against an +unbounded candidate set -- no investigation happens in this module. + +No LLM calls. No I/O. No environment reads. Pure data in, pure data out. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from utilities.autopatcher.repository_grounding_models import ( + RepositoryCandidate, + RepositoryGroundingResult, +) + +DEFAULT_MAX_CANDIDATES = 3 + + +def _is_structurally_valid(candidate: RepositoryCandidate) -> bool: + """Defensive validity check -- not a strength/tier judgment. + + Real ``ground_repository()`` output should never fail this. It exists + so ``excluded_by_policy`` has an honest, distinct meaning from "weak + evidence": a candidate whose only evidence is the weakest existing tier + (``cwe_keywords``) is still structurally valid and therefore eligible + -- see ``select_candidates()``'s docstring for why weak evidence is + never excluded outright, only ranked last. + """ + return bool(candidate.path) and bool(candidate.evidence) and candidate.best_tier is not None + + +@dataclass +class CandidateSelection: + """Complete accounting of one candidate-selection decision. + + Every list holds ``RepositoryCandidate`` objects directly -- no wrapper + model is introduced in this phase. If a later phase needs + investigation-specific metadata (e.g. a resolved symbol, a snippet), a + dedicated model should be introduced then, scoped to what that phase + can genuinely populate. + + Invariants (see tests): + generated == excluded_by_policy + eligible (as sets) + eligible == selected + excluded_by_cap (as sets) + len(selected) <= max_candidates + """ + + generated: list[RepositoryCandidate] + excluded_by_policy: list[RepositoryCandidate] + eligible: list[RepositoryCandidate] + selected: list[RepositoryCandidate] + excluded_by_cap: list[RepositoryCandidate] + max_candidates: int + + @property + def used_fallback(self) -> bool: + """True iff nothing was selected. + + The caller must fall back to today's existing grounding behavior + (``pipeline.run()``'s own internal ``ground_repository()`` call) + rather than fail the patch run or widen the search. + """ + return len(self.selected) == 0 + + +def select_candidates( + grounding: RepositoryGroundingResult, + max_candidates: int = DEFAULT_MAX_CANDIDATES, +) -> CandidateSelection: + """Select a bounded, deterministically-ordered subset of + ``grounding.candidates`` for later investigation. + + Does not investigate anything and does not call an LLM: this is pure + selection policy over data ``ground_repository()`` already computed. + + Ordering is by the existing ``best_tier`` descending (stronger evidence + first: ``explicit_path``=4 > ``symbol_definition``=3 > + ``symbol_search``=2 > ``cwe_keywords``=1 -- see repo_locator.py's tier + constants), then by ``path`` ascending as a pure, deterministic + tie-break. Neither key depends on dict insertion order, filesystem + traversal order, or time -- two calls against the same + ``RepositoryGroundingResult`` always produce the same order. + + Weak evidence (e.g. a ``cwe_keywords``-only match) is never excluded + outright: it is ranked last and only enters ``selected`` if capacity + remains after every stronger candidate has been placed. This is + deliberate: a broad/noisy advisory should not let weak candidates + crowd out strong ones, but a sparse advisory with no explicit-path or + symbol evidence should still be able to select its best available + (weak) candidate rather than select nothing. + + ``max_candidates`` bounds how many candidates are ever selected, + regardless of how many are eligible -- this is what prevents a broad + or noisy advisory from producing unbounded downstream cost, latency, + or report noise in a later phase. ``0`` is valid (selects nothing; a + legitimate dry-run/audit mode -- ``used_fallback`` correctly reads + True). Negative values have no sensible meaning and raise + ``ValueError``. + + Raises: + ValueError: if max_candidates < 0. + """ + if max_candidates < 0: + raise ValueError(f"max_candidates must be >= 0, got {max_candidates}") + + generated = list(grounding.candidates) + eligible = [c for c in generated if _is_structurally_valid(c)] + excluded_by_policy = [c for c in generated if not _is_structurally_valid(c)] + + ordered = sorted(eligible, key=lambda c: (-c.best_tier, c.path)) + selected = ordered[:max_candidates] + excluded_by_cap = ordered[max_candidates:] + + return CandidateSelection( + generated=generated, + excluded_by_policy=excluded_by_policy, + eligible=eligible, + selected=selected, + excluded_by_cap=excluded_by_cap, + max_candidates=max_candidates, + ) diff --git a/libs/openant-core/utilities/autopatcher/confidence_scorer.py b/libs/openant-core/utilities/autopatcher/confidence_scorer.py new file mode 100644 index 00000000..4435b0eb --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/confidence_scorer.py @@ -0,0 +1,63 @@ +""" +Confidence scorer stage. + +Loads the confidence_scorer prompt, sends the full context (vulnerability + +patch + review) to the LLM, and returns a structured confidence assessment. +""" + +from __future__ import annotations + +from pathlib import Path + +from .llm_client import LLMClient + +_PROMPT_PATH = Path(__file__).parent / "prompts" / "confidence_scorer.md" + + +def score_confidence( + vulnerability_text: str, + patch: str, + review: str, + llm: LLMClient, + code_context: str = "", +) -> str: + """ + Assign a confidence score to the generated patch. + + Parameters + ---------- + vulnerability_text: + The original vulnerability description and code context. + patch: + The unified diff patch. + review: + The structured patch review (explanation, affected areas, validation + notes). + llm: + An initialised :class:`LLMClient` instance. + code_context: + Optional repository evidence selected by static analysis. When + provided it is prepended to the user message so the scorer reasons + from the same evidence the patch generator used. + + Returns + ------- + str + The raw LLM response containing the confidence score and reasons. + """ + system_prompt = _PROMPT_PATH.read_text(encoding="utf-8") + context_section = ( + "## Repository evidence (selected by static analysis)\n\n" + + code_context + + "\n\n" + ) if code_context else "" + user_message = ( + context_section + + "## Vulnerability report\n\n" + + vulnerability_text + + "\n\n## Proposed patch\n\n" + + patch + + "\n\n## Patch review\n\n" + + review + ) + return llm.complete(system_prompt, user_message, stage="confidence_scorer") diff --git a/libs/openant-core/utilities/autopatcher/cve_converter.py b/libs/openant-core/utilities/autopatcher/cve_converter.py new file mode 100644 index 00000000..e0af57a5 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/cve_converter.py @@ -0,0 +1,165 @@ +"""Convert a parsed NVD CVE record dict into the vulnerability text format +consumed by the Auto Patcher pipeline. Mirrors the standalone Auto Patcher +project's GHSA path (``advisory_converter.py``), so both sources reach +``pipeline.run()`` through the same rendered-Markdown shape. + +Ported from the standalone Auto Patcher project's ``cve_converter.py``. +``extract_description``/``extract_cwes``/``extract_cvss``/ +``extract_affected_products``/``first_sentence`` are public (not +leading-underscore) so a future InvestigationCase adapter can reuse them to +populate ``ProblemClaims`` without re-deriving the same fields from the raw +NVD shape a second time -- that adapter is not part of this commit. + +Pure formatting only: no network access, no filesystem access, no repository +inspection. The rendered Markdown is unchanged from the reference +implementation's template -- including the "Affected products" section, +which lists only what the advisory itself claims. The closing Note already +states plainly that no source code was inspected; see +extract_affected_products' docstring for why this data must not be treated +as repository-verified. +""" + +from __future__ import annotations + +import html as _html +import re + + +def cve_to_vuln_text(cve: dict) -> str: + """Format an NVD CVE record dict as a Markdown vulnerability description. + + The output structure matches what ``advisory_converter.ghsa_to_vuln_text`` + produces upstream, so both sources reach the pipeline through the same + shape. When no source code snippet is available (always true for a bare + CVE record), this is stated explicitly so downstream stages -- and a + human reviewer -- can adjust their expectations accordingly. + """ + cve_id = cve.get("id") or "unknown" + description = _clean(extract_description(cve)) or "No description available" + summary = _clean(first_sentence(description)) or cve_id + + cwes = extract_cwes(cve) + cwe_str = ", ".join(cwes) if cwes else "Unknown" + + score_str, severity_str = extract_cvss(cve) + + products = extract_affected_products(cve) + product_section = "\n".join(f"- {p}" for p in products) if products else "- (not specified)" + + refs = _extract_references(cve) + ref_section = "\n".join(f"- {r}" for r in refs) if refs else "- (none)" + + return f"""\ +# {summary} + +## Vulnerability description + +**Advisory:** {cve_id} +**Severity:** {severity_str} (CVSS: {score_str}) +**Type:** {cwe_str} + +{description} + +## Affected products + +{product_section} + +## References + +{ref_section} + +## Note + +No source code snippet is available from this advisory. \ +The patch generator will produce a best-effort patch based on the description above. \ +Use --repo-root to enable impact analysis and test discovery on the actual codebase. +""" + + +def extract_description(cve: dict) -> str: + """Return the English-language description text, or "" if absent.""" + for entry in cve.get("descriptions") or []: + if entry.get("lang") == "en" and entry.get("value"): + return entry["value"] + return "" + + +def extract_cwes(cve: dict) -> list[str]: + """Return the deduplicated list of English-language CWE labels (e.g. "CWE-89").""" + cwes: list[str] = [] + for weakness in cve.get("weaknesses") or []: + for entry in weakness.get("description") or []: + value = entry.get("value") + if entry.get("lang") == "en" and value and value not in cwes: + cwes.append(value) + return cwes + + +_CVSS_METRIC_PREFERENCE = ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2") + + +def extract_cvss(cve: dict) -> tuple[str, str]: + """Return (score_str, severity_str), preferring the newest CVSS version present. + + Falls back to ("N/A", "UNKNOWN") when no CVSS metric of any known + version is present. + """ + metrics = cve.get("metrics") or {} + for key in _CVSS_METRIC_PREFERENCE: + entries = metrics.get(key) or [] + if not entries: + continue + metric = entries[0] + cvss_data = metric.get("cvssData") or {} + score = cvss_data.get("baseScore") + severity = cvss_data.get("baseSeverity") or metric.get("baseSeverity") + score_str = str(score) if score is not None else "N/A" + severity_str = (severity or "UNKNOWN").upper() + return score_str, severity_str + return "N/A", "UNKNOWN" + + +def extract_affected_products(cve: dict) -> list[str]: + """Return up to 5 vulnerable CPE criteria strings the advisory itself claims are affected. + + This reflects only what NVD's ``configurations`` block asserts -- it is + not cross-checked against any repository's actual dependency versions. + Callers (and any future consumer of this list, e.g. an InvestigationCase + adapter) must not present this as a verified affected-version match. + """ + products: list[str] = [] + for config in cve.get("configurations") or []: + for node in config.get("nodes") or []: + for match in node.get("cpeMatch") or []: + if match.get("vulnerable") and match.get("criteria"): + if match["criteria"] not in products: + products.append(match["criteria"]) + if len(products) >= 5: + return products + return products + + +def _extract_references(cve: dict) -> list[str]: + """Return up to 5 reference URLs from the advisory.""" + refs: list[str] = [] + for entry in cve.get("references") or []: + url = entry.get("url") + if url: + refs.append(url) + if len(refs) >= 5: + break + return refs + + +def first_sentence(text: str) -> str: + """Return the first sentence of text, truncated to 120 chars.""" + if not text: + return "" + match = re.search(r"^(.*?[.!?])(\s|$)", text.strip()) + sentence = match.group(1) if match else text.strip() + return sentence[:120].rstrip() + + +def _clean(text: str) -> str: + """Unescape HTML entities and strip whitespace.""" + return _html.unescape(text).strip() diff --git a/libs/openant-core/utilities/autopatcher/cve_fetcher.py b/libs/openant-core/utilities/autopatcher/cve_fetcher.py new file mode 100644 index 00000000..9d0acff6 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/cve_fetcher.py @@ -0,0 +1,99 @@ +"""Fetch a CVE record by ID from the NVD REST API (v2.0). + +Ported from the standalone Auto Patcher project's ``cve_fetcher.py``. Uses +only stdlib -- no third-party deps. Set ``NVD_API_KEY`` in the environment to +raise NVD's rate limits; the key is only ever placed in an outgoing request +header, never logged or included in any exception message. + +Unlike the original, "not found" and "fetch failure" are distinct exception +types (``CVENotFoundError`` / ``CVEFetchError``) rather than a single bare +``ValueError``, so callers -- and the eventual CLI error messages -- can tell +a typo'd or unknown CVE id apart from a network/parsing problem. Both +subclass ``ValueError`` so existing ``except ValueError`` handling still +catches either. + +No retries here by design: a caller that wants retry behavior adds it at a +higher layer, where it can also decide whether a retry is worthwhile for a +given failure kind (retrying a CVENotFoundError is never useful). +""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + +_API_URL = "https://services.nvd.nist.gov/rest/json/cves/2.0" + + +class CVENotFoundError(ValueError): + """NVD has no record matching the given CVE id.""" + + +class CVEFetchError(ValueError): + """Network, HTTP, or parsing failure while contacting NVD.""" + + +def fetch_cve(cve_id: str, timeout: int = 15) -> dict: + """Fetch a CVE record from the NVD API. + + Parameters + ---------- + cve_id: + Full CVE identifier, e.g. "CVE-2021-12345". + timeout: + Socket timeout in seconds for the NVD request. + + Returns + ------- + dict + The single CVE object (NVD's ``vulnerabilities[0]["cve"]``), not the + NVD response envelope. + + Raises + ------ + CVENotFoundError + NVD returned HTTP 404, or a 200 response with no matching record. + CVEFetchError + Any other HTTP error, a network/timeout failure, or an unparseable + or malformed response body. + """ + url = f"{_API_URL}?cveId={cve_id}" + headers = {} + api_key = os.environ.get("NVD_API_KEY", "") + if api_key: + headers["apiKey"] = api_key + + req = urllib.request.Request(url, headers=headers) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8") + except urllib.error.HTTPError as exc: + if exc.code == 404: + raise CVENotFoundError(f"CVE {cve_id} not found in NVD (HTTP 404)") from exc + raise CVEFetchError( + f"Failed to fetch {cve_id} from NVD: HTTP {exc.code} {exc.reason}" + ) from exc + except (urllib.error.URLError, TimeoutError) as exc: + # HTTPError is a URLError subclass, so it's already handled above by + # the time we get here. A bare TimeoutError can reach us directly + # when the socket layer raises it without urllib wrapping it first + # (e.g. when a caller mocks urlopen itself, bypassing that wrapping). + raise CVEFetchError(f"Failed to fetch {cve_id} from NVD: {exc}") from exc + + try: + payload = json.loads(body) + except json.JSONDecodeError as exc: + raise CVEFetchError(f"NVD returned an unparseable response for {cve_id}") from exc + + vulnerabilities = payload.get("vulnerabilities") or [] + if not vulnerabilities: + raise CVENotFoundError(f"NVD returned no matching record for {cve_id}") + + cve = vulnerabilities[0].get("cve") + if not cve: + raise CVEFetchError( + f"NVD returned a malformed response for {cve_id} (missing cve object)" + ) + return cve diff --git a/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py b/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py new file mode 100644 index 00000000..ad9375b2 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/diff_hunk_repair.py @@ -0,0 +1,205 @@ +"""Deterministic unified diff hunk header repair. + +LLMs frequently generate unified diffs with arithmetically wrong @@ -a,b +c,d @@ +counts. This module recomputes b and d from the actual hunk body lines, and +recomputes c as old_start + cumulative_prior_net_delta within the same file. +The old start line (a) is preserved as-is — it is the LLM's positional anchor +and requires knowledge of the original file to validate independently. + +Public API: + repair_hunk_headers(patch: str) -> tuple[str, RepairResult] + +RepairResult fields: + normalization_applied : bool — True if any @@ line was changed + hunks_rewritten : int — number of @@ headers with corrected values + files_rewritten : int — number of distinct filenames with ≥1 rewrite +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + + +_HUNK_RE = re.compile( + r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@(.*)" +) +_FENCE_OPEN_RE = re.compile(r"^```") + + +@dataclass +class RepairResult: + normalization_applied: bool = False + hunks_rewritten: int = 0 + files_rewritten: int = 0 + + +def repair_hunk_headers(patch: str) -> tuple[str, RepairResult]: + """Recompute unified diff hunk header counts from body content. + + Returns (repaired_patch, RepairResult). + Never raises — on any unexpected error returns (original_patch, RepairResult()). + """ + meta = RepairResult() + if not patch or not patch.strip(): + return patch, meta + try: + open_fence, clean, close_fence = _strip_md_fences(patch) + repaired, meta = _repair(clean, meta) + return open_fence + repaired + close_fence, meta + except Exception: + return patch, meta + + +# --------------------------------------------------------------------------- +# Internal +# --------------------------------------------------------------------------- + +def _strip_md_fences(patch: str) -> tuple[str, str, str]: + """Extract markdown code fences surrounding a patch string. + + Returns (open_fence, clean_patch, close_fence). + Concatenating the three parts always reconstructs the original string. + """ + lines = patch.splitlines(keepends=True) + open_fence = "" + close_fence = "" + if lines and _FENCE_OPEN_RE.match(lines[0]): + open_fence = lines.pop(0) + if lines and lines[-1].rstrip() in ("```", "~~~"): + close_fence = lines.pop() + return open_fence, "".join(lines), close_fence + + +def _repair(patch: str, meta: RepairResult) -> tuple[str, RepairResult]: + lines = patch.splitlines(keepends=True) + output: list[str] = [] + + # Per-file accumulator: sum of (new_count - old_count) for prior hunks + file_delta: int = 0 + current_file: str | None = None + files_touched: set[str] = set() + + # Per-hunk state + hunk_orig_header: str | None = None + hunk_old_start: int = 0 + hunk_claimed_new_start: int = 0 + hunk_suffix: str = "" + hunk_body: list[str] = [] + in_hunk: bool = False + + def flush_hunk() -> None: + nonlocal file_delta, in_hunk, hunk_orig_header, hunk_body + + if not in_hunk: + return + + old_count, new_count = _count_body(hunk_body) + + # New-file sentinel: old_start=0 means the old file doesn't exist. + # The delta formula doesn't apply; preserve the original new_start + # (conventionally 1 for a new non-empty file, 0 for an empty one). + if hunk_old_start == 0: + correct_new_start = hunk_claimed_new_start + else: + correct_new_start = hunk_old_start + file_delta + rewritten = ( + f"@@ -{hunk_old_start},{old_count}" + f" +{correct_new_start},{new_count}" + f" @@{hunk_suffix}\n" + ) + + if rewritten != hunk_orig_header: + meta.hunks_rewritten += 1 + meta.normalization_applied = True + if current_file is not None: + files_touched.add(current_file) + + output.append(rewritten) + output.extend(hunk_body) + + file_delta += new_count - old_count + in_hunk = False + hunk_orig_header = None + hunk_body = [] + + i = 0 + n = len(lines) + while i < n: + line = lines[i] + stripped = line.rstrip("\n") + + if stripped.startswith("@@ "): + flush_hunk() + m = _HUNK_RE.match(stripped) + if not m: + output.append(line) # malformed — pass through unchanged + i += 1 + continue + hunk_old_start = int(m.group(1)) + hunk_claimed_new_start = int(m.group(3)) + hunk_suffix = m.group(5) # text after second @@, e.g. " function setKey" + hunk_orig_header = line + hunk_body = [] + in_hunk = True + i += 1 + continue + + # A real file header is a "--- "/"+++ " PAIR on adjacent lines, not + # merely a line that starts with one of those prefixes — a + # removed/added hunk-body line whose text is "-- foo" or "++ foo" + # produces the raw line "--- foo" / "+++ foo" too. Requiring the + # very next line to complete the pair is what tells apart a genuine + # file boundary from coincidental body content: two unrelated body + # lines almost never line up to form both halves of the pair. + if ( + stripped.startswith("--- ") + and i + 1 < n + and lines[i + 1].rstrip("\n").startswith("+++ ") + ): + flush_hunk() + file_delta = 0 + # Track filename for metadata (strip a/ prefix when present) + raw = stripped[4:].split("\t")[0].strip() + current_file = raw[2:] if raw.startswith("a/") else raw + output.append(line) + output.append(lines[i + 1]) + i += 2 + continue + + if in_hunk: + hunk_body.append(line) + i += 1 + continue + + output.append(line) # preamble, diff --git lines, stray +++, etc. + i += 1 + + flush_hunk() + meta.files_rewritten = len(files_touched) + return "".join(output), meta + + +def _count_body(body: list[str]) -> tuple[int, int]: + """Return (old_count, new_count) by walking hunk body lines. + + old_count = context lines + removed lines + new_count = context lines + added lines + The '\\' No newline marker is excluded from both counts. + """ + old_count = 0 + new_count = 0 + for raw in body: + line = raw.rstrip("\n") + if line.startswith("\\"): + # \\ No newline at end of file — metadata marker, not a content line + continue + if line.startswith("-") and not line.startswith("---"): + old_count += 1 + elif line.startswith("+") and not line.startswith("+++"): + new_count += 1 + else: + # Context: leading space, empty line, or any other non-marker content + old_count += 1 + new_count += 1 + return old_count, new_count diff --git a/libs/openant-core/utilities/autopatcher/diff_parsing.py b/libs/openant-core/utilities/autopatcher/diff_parsing.py new file mode 100644 index 00000000..37638afb --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/diff_parsing.py @@ -0,0 +1,68 @@ +"""Generic unified-diff parsing. + +Extracted from impact_surface.py: this parser understands only the +language-agnostic unified-diff conventions (`--- a/...`, `+++ b/...`, `@@ ... @@` +hunk headers, and ' '/'+'/'-' prefixed body lines). It has no Python-specific +behavior — symbol resolution, AST parsing, and everything else that depends on +a particular language stays in impact_surface.py and consumes this module's +output. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Tuple + + +@dataclass(frozen=True) +class DiffHunk: + """One hunk's raw body lines, each still prefixed with its diff marker + (' ', '+', or '-'). `new_start`/`new_count` are kept only for + diagnostic/debugging purposes — symbol resolution does not use them, + by design (see impact_surface.py's module docstring).""" + new_start: int + new_count: int + lines: List[str] = field(default_factory=list) + + +def parse_diff(diff: str) -> Tuple[List[str], Dict[str, List[DiffHunk]]]: + """Parse a unified diff and return list of changed files and hunks per file. + + Unlike a purely line-range-based parse, this keeps each hunk's actual + body lines (context/added/removed), which symbol resolution needs to + relocate the hunk by content rather than by trusting its header. + """ + changed_files: List[str] = [] + file_hunks: Dict[str, List[DiffHunk]] = {} + cur_file: Optional[str] = None + cur_hunk: Optional[DiffHunk] = None + + def flush() -> None: + nonlocal cur_hunk + if cur_hunk is not None and cur_file is not None: + file_hunks[cur_file].append(cur_hunk) + cur_hunk = None + + for line in diff.splitlines(): + if line.startswith("--- "): + flush() + continue + if line.startswith("+++ b/"): + flush() + cur_file = line[6:].strip() + if cur_file not in changed_files: + changed_files.append(cur_file) + file_hunks[cur_file] = [] + continue + if line.startswith("@@") and cur_file is not None: + flush() + m = re.search(r"\+([0-9]+)(?:,([0-9]+))?", line) + if m: + start = int(m.group(1)) + count = int(m.group(2)) if m.group(2) else 1 + cur_hunk = DiffHunk(new_start=start, new_count=count) + continue + if cur_hunk is not None and line[:1] in (" ", "+", "-"): + cur_hunk.lines.append(line) + flush() + return changed_files, file_hunks diff --git a/libs/openant-core/utilities/autopatcher/evidence_fusion.py b/libs/openant-core/utilities/autopatcher/evidence_fusion.py new file mode 100644 index 00000000..532aa24f --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/evidence_fusion.py @@ -0,0 +1,559 @@ +"""Deterministic Evidence Fusion. + +Fuses every selected candidate's enrichment (see candidate_enrichment.py) +into a RepositoryUnderstanding that preserves candidate identity rather +than flattening candidates into global summary lists. Every +RepositoryCandidate already carries its own grounding evidence +(.evidence/.best_tier) and enrichment (.enrichment) -- this module adds +exactly two things beyond passthrough: relationships between candidates, +and a short fusion_notes trail recording what fusion noticed. + +No LLM calls, no I/O, no parsing, no vulnerability judgement anywhere in +this module. Pure data in (a CandidateSelection, already produced by +candidate_selection.py/candidate_enrichment.py), pure data out. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from utilities.autopatcher.candidate_selection import CandidateSelection +from utilities.autopatcher.repository_grounding_models import RepositoryCandidate + +# best_tier values strong enough that a failure to resolve a function is +# worth flagging as a divergence -- explicit_path (4) and symbol_definition +# (3), per repo_locator.py's tier constants. Weaker tiers (symbol_search=2, +# cwe_keywords=1) matching no function is expected/unremarkable, not +# flagged. +_STRONG_TIER_THRESHOLD = 3 + + +@dataclass +class CandidateRelationship: + """A directly-observed structural link between two selected + candidates. Flat -- not a graph edge type, no traversal, no + transitive closure, nothing scored.""" + + from_path: str + to_path: str + kind: str # "calls" -- the only kind in this phase + detail: str # the func_id that evidences this link + + +@dataclass +class RepositoryUnderstanding: + """Deterministic fusion of every selected candidate's enrichment. + + ``candidate_evidence`` holds the exact same ``RepositoryCandidate`` + objects produced by candidate_selection.py/candidate_enrichment.py -- + never copies, never reconstructs, never wraps them. No LLM output, no + vulnerability verdict, no confidence score anywhere here. + """ + + candidate_evidence: list[RepositoryCandidate] + relationships: list[CandidateRelationship] = field(default_factory=list) + fusion_notes: list[str] = field(default_factory=list) + investigation_context_available: bool = True + + +def fuse_evidence( + selection: CandidateSelection, + investigation_context_available: bool, +) -> RepositoryUnderstanding: + """Fuse every selected candidate's enrichment into a + RepositoryUnderstanding. + + Pure and deterministic: no LLM calls, no I/O, no parsing. Reads only + fields already populated by select_candidates()/enrich_candidates(). + Never mutates ``selection`` or any ``RepositoryCandidate``/ + ``CandidateEnrichment`` it reads. + """ + candidate_evidence = list(selection.selected) + relationships = _find_call_relationships(candidate_evidence) + fusion_notes = _build_fusion_notes( + candidate_evidence, investigation_context_available, relationships + ) + return RepositoryUnderstanding( + candidate_evidence=candidate_evidence, + relationships=relationships, + fusion_notes=fusion_notes, + investigation_context_available=investigation_context_available, + ) + + +def _file_part(func_id: str) -> str: + """Extract the file portion of a func_id (``"file/path.py:funcName"`` + -> ``"file/path.py"``), matching ``RepositoryIndex._build_index``'s + own convention (split on the last colon).""" + colon_idx = func_id.rfind(":") + if colon_idx <= 0: + return func_id + return func_id[:colon_idx] + + +def _find_call_relationships( + candidate_evidence: list[RepositoryCandidate], +) -> list[CandidateRelationship]: + """Structural call-graph adjacency between selected candidates only. + + Uses only ``callees``/``callers_by_call_graph`` (deterministic, from + the parser's own call graph) -- never ``callers_by_text_search``, + which is regex-based and noisier; asserting a structural relationship + from it would claim more than that signal supports. + + The same real edge can be discovered from either candidate's + perspective (A's ``callees`` names B, or B's ``callers_by_call_graph`` + names A) -- deduplicated by ``(from_path, to_path, kind)`` so it is + reported once. + """ + by_path = {c.path: c for c in candidate_evidence} + seen: set[tuple[str, str, str]] = set() + relationships: list[CandidateRelationship] = [] + + for candidate in candidate_evidence: + if candidate.enrichment is None: + continue + + for callee_id in candidate.enrichment.callees: + callee_file = _file_part(callee_id) + if callee_file == candidate.path or callee_file not in by_path: + continue + key = (candidate.path, callee_file, "calls") + if key not in seen: + seen.add(key) + relationships.append( + CandidateRelationship( + from_path=candidate.path, + to_path=callee_file, + kind="calls", + detail=callee_id, + ) + ) + + for caller_id in candidate.enrichment.callers_by_call_graph: + caller_file = _file_part(caller_id) + if caller_file == candidate.path or caller_file not in by_path: + continue + key = (caller_file, candidate.path, "calls") + if key not in seen: + seen.add(key) + relationships.append( + CandidateRelationship( + from_path=caller_file, + to_path=candidate.path, + kind="calls", + detail=caller_id, + ) + ) + + return relationships + + +def _build_fusion_notes( + candidate_evidence: list[RepositoryCandidate], + investigation_context_available: bool, + relationships: list[CandidateRelationship], +) -> list[str]: + """Deterministic, human-readable trail of what fusion noticed. Never + a verdict, never a score -- just an honest record of divergences, + degraded-mode operation, enrichment failures, and the relationships + found. Each candidate contributes at most one divergence/error note, + never both, to avoid redundant reporting of the same candidate.""" + notes: list[str] = [] + + if not investigation_context_available: + notes.append( + "investigation context unavailable -- candidates enriched in " + "degraded, file/test/sink-only mode (no parse/call-graph/reachability)" + ) + + for candidate in candidate_evidence: + enrichment = candidate.enrichment + if enrichment is None: + continue + + is_strong = ( + candidate.best_tier is not None + and candidate.best_tier >= _STRONG_TIER_THRESHOLD + ) + unresolved = enrichment.resolved_function is None + has_error = bool(enrichment.enrichment_errors) + + if is_strong and (unresolved or has_error): + notes.append( + f"{candidate.path}: strong grounding (best_tier={candidate.best_tier}) " + f"but enrichment could not confirm a function " + f"(resolved_function={enrichment.resolved_function!r}, " + f"enrichment_errors={enrichment.enrichment_errors!r})" + ) + elif has_error: + # Not a "strong grounding" divergence, but errors must never + # be silently swallowed regardless of grounding strength. + notes.append(f"{candidate.path}: enrichment_errors={enrichment.enrichment_errors!r}") + + for rel in relationships: + notes.append(_relationship_note(rel)) + + return notes + + +def _relationship_note(rel: CandidateRelationship) -> str: + """The exact fusion-note text for one relationship. Factored out so the + renderer below can recognise (and skip) these lines when rendering + fusion_notes separately from the Structural relationships section -- + same wording either place, computed once.""" + return f"{rel.from_path} {rel.kind} {rel.to_path} (via {rel.detail})" + + +# --------------------------------------------------------------------------- +# Deterministic Markdown rendering of RepositoryUnderstanding. +# +# This is the sole consumption boundary this phase adds: a pure function +# from RepositoryUnderstanding to a Markdown string, meant to be appended +# into pipeline.run()'s existing `code_context` string alongside the repo +# code / vulnerability-pattern-guidance blocks it already builds. Nothing +# here calls that pipeline -- this module remains dormant until a later, +# separate wiring phase. +# +# No LLM calls, no I/O, no verdicts, no confidence scores. Every line is +# either a literal passthrough of an existing field or an explicit "not +# resolved / not evaluated" statement -- never a positive claim inferred +# from missing evidence. +# --------------------------------------------------------------------------- + +DEFAULT_MAX_CHARS = 4_000 +"""Matches repo_locator.py's _MAX_CONTEXT_CHARS. Candidate selection is +already bounded to at most DEFAULT_MAX_CANDIDATES (3) candidates +(candidate_selection.py), so this budget is sized to normally preserve the +complete deterministic understanding for all of them, not to further +ration an already-small set.""" + +_MAX_LIST_ITEMS = 5 +"""Per-list cap (callees, callers, tests, sinks, relationships, notes) +before a deterministic "(+N more)" note. Keeps any single candidate's +block bounded regardless of how noisy its enrichment is.""" + +_TRUNCATION_MARKER = "\n\n*(truncated to fit the character budget)*\n" + +_HEADING = "## Repository Understanding" + +_PREAMBLE = ( + "*Deterministic repository analysis, not a vulnerability verdict. These " + "are structural facts (parsing, call graph, reachability, tests) -- not " + "confirmation that a candidate is vulnerable, exploitable, or on the " + "attack path. Missing evidence is reported as missing, never as a " + "negative finding.*" +) + + +def render_repository_understanding( + understanding: RepositoryUnderstanding, + *, + max_chars: int = DEFAULT_MAX_CHARS, +) -> str: + """Render a RepositoryUnderstanding into one deterministic Markdown + block, starting with a top-level ``## Repository Understanding`` + heading. + + Candidates are rendered in the exact order already given by + ``understanding.candidate_evidence`` (candidate_selection.py's + tier-descending, path-ascending order) -- this function never + re-sorts or re-selects. If the character budget cannot fit every + candidate, weaker (later) candidates are dropped first and an explicit + note names what was omitted; malformed truncation mid-candidate never + happens -- a candidate's block is included whole or not at all. + + Never mutates ``understanding`` or anything it references. No LLM + calls, no I/O, no parsing. + + The returned string never exceeds ``max_chars`` -- a final safety-net + truncation (at a line boundary, with an explicit marker) applies in the + unlikely case that even omitting every candidate can't make the fixed + sections (heading, preamble, relationships, notes, investigation-context + line) fit. + """ + candidate_blocks = [ + _render_candidate(c, role) + for c, role in zip( + understanding.candidate_evidence, + _candidate_roles(understanding.candidate_evidence, understanding.relationships), + ) + ] + relationships_block = _render_relationships(understanding.relationships) + notes_block = _render_notes(understanding) + context_block = _render_investigation_context(understanding.investigation_context_available) + + header = _HEADING + "\n\n" + _PREAMBLE + "\n" + + if not candidate_blocks: + body = "\nNo repository candidates were selected for investigation.\n" + else: + fixed_cost = len(header) + len(relationships_block) + len(notes_block) + len(context_block) + budget_for_candidates = max(max_chars - fixed_cost, 0) + + included: list[str] = [] + omitted_paths: list[str] = [] + running = 0 + for candidate, block in zip(understanding.candidate_evidence, candidate_blocks): + if running + len(block) <= budget_for_candidates: + included.append(block) + running += len(block) + else: + omitted_paths.append(candidate.path) + + body = "\n" + "\n".join(included) + if omitted_paths: + body += ( + f"\n\n*{len(omitted_paths)} candidate(s) omitted to stay within the " + f"{max_chars}-character budget: {', '.join(omitted_paths)}.*\n" + ) + + rendered = header + body + "\n" + relationships_block + "\n" + notes_block + "\n" + context_block + + if len(rendered) > max_chars: + rendered = _hard_clamp(rendered, max_chars) + + return rendered + + +def _hard_clamp(rendered: str, max_chars: int) -> str: + """Absolute backstop: truncate at the last full line boundary that fits, + so the result is never split mid-item, and append an explicit marker. + Guarantees len(result) <= max_chars.""" + limit = max_chars - len(_TRUNCATION_MARKER) + if limit <= 0: + return _TRUNCATION_MARKER[:max_chars] + cut = rendered.rfind("\n", 0, limit) + if cut <= 0: + cut = limit + return rendered[:cut] + _TRUNCATION_MARKER + + +_ROLE_LABELS = { + "primary": ( + "**Primary evidence** -- the strongest-grounded candidate; not " + "necessarily the only location that needs to change." + ), + "supporting": ( + "**Supporting evidence** -- connected to the primary candidate by a " + "direct call-graph relationship (see Structural relationships, " + "below); may be part of the same remediation flow." + ), + "independent": ( + "**Additional candidate** -- matched independently during " + "grounding; not confirmed to be related to the primary candidate." + ), +} + + +def _candidate_roles( + candidate_evidence: list[RepositoryCandidate], + relationships: list[CandidateRelationship], +) -> list[str]: + """One role per candidate, same order as ``candidate_evidence``. + + Positional, not re-ranked: index 0 is "primary" because + candidate_selection.py has already sorted the list tier-descending/ + path-ascending before fusion ever sees it -- this function trusts that + order rather than recomputing it. Every other candidate is "supporting" + only if a real call-graph edge (from ``relationships``, never the + noisier ``callers_by_text_search``) connects it to the primary; that is + the one existing signal for "same remediation flow." Everything else is + "independent" -- evidence found on its own, not claimed to be related. + """ + if not candidate_evidence: + return [] + + primary_path = candidate_evidence[0].path + connected_to_primary = { + rel.to_path if rel.from_path == primary_path else rel.from_path + for rel in relationships + if rel.from_path == primary_path or rel.to_path == primary_path + } + + roles = ["primary"] + for candidate in candidate_evidence[1:]: + roles.append("supporting" if candidate.path in connected_to_primary else "independent") + return roles + + +def _render_candidate(candidate: RepositoryCandidate, role: str) -> str: + lines = [f"### `{candidate.path}`", "", _ROLE_LABELS[role], "", _render_grounding_line(candidate)] + + enrichment = candidate.enrichment + if enrichment is None: + lines.append("- Enrichment: not attempted for this candidate") + return "\n".join(lines) + "\n" + + lines.append(_render_resolution_line(enrichment)) + lines.append(_render_list_line("Direct callees", enrichment.callees, "none found")) + lines.append( + _render_list_line("Direct callers (call graph)", enrichment.callers_by_call_graph, "none found") + ) + lines.append(_render_reachability_line(enrichment)) + lines.append(_render_test_support_lines(enrichment)) + lines.append(_render_sink_matches_lines(enrichment)) + lines.append(_render_enrichment_errors_line(enrichment)) + + return "\n".join(lines) + "\n" + + +def _render_grounding_line(candidate: RepositoryCandidate) -> str: + ordered = sorted( + candidate.evidence, + key=lambda e: (-(e.tier if e.tier is not None else -1), e.pass_name), + ) + passes = ", ".join( + f"{e.pass_name} (tier {e.tier})" if e.tier is not None else f"{e.pass_name} (tier unknown)" + for e in ordered + ) + best = candidate.best_tier if candidate.best_tier is not None else "unknown" + return f"- Grounding: best tier {best}; evidence passes: {passes or 'none'}" + + +def _render_resolution_line(enrichment: CandidateEnrichment) -> str: + fn = enrichment.resolved_function + if fn is None: + reason = enrichment.resolution_note or "not attempted" + return f"- No function was resolved (reason: {reason})" + name = fn.get("name", "?") + func_id = fn.get("id", "?") + start = fn.get("startLine") + end = fn.get("endLine") + span = f"lines {start}-{end}" if start is not None and end is not None else "line range unknown" + note = f" (note: {enrichment.resolution_note})" if enrichment.resolution_note else "" + return f"- Resolved near grounding evidence: `{name}` (id: `{func_id}`, {span}){note}" + + +def _render_list_line(label: str, items: list[str], empty_text: str) -> str: + if not items: + return f"- {label}: {empty_text}" + shown = items[:_MAX_LIST_ITEMS] + remainder = len(items) - len(shown) + text = ", ".join(f"`{i}`" for i in shown) + if remainder > 0: + text += f" (+{remainder} more)" + return f"- {label}: {text}" + + +def _render_reachability_line(enrichment: CandidateEnrichment) -> str: + reachable = enrichment.is_reachable_from_entry_point + if reachable is None: + return "- Reachability: not evaluated (no investigation context available)" + if reachable is False: + return "- Reachability: detected as not reachable by current entry-point heuristics" + + path = enrichment.entry_point_path or [] + shown = path[:_MAX_LIST_ITEMS] + remainder = len(path) - len(shown) + path_text = " → ".join(f"`{p}`" for p in shown) if shown else "path unavailable" + if remainder > 0: + path_text += f" (+{remainder} more hop(s))" + return f"- Reachability: detected as reachable by current entry-point heuristics (path: {path_text})" + + +def _render_test_support_lines(enrichment: CandidateEnrichment) -> str: + rating_tuple = enrichment.test_support_rating + tests = enrichment.related_tests or [] + if rating_tuple is None: + return "- Test support: not evaluated (see enrichment errors, if any)" + + rating = rating_tuple[0] if rating_tuple else "unknown" + lines = [f"- Test support: {rating} ({len(tests)} related test file(s))"] + shown = tests[:_MAX_LIST_ITEMS] + remainder = len(tests) - len(shown) + for t in shown: + lines.append(f" - `{t.get('path', '?')}` ({t.get('proximity', '?')})") + if remainder > 0: + lines.append(f" - (+{remainder} more test file(s))") + return "\n".join(lines) + + +def _render_sink_matches_lines(enrichment: CandidateEnrichment) -> str: + sinks = enrichment.sink_matches + if sinks is None: + return "- Sink matches: not evaluated (vulnerability class not recognized, or not attempted)" + if not sinks: + return "- Sink matches: none found" + + lines = [f"- Sink matches: {len(sinks)} found"] + shown = sinks[:_MAX_LIST_ITEMS] + remainder = len(sinks) - len(shown) + for s in shown: + method = s.get("method") or "module level" + lines.append(f" - `{s.get('file', '?')}`:{s.get('line', '?')} in `{method}`") + if remainder > 0: + lines.append(f" - (+{remainder} more sink match(es))") + return "\n".join(lines) + + +def _render_enrichment_errors_line(enrichment: CandidateEnrichment) -> str: + errors = enrichment.enrichment_errors + if not errors: + return "- Enrichment errors: none" + + shown = errors[:_MAX_LIST_ITEMS] + remainder = len(errors) - len(shown) + lines = ["- Enrichment errors:"] + for e in shown: + lines.append(f" - {e}") + if remainder > 0: + lines.append(f" - (+{remainder} more)") + return "\n".join(lines) + + +def _render_relationships(relationships: list[CandidateRelationship]) -> str: + lines = ["### Structural relationships", ""] + if not relationships: + lines.append("None detected.") + return "\n".join(lines) + "\n" + + shown = relationships[:_MAX_LIST_ITEMS] + remainder = len(relationships) - len(shown) + for rel in shown: + lines.append( + f"- `{rel.from_path}` → `{rel.to_path}` -- direct call-graph relationship " + f"(via `{rel.detail}`)" + ) + if remainder > 0: + lines.append(f"- (+{remainder} more relationship(s))") + return "\n".join(lines) + "\n" + + +def _render_notes(understanding: RepositoryUnderstanding) -> str: + """Renders fusion_notes minus the entries already covered by their own + dedicated sections -- relationship echoes (see Structural relationships, + above) and the investigation-context-unavailable note (see Investigation + context, below) -- so the same fact is never stated twice in one + rendered block.""" + relationship_texts = {_relationship_note(r) for r in understanding.relationships} + other_notes = [ + n + for n in understanding.fusion_notes + if n not in relationship_texts and not n.startswith("investigation context unavailable") + ] + + lines = ["### Notes", ""] + if not other_notes: + lines.append("None.") + return "\n".join(lines) + "\n" + + shown = other_notes[:_MAX_LIST_ITEMS] + remainder = len(other_notes) - len(shown) + for n in shown: + lines.append(f"- {n}") + if remainder > 0: + lines.append(f"- (+{remainder} more)") + return "\n".join(lines) + "\n" + + +def _render_investigation_context(available: bool) -> str: + lines = ["### Investigation context", ""] + if available: + lines.append("Investigation context was available for this run.") + else: + lines.append( + "Investigation context unavailable -- candidates were enriched in degraded, " + "file/test/sink-only mode (no parse/call-graph/reachability)." + ) + return "\n".join(lines) + "\n" diff --git a/libs/openant-core/utilities/autopatcher/finding_calibration.py b/libs/openant-core/utilities/autopatcher/finding_calibration.py new file mode 100644 index 00000000..00aac2f9 --- /dev/null +++ b/libs/openant-core/utilities/autopatcher/finding_calibration.py @@ -0,0 +1,110 @@ +""" +Finding calibration — an LLM post-processing stage that classifies and +rewords challenger findings for calibrated certainty before they reach the +report. + +Provides `calibrate_findings(vulnerability_text, patch, findings, llm, +code_context)`, which returns one entry per input finding: which of three +epistemic groups it belongs to (Observed / Hypothesis / Hardening), and a +reworded version whose certainty matches that group. + +This is additive to the existing challenger/classifier: it does not change +`_classify_finding`'s categories or counts, and its output is read only by +report presentation (`_build_known_findings` / `_render_known_findings`) — +never by `_compute_trust_signals` or `_build_recommendation_v1`. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import List, Dict + +_PROMPT_PATH = Path(__file__).parent / "prompts" / "finding_calibration.md" + +_VALID_GROUPS = {"observed", "hypothesis", "hardening"} + +# Matches "N. Group: