diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index 27695be..5b3c24f 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -7,13 +7,32 @@ GITHUB_TOKEN=ghp_xxx python local_pr_test.py # explicit token Token resolution: GITHUB_TOKEN env var → `gh auth token` CLI fallback. + +Layer 3 runs with no coverage report (coverage_files=[]), so files reach the +AI gates instead of being resolved by coverage shortcuts — this is the way to +exercise the AI path end to end. It needs a provider key: + + GEMINI_API_KEY=... python local_pr_test.py + +Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key, +AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_OUTPUT_TOKENS, +AI_MAX_INPUT_TOKENS. +Defaults target gemini-3.1-flash-lite at temperature 1.0 with low thinking, +per Google's guidance for Gemini 3.x. Without a key the AI phase is skipped. """ from __future__ import annotations +import os import subprocess import sys +from pathlib import Path + +# Running this file directly puts .dev/ on sys.path, not the repo root, so the +# src.* imports below fail. In CI action.yml sets PYTHONPATH; here we bootstrap +# it so `python .dev/local_pr_test.py` works from anywhere. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from src.config import Config, _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS +from src.config import _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS, Config from src.github_api import create_session from src.github_client import format_report from src.layer1_coverage import run_layer1 @@ -28,7 +47,6 @@ def _resolve_token() -> str: - import os token = os.environ.get("GITHUB_TOKEN") if token: return token @@ -41,6 +59,25 @@ def _resolve_token() -> str: sys.exit(1) +def _resolve_ai_api_key() -> str: + """Provider key for Layer 3, from the first env var that is set. + + Deliberately not GITHUB_TOKEN: GitHub Models was retired on 2026-07-30 and + a GitHub token authenticates no inference endpoint. Without a key the AI + phase is skipped and only the shortcut gates run. + """ + for name in ("AI_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY"): + value = os.environ.get(name, "").strip() + if value: + return value + print( + "WARNING: no AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY set — " + "Layer 3 will run shortcut gates only, no AI call.", + file=sys.stderr, + ) + return "" + + def run_local(repo: str, pr_number: int) -> None: token = _resolve_token() @@ -55,8 +92,17 @@ def run_local(repo: str, pr_number: int) -> None: test_patterns=_DEFAULT_TEST_PATTERNS, exclude_patterns=exclude_patterns, ai_enabled=True, - ai_model="openai/gpt-5-mini", + ai_model=os.environ.get("AI_MODEL", "gemini-3.1-flash-lite"), ai_confidence_threshold=0.7, + ai_base_url=os.environ.get( + "AI_BASE_URL", + "https://generativelanguage.googleapis.com/v1beta/openai/", + ), + ai_api_key=_resolve_ai_api_key(), + ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"), + ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), + ai_max_output_tokens=int(os.environ.get("AI_MAX_OUTPUT_TOKENS", "8192")), + ai_max_input_tokens=int(os.environ.get("AI_MAX_INPUT_TOKENS", "32000")), ) session = create_session(token) @@ -122,7 +168,12 @@ def run_local(repo: str, pr_number: int) -> None: coverage_details=l1.coverage_details, coverage_threshold=config.coverage_threshold, model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, + reasoning_effort=config.ai_reasoning_effort, + temperature=config.ai_temperature, + max_output_tokens=config.ai_max_output_tokens, + max_input_tokens=config.ai_max_input_tokens, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 1bb3a33..fe251b9 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -1,7 +1,8 @@ name: Test Guard (Dogfood) on: pull_request: - types: [opened, synchronize] + # `labeled` lets the dogfood-ai label below start a run without a push. + types: [opened, synchronize, labeled] # Cancel superseded runs: only the newest push per PR branch keeps running. concurrency: @@ -12,7 +13,7 @@ permissions: contents: read pull-requests: write statuses: write - models: read + # No `models: read` — it only ever gated GitHub Models, retired 2026-07-30. jobs: test-guard: @@ -31,12 +32,72 @@ jobs: pip install -r requirements.txt pytest tests/ --cov=src --cov-report=xml + # Two steps rather than one with a conditional coverage-file: in GitHub + # expressions `cond && '' || 'x'` always yields 'x', because the empty + # string is falsy and short-circuits to the fallback. A ternary cannot + # produce an empty value, so gate whole steps instead. - name: Run Test Guard on itself + if: "!contains(github.event.pull_request.labels.*.name, 'dogfood-ai')" uses: ./ with: coverage-file: coverage.xml coverage-threshold: '100' - ai-model: 'openai/gpt-4.1-mini' + # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and + # now 404s. flash-lite defaults to minimal thinking, which suits the + # 2048-token output cap. + ai-model: 'gemini-3.1-flash-lite' + # Trailing slash is required — without it the compat layer 404s. + ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/' + # Layer 3 needs a provider key. Absent the secret, the gate degrades + # to Layer 1 + Layer 2 with a warning instead of failing. + ai-api-key: ${{ secrets.GEMINI_API_KEY }} + # Gemini 3.x has no "off" for thinking. Its published intelligence + # scores are reasoning-enabled, and Google recommends thinking for + # code reasoning, so don't floor this at minimal — the 4096-token + # default output cap leaves room for a short thought trace. + ai-reasoning-effort: 'low' + # Gemini 3.x warns that lowering temperature causes looping and + # degraded reasoning, so use its recommended default instead of 0.1. + ai-temperature: '1.0' + # 8000 was sized for the retired GitHub Models 8K input cap. + # Gemini has a 1M context, and at 8000 a large test diff is shed, + # which makes the model warn about code it was never shown. + ai-max-input-tokens: '32000' + # Repo-specific: benchmarks/ is a standalone perf harness, not product + # code under test. Kept OUT of the shipped defaults on purpose — a + # test-adequacy gate should not silently skip a code tree for every + # user. Unioned onto the defaults, so no need to re-list them. + extra-exclude-patterns: 'benchmarks/**' + + # `dogfood-ai` label: withhold the coverage report so files fall through + # to Layer 3's AI gates (6-8) instead of being resolved by coverage + # shortcuts, which is the only way this workflow exercises the model. + - name: Run Test Guard on itself (AI path, no coverage) + if: contains(github.event.pull_request.labels.*.name, 'dogfood-ai') + uses: ./ + with: + coverage-threshold: '100' + # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and + # now 404s. flash-lite defaults to minimal thinking, which suits the + # 2048-token output cap. + ai-model: 'gemini-3.1-flash-lite' + # Trailing slash is required — without it the compat layer 404s. + ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/' + # Layer 3 needs a provider key. Absent the secret, the gate degrades + # to Layer 1 + Layer 2 with a warning instead of failing. + ai-api-key: ${{ secrets.GEMINI_API_KEY }} + # Gemini 3.x has no "off" for thinking. Its published intelligence + # scores are reasoning-enabled, and Google recommends thinking for + # code reasoning, so don't floor this at minimal — the 4096-token + # default output cap leaves room for a short thought trace. + ai-reasoning-effort: 'low' + # Gemini 3.x warns that lowering temperature causes looping and + # degraded reasoning, so use its recommended default instead of 0.1. + ai-temperature: '1.0' + # 8000 was sized for the retired GitHub Models 8K input cap. + # Gemini has a 1M context, and at 8000 a large test diff is shed, + # which makes the model warn about code it was never shown. + ai-max-input-tokens: '32000' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every diff --git a/README.md b/README.md index 03c0ad3..2bfa587 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,9 @@ Priority within a layer: **FAIL > WARNING > PASS > SKIP**. ## Quick Start -Test-Guard uses the GitHub Models API with your standard `GITHUB_TOKEN`. No external API keys required. +Layer 3 calls any OpenAI-compatible inference endpoint, so it needs one provider API key. + +> **Breaking change (v2).** Test-Guard used to reach GitHub Models with your `GITHUB_TOKEN` and no external key. [GitHub retired GitHub Models on 30 July 2026](https://github.blog/changelog/2026-07-30-github-models-is-now-retired/) — the playground, catalog, and inference API are gone for all customers, and a `GITHUB_TOKEN` no longer authenticates any model endpoint. Set `ai-api-key` (and `ai-base-url` if you are not on OpenAI). Without a key, Layer 3 is skipped and the gate runs on Layer 1 + Layer 2. ```yaml name: Test-Guard @@ -122,7 +124,6 @@ permissions: contents: read pull-requests: write checks: write - models: read # Required for Layer 3 AI analysis jobs: test-guard: @@ -139,9 +140,70 @@ jobs: uses: ostico/test-guard@v1 with: coverage-file: coverage.xml + ai-api-key: ${{ secrets.OPENAI_API_KEY }} ``` -> The `models: read` permission is required for AI analysis. If you set `ai-enabled: 'false'`, you can omit it. +> No special GitHub permission is needed for AI analysis — the old `models: read` scope only ever gated GitHub Models and is now inert. What Layer 3 needs is `ai-api-key`. Set `ai-enabled: 'false'` to run Layers 1–2 only. + +### Provider settings at a glance + +Every AI input in one place. Copy the column for your provider. + +| Setting | OpenAI (default) | Google Gemini | Azure AI Foundry | Groq (free) | +|:--------|:-----------------|:--------------|:-----------------|:------------| +| `ai-base-url` | `https://api.openai.com/v1` | `https://generativelanguage.googleapis.com/v1beta/openai/` | `https://.services.ai.azure.com/openai/v1` | `https://api.groq.com/openai/v1` | +| `ai-model` | `gpt-4.1-mini` | `gemini-3.1-flash-lite` | your deployment name | `openai/gpt-oss-120b` | +| `ai-api-key` | `OPENAI_API_KEY` | `GEMINI_API_KEY` | Foundry key | `GROQ_API_KEY` | +| `ai-reasoning-effort` | `none` (stripped automatically) | `low` | depends on model | `low` | +| `ai-temperature` | `0.1` | **`1.0`** | `0.1` | `0.1` | +| `ai-max-output-tokens` | `8192` | `8192` | `8192` | `8192` | +| `ai-max-input-tokens` | `8000` | `32000` | `32000` | **`8000`** | +| Strict `json_schema` | yes | yes | yes | gpt-oss only | + +Why the non-obvious values: + +- **Gemini `ai-temperature: '1.0'`** — Google documents its default as strongly recommended and warns that lowering it causes looping or degraded reasoning. The `0.1` default suits OpenAI-style models and is actively wrong here. +- **Gemini `ai-max-input-tokens: '32000'`** — a 1M-token context window makes the `8000` default needlessly tight, and a shed test diff produces false warnings (see below). +- **Groq stays at `8000`** — its free tier allows only 8K tokens *per minute*, so a larger prompt trips 429s, and Layer 3 has no 429 backoff. +- **OpenAI `ai-reasoning-effort`** — `gpt-4.1` rejects the parameter with a 400; Layer 3 detects that, retries once without it, and remembers the endpoint/model pair. + +### Pointing at another provider + +`ai-base-url` takes any OpenAI-compatible `/v1` endpoint. Use the model ID *that provider* expects. + +| Provider | `ai-base-url` | `ai-model` | +|:---------|:--------------|:-----------| +| OpenAI (default) | `https://api.openai.com/v1` | `gpt-4.1-mini` | +| Azure AI Foundry | `https://.services.ai.azure.com/openai/v1` | your deployment name | +| OpenRouter | `https://openrouter.ai/api/v1` | `openai/gpt-4.1-mini` | + +Migrating from a pre-retirement config? Drop the `openai/` prefix from `ai-model` unless your new provider namespaces by publisher (OpenRouter does; OpenAI and Azure do not). + +### Free options for open-source repos + +GitHub no longer offers free inference. Copilot Pro remains [free for verified open-source maintainers, students and teachers](https://docs.github.com/en/copilot/how-tos/manage-your-account/getting-free-access-to-copilot-pro-as-a-student-teacher-or-maintainer), but that is an IDE/CLI entitlement — it exposes no OpenAI-compatible endpoint, so this action cannot use it. Azure AI Foundry, the official migration target, needs an Azure subscription and bills per token. + +Third-party free tiers do work here: + +**Layer 3 requires strict structured output.** It sends `response_format=json_schema` with `strict: true`. A model that does not support strict constrained decoding rejects the request, and because Layer 3 only retries on 413 and only falls back on 403, the batch is abandoned with a `::warning::` and those files defer to Layer 1 + Layer 2 — a working gate with no AI analysis. Check strict support before picking a free model. + +| Provider | `ai-base-url` | `ai-model` | Free-tier limits | +|:---------|:--------------|:-----------|:-----------------| +| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-3.1-flash-lite` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | +| Groq | `https://api.groq.com/openai/v1` | `openai/gpt-oss-120b` | 30 RPM, 1,000 req/day, **8K tokens/min**, 200K tokens/day | + +Notes that decide whether these actually work: + +- **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. +- **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. +- **Model IDs go stale — verify before trusting one.** Gemini 2.5 was shut down ahead of its documented October 2026 date, and a retired ID returns a generic `404 models/ is not found for API version v1beta`, which reads like a URL bug. List what your key can actually reach: `curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY"`. +- **Gemini 3.x cannot disable thinking — and you should not want to.** `minimal` is the floor and only "matches the no-thinking setting for most queries". Published intelligence scores for these models are *reasoning-enabled*: `gemini-3.1-flash-lite` scores 25 on the Artificial Analysis Intelligence Index versus 15 for `gpt-4.1-mini`, the baseline this prompt was tuned on — but that figure does not describe a minimal-thinking run, and Google recommends thinking for code reasoning specifically. Layer 3 is code judgment, so prefer `low` or higher and give it room via `ai-max-output-tokens` rather than flooring the effort. +- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-output-tokens` defaults to `8192`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. +- **Gemini 3.x wants `ai-temperature: '1.0'`.** Google documents its default as strongly recommended and warns that lowering temperature can cause looping or degraded reasoning. The action default of `0.1` suits OpenAI-style models but is actively wrong here. +- **Superseded guidance, kept for context: `gemini-2.5-flash` over `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. +- **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_output_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. +- **Providers that reject the parameter are handled automatically.** OpenAI's `gpt-4.1` family returns a 400 for `reasoning_effort`. Layer 3 detects that specific rejection, retries once without the parameter, and remembers the endpoint/model pair — so the default OpenAI setup costs one extra request per run, not per batch. +- **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. --- @@ -155,7 +217,13 @@ jobs: | `exclude-patterns` | _(see below)_ | Comma-separated glob patterns to skip. Setting this **replaces** the default list. See [Excluding files](#excluding-files). | | `extra-exclude-patterns` | _(empty)_ | Comma-separated glob patterns to exclude **in addition** to `exclude-patterns` (unioned, deduped). Add repo-specific excludes here without re-listing the defaults. | | `ai-enabled` | `true` | Enable Layer 3 AI analysis. Truthy values: `true`, `1`, `yes` (case-insensitive); anything else disables it. | -| `ai-model` | `openai/gpt-4.1-mini` | GitHub Models model ID. | +| `ai-model` | `gpt-4.1-mini` | Model ID, exactly as `ai-base-url` expects it. | +| `ai-base-url` | `https://api.openai.com/v1` | OpenAI-compatible inference endpoint. | +| `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | +| `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. Gemini 3.x has no `none` — use `minimal` or above. | +| `ai-temperature` | `0.1` | Sampling temperature. Gemini 3.x wants `1.0`; lowering it there causes looping per Google's docs. | +| `ai-max-output-tokens` | `8192` | Output cap per request. Raise alongside `ai-reasoning-effort`. | +| `ai-max-input-tokens` | `8000` | Prompt budget: drives batching and how much diff evidence survives. Raise on large-context providers. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** @@ -232,23 +300,28 @@ For `GetSearchController.php` this matches **both** `GetSearchControllerTest.php --- -## GitHub Models Setup +## AI Provider Setup -Layer 3 uses the [GitHub Models](https://github.com/marketplace/models) inference API. This works with your existing `GITHUB_TOKEN` — no external API keys needed. +Layer 3 calls an OpenAI-compatible inference endpoint over the `openai` Python SDK. It previously used GitHub Models, which GitHub retired on 30 July 2026; there is no `GITHUB_TOKEN`-authenticated replacement, so you now supply a provider key. ### Requirements -1. **`models: read` permission** in your workflow (see Quick Start above). -2. **GitHub Models enabled** for your account or organization: - - **Personal repos:** Go to [github.com/marketplace/models](https://github.com/marketplace/models) and accept the terms. The free tier is sufficient. - - **Organization repos:** An organization owner must enable GitHub Models at the org level. Go to **Organization Settings → Copilot → Policies** and enable model access. -3. **Free tier limits:** GitHub Models free tier allows ~150 requests/day with up to 8K input tokens per request. Test-Guard's smart batching is designed to stay within these limits. +1. **`ai-api-key`** set to a provider key, passed as a repository or organization secret. +2. **`ai-base-url`** matching that provider (defaults to OpenAI). Azure AI Foundry is GitHub's own recommended destination for retired GitHub Models workloads. +3. **`ai-model`** using the ID that provider expects — see the provider table in Quick Start. + +Batching still targets ~8K input tokens per request, which keeps per-run cost low on metered providers. + +> Requests are billed by whichever provider you configure, not by GitHub. GitHub AI Credits apply to Copilot, and Copilot exposes no OpenAI-compatible endpoint this action can call. ### Troubleshooting | Symptom | Cause | Fix | |:--------|:------|:----| -| 403 "Model not accessible" | GitHub Models not enabled | Enable at org or personal level (step 2 above) | +| 410 `github_models_retirement_brownout` | Config still points at `models.github.ai` | Set `ai-base-url` + `ai-api-key`; that service was retired 30 Jul 2026 | +| Layer 3 skipped with an `ai-api-key` warning | No provider key configured | Set `ai-api-key` to a secret, or `ai-enabled: 'false'` to silence it | +| 401 / 403 "Incorrect API key" | Key does not match `ai-base-url`, or `GITHUB_TOKEN` was passed | Use a key issued by that provider | +| 404 "model does not exist" | Model ID carries the wrong prefix | OpenAI/Azure: `gpt-4.1-mini`. OpenRouter: `openai/gpt-4.1-mini` | | 413 "Request body too large" | Diff exceeds model token limit | Automatic — smart batching handles this | | Intermittent 429 errors | Rate limit exceeded | Reduce PR size or use `ai-enabled: 'false'` for low-priority PRs | @@ -256,7 +329,7 @@ Layer 3 uses the [GitHub Models](https://github.com/marketplace/models) inferenc ## AI Architecture -GitHub Models' free tier caps requests at **8K input tokens**. A naive implementation hits that wall constantly on real PRs — Test-Guard avoids it with three layers working together: **compaction** (shrink each diff without losing signal), **batching** (group files so a call never exceeds the cap), and **matching** (only attach the test diffs that actually belong to this batch). +Layer 3 budgets each request to **8K input tokens**. A naive implementation hits that wall constantly on real PRs — Test-Guard avoids it with three layers working together: **compaction** (shrink each diff without losing signal), **batching** (group files so a call never exceeds the cap), and **matching** (only attach the test diffs that actually belong to this batch). ```mermaid flowchart LR diff --git a/action.yml b/action.yml index 2da9484..3ec7bfb 100644 --- a/action.yml +++ b/action.yml @@ -27,8 +27,31 @@ inputs: description: 'Enable AI Layer 3 analysis (true/false).' default: 'true' ai-model: - description: 'GitHub Models model ID for AI analysis.' - default: 'openai/gpt-4.1-mini' + description: 'Model ID for AI analysis, exactly as the configured endpoint expects (OpenAI: "gpt-4.1-mini"; OpenRouter: "openai/gpt-4.1-mini").' + default: 'gpt-4.1-mini' + ai-base-url: + description: 'OpenAI-compatible inference endpoint. Examples: https://api.openai.com/v1, https://.services.ai.azure.com/openai/v1, https://openrouter.ai/api/v1. GitHub Models (models.github.ai) was retired on 2026-07-30 and no longer works.' + default: 'https://api.openai.com/v1' + ai-api-key: + # Do not write a ${{ ... }} expression anywhere in this file, including + # descriptions: GitHub evaluates them when loading the manifest, and + # `secrets` is not in scope for a composite action, so the whole action + # fails with "Unrecognized named-value: 'secrets'". + description: 'API key for ai-base-url; pass it from a repository secret. Required for Layer 3 — a GITHUB_TOKEN is not accepted by any of these providers. When empty, Layer 3 is skipped and the gate falls back to Layer 1 + Layer 2.' + required: false + default: '' + ai-reasoning-effort: + description: 'Thinking budget for reasoning models: none, minimal, low, medium, high. Defaults to none so a thought trace cannot consume the output budget and truncate the JSON verdict. Set to an empty string to omit the parameter entirely; providers that reject it (OpenAI gpt-4.1 family) have it stripped automatically.' + default: 'none' + ai-temperature: + description: 'Sampling temperature. 0.1 suits OpenAI-style models for a deterministic verdict. Gemini 3.x recommends its default of 1.0 and warns that lowering it can cause looping or degraded reasoning — set 1.0 for those models.' + default: '0.1' + ai-max-output-tokens: + description: 'Output token cap per request. Thought tokens are charged against it, and exhausting it returns finish_reason=length with empty content, which is reported as a warning. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' + default: '8192' + ai-max-input-tokens: + description: 'Budget for the prompt Layer 3 sends, which drives batching and how much diff evidence survives. The default 8000 was sized for the retired GitHub Models 8K input cap; raise it (e.g. 32000) on a large-context provider so big test diffs are not shed, at the cost of more tokens per minute. Also scales the per-diff character cap.' + default: '8000' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -56,6 +79,12 @@ runs: INPUT_EXTRA-EXCLUDE-PATTERNS: ${{ inputs.extra-exclude-patterns }} INPUT_AI-ENABLED: ${{ inputs.ai-enabled }} INPUT_AI-MODEL: ${{ inputs.ai-model }} + INPUT_AI-BASE-URL: ${{ inputs.ai-base-url }} + INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} + INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} + INPUT_AI-TEMPERATURE: ${{ inputs.ai-temperature }} + INPUT_AI-MAX-OUTPUT-TOKENS: ${{ inputs.ai-max-output-tokens }} + INPUT_AI-MAX-INPUT-TOKENS: ${{ inputs.ai-max-input-tokens }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 0a3be55..74507a7 100644 --- a/src/config.py +++ b/src/config.py @@ -24,7 +24,36 @@ "build.rs" ) _DEFAULT_COVERAGE_THRESHOLD = 80 -_DEFAULT_AI_MODEL = "openai/gpt-4.1-mini" +_DEFAULT_AI_MODEL = "gpt-4.1-mini" +# Any OpenAI-compatible /v1 endpoint. GitHub Models (models.github.ai) was +# retired on 2026-07-30 and is no longer a valid target. +DEFAULT_AI_BASE_URL = "https://api.openai.com/v1" +# "none" keeps thinking models from spending the output budget on a thought +# trace and truncating the JSON verdict. Providers that reject the parameter +# get it stripped automatically (see layer3_ai). Empty string never sends it. +DEFAULT_AI_REASONING_EFFORT = "none" +# 0.1 suits OpenAI-style models for a deterministic classification. Gemini 3.x +# documents the opposite: it recommends its default of 1.0 and warns that +# lowering temperature can cause looping or degraded reasoning, so that +# provider needs this raised. +DEFAULT_AI_TEMPERATURE = 0.1 +# A verdict payload is a few hundred tokens; the rest of this budget exists so +# a reasoning model's thought trace cannot squeeze the JSON out. Thought tokens +# are charged against the output cap, and when they exhaust it the API returns +# finish_reason="length" with empty content — which parses as SKIP at +# confidence 0.0. 8192 is the community-reported floor that avoids that with +# thinking enabled; the cap is not a reservation, so unused headroom is free. +# It stays well under the 65536 Gemini 3.x allows, because thought tokens are +# billed as output and burn free-tier tokens/minute. +DEFAULT_AI_MAX_OUTPUT_TOKENS = 8192 +# Client-side budget for the prompt we send, which drives batching and decides +# how much diff evidence survives. 8000 was sized for GitHub Models' retired 8K +# input cap, and it is now the binding constraint on evidence quality: a test +# diff larger than the budget gets shed, and the prompt instructs the model to +# treat omitted code as untested, producing false warnings. It stays +# conservative by default because providers meter tokens per minute — Groq's +# free tier allows only 8K/min — so raise it deliberately per provider. +DEFAULT_AI_MAX_INPUT_TOKENS = 8000 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -161,7 +190,15 @@ class Config: test_patterns: Language-specific source-to-test file mappings. exclude_patterns: Glob patterns to skip (config files, docs, etc.). ai_enabled: Whether Layer 3 AI analysis is enabled. - ai_model: GitHub Models model ID (e.g., "openai/gpt-4.1-mini"). + ai_model: Provider model ID (e.g., "gpt-4.1-mini"). Use the exact ID + the configured endpoint expects — OpenAI wants "gpt-4.1-mini", + OpenRouter wants "openai/gpt-4.1-mini". + ai_base_url: OpenAI-compatible inference endpoint. + ai_api_key: API key for that endpoint. Empty disables Layer 3's AI + phase (shortcut gates still run). + ai_reasoning_effort: Thinking budget hint sent to reasoning models + ("none", "minimal", "low", "medium", "high"). Empty sends nothing; + endpoints that reject the parameter get it stripped on retry. ai_confidence_threshold: AI FAIL verdicts below this become WARNING (0.0-1.0). """ @@ -183,6 +220,12 @@ class Config: ai_enabled: bool ai_model: str ai_confidence_threshold: float # 0.0-1.0 + ai_base_url: str = DEFAULT_AI_BASE_URL + ai_api_key: str = "" + ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT + ai_temperature: float = DEFAULT_AI_TEMPERATURE + ai_max_output_tokens: int = DEFAULT_AI_MAX_OUTPUT_TOKENS + ai_max_input_tokens: int = DEFAULT_AI_MAX_INPUT_TOKENS def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -305,6 +348,48 @@ def parse_config() -> Config: # Layer 3: Parse AI configuration. ai_enabled = _env("AI-ENABLED", "true").lower() in _DEFAULT_AI_ENABLED_VALUES ai_model = _env("AI-MODEL", _DEFAULT_AI_MODEL) + ai_base_url = _env("AI-BASE-URL", DEFAULT_AI_BASE_URL) + # Provider API key. Deliberately NOT GITHUB_TOKEN: GitHub Models was + # retired on 2026-07-30 and a GitHub token authenticates nothing else. + ai_api_key = _env("AI-API-KEY", "") + ai_reasoning_effort = _env("AI-REASONING-EFFORT", DEFAULT_AI_REASONING_EFFORT) + temperature_raw = _env("AI-TEMPERATURE", str(DEFAULT_AI_TEMPERATURE)) + try: + ai_temperature = float(temperature_raw) + except ValueError: + print( + f"::warning::Invalid ai-temperature '{temperature_raw}' — " + f"falling back to {DEFAULT_AI_TEMPERATURE}." + ) + ai_temperature = DEFAULT_AI_TEMPERATURE + + max_output_tokens_raw = _env("AI-MAX-OUTPUT-TOKENS", str(DEFAULT_AI_MAX_OUTPUT_TOKENS)) + try: + ai_max_output_tokens = int(max_output_tokens_raw) + except ValueError: + print( + f"::warning::Invalid ai-max-output-tokens '{max_output_tokens_raw}' — " + f"falling back to {DEFAULT_AI_MAX_OUTPUT_TOKENS}." + ) + ai_max_output_tokens = DEFAULT_AI_MAX_OUTPUT_TOKENS + + input_limit_raw = _env("AI-MAX-INPUT-TOKENS", str(DEFAULT_AI_MAX_INPUT_TOKENS)) + try: + ai_max_input_tokens = int(input_limit_raw) + except ValueError: + print( + f"::warning::Invalid ai-max-input-tokens '{input_limit_raw}' — " + f"falling back to {DEFAULT_AI_MAX_INPUT_TOKENS}." + ) + ai_max_input_tokens = DEFAULT_AI_MAX_INPUT_TOKENS + if ai_enabled and not ai_api_key: + print( + "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " + "AI analysis disabled, falling back to Layer 1 + Layer 2. GitHub " + "Models was retired on 2026-07-30; set ai-api-key (plus " + "ai-base-url for non-OpenAI providers) to re-enable." + ) + ai_enabled = False confidence_raw = _env( "AI-CONFIDENCE-THRESHOLD", str(_DEFAULT_AI_CONFIDENCE_THRESHOLD) ) @@ -330,5 +415,11 @@ def parse_config() -> Config: exclude_patterns=exclude_patterns, ai_enabled=ai_enabled, ai_model=ai_model, + ai_base_url=ai_base_url, + ai_api_key=ai_api_key, + ai_reasoning_effort=ai_reasoning_effort, + ai_temperature=ai_temperature, + ai_max_output_tokens=ai_max_output_tokens, + ai_max_input_tokens=ai_max_input_tokens, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index d663183..5a052e5 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -2,7 +2,7 @@ Evaluates each source file through an 8-gate shortcut truth table using coverage data from L1 and test-match data from L2. Files that cannot be -resolved deterministically fall through to the GitHub Models AI for a +resolved deterministically fall through to the configured AI provider for a structured JSON verdict with confidence-based downgrade. Flow: @@ -31,6 +31,11 @@ from unidiff import PatchSet from unidiff.errors import UnidiffParseError +from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL +from src.config import DEFAULT_AI_MAX_INPUT_TOKENS as _DEFAULT_AI_MAX_INPUT_TOKENS +from src.config import DEFAULT_AI_MAX_OUTPUT_TOKENS as _DEFAULT_AI_MAX_OUTPUT_TOKENS +from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT +from src.config import DEFAULT_AI_TEMPERATURE as _DEFAULT_AI_TEMPERATURE from src.diff_utils import is_trivial_diff from src.models import FileVerdict, LayerResult, Verdict @@ -237,7 +242,11 @@ def _build_ai_prompt( coverage_details: dict[str, float] | None, coverage_threshold: float, matched_tests: dict[str, str | list[str] | None], - max_diff_chars: int = 10_000, + # Both resolved at call time rather than as def-time defaults: max_diff_chars + # is derived from token_budget, and the budget constant is defined further + # down this module, so a def-time default would NameError. + max_diff_chars: int | None = None, + token_budget: int | None = None, ) -> str: """Build the structured Markdown user prompt for the AI batch. @@ -253,6 +262,17 @@ def _build_ai_prompt( if not files_for_ai: return "" + if token_budget is None: + token_budget = _user_prompt_budget() + # Two independent truncations guard the prompt: this per-diff character cap + # and the whole-diff shedding below. Raising only the token budget changes + # nothing while the char cap still clips every large diff, so derive the cap + # from the budget — half of it, leaving room for other files in the batch, + # and never below the historical 10k floor. An explicit value always wins: + # the 413 retry passes a deliberately smaller cap to shrink the request. + if max_diff_chars is None: + max_diff_chars = max(10_000, token_budget * _CHARS_PER_TOKEN // 2) + # Build a reverse map: test file → list of source files it is matched to. # Used to annotate each test diff in the prompt so the AI knows which # source files a given test is authoritative for. @@ -323,7 +343,7 @@ def assemble(entries: list[dict[str, object]]) -> str: # input cap even when the aggregate test payload is large. The evidence # banner (recomputed in assemble) tells the model what was dropped. kept = list(test_entries) - while kept and _estimate_tokens(assemble(kept)) > _USER_PROMPT_TOKEN_BUDGET: + while kept and _estimate_tokens(assemble(kept)) > token_budget: victim = max(kept, key=lambda e: (bool(e["is_candidate"]), int(e["cost"]))) kept.remove(victim) return assemble(kept) @@ -601,18 +621,34 @@ def _sanitize_diff( # Default model chain: try gpt-4.1-mini first, fall back to gpt-4.1-nano on 403. # When the user explicitly sets ai-model to something other than the chain head, # a single attempt is made with that model (no fallback). +# +# IDs are unprefixed (OpenAI style). Providers that namespace by publisher +# (OpenRouter: "openai/gpt-4.1-mini") need ai-model set explicitly. _DEFAULT_FALLBACK_CHAIN: tuple[str, ...] = ( - "openai/gpt-4.1-mini", - "openai/gpt-4.1-nano", + "gpt-4.1-mini", + "gpt-4.1-nano", ) +# Endpoint/model pairs that rejected `reasoning_effort`. Populated at runtime +# so the fallback costs one extra request per pair, not one per batch. +_REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() + _CHARS_PER_TOKEN = 3 -_INPUT_TOKEN_LIMIT = 8000 +_MAX_INPUT_TOKENS = _DEFAULT_AI_MAX_INPUT_TOKENS _SYSTEM_OVERHEAD_TOKENS = 800 # system prompt + JSON schema overhead _SAFETY_FACTOR = 0.85 -_USER_PROMPT_TOKEN_BUDGET = int( - (_INPUT_TOKEN_LIMIT - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR -) # = 6120 tokens + + +def _user_prompt_budget(max_input_tokens: int = _MAX_INPUT_TOKENS) -> int: + """Tokens available for the user prompt at a given input budget. + + Subtracts the system prompt and schema, then applies a safety factor + because token counts here are estimated from characters, not measured. + """ + return int((max_input_tokens - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR) + + +_USER_PROMPT_TOKEN_BUDGET = _user_prompt_budget() # = 6120 tokens at 8000 _FILE_ENTRY_OVERHEAD_TOKENS = 25 # markdown headers, code fences per file _BATCH_OVERHEAD_TOKENS = 30 # section headers per batch prompt _RETRY_MAX_DIFF_CHARS = 3000 # tighter truncation on 413 retry @@ -755,6 +791,19 @@ def _is_retryable_size_error(exc: Exception) -> bool: return False +def _is_unsupported_reasoning_effort_error(exc: Exception) -> bool: + """True when the endpoint rejected the ``reasoning_effort`` parameter. + + Non-reasoning models reject it in provider-specific ways — OpenAI answers + "Unsupported value", Azure "Unrecognized request argument", others + "Unknown parameter" — but every variant names the parameter, so match on + that rather than on a message template. + """ + if isinstance(exc, APIStatusError) and exc.status_code in (400, 404, 422): + return "reasoning_effort" in str(exc).lower() + return False + + def _validate_batch_verdicts( verdicts: list[FileVerdict], batch_files: list[str], @@ -804,6 +853,11 @@ def _call_ai_for_batch( model: str, system_prompt: str, token: str, + base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, + max_input_tokens: int = _DEFAULT_AI_MAX_INPUT_TOKENS, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -814,12 +868,17 @@ def _call_ai_for_batch( batch_test_diffs = _filter_test_diffs_for_batch( batch_files, test_diffs, matched_tests, ) + token_budget = _user_prompt_budget(max_input_tokens) user_prompt = _build_ai_prompt( batch_files, source_diffs, batch_test_diffs, coverage_details, coverage_threshold, matched_tests, + token_budget=token_budget, ) try: - raw = _call_github_models(model, system_prompt, user_prompt, token) + raw = _call_ai_provider( + model, system_prompt, user_prompt, token, base_url, + reasoning_effort, temperature, max_output_tokens, + ) return raw, None except Exception as exc: if _is_retryable_size_error(exc): @@ -827,10 +886,11 @@ def _call_ai_for_batch( batch_files, source_diffs, batch_test_diffs, coverage_details, coverage_threshold, matched_tests, max_diff_chars=_RETRY_MAX_DIFF_CHARS, + token_budget=token_budget, ) try: - raw = _call_github_models( - model, system_prompt, user_prompt, token, + raw = _call_ai_provider( + model, system_prompt, user_prompt, token, base_url, ) return raw, None except Exception as retry_exc: @@ -838,26 +898,82 @@ def _call_ai_for_batch( return None, exc -def _call_github_models( +def _call_ai_provider( model: str, system_prompt: str, user_prompt: str, token: str, + base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, ) -> str: - """Call GitHub Models API and return the raw response text.""" + """Call an OpenAI-compatible inference endpoint, return raw response text. + + ``base_url`` is provider-agnostic: OpenAI, Azure AI Foundry, OpenRouter, + or any local gateway speaking the same protocol. GitHub Models used to be + hardcoded here; it was retired on 2026-07-30. + + ``reasoning_effort`` defaults to ``"none"`` to keep thinking models from + spending the 2048-token output budget on a thought trace and truncating + the JSON verdict. Non-reasoning models (OpenAI's gpt-4.1 family) reject + the parameter outright, so an unsupported-parameter rejection is retried + once without it and the (endpoint, model) pair is remembered. + """ client = OpenAI( - base_url="https://models.github.ai/inference", + base_url=base_url, api_key=token, ) messages = [ ChatCompletionSystemMessageParam(role="system", content=system_prompt), ChatCompletionUserMessageParam(role="user", content=user_prompt), ] - response = client.chat.completions.create( + extra: dict[str, object] = {} + if reasoning_effort and (base_url, model) not in _REASONING_EFFORT_UNSUPPORTED: + extra["reasoning_effort"] = reasoning_effort + + try: + response = _create_completion( + client, model, messages, extra, temperature, max_output_tokens, + ) + except Exception as exc: + if not extra or not _is_unsupported_reasoning_effort_error(exc): + raise + _REASONING_EFFORT_UNSUPPORTED.add((base_url, model)) + response = _create_completion(client, model, messages, {}, temperature, max_output_tokens) + + if not response.choices: + return "" + + choice = response.choices[0] + content = choice.message.content or "" + # A reasoning model can spend the whole output cap on its thought trace and + # return finish_reason="length" with empty content. That parses as SKIP at + # confidence 0.0, which is indistinguishable from a model that simply had + # nothing to say — so name the cause instead of degrading quietly. + if getattr(choice, "finish_reason", None) == "length" and not content.strip(): + print( + f"::warning::{model} hit the {max_output_tokens}-token output cap before " + f"emitting a verdict — most likely the thought trace consumed it. " + f"Raise ai-max-output-tokens or lower ai-reasoning-effort." + ) + return content + + +def _create_completion( + client: OpenAI, + model: str, + messages: list[object], + extra: dict[str, object], + temperature: float = _DEFAULT_AI_TEMPERATURE, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, +) -> object: + """Issue the chat-completion request, with ``extra`` merged into kwargs.""" + return client.chat.completions.create( model=model, messages=messages, - temperature=0.1, - max_tokens=2048, + temperature=temperature, + max_tokens=max_output_tokens, response_format=ResponseFormatJSONSchema( type="json_schema", json_schema=JSONSchema( @@ -866,10 +982,8 @@ def _call_github_models( schema=_VERDICT_SCHEMA, ), ), + **extra, ) - if not response.choices: - return "" - return response.choices[0].message.content or "" def _parse_ai_response( @@ -918,6 +1032,11 @@ def run_layer3( token: str, confidence_threshold: float, unmeasurable_files: set[str] | None = None, + base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, + max_input_tokens: int = _DEFAULT_AI_MAX_INPUT_TOKENS, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1030,6 +1149,8 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, + base_url, reasoning_effort, temperature, max_output_tokens, + max_input_tokens, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index b3a137b..ca1b59a 100644 --- a/src/main.py +++ b/src/main.py @@ -142,7 +142,12 @@ def run_pipeline(config: Config) -> Report: coverage_details=l1.coverage_details, coverage_threshold=config.coverage_threshold, model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, + reasoning_effort=config.ai_reasoning_effort, + temperature=config.ai_temperature, + max_output_tokens=config.ai_max_output_tokens, + max_input_tokens=config.ai_max_input_tokens, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. @@ -164,6 +169,8 @@ def _attach_summary( """Generate and attach summary to report if verdict is WARNING/FAIL.""" if report.overall_verdict in (Verdict.PASS, Verdict.SKIP): return + if not config.ai_api_key: + return report.summary = generate_summary( report=report, @@ -171,7 +178,8 @@ def _attach_summary( test_files_in_pr=test_files_in_pr, coverage_files_provided=bool(config.coverage_files), model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, ) diff --git a/src/summary.py b/src/summary.py index 65295e0..0803311 100644 --- a/src/summary.py +++ b/src/summary.py @@ -15,6 +15,7 @@ ChatCompletionUserMessageParam, ) +from src.config import DEFAULT_AI_BASE_URL from src.models import Report, Verdict _PROMPT_PATH = Path(__file__).parent.parent / "prompts" / "summary_explainer.txt" @@ -97,6 +98,7 @@ def generate_summary( coverage_files_provided: bool, model: str, token: str, + base_url: str = DEFAULT_AI_BASE_URL, ) -> str | None: """Generate a developer-facing explanation for WARNING/FAIL verdicts. @@ -117,7 +119,7 @@ def generate_summary( try: client = OpenAI( - base_url="https://models.github.ai/inference", + base_url=base_url, api_key=token, ) response = client.chat.completions.create( diff --git a/tests/test_config.py b/tests/test_config.py index 19db0e6..f2c6117 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,6 +16,8 @@ def test_defaults(self, monkeypatch): monkeypatch.delenv("INPUT_AI-ENABLED", raising=False) monkeypatch.delenv("INPUT_AI-MODEL", raising=False) monkeypatch.delenv("INPUT_AI-CONFIDENCE-THRESHOLD", raising=False) + monkeypatch.delenv("INPUT_AI-BASE-URL", raising=False) + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") @@ -24,7 +26,9 @@ def test_defaults(self, monkeypatch): assert cfg.coverage_files == [] assert cfg.coverage_threshold == 80 assert cfg.ai_enabled is True - assert cfg.ai_model == "openai/gpt-4.1-mini" + assert cfg.ai_model == "gpt-4.1-mini" + assert cfg.ai_base_url == "https://api.openai.com/v1" + assert cfg.ai_api_key == "sk-fake" assert cfg.ai_confidence_threshold == 0.7 assert cfg.github_token == "ghp_fake123" assert cfg.repo == "owner/repo" @@ -48,6 +52,104 @@ def test_custom_values(self, monkeypatch): assert cfg.ai_confidence_threshold == 0.9 assert cfg.exclude_patterns == ["*.md", "docs/**"] + def test_custom_endpoint_and_key(self, monkeypatch): + """ai-base-url / ai-api-key point Layer 3 at any OpenAI-compatible provider.""" + monkeypatch.setenv("INPUT_AI-BASE-URL", "https://openrouter.ai/api/v1") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-or-abc") + monkeypatch.setenv("INPUT_AI-MODEL", "openai/gpt-4.1-mini") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_base_url == "https://openrouter.ai/api/v1" + assert cfg.ai_api_key == "sk-or-abc" + # Publisher-prefixed IDs pass through untouched — OpenRouter needs them. + assert cfg.ai_model == "openai/gpt-4.1-mini" + assert cfg.ai_enabled is True + + def test_invalid_temperature_falls_back(self, monkeypatch, capsys): + """A bad temperature must not abort the run — warn and use the default.""" + monkeypatch.setenv("INPUT_AI-TEMPERATURE", "not-a-number") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_temperature == 0.1 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-temperature" in out + + def test_max_input_tokens_is_configurable(self, monkeypatch): + """Raising the prompt budget is how large test diffs stop being shed.""" + monkeypatch.setenv("INPUT_AI-MAX-INPUT-TOKENS", "32000") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + assert parse_config().ai_max_input_tokens == 32000 + + def test_invalid_max_input_tokens_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-MAX-INPUT-TOKENS", "huge") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_max_input_tokens == 8000 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-max-input-tokens" in out + + def test_invalid_max_output_tokens_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-MAX-OUTPUT-TOKENS", "lots") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_max_output_tokens == 8192 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-max-output-tokens" in out + + def test_valid_temperature_and_max_output_tokens_are_used(self, monkeypatch): + monkeypatch.setenv("INPUT_AI-TEMPERATURE", "1.0") + monkeypatch.setenv("INPUT_AI-MAX-OUTPUT-TOKENS", "16384") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_temperature == 1.0 + assert cfg.ai_max_output_tokens == 16384 + + def test_missing_api_key_disables_ai(self, monkeypatch, capsys): + """No key means no AI phase: GITHUB_TOKEN is not a provider credential. + + GitHub Models was retired 2026-07-30, so there is no endpoint a bare + GitHub token authenticates against. Degrade to L1+L2 with a warning + instead of failing every run with a 401/410. + """ + monkeypatch.setenv("INPUT_AI-ENABLED", "true") + monkeypatch.delenv("INPUT_AI-API-KEY", raising=False) + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_enabled is False + assert cfg.ai_api_key == "" + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-api-key" in out + def _base_env(self, monkeypatch): monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 1b0486c..ec41182 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -3,16 +3,18 @@ import json from unittest.mock import MagicMock, patch +import pytest from openai import APIStatusError import src.layer3_ai as layer3_ai from src.layer3_ai import ( + _REASONING_EFFORT_UNSUPPORTED, Layer3Result, Relevance, _batch_files, _build_ai_prompt, _call_ai_for_batch, - _call_github_models, + _call_ai_provider, _compact_diff, _context_ladder, _count_change_hunks, @@ -359,7 +361,7 @@ def test_skip_and_pass_returns_pass(self): class TestRunLayer3: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_all_shortcuts_no_ai_called(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/trivial.py": "+ # comment"}, @@ -375,7 +377,7 @@ def test_all_shortcuts_no_ai_called(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_deleted_files_produce_pass(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/old.py": "+ code"}, @@ -391,7 +393,7 @@ def test_deleted_files_produce_pass(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_fallthrough_calls_ai(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "pass", @@ -412,7 +414,7 @@ def test_fallthrough_calls_ai(self, mock_call: MagicMock): mock_call.assert_called_once() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_ai_failure_returns_skip(self, mock_call: MagicMock): mock_call.side_effect = Exception("API down") result = run_layer3( @@ -429,7 +431,7 @@ def test_ai_failure_returns_skip(self, mock_call: MagicMock): assert result.verdict == Verdict.SKIP assert "API down" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_ai_failure_with_shortcuts_preserves_shortcut_verdicts(self, mock_call: MagicMock): mock_call.side_effect = Exception("API down") result = run_layer3( @@ -469,7 +471,7 @@ def test_empty_source_diffs_returns_pass(self): ) assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_mixed_shortcut_and_ai_fail(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -492,7 +494,7 @@ def test_mixed_shortcut_and_ai_fail(self, mock_call: MagicMock): ) assert result.verdict == Verdict.FAIL - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_gate4_fail_without_ai(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/billing.py": "+ bill()"}, @@ -508,7 +510,7 @@ def test_gate4_fail_without_ai(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.FAIL - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_low_confidence_ai_downgrades_to_warning(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -528,7 +530,7 @@ def test_low_confidence_ai_downgrades_to_warning(self, mock_call: MagicMock): ) assert result.verdict == Verdict.WARNING - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_coverage_ok_shortcut_passes(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/user.py": "+ real_code()"}, @@ -544,7 +546,7 @@ def test_coverage_ok_shortcut_passes(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_coverage_below_threshold_with_relevant_tests_fails_shortcut( self, mock_call: MagicMock ): @@ -567,7 +569,7 @@ def test_coverage_below_threshold_with_relevant_tests_fails_shortcut( fv = {v.file: v for v in result.file_verdicts}["src/user.py"] assert "relevant tests exist but insufficient" in fv.reason - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_unmeasurable_file_skips_instead_of_failing_gate_4( self, mock_call: MagicMock ): @@ -593,7 +595,7 @@ def test_unmeasurable_file_skips_instead_of_failing_gate_4( assert fv.verdict == Verdict.SKIP assert "no executable lines changed" in fv.reason - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_file_not_marked_unmeasurable_still_fails_gate_4( self, mock_call: MagicMock ): @@ -616,7 +618,315 @@ def test_file_not_marked_unmeasurable_still_fails_gate_4( assert fv.verdict == Verdict.FAIL -class TestCallGithubModels: +class TestReasoningEffort: + """`reasoning_effort` defaults to "none" but must not break providers + that reject the parameter (OpenAI's gpt-4.1 family).""" + + def setup_method(self): + _REASONING_EFFORT_UNSUPPORTED.clear() + + def teardown_method(self): + _REASONING_EFFORT_UNSUPPORTED.clear() + + @patch("src.layer3_ai.OpenAI") + def test_forwards_reasoning_effort(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-2.5-flash", + system_prompt="system", + user_prompt="user", + token="sk-fake", + base_url="https://generativelanguage.googleapis.com/v1beta/openai/", + reasoning_effort="none", + ) + + kwargs = mock_client.chat.completions.create.call_args.kwargs + assert kwargs["reasoning_effort"] == "none" + + @patch("src.layer3_ai.OpenAI") + def test_empty_effort_omits_parameter(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="", + ) + + kwargs = mock_client.chat.completions.create.call_args.kwargs + assert "reasoning_effort" not in kwargs + + @patch("src.layer3_ai.OpenAI") + def test_rejection_retries_without_parameter(self, mock_openai: MagicMock): + """A model that rejects the parameter still gets its verdict.""" + ok = MagicMock() + ok.choices = [MagicMock()] + ok.choices[0].message.content = '{"verdict": "pass"}' + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = [ + _make_api_error(400, "Unsupported value: 'reasoning_effort' ..."), + ok, + ] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + assert result == '{"verdict": "pass"}' + assert mock_client.chat.completions.create.call_count == 2 + retry_kwargs = mock_client.chat.completions.create.call_args.kwargs + assert "reasoning_effort" not in retry_kwargs + + @patch("src.layer3_ai.OpenAI") + def test_rejection_is_remembered_per_endpoint_and_model( + self, mock_openai: MagicMock, + ): + """The retry costs one extra request per pair, not one per batch.""" + ok = MagicMock() + ok.choices = [] + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = [ + _make_api_error(400, "Unknown parameter: 'reasoning_effort'"), + ok, + ok, + ] + mock_openai.return_value = mock_client + + for _ in range(2): + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + # 2 for the first call (reject + retry), 1 for the second — not 4. + assert mock_client.chat.completions.create.call_count == 3 + assert ( + "reasoning_effort" + not in mock_client.chat.completions.create.call_args.kwargs + ) + + @patch("src.layer3_ai.OpenAI") + def test_unrelated_400_is_not_swallowed(self, mock_openai: MagicMock): + """Only parameter rejections trigger the retry — real errors propagate.""" + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = _make_api_error( + 400, "Incorrect API key provided", + ) + mock_openai.return_value = mock_client + + with pytest.raises(APIStatusError): + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + assert mock_client.chat.completions.create.call_count == 1 + + +class TestTemperature: + @patch("src.layer3_ai.OpenAI") + def test_forwards_temperature(self, mock_openai: MagicMock): + """Gemini 3.x needs 1.0 — lower values cause looping per Google's docs.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + temperature=1.0, + ) + + assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 1.0 + + @patch("src.layer3_ai.OpenAI") + def test_defaults_to_deterministic_temperature(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 0.1 + + +class TestInputTokenBudget: + """The input budget decides how much test evidence survives into the prompt. + + Shed test diffs make the prompt's evidence-completeness rule kick in, and + the model correctly returns a warning for code it cannot see tested — so a + budget that is too small manufactures false warnings. + """ + + def test_budget_scales_with_the_limit(self): + assert layer3_ai._user_prompt_budget(8000) == 6120 + assert layer3_ai._user_prompt_budget(32000) == 26520 + + def test_raised_limit_keeps_a_large_test_diff(self): + big_test_diff = "+ assert something\n" * 1200 # ~22k chars, ~7.6k tokens + + shed = layer3_ai._build_ai_prompt( + ["src/thing.py"], + {"src/thing.py": "+ def thing(): pass"}, + {"tests/test_thing.py": big_test_diff}, + None, 80, {"src/thing.py": "tests/test_thing.py"}, + token_budget=layer3_ai._user_prompt_budget(8000), + ) + kept = layer3_ai._build_ai_prompt( + ["src/thing.py"], + {"src/thing.py": "+ def thing(): pass"}, + {"tests/test_thing.py": big_test_diff}, + None, 80, {"src/thing.py": "tests/test_thing.py"}, + token_budget=layer3_ai._user_prompt_budget(32000), + ) + + # The raised budget must retain strictly more evidence. + assert len(kept) > len(shed) + + +class TestTruncationWarning: + """A thought trace that eats the output cap must not degrade silently.""" + + @patch("src.layer3_ai.OpenAI") + def test_warns_when_cap_consumed_before_verdict( + self, mock_openai: MagicMock, capsys, + ): + choice = MagicMock() + choice.finish_reason = "length" + choice.message.content = "" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [choice] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + max_output_tokens=8192, + ) + + assert result == "" + out = capsys.readouterr().out + assert "::warning::" in out + assert "8192-token output cap" in out + assert "ai-max-output-tokens" in out + + @patch("src.layer3_ai.OpenAI") + def test_no_warning_on_a_normal_verdict(self, mock_openai: MagicMock, capsys): + choice = MagicMock() + choice.finish_reason = "stop" + choice.message.content = '{"verdict": "pass"}' + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [choice] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert result == '{"verdict": "pass"}' + assert "::warning::" not in capsys.readouterr().out + + +class TestMaxTokens: + @patch("src.layer3_ai.OpenAI") + def test_forwards_max_output_tokens(self, mock_openai: MagicMock): + """Headroom for a thought trace is configurable, not hardcoded.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + max_output_tokens=16384, + ) + + assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 16384 + + @patch("src.layer3_ai.OpenAI") + def test_default_leaves_room_for_thinking(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 8192 + + +class TestCallAiProvider: + @patch("src.layer3_ai.OpenAI") + def test_uses_configured_base_url(self, mock_openai: MagicMock): + """The endpoint is caller-supplied, not the retired GitHub Models host.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + base_url="https://myresource.services.ai.azure.com/openai/v1", + ) + + mock_openai.assert_called_once_with( + base_url="https://myresource.services.ai.azure.com/openai/v1", + api_key="sk-fake", + ) + + @patch("src.layer3_ai.OpenAI") + def test_defaults_to_openai_endpoint(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_openai.call_args.kwargs["base_url"] == "https://api.openai.com/v1" + @patch("src.layer3_ai.OpenAI") def test_empty_choices_returns_empty_string(self, mock_openai: MagicMock): mock_client = MagicMock() @@ -625,7 +935,7 @@ def test_empty_choices_returns_empty_string(self, mock_openai: MagicMock): mock_client.chat.completions.create.return_value = mock_response mock_openai.return_value = mock_client - raw = _call_github_models( + raw = _call_ai_provider( model="openai/gpt-5-mini", system_prompt="system", user_prompt="user", @@ -1106,7 +1416,7 @@ def test_example3_deleted_file_plus_covered_change(self): ) assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example4_unrelated_test_ai_judges_fail(self, mock_ai): """Ex4: no coverage + unknown relevance → AI judges; AI says FAIL.""" mock_ai.return_value = json.dumps({ @@ -1130,7 +1440,7 @@ def test_example4_unrelated_test_ai_judges_fail(self, mock_ai): assert result.verdict == Verdict.FAIL mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example5_ai_fallthrough_with_relevant_tests(self, mock_ai): """Ex5: no coverage + YES relevance → AI judges adequacy.""" mock_ai.return_value = json.dumps({ @@ -1154,7 +1464,7 @@ def test_example5_ai_fallthrough_with_relevant_tests(self, mock_ai): assert result.verdict == Verdict.PASS mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example6_ai_api_failure(self, mock_ai): """Ex6: AI fails → execution_status=ERROR → verdict=SKIP.""" mock_ai.side_effect = RuntimeError("HTTP 500") @@ -1172,7 +1482,7 @@ def test_example6_ai_api_failure(self, mock_ai): assert result.verdict == Verdict.SKIP assert "HTTP 500" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example7_unknown_relevance_below_threshold_ai_judges(self, mock_ai): """Ex7: coverage below threshold + UNKNOWN relevance → AI judges.""" mock_ai.return_value = json.dumps({ @@ -1200,7 +1510,7 @@ def test_example7_unknown_relevance_below_threshold_ai_judges(self, mock_ai): assert result.verdict == Verdict.WARNING mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example8_ai_failure_after_unknown_relevance(self, mock_ai): """Ex8: no coverage + UNKNOWN → AI, but AI fails → SKIP.""" mock_ai.side_effect = TimeoutError("timeout") @@ -1525,18 +1835,18 @@ def test_regular_exception(self): class TestResolveModels: def test_default_model_returns_full_chain(self): - models = _resolve_models("openai/gpt-4.1-mini") - assert models == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano"] + models = _resolve_models("gpt-4.1-mini") + assert models == ["gpt-4.1-mini", "gpt-4.1-nano"] def test_custom_model_returns_single(self): assert _resolve_models("openai/gpt-5-mini") == ["openai/gpt-5-mini"] def test_nano_alone_returns_single(self): - assert _resolve_models("openai/gpt-4.1-nano") == ["openai/gpt-4.1-nano"] + assert _resolve_models("gpt-4.1-nano") == ["gpt-4.1-nano"] class TestCallAiForBatch: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_success_on_first_try(self, mock_call: MagicMock): mock_call.return_value = '{"verdict":"pass","confidence":0.9,"files":[]}' raw, exc = _call_ai_for_batch( @@ -1546,7 +1856,7 @@ def test_success_on_first_try(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1554,7 +1864,7 @@ def test_success_on_first_try(self, mock_call: MagicMock): assert exc is None mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): error_413 = _make_api_error(413, "Request body too large for model") mock_call.side_effect = [ @@ -1568,7 +1878,7 @@ def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1576,7 +1886,7 @@ def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): assert exc is None assert mock_call.call_count == 2 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retry_also_fails(self, mock_call: MagicMock): error_413 = _make_api_error(413, "Request body too large for model") mock_call.side_effect = [error_413, error_413] @@ -1587,14 +1897,14 @@ def test_413_retry_also_fails(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) assert raw is None assert exc is not None - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): mock_call.side_effect = RuntimeError("connection lost") raw, exc = _call_ai_for_batch( @@ -1604,7 +1914,7 @@ def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1613,7 +1923,7 @@ def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): assert "connection lost" in str(exc) mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: MagicMock): mock_call.return_value = '{"verdict":"pass","confidence":0.9,"files":[]}' _call_ai_for_batch( @@ -1629,7 +1939,7 @@ def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: Magi "src/a.py": "tests/test_a.py", "src/b.py": "tests/test_b.py", }, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1637,7 +1947,7 @@ def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: Magi assert "matched_diff" in user_prompt # matched to in-batch source assert "outside_batch_diff" not in user_prompt # matched to another batch - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retry_returns_size_error_for_caller(self, mock_call: MagicMock): """When both normal and truncated attempts fail with 413, the returned exception should be a retryable size error @@ -1651,7 +1961,7 @@ def test_413_retry_returns_size_error_for_caller(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1671,7 +1981,7 @@ def test_prompt_concise_instruction_present(self): class TestRunLayer3Batching: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_403_triggers_model_fallback(self, mock_call: MagicMock): error_403 = _make_api_error(403, "Forbidden") mock_call.side_effect = [ @@ -1689,16 +1999,16 @@ def test_403_triggers_model_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 2 - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_403_no_fallback_for_custom_model(self, mock_call: MagicMock): mock_call.side_effect = _make_api_error(403, "Forbidden") result = run_layer3( @@ -1715,7 +2025,7 @@ def test_403_no_fallback_for_custom_model(self, mock_call: MagicMock): assert result.verdict == Verdict.SKIP mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_all_models_exhausted_returns_skip(self, mock_call: MagicMock): mock_call.side_effect = _make_api_error(403, "Forbidden") result = run_layer3( @@ -1725,14 +2035,14 @@ def test_all_models_exhausted_returns_skip(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.SKIP assert mock_call.call_count == 2 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_batch_count_in_details_single(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "pass", @@ -1746,13 +2056,13 @@ def test_batch_count_in_details_single(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert "1 batch" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") @patch("src.layer3_ai._batch_files") def test_model_escalation_carries_across_batches( self, mock_batch: MagicMock, mock_call: MagicMock @@ -1779,17 +2089,17 @@ def test_model_escalation_carries_across_batches( l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 3 - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-nano" - assert mock_call.call_args_list[2][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-nano" + assert mock_call.call_args_list[2][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") @patch("src.layer3_ai._batch_files") def test_remaining_batches_skip_when_models_exhausted( self, mock_batch: MagicMock, mock_call: MagicMock @@ -1803,7 +2113,7 @@ def test_remaining_batches_skip_when_models_exhausted( l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -1813,7 +2123,7 @@ def test_remaining_batches_skip_when_models_exhausted( assert file_map["src/b.py"].verdict == Verdict.SKIP assert "deferred" in file_map["src/b.py"].reason.lower() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_triggers_model_fallback(self, mock_call: MagicMock): """When gpt-4.1-mini returns 413 on both normal and truncated attempts, the model loop should escalate to gpt-4.1-nano (just like 403).""" @@ -1837,18 +2147,18 @@ def test_413_triggers_model_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/big.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 3 # Verify model escalation: mini → mini (retry) → nano - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[2][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[2][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_all_models_exhausted_returns_skip(self, mock_call: MagicMock): """When all models fail with 413 (both attempts each), result is SKIP.""" error_413 = _make_api_error(413, "Request body too large for model") @@ -1861,14 +2171,14 @@ def test_413_all_models_exhausted_returns_skip(self, mock_call: MagicMock): l2_matched_tests={"src/big.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.SKIP assert mock_call.call_count == 4 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_no_fallback_for_custom_model(self, mock_call: MagicMock): """Custom models have no fallback chain — 413 means SKIP immediately.""" error_413 = _make_api_error(413, "Request body too large for model") @@ -1891,8 +2201,8 @@ def test_413_no_fallback_for_custom_model(self, mock_call: MagicMock): class TestTokenBudgetConstants: """Pin the token budget constants to their corrected values.""" - def test_input_token_limit(self): - assert layer3_ai._INPUT_TOKEN_LIMIT == 8000 + def test_max_input_tokens(self): + assert layer3_ai._MAX_INPUT_TOKENS == 8000 def test_chars_per_token(self): assert layer3_ai._CHARS_PER_TOKEN == 3 @@ -1909,7 +2219,7 @@ def test_user_prompt_token_budget(self): def test_budget_leaves_headroom_for_system_prompt(self): total = layer3_ai._SYSTEM_OVERHEAD_TOKENS + layer3_ai._USER_PROMPT_TOKEN_BUDGET - assert total < layer3_ai._INPUT_TOKEN_LIMIT + assert total < layer3_ai._MAX_INPUT_TOKENS # --------------------------------------------------------------------------- @@ -1959,7 +2269,7 @@ def test_all_hallucinated_returns_none(self): class TestRunLayer3EmptyAiResponse: """BUG 1: Empty AI response must not be treated as success.""" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): """When AI returns empty content, batch files should get SKIP and fall back to L1+L2, not silently disappear.""" @@ -1971,7 +2281,7 @@ def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -1983,7 +2293,7 @@ def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): class TestRunLayer3HallucinatedFiles: """BUG 2: AI hallucinated files must not appear in final results.""" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -2000,7 +2310,7 @@ def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2008,7 +2318,7 @@ def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): assert "src/HALLUCINATED.py" not in file_names assert "src/a.py" in file_names - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_missing_batch_file_treated_as_failure(self, mock_call: MagicMock): """AI omits src/b.py from response — it should get SKIP, not vanish.""" mock_call.return_value = json.dumps({ @@ -2025,7 +2335,7 @@ def test_missing_batch_file_treated_as_failure(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2050,7 +2360,7 @@ def test_missing_prompt_degrades_to_skip(self, mock_path: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2060,7 +2370,7 @@ def test_missing_prompt_degrades_to_skip(self, mock_path: MagicMock): class TestRunLayer3ParseFailure: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_schema_drift_degrades_to_skip(self, mock_call: MagicMock): mock_call.return_value = '{"verdict": "pass"}' result = run_layer3( @@ -2070,7 +2380,7 @@ def test_schema_drift_degrades_to_skip(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) diff --git a/tests/test_main.py b/tests/test_main.py index 4e24657..2618ffd 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -26,6 +26,8 @@ def base_config(): ai_enabled=True, ai_model="openai/gpt-5-mini", ai_confidence_threshold=0.7, + ai_base_url="https://api.openai.com/v1", + ai_api_key="sk-fake", ) @@ -238,7 +240,14 @@ def test_full_pipeline_classifies_files_for_l3( coverage_details={"src/auth.py": 75.0}, coverage_threshold=80, model="openai/gpt-5-mini", - token="ghp_fake", + # The provider key, never the GitHub token — the two are unrelated + # now that GitHub Models is retired. + token="sk-fake", + base_url="https://api.openai.com/v1", + reasoning_effort="none", + temperature=0.1, + max_output_tokens=8192, + max_input_tokens=8000, confidence_threshold=0.7, unmeasurable_files=set(), ) diff --git a/tests/test_summary.py b/tests/test_summary.py index 380e121..447dd3e 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -133,13 +133,14 @@ def test_calls_model_on_warning(self, mock_openai_cls, warning_report): mock_client.chat.completions.create.return_value = MagicMock(choices=[mock_choice]) result = generate_summary( - warning_report, ["lib/Utils.php"], [], True, "openai/gpt-4.1-mini", "fake-token", + warning_report, ["lib/Utils.php"], [], True, "gpt-4.1-mini", "fake-key", + "https://example.test/v1", ) assert result == "**Why this WARNING?**\n- Explanation here." mock_openai_cls.assert_called_once_with( - base_url="https://models.github.ai/inference", - api_key="fake-token", + base_url="https://example.test/v1", + api_key="fake-key", ) mock_client.chat.completions.create.assert_called_once()