From 1a020c9ae49665cc8e412a2d635fcd1108c72b25 Mon Sep 17 00:00:00 2001 From: domenico Date: Fri, 31 Jul 2026 23:31:35 +0200 Subject: [PATCH 01/16] fix(layer3): replace retired GitHub Models with any OpenAI-compatible endpoint GitHub Models was fully retired on 2026-07-30 (playground, catalog, inference API, BYOK). Runs now fail with: 410 {'code': 'github_models_retirement_brownout'} The endpoint was hardcoded in two places and authenticated with GITHUB_TOKEN. There is no GITHUB_TOKEN-authenticated replacement: `models: read` only ever gated GitHub Models, and Copilot (now billed as AI Credits) exposes no OpenAI-compatible endpoint an Action can call. Layer 3 therefore takes a provider key: - new `ai-base-url` input (default https://api.openai.com/v1) and `ai-api-key` input, threaded through run_layer3 and generate_summary - `_call_github_models` -> `_call_ai_provider`, base_url is a parameter - default model IDs lose the GitHub Models publisher prefix (`openai/gpt-4.1-mini` -> `gpt-4.1-mini`); explicit IDs pass through untouched so OpenRouter-style namespacing still works - ai-enabled with no key degrades to L1+L2 with a ::warning:: instead of failing every run on a 401 - drop the now-inert `models: read` permission from dogfood - README: breaking-change note, provider table (OpenAI / Azure AI Foundry / OpenRouter), migration and troubleshooting rows 425 tests pass; ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dogfood.yml | 7 +- README.md | 47 +++++++--- action.yml | 13 ++- src/config.py | 28 +++++- src/layer3_ai.py | 33 +++++-- src/main.py | 8 +- src/summary.py | 4 +- tests/test_config.py | 42 ++++++++- tests/test_layer3_ai.py | 171 ++++++++++++++++++++-------------- tests/test_main.py | 7 +- tests/test_summary.py | 7 +- 11 files changed, 263 insertions(+), 104 deletions(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 1bb3a33..d5ccee7 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -12,7 +12,7 @@ permissions: contents: read pull-requests: write statuses: write - models: read + # No `models: read` — it only ever gated GitHub Models, retired 2026-07-30. jobs: test-guard: @@ -36,7 +36,10 @@ jobs: with: coverage-file: coverage.xml coverage-threshold: '100' - ai-model: 'openai/gpt-4.1-mini' + ai-model: 'gpt-4.1-mini' + # Layer 3 needs a provider key. Absent the secret, the gate degrades + # to Layer 1 + Layer 2 with a warning instead of failing. + ai-api-key: ${{ secrets.OPENAI_API_KEY }} # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every diff --git a/README.md b/README.md index 03c0ad3..7e1995a 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,9 @@ Priority within a layer: **FAIL > WARNING > PASS > SKIP**. ## Quick Start -Test-Guard uses the GitHub Models API with your standard `GITHUB_TOKEN`. No external API keys required. +Layer 3 calls any OpenAI-compatible inference endpoint, so it needs one provider API key. + +> **Breaking change (v2).** Test-Guard used to reach GitHub Models with your `GITHUB_TOKEN` and no external key. [GitHub retired GitHub Models on 30 July 2026](https://github.blog/changelog/2026-07-30-github-models-is-now-retired/) — the playground, catalog, and inference API are gone for all customers, and a `GITHUB_TOKEN` no longer authenticates any model endpoint. Set `ai-api-key` (and `ai-base-url` if you are not on OpenAI). Without a key, Layer 3 is skipped and the gate runs on Layer 1 + Layer 2. ```yaml name: Test-Guard @@ -122,7 +124,6 @@ permissions: contents: read pull-requests: write checks: write - models: read # Required for Layer 3 AI analysis jobs: test-guard: @@ -139,9 +140,22 @@ jobs: uses: ostico/test-guard@v1 with: coverage-file: coverage.xml + ai-api-key: ${{ secrets.OPENAI_API_KEY }} ``` -> The `models: read` permission is required for AI analysis. If you set `ai-enabled: 'false'`, you can omit it. +> No special GitHub permission is needed for AI analysis — the old `models: read` scope only ever gated GitHub Models and is now inert. What Layer 3 needs is `ai-api-key`. Set `ai-enabled: 'false'` to run Layers 1–2 only. + +### Pointing at another provider + +`ai-base-url` takes any OpenAI-compatible `/v1` endpoint. Use the model ID *that provider* expects. + +| Provider | `ai-base-url` | `ai-model` | +|:---------|:--------------|:-----------| +| OpenAI (default) | `https://api.openai.com/v1` | `gpt-4.1-mini` | +| Azure AI Foundry | `https://.services.ai.azure.com/openai/v1` | your deployment name | +| OpenRouter | `https://openrouter.ai/api/v1` | `openai/gpt-4.1-mini` | + +Migrating from a pre-retirement config? Drop the `openai/` prefix from `ai-model` unless your new provider namespaces by publisher (OpenRouter does; OpenAI and Azure do not). --- @@ -155,7 +169,9 @@ jobs: | `exclude-patterns` | _(see below)_ | Comma-separated glob patterns to skip. Setting this **replaces** the default list. See [Excluding files](#excluding-files). | | `extra-exclude-patterns` | _(empty)_ | Comma-separated glob patterns to exclude **in addition** to `exclude-patterns` (unioned, deduped). Add repo-specific excludes here without re-listing the defaults. | | `ai-enabled` | `true` | Enable Layer 3 AI analysis. Truthy values: `true`, `1`, `yes` (case-insensitive); anything else disables it. | -| `ai-model` | `openai/gpt-4.1-mini` | GitHub Models model ID. | +| `ai-model` | `gpt-4.1-mini` | Model ID, exactly as `ai-base-url` expects it. | +| `ai-base-url` | `https://api.openai.com/v1` | OpenAI-compatible inference endpoint. | +| `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** @@ -232,23 +248,28 @@ For `GetSearchController.php` this matches **both** `GetSearchControllerTest.php --- -## GitHub Models Setup +## AI Provider Setup -Layer 3 uses the [GitHub Models](https://github.com/marketplace/models) inference API. This works with your existing `GITHUB_TOKEN` — no external API keys needed. +Layer 3 calls an OpenAI-compatible inference endpoint over the `openai` Python SDK. It previously used GitHub Models, which GitHub retired on 30 July 2026; there is no `GITHUB_TOKEN`-authenticated replacement, so you now supply a provider key. ### Requirements -1. **`models: read` permission** in your workflow (see Quick Start above). -2. **GitHub Models enabled** for your account or organization: - - **Personal repos:** Go to [github.com/marketplace/models](https://github.com/marketplace/models) and accept the terms. The free tier is sufficient. - - **Organization repos:** An organization owner must enable GitHub Models at the org level. Go to **Organization Settings → Copilot → Policies** and enable model access. -3. **Free tier limits:** GitHub Models free tier allows ~150 requests/day with up to 8K input tokens per request. Test-Guard's smart batching is designed to stay within these limits. +1. **`ai-api-key`** set to a provider key, passed as a repository or organization secret. +2. **`ai-base-url`** matching that provider (defaults to OpenAI). Azure AI Foundry is GitHub's own recommended destination for retired GitHub Models workloads. +3. **`ai-model`** using the ID that provider expects — see the provider table in Quick Start. + +Batching still targets ~8K input tokens per request, which keeps per-run cost low on metered providers. + +> Requests are billed by whichever provider you configure, not by GitHub. GitHub AI Credits apply to Copilot, and Copilot exposes no OpenAI-compatible endpoint this action can call. ### Troubleshooting | Symptom | Cause | Fix | |:--------|:------|:----| -| 403 "Model not accessible" | GitHub Models not enabled | Enable at org or personal level (step 2 above) | +| 410 `github_models_retirement_brownout` | Config still points at `models.github.ai` | Set `ai-base-url` + `ai-api-key`; that service was retired 30 Jul 2026 | +| Layer 3 skipped with an `ai-api-key` warning | No provider key configured | Set `ai-api-key` to a secret, or `ai-enabled: 'false'` to silence it | +| 401 / 403 "Incorrect API key" | Key does not match `ai-base-url`, or `GITHUB_TOKEN` was passed | Use a key issued by that provider | +| 404 "model does not exist" | Model ID carries the wrong prefix | OpenAI/Azure: `gpt-4.1-mini`. OpenRouter: `openai/gpt-4.1-mini` | | 413 "Request body too large" | Diff exceeds model token limit | Automatic — smart batching handles this | | Intermittent 429 errors | Rate limit exceeded | Reduce PR size or use `ai-enabled: 'false'` for low-priority PRs | @@ -256,7 +277,7 @@ Layer 3 uses the [GitHub Models](https://github.com/marketplace/models) inferenc ## AI Architecture -GitHub Models' free tier caps requests at **8K input tokens**. A naive implementation hits that wall constantly on real PRs — Test-Guard avoids it with three layers working together: **compaction** (shrink each diff without losing signal), **batching** (group files so a call never exceeds the cap), and **matching** (only attach the test diffs that actually belong to this batch). +Layer 3 budgets each request to **8K input tokens**. A naive implementation hits that wall constantly on real PRs — Test-Guard avoids it with three layers working together: **compaction** (shrink each diff without losing signal), **batching** (group files so a call never exceeds the cap), and **matching** (only attach the test diffs that actually belong to this batch). ```mermaid flowchart LR diff --git a/action.yml b/action.yml index 2da9484..42d9a71 100644 --- a/action.yml +++ b/action.yml @@ -27,8 +27,15 @@ inputs: description: 'Enable AI Layer 3 analysis (true/false).' default: 'true' ai-model: - description: 'GitHub Models model ID for AI analysis.' - default: 'openai/gpt-4.1-mini' + description: 'Model ID for AI analysis, exactly as the configured endpoint expects (OpenAI: "gpt-4.1-mini"; OpenRouter: "openai/gpt-4.1-mini").' + default: 'gpt-4.1-mini' + ai-base-url: + description: 'OpenAI-compatible inference endpoint. Examples: https://api.openai.com/v1, https://.services.ai.azure.com/openai/v1, https://openrouter.ai/api/v1. GitHub Models (models.github.ai) was retired on 2026-07-30 and no longer works.' + default: 'https://api.openai.com/v1' + ai-api-key: + description: 'API key for ai-base-url. Pass a secret, e.g. ${{ secrets.OPENAI_API_KEY }}. Required for Layer 3 — a GITHUB_TOKEN is not accepted by any of these providers. When empty, Layer 3 is skipped and the gate falls back to Layer 1 + Layer 2.' + required: false + default: '' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -56,6 +63,8 @@ runs: INPUT_EXTRA-EXCLUDE-PATTERNS: ${{ inputs.extra-exclude-patterns }} INPUT_AI-ENABLED: ${{ inputs.ai-enabled }} INPUT_AI-MODEL: ${{ inputs.ai-model }} + INPUT_AI-BASE-URL: ${{ inputs.ai-base-url }} + INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 0a3be55..0f41dc9 100644 --- a/src/config.py +++ b/src/config.py @@ -24,7 +24,10 @@ "build.rs" ) _DEFAULT_COVERAGE_THRESHOLD = 80 -_DEFAULT_AI_MODEL = "openai/gpt-4.1-mini" +_DEFAULT_AI_MODEL = "gpt-4.1-mini" +# Any OpenAI-compatible /v1 endpoint. GitHub Models (models.github.ai) was +# retired on 2026-07-30 and is no longer a valid target. +DEFAULT_AI_BASE_URL = "https://api.openai.com/v1" _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -161,7 +164,12 @@ class Config: test_patterns: Language-specific source-to-test file mappings. exclude_patterns: Glob patterns to skip (config files, docs, etc.). ai_enabled: Whether Layer 3 AI analysis is enabled. - ai_model: GitHub Models model ID (e.g., "openai/gpt-4.1-mini"). + ai_model: Provider model ID (e.g., "gpt-4.1-mini"). Use the exact ID + the configured endpoint expects — OpenAI wants "gpt-4.1-mini", + OpenRouter wants "openai/gpt-4.1-mini". + ai_base_url: OpenAI-compatible inference endpoint. + ai_api_key: API key for that endpoint. Empty disables Layer 3's AI + phase (shortcut gates still run). ai_confidence_threshold: AI FAIL verdicts below this become WARNING (0.0-1.0). """ @@ -183,6 +191,8 @@ class Config: ai_enabled: bool ai_model: str ai_confidence_threshold: float # 0.0-1.0 + ai_base_url: str = DEFAULT_AI_BASE_URL + ai_api_key: str = "" def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -305,6 +315,18 @@ def parse_config() -> Config: # Layer 3: Parse AI configuration. ai_enabled = _env("AI-ENABLED", "true").lower() in _DEFAULT_AI_ENABLED_VALUES ai_model = _env("AI-MODEL", _DEFAULT_AI_MODEL) + ai_base_url = _env("AI-BASE-URL", DEFAULT_AI_BASE_URL) + # Provider API key. Deliberately NOT GITHUB_TOKEN: GitHub Models was + # retired on 2026-07-30 and a GitHub token authenticates nothing else. + ai_api_key = _env("AI-API-KEY", "") + if ai_enabled and not ai_api_key: + print( + "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " + "AI analysis disabled, falling back to Layer 1 + Layer 2. GitHub " + "Models was retired on 2026-07-30; set ai-api-key (plus " + "ai-base-url for non-OpenAI providers) to re-enable." + ) + ai_enabled = False confidence_raw = _env( "AI-CONFIDENCE-THRESHOLD", str(_DEFAULT_AI_CONFIDENCE_THRESHOLD) ) @@ -330,5 +352,7 @@ def parse_config() -> Config: exclude_patterns=exclude_patterns, ai_enabled=ai_enabled, ai_model=ai_model, + ai_base_url=ai_base_url, + ai_api_key=ai_api_key, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index d663183..ddbf4f5 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -2,7 +2,7 @@ Evaluates each source file through an 8-gate shortcut truth table using coverage data from L1 and test-match data from L2. Files that cannot be -resolved deterministically fall through to the GitHub Models AI for a +resolved deterministically fall through to the configured AI provider for a structured JSON verdict with confidence-based downgrade. Flow: @@ -31,6 +31,7 @@ from unidiff import PatchSet from unidiff.errors import UnidiffParseError +from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL from src.diff_utils import is_trivial_diff from src.models import FileVerdict, LayerResult, Verdict @@ -601,9 +602,12 @@ def _sanitize_diff( # Default model chain: try gpt-4.1-mini first, fall back to gpt-4.1-nano on 403. # When the user explicitly sets ai-model to something other than the chain head, # a single attempt is made with that model (no fallback). +# +# IDs are unprefixed (OpenAI style). Providers that namespace by publisher +# (OpenRouter: "openai/gpt-4.1-mini") need ai-model set explicitly. _DEFAULT_FALLBACK_CHAIN: tuple[str, ...] = ( - "openai/gpt-4.1-mini", - "openai/gpt-4.1-nano", + "gpt-4.1-mini", + "gpt-4.1-nano", ) _CHARS_PER_TOKEN = 3 @@ -804,6 +808,7 @@ def _call_ai_for_batch( model: str, system_prompt: str, token: str, + base_url: str = _DEFAULT_AI_BASE_URL, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -819,7 +824,9 @@ def _call_ai_for_batch( coverage_details, coverage_threshold, matched_tests, ) try: - raw = _call_github_models(model, system_prompt, user_prompt, token) + raw = _call_ai_provider( + model, system_prompt, user_prompt, token, base_url, + ) return raw, None except Exception as exc: if _is_retryable_size_error(exc): @@ -829,8 +836,8 @@ def _call_ai_for_batch( max_diff_chars=_RETRY_MAX_DIFF_CHARS, ) try: - raw = _call_github_models( - model, system_prompt, user_prompt, token, + raw = _call_ai_provider( + model, system_prompt, user_prompt, token, base_url, ) return raw, None except Exception as retry_exc: @@ -838,15 +845,21 @@ def _call_ai_for_batch( return None, exc -def _call_github_models( +def _call_ai_provider( model: str, system_prompt: str, user_prompt: str, token: str, + base_url: str = _DEFAULT_AI_BASE_URL, ) -> str: - """Call GitHub Models API and return the raw response text.""" + """Call an OpenAI-compatible inference endpoint, return raw response text. + + ``base_url`` is provider-agnostic: OpenAI, Azure AI Foundry, OpenRouter, + or any local gateway speaking the same protocol. GitHub Models used to be + hardcoded here; it was retired on 2026-07-30. + """ client = OpenAI( - base_url="https://models.github.ai/inference", + base_url=base_url, api_key=token, ) messages = [ @@ -918,6 +931,7 @@ def run_layer3( token: str, confidence_threshold: float, unmeasurable_files: set[str] | None = None, + base_url: str = _DEFAULT_AI_BASE_URL, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1030,6 +1044,7 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, + base_url, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index b3a137b..d78a036 100644 --- a/src/main.py +++ b/src/main.py @@ -142,7 +142,8 @@ def run_pipeline(config: Config) -> Report: coverage_details=l1.coverage_details, coverage_threshold=config.coverage_threshold, model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. @@ -164,6 +165,8 @@ def _attach_summary( """Generate and attach summary to report if verdict is WARNING/FAIL.""" if report.overall_verdict in (Verdict.PASS, Verdict.SKIP): return + if not config.ai_api_key: + return report.summary = generate_summary( report=report, @@ -171,7 +174,8 @@ def _attach_summary( test_files_in_pr=test_files_in_pr, coverage_files_provided=bool(config.coverage_files), model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, ) diff --git a/src/summary.py b/src/summary.py index 65295e0..0803311 100644 --- a/src/summary.py +++ b/src/summary.py @@ -15,6 +15,7 @@ ChatCompletionUserMessageParam, ) +from src.config import DEFAULT_AI_BASE_URL from src.models import Report, Verdict _PROMPT_PATH = Path(__file__).parent.parent / "prompts" / "summary_explainer.txt" @@ -97,6 +98,7 @@ def generate_summary( coverage_files_provided: bool, model: str, token: str, + base_url: str = DEFAULT_AI_BASE_URL, ) -> str | None: """Generate a developer-facing explanation for WARNING/FAIL verdicts. @@ -117,7 +119,7 @@ def generate_summary( try: client = OpenAI( - base_url="https://models.github.ai/inference", + base_url=base_url, api_key=token, ) response = client.chat.completions.create( diff --git a/tests/test_config.py b/tests/test_config.py index 19db0e6..bdcb756 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -16,6 +16,8 @@ def test_defaults(self, monkeypatch): monkeypatch.delenv("INPUT_AI-ENABLED", raising=False) monkeypatch.delenv("INPUT_AI-MODEL", raising=False) monkeypatch.delenv("INPUT_AI-CONFIDENCE-THRESHOLD", raising=False) + monkeypatch.delenv("INPUT_AI-BASE-URL", raising=False) + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") @@ -24,7 +26,9 @@ def test_defaults(self, monkeypatch): assert cfg.coverage_files == [] assert cfg.coverage_threshold == 80 assert cfg.ai_enabled is True - assert cfg.ai_model == "openai/gpt-4.1-mini" + assert cfg.ai_model == "gpt-4.1-mini" + assert cfg.ai_base_url == "https://api.openai.com/v1" + assert cfg.ai_api_key == "sk-fake" assert cfg.ai_confidence_threshold == 0.7 assert cfg.github_token == "ghp_fake123" assert cfg.repo == "owner/repo" @@ -48,6 +52,42 @@ def test_custom_values(self, monkeypatch): assert cfg.ai_confidence_threshold == 0.9 assert cfg.exclude_patterns == ["*.md", "docs/**"] + def test_custom_endpoint_and_key(self, monkeypatch): + """ai-base-url / ai-api-key point Layer 3 at any OpenAI-compatible provider.""" + monkeypatch.setenv("INPUT_AI-BASE-URL", "https://openrouter.ai/api/v1") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-or-abc") + monkeypatch.setenv("INPUT_AI-MODEL", "openai/gpt-4.1-mini") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_base_url == "https://openrouter.ai/api/v1" + assert cfg.ai_api_key == "sk-or-abc" + # Publisher-prefixed IDs pass through untouched — OpenRouter needs them. + assert cfg.ai_model == "openai/gpt-4.1-mini" + assert cfg.ai_enabled is True + + def test_missing_api_key_disables_ai(self, monkeypatch, capsys): + """No key means no AI phase: GITHUB_TOKEN is not a provider credential. + + GitHub Models was retired 2026-07-30, so there is no endpoint a bare + GitHub token authenticates against. Degrade to L1+L2 with a warning + instead of failing every run with a 401/410. + """ + monkeypatch.setenv("INPUT_AI-ENABLED", "true") + monkeypatch.delenv("INPUT_AI-API-KEY", raising=False) + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_enabled is False + assert cfg.ai_api_key == "" + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-api-key" in out + def _base_env(self, monkeypatch): monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 1b0486c..8819e7b 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -12,7 +12,7 @@ _batch_files, _build_ai_prompt, _call_ai_for_batch, - _call_github_models, + _call_ai_provider, _compact_diff, _context_ladder, _count_change_hunks, @@ -359,7 +359,7 @@ def test_skip_and_pass_returns_pass(self): class TestRunLayer3: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_all_shortcuts_no_ai_called(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/trivial.py": "+ # comment"}, @@ -375,7 +375,7 @@ def test_all_shortcuts_no_ai_called(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_deleted_files_produce_pass(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/old.py": "+ code"}, @@ -391,7 +391,7 @@ def test_deleted_files_produce_pass(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_fallthrough_calls_ai(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "pass", @@ -412,7 +412,7 @@ def test_fallthrough_calls_ai(self, mock_call: MagicMock): mock_call.assert_called_once() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_ai_failure_returns_skip(self, mock_call: MagicMock): mock_call.side_effect = Exception("API down") result = run_layer3( @@ -429,7 +429,7 @@ def test_ai_failure_returns_skip(self, mock_call: MagicMock): assert result.verdict == Verdict.SKIP assert "API down" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_ai_failure_with_shortcuts_preserves_shortcut_verdicts(self, mock_call: MagicMock): mock_call.side_effect = Exception("API down") result = run_layer3( @@ -469,7 +469,7 @@ def test_empty_source_diffs_returns_pass(self): ) assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_mixed_shortcut_and_ai_fail(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -492,7 +492,7 @@ def test_mixed_shortcut_and_ai_fail(self, mock_call: MagicMock): ) assert result.verdict == Verdict.FAIL - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_gate4_fail_without_ai(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/billing.py": "+ bill()"}, @@ -508,7 +508,7 @@ def test_gate4_fail_without_ai(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.FAIL - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_low_confidence_ai_downgrades_to_warning(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -528,7 +528,7 @@ def test_low_confidence_ai_downgrades_to_warning(self, mock_call: MagicMock): ) assert result.verdict == Verdict.WARNING - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_coverage_ok_shortcut_passes(self, mock_call: MagicMock): result = run_layer3( source_diffs={"src/user.py": "+ real_code()"}, @@ -544,7 +544,7 @@ def test_coverage_ok_shortcut_passes(self, mock_call: MagicMock): mock_call.assert_not_called() assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_coverage_below_threshold_with_relevant_tests_fails_shortcut( self, mock_call: MagicMock ): @@ -567,7 +567,7 @@ def test_coverage_below_threshold_with_relevant_tests_fails_shortcut( fv = {v.file: v for v in result.file_verdicts}["src/user.py"] assert "relevant tests exist but insufficient" in fv.reason - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_unmeasurable_file_skips_instead_of_failing_gate_4( self, mock_call: MagicMock ): @@ -593,7 +593,7 @@ def test_unmeasurable_file_skips_instead_of_failing_gate_4( assert fv.verdict == Verdict.SKIP assert "no executable lines changed" in fv.reason - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_file_not_marked_unmeasurable_still_fails_gate_4( self, mock_call: MagicMock ): @@ -616,7 +616,42 @@ def test_file_not_marked_unmeasurable_still_fails_gate_4( assert fv.verdict == Verdict.FAIL -class TestCallGithubModels: +class TestCallAiProvider: + @patch("src.layer3_ai.OpenAI") + def test_uses_configured_base_url(self, mock_openai: MagicMock): + """The endpoint is caller-supplied, not the retired GitHub Models host.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + base_url="https://myresource.services.ai.azure.com/openai/v1", + ) + + mock_openai.assert_called_once_with( + base_url="https://myresource.services.ai.azure.com/openai/v1", + api_key="sk-fake", + ) + + @patch("src.layer3_ai.OpenAI") + def test_defaults_to_openai_endpoint(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_openai.call_args.kwargs["base_url"] == "https://api.openai.com/v1" + @patch("src.layer3_ai.OpenAI") def test_empty_choices_returns_empty_string(self, mock_openai: MagicMock): mock_client = MagicMock() @@ -625,7 +660,7 @@ def test_empty_choices_returns_empty_string(self, mock_openai: MagicMock): mock_client.chat.completions.create.return_value = mock_response mock_openai.return_value = mock_client - raw = _call_github_models( + raw = _call_ai_provider( model="openai/gpt-5-mini", system_prompt="system", user_prompt="user", @@ -1106,7 +1141,7 @@ def test_example3_deleted_file_plus_covered_change(self): ) assert result.verdict == Verdict.PASS - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example4_unrelated_test_ai_judges_fail(self, mock_ai): """Ex4: no coverage + unknown relevance → AI judges; AI says FAIL.""" mock_ai.return_value = json.dumps({ @@ -1130,7 +1165,7 @@ def test_example4_unrelated_test_ai_judges_fail(self, mock_ai): assert result.verdict == Verdict.FAIL mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example5_ai_fallthrough_with_relevant_tests(self, mock_ai): """Ex5: no coverage + YES relevance → AI judges adequacy.""" mock_ai.return_value = json.dumps({ @@ -1154,7 +1189,7 @@ def test_example5_ai_fallthrough_with_relevant_tests(self, mock_ai): assert result.verdict == Verdict.PASS mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example6_ai_api_failure(self, mock_ai): """Ex6: AI fails → execution_status=ERROR → verdict=SKIP.""" mock_ai.side_effect = RuntimeError("HTTP 500") @@ -1172,7 +1207,7 @@ def test_example6_ai_api_failure(self, mock_ai): assert result.verdict == Verdict.SKIP assert "HTTP 500" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example7_unknown_relevance_below_threshold_ai_judges(self, mock_ai): """Ex7: coverage below threshold + UNKNOWN relevance → AI judges.""" mock_ai.return_value = json.dumps({ @@ -1200,7 +1235,7 @@ def test_example7_unknown_relevance_below_threshold_ai_judges(self, mock_ai): assert result.verdict == Verdict.WARNING mock_ai.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_example8_ai_failure_after_unknown_relevance(self, mock_ai): """Ex8: no coverage + UNKNOWN → AI, but AI fails → SKIP.""" mock_ai.side_effect = TimeoutError("timeout") @@ -1525,18 +1560,18 @@ def test_regular_exception(self): class TestResolveModels: def test_default_model_returns_full_chain(self): - models = _resolve_models("openai/gpt-4.1-mini") - assert models == ["openai/gpt-4.1-mini", "openai/gpt-4.1-nano"] + models = _resolve_models("gpt-4.1-mini") + assert models == ["gpt-4.1-mini", "gpt-4.1-nano"] def test_custom_model_returns_single(self): assert _resolve_models("openai/gpt-5-mini") == ["openai/gpt-5-mini"] def test_nano_alone_returns_single(self): - assert _resolve_models("openai/gpt-4.1-nano") == ["openai/gpt-4.1-nano"] + assert _resolve_models("gpt-4.1-nano") == ["gpt-4.1-nano"] class TestCallAiForBatch: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_success_on_first_try(self, mock_call: MagicMock): mock_call.return_value = '{"verdict":"pass","confidence":0.9,"files":[]}' raw, exc = _call_ai_for_batch( @@ -1546,7 +1581,7 @@ def test_success_on_first_try(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1554,7 +1589,7 @@ def test_success_on_first_try(self, mock_call: MagicMock): assert exc is None mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): error_413 = _make_api_error(413, "Request body too large for model") mock_call.side_effect = [ @@ -1568,7 +1603,7 @@ def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1576,7 +1611,7 @@ def test_413_retries_with_tighter_truncation(self, mock_call: MagicMock): assert exc is None assert mock_call.call_count == 2 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retry_also_fails(self, mock_call: MagicMock): error_413 = _make_api_error(413, "Request body too large for model") mock_call.side_effect = [error_413, error_413] @@ -1587,14 +1622,14 @@ def test_413_retry_also_fails(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) assert raw is None assert exc is not None - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): mock_call.side_effect = RuntimeError("connection lost") raw, exc = _call_ai_for_batch( @@ -1604,7 +1639,7 @@ def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1613,7 +1648,7 @@ def test_non_retryable_error_returns_immediately(self, mock_call: MagicMock): assert "connection lost" in str(exc) mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: MagicMock): mock_call.return_value = '{"verdict":"pass","confidence":0.9,"files":[]}' _call_ai_for_batch( @@ -1629,7 +1664,7 @@ def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: Magi "src/a.py": "tests/test_a.py", "src/b.py": "tests/test_b.py", }, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1637,7 +1672,7 @@ def test_includes_only_relevant_test_diffs_in_batch_prompt(self, mock_call: Magi assert "matched_diff" in user_prompt # matched to in-batch source assert "outside_batch_diff" not in user_prompt # matched to another batch - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_retry_returns_size_error_for_caller(self, mock_call: MagicMock): """When both normal and truncated attempts fail with 413, the returned exception should be a retryable size error @@ -1651,7 +1686,7 @@ def test_413_retry_returns_size_error_for_caller(self, mock_call: MagicMock): coverage_details=None, coverage_threshold=80.0, matched_tests={"src/a.py": None}, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", system_prompt="system", token="ghp_fake", ) @@ -1671,7 +1706,7 @@ def test_prompt_concise_instruction_present(self): class TestRunLayer3Batching: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_403_triggers_model_fallback(self, mock_call: MagicMock): error_403 = _make_api_error(403, "Forbidden") mock_call.side_effect = [ @@ -1689,16 +1724,16 @@ def test_403_triggers_model_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 2 - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_403_no_fallback_for_custom_model(self, mock_call: MagicMock): mock_call.side_effect = _make_api_error(403, "Forbidden") result = run_layer3( @@ -1715,7 +1750,7 @@ def test_403_no_fallback_for_custom_model(self, mock_call: MagicMock): assert result.verdict == Verdict.SKIP mock_call.assert_called_once() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_all_models_exhausted_returns_skip(self, mock_call: MagicMock): mock_call.side_effect = _make_api_error(403, "Forbidden") result = run_layer3( @@ -1725,14 +1760,14 @@ def test_all_models_exhausted_returns_skip(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.SKIP assert mock_call.call_count == 2 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_batch_count_in_details_single(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "pass", @@ -1746,13 +1781,13 @@ def test_batch_count_in_details_single(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert "1 batch" in result.details - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") @patch("src.layer3_ai._batch_files") def test_model_escalation_carries_across_batches( self, mock_batch: MagicMock, mock_call: MagicMock @@ -1779,17 +1814,17 @@ def test_model_escalation_carries_across_batches( l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 3 - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-nano" - assert mock_call.call_args_list[2][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-nano" + assert mock_call.call_args_list[2][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") @patch("src.layer3_ai._batch_files") def test_remaining_batches_skip_when_models_exhausted( self, mock_batch: MagicMock, mock_call: MagicMock @@ -1803,7 +1838,7 @@ def test_remaining_batches_skip_when_models_exhausted( l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -1813,7 +1848,7 @@ def test_remaining_batches_skip_when_models_exhausted( assert file_map["src/b.py"].verdict == Verdict.SKIP assert "deferred" in file_map["src/b.py"].reason.lower() - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_triggers_model_fallback(self, mock_call: MagicMock): """When gpt-4.1-mini returns 413 on both normal and truncated attempts, the model loop should escalate to gpt-4.1-nano (just like 403).""" @@ -1837,18 +1872,18 @@ def test_413_triggers_model_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/big.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.PASS assert mock_call.call_count == 3 # Verify model escalation: mini → mini (retry) → nano - assert mock_call.call_args_list[0][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[1][0][0] == "openai/gpt-4.1-mini" - assert mock_call.call_args_list[2][0][0] == "openai/gpt-4.1-nano" + assert mock_call.call_args_list[0][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[1][0][0] == "gpt-4.1-mini" + assert mock_call.call_args_list[2][0][0] == "gpt-4.1-nano" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_all_models_exhausted_returns_skip(self, mock_call: MagicMock): """When all models fail with 413 (both attempts each), result is SKIP.""" error_413 = _make_api_error(413, "Request body too large for model") @@ -1861,14 +1896,14 @@ def test_413_all_models_exhausted_returns_skip(self, mock_call: MagicMock): l2_matched_tests={"src/big.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) assert result.verdict == Verdict.SKIP assert mock_call.call_count == 4 - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_413_no_fallback_for_custom_model(self, mock_call: MagicMock): """Custom models have no fallback chain — 413 means SKIP immediately.""" error_413 = _make_api_error(413, "Request body too large for model") @@ -1959,7 +1994,7 @@ def test_all_hallucinated_returns_none(self): class TestRunLayer3EmptyAiResponse: """BUG 1: Empty AI response must not be treated as success.""" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): """When AI returns empty content, batch files should get SKIP and fall back to L1+L2, not silently disappear.""" @@ -1971,7 +2006,7 @@ def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): l2_matched_tests={"src/new.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -1983,7 +2018,7 @@ def test_empty_response_defers_to_fallback(self, mock_call: MagicMock): class TestRunLayer3HallucinatedFiles: """BUG 2: AI hallucinated files must not appear in final results.""" - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): mock_call.return_value = json.dumps({ "verdict": "fail", @@ -2000,7 +2035,7 @@ def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2008,7 +2043,7 @@ def test_hallucinated_file_not_in_results(self, mock_call: MagicMock): assert "src/HALLUCINATED.py" not in file_names assert "src/a.py" in file_names - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_missing_batch_file_treated_as_failure(self, mock_call: MagicMock): """AI omits src/b.py from response — it should get SKIP, not vanish.""" mock_call.return_value = json.dumps({ @@ -2025,7 +2060,7 @@ def test_missing_batch_file_treated_as_failure(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None, "src/b.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2050,7 +2085,7 @@ def test_missing_prompt_degrades_to_skip(self, mock_path: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) @@ -2060,7 +2095,7 @@ def test_missing_prompt_degrades_to_skip(self, mock_path: MagicMock): class TestRunLayer3ParseFailure: - @patch("src.layer3_ai._call_github_models") + @patch("src.layer3_ai._call_ai_provider") def test_schema_drift_degrades_to_skip(self, mock_call: MagicMock): mock_call.return_value = '{"verdict": "pass"}' result = run_layer3( @@ -2070,7 +2105,7 @@ def test_schema_drift_degrades_to_skip(self, mock_call: MagicMock): l2_matched_tests={"src/a.py": None}, coverage_details=None, coverage_threshold=80.0, - model="openai/gpt-4.1-mini", + model="gpt-4.1-mini", token="ghp_fake", confidence_threshold=0.7, ) diff --git a/tests/test_main.py b/tests/test_main.py index 4e24657..83ba285 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -26,6 +26,8 @@ def base_config(): ai_enabled=True, ai_model="openai/gpt-5-mini", ai_confidence_threshold=0.7, + ai_base_url="https://api.openai.com/v1", + ai_api_key="sk-fake", ) @@ -238,7 +240,10 @@ def test_full_pipeline_classifies_files_for_l3( coverage_details={"src/auth.py": 75.0}, coverage_threshold=80, model="openai/gpt-5-mini", - token="ghp_fake", + # The provider key, never the GitHub token — the two are unrelated + # now that GitHub Models is retired. + token="sk-fake", + base_url="https://api.openai.com/v1", confidence_threshold=0.7, unmeasurable_files=set(), ) diff --git a/tests/test_summary.py b/tests/test_summary.py index 380e121..447dd3e 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -133,13 +133,14 @@ def test_calls_model_on_warning(self, mock_openai_cls, warning_report): mock_client.chat.completions.create.return_value = MagicMock(choices=[mock_choice]) result = generate_summary( - warning_report, ["lib/Utils.php"], [], True, "openai/gpt-4.1-mini", "fake-token", + warning_report, ["lib/Utils.php"], [], True, "gpt-4.1-mini", "fake-key", + "https://example.test/v1", ) assert result == "**Why this WARNING?**\n- Explanation here." mock_openai_cls.assert_called_once_with( - base_url="https://models.github.ai/inference", - api_key="fake-token", + base_url="https://example.test/v1", + api_key="fake-key", ) mock_client.chat.completions.create.assert_called_once() From e80b8e533c5bfe981ac2d5f4cebcbe807a0090a3 Mon Sep 17 00:00:00 2001 From: domenico Date: Fri, 31 Jul 2026 23:39:00 +0200 Subject: [PATCH 02/16] docs: document free-tier providers and the strict json_schema constraint GitHub offers no free inference after the GitHub Models retirement. Copilot Pro stays free for verified OSS maintainers, students and teachers, but it is an IDE/CLI entitlement with no OpenAI-compatible endpoint, so this action cannot reach it. Azure AI Foundry, the official migration target, requires an Azure subscription. Document the third-party free tiers that do work with ai-base-url (Groq, Gemini, OpenRouter) plus the two constraints that decide whether they are usable: Layer 3 requires response_format=json_schema with strict: true, and free tiers generally train on submitted prompts. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 7e1995a..08d54c7 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,23 @@ jobs: Migrating from a pre-retirement config? Drop the `openai/` prefix from `ai-model` unless your new provider namespaces by publisher (OpenRouter does; OpenAI and Azure do not). +### Free options for open-source repos + +GitHub no longer offers free inference. Copilot Pro remains [free for verified open-source maintainers, students and teachers](https://docs.github.com/en/copilot/how-tos/manage-your-account/getting-free-access-to-copilot-pro-as-a-student-teacher-or-maintainer), but that is an IDE/CLI entitlement — it exposes no OpenAI-compatible endpoint, so this action cannot use it. Azure AI Foundry, the official migration target, needs an Azure subscription and bills per token. + +Third-party free tiers do work here: + +| Provider | `ai-base-url` | Free tier | +|:---------|:--------------|:----------| +| Groq | `https://api.groq.com/openai/v1` | No card required; ~30 RPM, 1,000–14,400 requests/day depending on model | +| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | Low daily quota, tightened in Dec 2025 | +| OpenRouter | `https://openrouter.ai/api/v1` | `:free` model variants | + +Two caveats before you rely on one: + +- **Strict structured output is required.** Layer 3 requests `response_format=json_schema` with `strict: true`. Models without strict support reject the call, and the batch falls back to Layer 1 + Layer 2 — a working gate with no AI analysis. Groq supports it only on selected models; on OpenRouter it varies per model; Gemini supports it on synchronous chat completions. +- **Free tiers usually train on your prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. + --- ## Inputs From fcdb461e84f684fbf96737f7713e48e5eed05219 Mon Sep 17 00:00:00 2001 From: domenico Date: Fri, 31 Jul 2026 23:49:37 +0200 Subject: [PATCH 03/16] docs: correct Groq strict json_schema support and free-tier viability The previous note claimed Groq supported strict structured output on kimi-k2-instruct-0905. Groq's docs list strict: true support for openai/gpt-oss-20b and openai/gpt-oss-120b only; kimi-k2 is best-effort JSON / tool-use, so Layer 3's strict call fails on it. Also record the constraints that decide whether a free tier works here: - Groq free tier is 8K tokens/min while Layer 3 budgets up to 8K input tokens per request, so one full batch can exhaust the minute and the next 429s. Layer 3 retries only on 413 and falls back only on 403, so a 429 abandons the batch and defers those files to L1+L2. - gemini-2.5-flash has thinking on by default and thought tokens count as output; with max_tokens capped at 2048 the JSON can truncate, which parses as SKIP at confidence 0.0. Recommend gemini-2.5-flash-lite, which defaults to thinking off. - Gemini per-model free quotas are no longer published in the API docs; link AI Studio instead of quoting numbers. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 08d54c7..41e0beb 100644 --- a/README.md +++ b/README.md @@ -163,16 +163,19 @@ GitHub no longer offers free inference. Copilot Pro remains [free for verified o Third-party free tiers do work here: -| Provider | `ai-base-url` | Free tier | -|:---------|:--------------|:----------| -| Groq | `https://api.groq.com/openai/v1` | No card required; ~30 RPM, 1,000–14,400 requests/day depending on model | -| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | Low daily quota, tightened in Dec 2025 | -| OpenRouter | `https://openrouter.ai/api/v1` | `:free` model variants | +**Layer 3 requires strict structured output.** It sends `response_format=json_schema` with `strict: true`. A model that does not support strict constrained decoding rejects the request, and because Layer 3 only retries on 413 and only falls back on 403, the batch is abandoned with a `::warning::` and those files defer to Layer 1 + Layer 2 — a working gate with no AI analysis. Check strict support before picking a free model. -Two caveats before you rely on one: +| Provider | `ai-base-url` | `ai-model` | Free-tier limits | +|:---------|:--------------|:-----------|:-----------------| +| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash-lite` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | +| Groq | `https://api.groq.com/openai/v1` | `openai/gpt-oss-120b` | 30 RPM, 1,000 req/day, **8K tokens/min**, 200K tokens/day | -- **Strict structured output is required.** Layer 3 requests `response_format=json_schema` with `strict: true`. Models without strict support reject the call, and the batch falls back to Layer 1 + Layer 2 — a working gate with no AI analysis. Groq supports it only on selected models; on OpenRouter it varies per model; Gemini supports it on synchronous chat completions. -- **Free tiers usually train on your prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. +Notes that decide whether these actually work: + +- **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. +- **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. +- **Prefer a non-thinking Gemini model.** `gemini-2.5-flash` has thinking on by default and billing counts thought tokens as output; Layer 3 caps `max_tokens` at 2048, so a long thought trace risks truncating the JSON (which parses as SKIP, confidence 0.0). `gemini-2.5-flash-lite` defaults to thinking off. Layer 3 does not currently send `reasoning_effort`, so this is not tunable from the action. +- **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. --- From a9c4aa8c563992be3e9f2b66f11c46a1054af6d3 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:03:30 +0200 Subject: [PATCH 04/16] feat(layer3): add ai-reasoning-effort, defaulting to none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Layer 3 caps max_tokens at 2048 and thinking models count thought tokens as output, so an unbounded thought trace can truncate the JSON verdict — which _parse_ai_response reads as SKIP at confidence 0.0. Sending reasoning_effort: none by default disables thinking on Gemini 2.5 and Groq gpt-oss, making those models usable without tuning. Non-reasoning models reject the parameter: OpenAI answers "Unsupported value", Azure "Unrecognized request argument", others "Unknown parameter". Since the default config is OpenAI + gpt-4.1-mini, a blanket send would break the default path, so: - the parameter is only sent when non-empty (empty string omits it) - a 400/404/422 naming reasoning_effort triggers one retry without it - the rejecting (base_url, model) pair is remembered, so the fallback costs one extra request per run rather than per batch - unrelated 400s still propagate untouched 430 tests pass, ruff clean. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 4 +- action.yml | 4 ++ src/config.py | 10 ++++ src/layer3_ai.py | 60 +++++++++++++++++-- src/main.py | 1 + tests/test_layer3_ai.py | 125 ++++++++++++++++++++++++++++++++++++++++ tests/test_main.py | 1 + 7 files changed, 199 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 41e0beb..ed7e065 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,8 @@ Notes that decide whether these actually work: - **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. -- **Prefer a non-thinking Gemini model.** `gemini-2.5-flash` has thinking on by default and billing counts thought tokens as output; Layer 3 caps `max_tokens` at 2048, so a long thought trace risks truncating the JSON (which parses as SKIP, confidence 0.0). `gemini-2.5-flash-lite` defaults to thinking off. Layer 3 does not currently send `reasoning_effort`, so this is not tunable from the action. +- **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. +- **Providers that reject the parameter are handled automatically.** OpenAI's `gpt-4.1` family returns a 400 for `reasoning_effort`. Layer 3 detects that specific rejection, retries once without the parameter, and remembers the endpoint/model pair — so the default OpenAI setup costs one extra request per run, not per batch. - **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. --- @@ -192,6 +193,7 @@ Notes that decide whether these actually work: | `ai-model` | `gpt-4.1-mini` | Model ID, exactly as `ai-base-url` expects it. | | `ai-base-url` | `https://api.openai.com/v1` | OpenAI-compatible inference endpoint. | | `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | +| `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** diff --git a/action.yml b/action.yml index 42d9a71..5e52778 100644 --- a/action.yml +++ b/action.yml @@ -36,6 +36,9 @@ inputs: description: 'API key for ai-base-url. Pass a secret, e.g. ${{ secrets.OPENAI_API_KEY }}. Required for Layer 3 — a GITHUB_TOKEN is not accepted by any of these providers. When empty, Layer 3 is skipped and the gate falls back to Layer 1 + Layer 2.' required: false default: '' + ai-reasoning-effort: + description: 'Thinking budget for reasoning models: none, minimal, low, medium, high. Defaults to none so a thought trace cannot consume the output budget and truncate the JSON verdict. Set to an empty string to omit the parameter entirely; providers that reject it (OpenAI gpt-4.1 family) have it stripped automatically.' + default: 'none' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -65,6 +68,7 @@ runs: INPUT_AI-MODEL: ${{ inputs.ai-model }} INPUT_AI-BASE-URL: ${{ inputs.ai-base-url }} INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} + INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 0f41dc9..6a33f33 100644 --- a/src/config.py +++ b/src/config.py @@ -28,6 +28,10 @@ # Any OpenAI-compatible /v1 endpoint. GitHub Models (models.github.ai) was # retired on 2026-07-30 and is no longer a valid target. DEFAULT_AI_BASE_URL = "https://api.openai.com/v1" +# "none" keeps thinking models from spending the output budget on a thought +# trace and truncating the JSON verdict. Providers that reject the parameter +# get it stripped automatically (see layer3_ai). Empty string never sends it. +DEFAULT_AI_REASONING_EFFORT = "none" _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -170,6 +174,9 @@ class Config: ai_base_url: OpenAI-compatible inference endpoint. ai_api_key: API key for that endpoint. Empty disables Layer 3's AI phase (shortcut gates still run). + ai_reasoning_effort: Thinking budget hint sent to reasoning models + ("none", "minimal", "low", "medium", "high"). Empty sends nothing; + endpoints that reject the parameter get it stripped on retry. ai_confidence_threshold: AI FAIL verdicts below this become WARNING (0.0-1.0). """ @@ -193,6 +200,7 @@ class Config: ai_confidence_threshold: float # 0.0-1.0 ai_base_url: str = DEFAULT_AI_BASE_URL ai_api_key: str = "" + ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -319,6 +327,7 @@ def parse_config() -> Config: # Provider API key. Deliberately NOT GITHUB_TOKEN: GitHub Models was # retired on 2026-07-30 and a GitHub token authenticates nothing else. ai_api_key = _env("AI-API-KEY", "") + ai_reasoning_effort = _env("AI-REASONING-EFFORT", DEFAULT_AI_REASONING_EFFORT) if ai_enabled and not ai_api_key: print( "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " @@ -354,5 +363,6 @@ def parse_config() -> Config: ai_model=ai_model, ai_base_url=ai_base_url, ai_api_key=ai_api_key, + ai_reasoning_effort=ai_reasoning_effort, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index ddbf4f5..8621cb7 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -32,6 +32,7 @@ from unidiff.errors import UnidiffParseError from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL +from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT from src.diff_utils import is_trivial_diff from src.models import FileVerdict, LayerResult, Verdict @@ -610,6 +611,10 @@ def _sanitize_diff( "gpt-4.1-nano", ) +# Endpoint/model pairs that rejected `reasoning_effort`. Populated at runtime +# so the fallback costs one extra request per pair, not one per batch. +_REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() + _CHARS_PER_TOKEN = 3 _INPUT_TOKEN_LIMIT = 8000 _SYSTEM_OVERHEAD_TOKENS = 800 # system prompt + JSON schema overhead @@ -759,6 +764,19 @@ def _is_retryable_size_error(exc: Exception) -> bool: return False +def _is_unsupported_reasoning_effort_error(exc: Exception) -> bool: + """True when the endpoint rejected the ``reasoning_effort`` parameter. + + Non-reasoning models reject it in provider-specific ways — OpenAI answers + "Unsupported value", Azure "Unrecognized request argument", others + "Unknown parameter" — but every variant names the parameter, so match on + that rather than on a message template. + """ + if isinstance(exc, APIStatusError) and exc.status_code in (400, 404, 422): + return "reasoning_effort" in str(exc).lower() + return False + + def _validate_batch_verdicts( verdicts: list[FileVerdict], batch_files: list[str], @@ -809,6 +827,7 @@ def _call_ai_for_batch( system_prompt: str, token: str, base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -826,6 +845,7 @@ def _call_ai_for_batch( try: raw = _call_ai_provider( model, system_prompt, user_prompt, token, base_url, + reasoning_effort, ) return raw, None except Exception as exc: @@ -851,12 +871,19 @@ def _call_ai_provider( user_prompt: str, token: str, base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, ) -> str: """Call an OpenAI-compatible inference endpoint, return raw response text. ``base_url`` is provider-agnostic: OpenAI, Azure AI Foundry, OpenRouter, or any local gateway speaking the same protocol. GitHub Models used to be hardcoded here; it was retired on 2026-07-30. + + ``reasoning_effort`` defaults to ``"none"`` to keep thinking models from + spending the 2048-token output budget on a thought trace and truncating + the JSON verdict. Non-reasoning models (OpenAI's gpt-4.1 family) reject + the parameter outright, so an unsupported-parameter rejection is retried + once without it and the (endpoint, model) pair is remembered. """ client = OpenAI( base_url=base_url, @@ -866,7 +893,31 @@ def _call_ai_provider( ChatCompletionSystemMessageParam(role="system", content=system_prompt), ChatCompletionUserMessageParam(role="user", content=user_prompt), ] - response = client.chat.completions.create( + extra: dict[str, object] = {} + if reasoning_effort and (base_url, model) not in _REASONING_EFFORT_UNSUPPORTED: + extra["reasoning_effort"] = reasoning_effort + + try: + response = _create_completion(client, model, messages, extra) + except Exception as exc: + if not extra or not _is_unsupported_reasoning_effort_error(exc): + raise + _REASONING_EFFORT_UNSUPPORTED.add((base_url, model)) + response = _create_completion(client, model, messages, {}) + + if not response.choices: + return "" + return response.choices[0].message.content or "" + + +def _create_completion( + client: OpenAI, + model: str, + messages: list[object], + extra: dict[str, object], +) -> object: + """Issue the chat-completion request, with ``extra`` merged into kwargs.""" + return client.chat.completions.create( model=model, messages=messages, temperature=0.1, @@ -879,10 +930,8 @@ def _call_ai_provider( schema=_VERDICT_SCHEMA, ), ), + **extra, ) - if not response.choices: - return "" - return response.choices[0].message.content or "" def _parse_ai_response( @@ -932,6 +981,7 @@ def run_layer3( confidence_threshold: float, unmeasurable_files: set[str] | None = None, base_url: str = _DEFAULT_AI_BASE_URL, + reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1044,7 +1094,7 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, - base_url, + base_url, reasoning_effort, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index d78a036..e052548 100644 --- a/src/main.py +++ b/src/main.py @@ -144,6 +144,7 @@ def run_pipeline(config: Config) -> Report: model=config.ai_model, token=config.ai_api_key, base_url=config.ai_base_url, + reasoning_effort=config.ai_reasoning_effort, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 8819e7b..7003016 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -3,10 +3,12 @@ import json from unittest.mock import MagicMock, patch +import pytest from openai import APIStatusError import src.layer3_ai as layer3_ai from src.layer3_ai import ( + _REASONING_EFFORT_UNSUPPORTED, Layer3Result, Relevance, _batch_files, @@ -616,6 +618,129 @@ def test_file_not_marked_unmeasurable_still_fails_gate_4( assert fv.verdict == Verdict.FAIL +class TestReasoningEffort: + """`reasoning_effort` defaults to "none" but must not break providers + that reject the parameter (OpenAI's gpt-4.1 family).""" + + def setup_method(self): + _REASONING_EFFORT_UNSUPPORTED.clear() + + def teardown_method(self): + _REASONING_EFFORT_UNSUPPORTED.clear() + + @patch("src.layer3_ai.OpenAI") + def test_forwards_reasoning_effort(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-2.5-flash", + system_prompt="system", + user_prompt="user", + token="sk-fake", + base_url="https://generativelanguage.googleapis.com/v1beta/openai/", + reasoning_effort="none", + ) + + kwargs = mock_client.chat.completions.create.call_args.kwargs + assert kwargs["reasoning_effort"] == "none" + + @patch("src.layer3_ai.OpenAI") + def test_empty_effort_omits_parameter(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="", + ) + + kwargs = mock_client.chat.completions.create.call_args.kwargs + assert "reasoning_effort" not in kwargs + + @patch("src.layer3_ai.OpenAI") + def test_rejection_retries_without_parameter(self, mock_openai: MagicMock): + """A model that rejects the parameter still gets its verdict.""" + ok = MagicMock() + ok.choices = [MagicMock()] + ok.choices[0].message.content = '{"verdict": "pass"}' + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = [ + _make_api_error(400, "Unsupported value: 'reasoning_effort' ..."), + ok, + ] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + assert result == '{"verdict": "pass"}' + assert mock_client.chat.completions.create.call_count == 2 + retry_kwargs = mock_client.chat.completions.create.call_args.kwargs + assert "reasoning_effort" not in retry_kwargs + + @patch("src.layer3_ai.OpenAI") + def test_rejection_is_remembered_per_endpoint_and_model( + self, mock_openai: MagicMock, + ): + """The retry costs one extra request per pair, not one per batch.""" + ok = MagicMock() + ok.choices = [] + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = [ + _make_api_error(400, "Unknown parameter: 'reasoning_effort'"), + ok, + ok, + ] + mock_openai.return_value = mock_client + + for _ in range(2): + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + # 2 for the first call (reject + retry), 1 for the second — not 4. + assert mock_client.chat.completions.create.call_count == 3 + assert ( + "reasoning_effort" + not in mock_client.chat.completions.create.call_args.kwargs + ) + + @patch("src.layer3_ai.OpenAI") + def test_unrelated_400_is_not_swallowed(self, mock_openai: MagicMock): + """Only parameter rejections trigger the retry — real errors propagate.""" + mock_client = MagicMock() + mock_client.chat.completions.create.side_effect = _make_api_error( + 400, "Incorrect API key provided", + ) + mock_openai.return_value = mock_client + + with pytest.raises(APIStatusError): + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + reasoning_effort="none", + ) + + assert mock_client.chat.completions.create.call_count == 1 + + class TestCallAiProvider: @patch("src.layer3_ai.OpenAI") def test_uses_configured_base_url(self, mock_openai: MagicMock): diff --git a/tests/test_main.py b/tests/test_main.py index 83ba285..e1ead46 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -244,6 +244,7 @@ def test_full_pipeline_classifies_files_for_l3( # now that GitHub Models is retired. token="sk-fake", base_url="https://api.openai.com/v1", + reasoning_effort="none", confidence_threshold=0.7, unmeasurable_files=set(), ) From 5ccaddaa9d2112b833d740e8f6147147ef5f309f Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:06:50 +0200 Subject: [PATCH 05/16] docs: recommend gemini-2.5-flash over flash-lite for Layer 3 With ai-reasoning-effort defaulting to none, the reason to prefer flash-lite (thinking off by default) no longer applies, and the capability gap decides it instead. Layer 3 asks the model to map changed behaviours to test assertions across four dimensions and calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for gpt-4.1-mini, the model this action was tuned on. Flash-Lite without reasoning scores 7. Record why a weak model is dangerous here specifically: missed problems fail open to L1+L2, but a confidently wrong FAIL blocks the PR, because ai-confidence-threshold only softens FAIL to WARNING when the model reports low confidence. Weak models also omit files more often, and _validate_batch_verdicts discards the entire batch when that happens. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ed7e065..cd3edd4 100644 --- a/README.md +++ b/README.md @@ -167,13 +167,14 @@ Third-party free tiers do work here: | Provider | `ai-base-url` | `ai-model` | Free-tier limits | |:---------|:--------------|:-----------|:-----------------| -| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash-lite` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | +| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | | Groq | `https://api.groq.com/openai/v1` | `openai/gpt-oss-120b` | 30 RPM, 1,000 req/day, **8K tokens/min**, 200K tokens/day | Notes that decide whether these actually work: - **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. +- **Pick `gemini-2.5-flash`, not `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. - **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. - **Providers that reject the parameter are handled automatically.** OpenAI's `gpt-4.1` family returns a 400 for `reasoning_effort`. Layer 3 detects that specific rejection, retries once without the parameter, and remembers the endpoint/model pair — so the default OpenAI setup costs one extra request per run, not per batch. - **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. From b773c56140f3da9ac8fbe369ed771acdd1de557d Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:09:51 +0200 Subject: [PATCH 06/16] ci(dogfood): run Layer 3 on gemini-2.5-flash with thinking off Points the dogfood workflow at the Gemini OpenAI-compatible endpoint so the next PR exercises the new provider-agnostic path end to end. gemini-2.5-flash rather than flash-lite: without reasoning it scores 14 on the Artificial Analysis Intelligence Index against 15 for the gpt-4.1-mini baseline this prompt was tuned on, where flash-lite scores 7. reasoning-effort is set explicitly even though none is the default, since this workflow doubles as the reference example: thinking must stay off or a thought trace can consume the 2048-token output budget and truncate the JSON verdict. Requires a GEMINI_API_KEY repository secret. Without it Layer 3 is skipped with a warning and the gate falls back to Layer 1 + Layer 2, so fork PRs (which get no secrets) degrade rather than fail. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dogfood.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index d5ccee7..48fadf7 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -36,10 +36,15 @@ jobs: with: coverage-file: coverage.xml coverage-threshold: '100' - ai-model: 'gpt-4.1-mini' + ai-model: 'gemini-2.5-flash' + # Trailing slash is required — without it the compat layer 404s. + ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/' # Layer 3 needs a provider key. Absent the secret, the gate degrades # to Layer 1 + Layer 2 with a warning instead of failing. - ai-api-key: ${{ secrets.OPENAI_API_KEY }} + ai-api-key: ${{ secrets.GEMINI_API_KEY }} + # Explicit though it is the default: thinking must stay off, or a + # thought trace can eat the 2048-token budget and truncate the JSON. + ai-reasoning-effort: 'none' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every From dd6e393b537404b48f50f91f25b295301b2c18fb Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:19:22 +0200 Subject: [PATCH 07/16] fix(action): remove ${{ }} expression from an input description The dogfood run failed before executing anything: action.yml (Line: 36): Unrecognized named-value: 'secrets'. Located at position 1 within expression: secrets.OPENAI_API_KEY GitHub evaluates ${{ }} expressions anywhere in action.yml, including input descriptions, and `secrets` is not in scope for a composite action, so the manifest fails to load and every consumer of the action breaks. Reword the description in prose and leave a comment so the trap is not reintroduced. Co-Authored-By: Claude Opus 5 (1M context) --- action.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 5e52778..c46c5a5 100644 --- a/action.yml +++ b/action.yml @@ -33,7 +33,11 @@ inputs: description: 'OpenAI-compatible inference endpoint. Examples: https://api.openai.com/v1, https://.services.ai.azure.com/openai/v1, https://openrouter.ai/api/v1. GitHub Models (models.github.ai) was retired on 2026-07-30 and no longer works.' default: 'https://api.openai.com/v1' ai-api-key: - description: 'API key for ai-base-url. Pass a secret, e.g. ${{ secrets.OPENAI_API_KEY }}. Required for Layer 3 — a GITHUB_TOKEN is not accepted by any of these providers. When empty, Layer 3 is skipped and the gate falls back to Layer 1 + Layer 2.' + # Do not write a ${{ ... }} expression anywhere in this file, including + # descriptions: GitHub evaluates them when loading the manifest, and + # `secrets` is not in scope for a composite action, so the whole action + # fails with "Unrecognized named-value: 'secrets'". + description: 'API key for ai-base-url; pass it from a repository secret. Required for Layer 3 — a GITHUB_TOKEN is not accepted by any of these providers. When empty, Layer 3 is skipped and the gate falls back to Layer 1 + Layer 2.' required: false default: '' ai-reasoning-effort: From a39f93136be5b97498a2bb21953ca69046e6b79e Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:27:30 +0200 Subject: [PATCH 08/16] fix(dev): point the local PR harness at the configured AI provider .dev/local_pr_test.py still passed token=config.github_token with no base_url, so after the GitHub Models retirement it would call OpenAI with a GitHub token and 401. It was missed in the migration sweep because it lives outside src/ and tests/. It now reads the provider key from AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY and forwards base_url and reasoning_effort, defaulting to gemini-2.5-flash with thinking off. Without a key it warns and runs the shortcut gates only. This harness is how the AI path gets exercised at all: it runs with coverage_files=[], so files fall through to the AI gates (6-8) instead of being resolved by the coverage shortcuts that decide everything in the dogfood workflow at coverage-threshold 100. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 45 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index 27695be..cec3cc8 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -7,13 +7,24 @@ GITHUB_TOKEN=ghp_xxx python local_pr_test.py # explicit token Token resolution: GITHUB_TOKEN env var → `gh auth token` CLI fallback. + +Layer 3 runs with no coverage report (coverage_files=[]), so files reach the +AI gates instead of being resolved by coverage shortcuts — this is the way to +exercise the AI path end to end. It needs a provider key: + + GEMINI_API_KEY=... python local_pr_test.py + +Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key, +AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT. Defaults target gemini-2.5-flash +with thinking off. Without a key the AI phase is skipped. """ from __future__ import annotations +import os import subprocess import sys -from src.config import Config, _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS +from src.config import _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS, Config from src.github_api import create_session from src.github_client import format_report from src.layer1_coverage import run_layer1 @@ -28,7 +39,6 @@ def _resolve_token() -> str: - import os token = os.environ.get("GITHUB_TOKEN") if token: return token @@ -41,6 +51,25 @@ def _resolve_token() -> str: sys.exit(1) +def _resolve_ai_api_key() -> str: + """Provider key for Layer 3, from the first env var that is set. + + Deliberately not GITHUB_TOKEN: GitHub Models was retired on 2026-07-30 and + a GitHub token authenticates no inference endpoint. Without a key the AI + phase is skipped and only the shortcut gates run. + """ + for name in ("AI_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY"): + value = os.environ.get(name, "").strip() + if value: + return value + print( + "WARNING: no AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY set — " + "Layer 3 will run shortcut gates only, no AI call.", + file=sys.stderr, + ) + return "" + + def run_local(repo: str, pr_number: int) -> None: token = _resolve_token() @@ -55,8 +84,14 @@ def run_local(repo: str, pr_number: int) -> None: test_patterns=_DEFAULT_TEST_PATTERNS, exclude_patterns=exclude_patterns, ai_enabled=True, - ai_model="openai/gpt-5-mini", + ai_model=os.environ.get("AI_MODEL", "gemini-2.5-flash"), ai_confidence_threshold=0.7, + ai_base_url=os.environ.get( + "AI_BASE_URL", + "https://generativelanguage.googleapis.com/v1beta/openai/", + ), + ai_api_key=_resolve_ai_api_key(), + ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "none"), ) session = create_session(token) @@ -122,7 +157,9 @@ def run_local(repo: str, pr_number: int) -> None: coverage_details=l1.coverage_details, coverage_threshold=config.coverage_threshold, model=config.ai_model, - token=config.github_token, + token=config.ai_api_key, + base_url=config.ai_base_url, + reasoning_effort=config.ai_reasoning_effort, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) From 92d26d6d187724c52f4d0be1424c21fa2f101df2 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:35:05 +0200 Subject: [PATCH 09/16] fix(dev): bootstrap sys.path so the harness runs standalone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running `python .dev/local_pr_test.py` failed with ModuleNotFoundError: No module named 'src', because executing the file directly puts .dev/ on sys.path rather than the repo root. CI never hit this since action.yml sets PYTHONPATH. Insert the repo root explicitly so the documented invocation works from any working directory. Verified by running it from /tmp against Ostico/test-guard#5: the pipeline completes and Layer 3 reaches the Gemini endpoint, returning "Missing or invalid Authorization header" locally where no key is set — which confirms base_url and request shape are correct. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index cec3cc8..99c4d7e 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -23,6 +23,12 @@ import os import subprocess import sys +from pathlib import Path + +# Running this file directly puts .dev/ on sys.path, not the repo root, so the +# src.* imports below fail. In CI action.yml sets PYTHONPATH; here we bootstrap +# it so `python .dev/local_pr_test.py` works from anywhere. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) from src.config import _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS, Config from src.github_api import create_session From b8d6cabcbfc8d8b817cce08bbbed91672dcc27e3 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:46:30 +0200 Subject: [PATCH 10/16] fix(layer3): move to live Gemini 3 model ids, add ai-temperature Gemini 2.5 was shut down ahead of its documented Oct 16 2026 date, so gemini-2.5-flash now returns: 404 models/gemini-2.5-flash is not found for API version v1beta Two consequences beyond the ID change: - Gemini 3.x has no off switch for thinking. `minimal` is the floor and only "matches the no-thinking setting for most queries". flash-lite defaults to minimal, whereas Gemini 3 Flash defaults to high, which risks a thought trace eating the 2048-token output budget and truncating the JSON verdict. Default to gemini-3.1-flash-lite with reasoning-effort minimal. - Gemini 3.x documents temperature 1.0 as strongly recommended and warns that lowering it causes looping and degraded reasoning. Layer 3 hardcoded 0.1, so evaluating Gemini 3 with it would have produced a false negative on capability. Temperature is now the ai-temperature input, still defaulting to 0.1 for OpenAI-style models. Dogfood and the local harness both target gemini-3.1-flash-lite with minimal thinking and temperature 1.0. README documents how to list the models a key can actually reach, since a retired ID's 404 reads like a URL bug. 432 tests pass, ruff clean. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 6 ++++-- .github/workflows/dogfood.yml | 14 ++++++++++---- README.md | 7 +++++-- action.yml | 4 ++++ src/config.py | 16 ++++++++++++++++ src/layer3_ai.py | 15 ++++++++++----- src/main.py | 1 + tests/test_layer3_ai.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_main.py | 1 + 9 files changed, 85 insertions(+), 13 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index 99c4d7e..c7d3cb3 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -90,14 +90,15 @@ def run_local(repo: str, pr_number: int) -> None: test_patterns=_DEFAULT_TEST_PATTERNS, exclude_patterns=exclude_patterns, ai_enabled=True, - ai_model=os.environ.get("AI_MODEL", "gemini-2.5-flash"), + ai_model=os.environ.get("AI_MODEL", "gemini-3.1-flash-lite"), ai_confidence_threshold=0.7, ai_base_url=os.environ.get( "AI_BASE_URL", "https://generativelanguage.googleapis.com/v1beta/openai/", ), ai_api_key=_resolve_ai_api_key(), - ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "none"), + ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "minimal"), + ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), ) session = create_session(token) @@ -166,6 +167,7 @@ def run_local(repo: str, pr_number: int) -> None: token=config.ai_api_key, base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, + temperature=config.ai_temperature, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 48fadf7..8aba182 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -36,15 +36,21 @@ jobs: with: coverage-file: coverage.xml coverage-threshold: '100' - ai-model: 'gemini-2.5-flash' + # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and + # now 404s. flash-lite defaults to minimal thinking, which suits the + # 2048-token output cap. + ai-model: 'gemini-3.1-flash-lite' # Trailing slash is required — without it the compat layer 404s. ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/' # Layer 3 needs a provider key. Absent the secret, the gate degrades # to Layer 1 + Layer 2 with a warning instead of failing. ai-api-key: ${{ secrets.GEMINI_API_KEY }} - # Explicit though it is the default: thinking must stay off, or a - # thought trace can eat the 2048-token budget and truncate the JSON. - ai-reasoning-effort: 'none' + # Gemini 3.x has no "off" for thinking; minimal is the floor and + # keeps a thought trace from eating the output budget. + ai-reasoning-effort: 'minimal' + # Gemini 3.x warns that lowering temperature causes looping and + # degraded reasoning, so use its recommended default instead of 0.1. + ai-temperature: '1.0' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every diff --git a/README.md b/README.md index cd3edd4..35d193b 100644 --- a/README.md +++ b/README.md @@ -167,14 +167,17 @@ Third-party free tiers do work here: | Provider | `ai-base-url` | `ai-model` | Free-tier limits | |:---------|:--------------|:-----------|:-----------------| -| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-2.5-flash` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | +| Google Gemini | `https://generativelanguage.googleapis.com/v1beta/openai/` | `gemini-3.1-flash-lite` | Per-model quotas are shown in [AI Studio](https://aistudio.google.com/rate-limit); token/min is roomy | | Groq | `https://api.groq.com/openai/v1` | `openai/gpt-oss-120b` | 30 RPM, 1,000 req/day, **8K tokens/min**, 200K tokens/day | Notes that decide whether these actually work: - **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. -- **Pick `gemini-2.5-flash`, not `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. +- **Model IDs go stale — verify before trusting one.** Gemini 2.5 was shut down ahead of its documented October 2026 date, and a retired ID returns a generic `404 models/ is not found for API version v1beta`, which reads like a URL bug. List what your key can actually reach: `curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY"`. +- **Gemini 3.x cannot disable thinking.** `minimal` is the floor, and it "matches the no-thinking setting for most queries" without guaranteeing it. `gemini-3.1-flash-lite` defaults to `minimal`; Gemini 3 Flash defaults to `high`, which risks a thought trace consuming Layer 3's 2048-token output budget. +- **Gemini 3.x wants `ai-temperature: '1.0'`.** Google documents its default as strongly recommended and warns that lowering temperature can cause looping or degraded reasoning. The action default of `0.1` suits OpenAI-style models but is actively wrong here. +- **Superseded guidance, kept for context: `gemini-2.5-flash` over `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. - **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. - **Providers that reject the parameter are handled automatically.** OpenAI's `gpt-4.1` family returns a 400 for `reasoning_effort`. Layer 3 detects that specific rejection, retries once without the parameter, and remembers the endpoint/model pair — so the default OpenAI setup costs one extra request per run, not per batch. - **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. diff --git a/action.yml b/action.yml index c46c5a5..0967522 100644 --- a/action.yml +++ b/action.yml @@ -43,6 +43,9 @@ inputs: ai-reasoning-effort: description: 'Thinking budget for reasoning models: none, minimal, low, medium, high. Defaults to none so a thought trace cannot consume the output budget and truncate the JSON verdict. Set to an empty string to omit the parameter entirely; providers that reject it (OpenAI gpt-4.1 family) have it stripped automatically.' default: 'none' + ai-temperature: + description: 'Sampling temperature. 0.1 suits OpenAI-style models for a deterministic verdict. Gemini 3.x recommends its default of 1.0 and warns that lowering it can cause looping or degraded reasoning — set 1.0 for those models.' + default: '0.1' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -73,6 +76,7 @@ runs: INPUT_AI-BASE-URL: ${{ inputs.ai-base-url }} INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} + INPUT_AI-TEMPERATURE: ${{ inputs.ai-temperature }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 6a33f33..8175c77 100644 --- a/src/config.py +++ b/src/config.py @@ -32,6 +32,11 @@ # trace and truncating the JSON verdict. Providers that reject the parameter # get it stripped automatically (see layer3_ai). Empty string never sends it. DEFAULT_AI_REASONING_EFFORT = "none" +# 0.1 suits OpenAI-style models for a deterministic classification. Gemini 3.x +# documents the opposite: it recommends its default of 1.0 and warns that +# lowering temperature can cause looping or degraded reasoning, so that +# provider needs this raised. +DEFAULT_AI_TEMPERATURE = 0.1 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -201,6 +206,7 @@ class Config: ai_base_url: str = DEFAULT_AI_BASE_URL ai_api_key: str = "" ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT + ai_temperature: float = DEFAULT_AI_TEMPERATURE def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -328,6 +334,15 @@ def parse_config() -> Config: # retired on 2026-07-30 and a GitHub token authenticates nothing else. ai_api_key = _env("AI-API-KEY", "") ai_reasoning_effort = _env("AI-REASONING-EFFORT", DEFAULT_AI_REASONING_EFFORT) + temperature_raw = _env("AI-TEMPERATURE", str(DEFAULT_AI_TEMPERATURE)) + try: + ai_temperature = float(temperature_raw) + except ValueError: + print( + f"::warning::Invalid ai-temperature '{temperature_raw}' — " + f"falling back to {DEFAULT_AI_TEMPERATURE}." + ) + ai_temperature = DEFAULT_AI_TEMPERATURE if ai_enabled and not ai_api_key: print( "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " @@ -364,5 +379,6 @@ def parse_config() -> Config: ai_base_url=ai_base_url, ai_api_key=ai_api_key, ai_reasoning_effort=ai_reasoning_effort, + ai_temperature=ai_temperature, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index 8621cb7..9f2818f 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -33,6 +33,7 @@ from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT +from src.config import DEFAULT_AI_TEMPERATURE as _DEFAULT_AI_TEMPERATURE from src.diff_utils import is_trivial_diff from src.models import FileVerdict, LayerResult, Verdict @@ -828,6 +829,7 @@ def _call_ai_for_batch( token: str, base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -845,7 +847,7 @@ def _call_ai_for_batch( try: raw = _call_ai_provider( model, system_prompt, user_prompt, token, base_url, - reasoning_effort, + reasoning_effort, temperature, ) return raw, None except Exception as exc: @@ -872,6 +874,7 @@ def _call_ai_provider( token: str, base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, ) -> str: """Call an OpenAI-compatible inference endpoint, return raw response text. @@ -898,12 +901,12 @@ def _call_ai_provider( extra["reasoning_effort"] = reasoning_effort try: - response = _create_completion(client, model, messages, extra) + response = _create_completion(client, model, messages, extra, temperature) except Exception as exc: if not extra or not _is_unsupported_reasoning_effort_error(exc): raise _REASONING_EFFORT_UNSUPPORTED.add((base_url, model)) - response = _create_completion(client, model, messages, {}) + response = _create_completion(client, model, messages, {}, temperature) if not response.choices: return "" @@ -915,12 +918,13 @@ def _create_completion( model: str, messages: list[object], extra: dict[str, object], + temperature: float = _DEFAULT_AI_TEMPERATURE, ) -> object: """Issue the chat-completion request, with ``extra`` merged into kwargs.""" return client.chat.completions.create( model=model, messages=messages, - temperature=0.1, + temperature=temperature, max_tokens=2048, response_format=ResponseFormatJSONSchema( type="json_schema", @@ -982,6 +986,7 @@ def run_layer3( unmeasurable_files: set[str] | None = None, base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, + temperature: float = _DEFAULT_AI_TEMPERATURE, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1094,7 +1099,7 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, - base_url, reasoning_effort, + base_url, reasoning_effort, temperature, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index e052548..0f5f525 100644 --- a/src/main.py +++ b/src/main.py @@ -145,6 +145,7 @@ def run_pipeline(config: Config) -> Report: token=config.ai_api_key, base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, + temperature=config.ai_temperature, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 7003016..6724f94 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -741,6 +741,40 @@ def test_unrelated_400_is_not_swallowed(self, mock_openai: MagicMock): assert mock_client.chat.completions.create.call_count == 1 +class TestTemperature: + @patch("src.layer3_ai.OpenAI") + def test_forwards_temperature(self, mock_openai: MagicMock): + """Gemini 3.x needs 1.0 — lower values cause looping per Google's docs.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + temperature=1.0, + ) + + assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 1.0 + + @patch("src.layer3_ai.OpenAI") + def test_defaults_to_deterministic_temperature(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 0.1 + + class TestCallAiProvider: @patch("src.layer3_ai.OpenAI") def test_uses_configured_base_url(self, mock_openai: MagicMock): diff --git a/tests/test_main.py b/tests/test_main.py index e1ead46..41a0430 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -245,6 +245,7 @@ def test_full_pipeline_classifies_files_for_l3( token="sk-fake", base_url="https://api.openai.com/v1", reasoning_effort="none", + temperature=0.1, confidence_threshold=0.7, unmeasurable_files=set(), ) From fd5bc85a8245768f7336093777540949ae555e76 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:52:49 +0200 Subject: [PATCH 11/16] feat(layer3): add ai-max-tokens, stop flooring Gemini 3 thinking at minimal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thinking was pinned to minimal to protect a hardcoded 2048-token output cap. That is backwards: the published intelligence scores for Gemini 3.x are reasoning-enabled (gemini-3.1-flash-lite scores 25 on the Artificial Analysis Intelligence Index against 15 for the gpt-4.1-mini baseline this prompt was tuned on), and Google recommends thinking for code reasoning — which is exactly what Layer 3 does. Flooring the effort discards the capability the model is chosen for. Move the cap instead: - max_tokens becomes the ai-max-tokens input, default 4096. It is a cap rather than a reservation, so unused headroom costs nothing, and it keeps a thought trace from squeezing out the JSON verdict (truncated JSON parses as SKIP at confidence 0.0). - dogfood and the local harness move from minimal to low thinking. - Gemini 3.x permits up to 65536 output tokens, so the model is not the binding constraint; free-tier tokens/minute is. 434 tests pass, ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 9 ++++++--- .github/workflows/dogfood.yml | 8 +++++--- README.md | 7 +++++-- action.yml | 4 ++++ src/config.py | 16 ++++++++++++++++ src/layer3_ai.py | 15 ++++++++++----- src/main.py | 1 + tests/test_layer3_ai.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_main.py | 1 + 9 files changed, 82 insertions(+), 13 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index c7d3cb3..37c372b 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -15,8 +15,9 @@ GEMINI_API_KEY=... python local_pr_test.py Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key, -AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT. Defaults target gemini-2.5-flash -with thinking off. Without a key the AI phase is skipped. +AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_TOKENS. +Defaults target gemini-3.1-flash-lite at temperature 1.0 with low thinking, +per Google's guidance for Gemini 3.x. Without a key the AI phase is skipped. """ from __future__ import annotations @@ -97,8 +98,9 @@ def run_local(repo: str, pr_number: int) -> None: "https://generativelanguage.googleapis.com/v1beta/openai/", ), ai_api_key=_resolve_ai_api_key(), - ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "minimal"), + ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"), ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), + ai_max_tokens=int(os.environ.get("AI_MAX_TOKENS", "4096")), ) session = create_session(token) @@ -168,6 +170,7 @@ def run_local(repo: str, pr_number: int) -> None: base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, + max_tokens=config.ai_max_tokens, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 8aba182..e1cdfe0 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -45,9 +45,11 @@ jobs: # Layer 3 needs a provider key. Absent the secret, the gate degrades # to Layer 1 + Layer 2 with a warning instead of failing. ai-api-key: ${{ secrets.GEMINI_API_KEY }} - # Gemini 3.x has no "off" for thinking; minimal is the floor and - # keeps a thought trace from eating the output budget. - ai-reasoning-effort: 'minimal' + # Gemini 3.x has no "off" for thinking. Its published intelligence + # scores are reasoning-enabled, and Google recommends thinking for + # code reasoning, so don't floor this at minimal — the 4096-token + # default output cap leaves room for a short thought trace. + ai-reasoning-effort: 'low' # Gemini 3.x warns that lowering temperature causes looping and # degraded reasoning, so use its recommended default instead of 0.1. ai-temperature: '1.0' diff --git a/README.md b/README.md index 35d193b..f59830f 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,8 @@ Notes that decide whether these actually work: - **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. - **Model IDs go stale — verify before trusting one.** Gemini 2.5 was shut down ahead of its documented October 2026 date, and a retired ID returns a generic `404 models/ is not found for API version v1beta`, which reads like a URL bug. List what your key can actually reach: `curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY"`. -- **Gemini 3.x cannot disable thinking.** `minimal` is the floor, and it "matches the no-thinking setting for most queries" without guaranteeing it. `gemini-3.1-flash-lite` defaults to `minimal`; Gemini 3 Flash defaults to `high`, which risks a thought trace consuming Layer 3's 2048-token output budget. +- **Gemini 3.x cannot disable thinking — and you should not want to.** `minimal` is the floor and only "matches the no-thinking setting for most queries". Published intelligence scores for these models are *reasoning-enabled*: `gemini-3.1-flash-lite` scores 25 on the Artificial Analysis Intelligence Index versus 15 for `gpt-4.1-mini`, the baseline this prompt was tuned on — but that figure does not describe a minimal-thinking run, and Google recommends thinking for code reasoning specifically. Layer 3 is code judgment, so prefer `low` or higher and give it room via `ai-max-tokens` rather than flooring the effort. +- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-tokens` defaults to `4096`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. - **Gemini 3.x wants `ai-temperature: '1.0'`.** Google documents its default as strongly recommended and warns that lowering temperature can cause looping or degraded reasoning. The action default of `0.1` suits OpenAI-style models but is actively wrong here. - **Superseded guidance, kept for context: `gemini-2.5-flash` over `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. - **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. @@ -197,7 +198,9 @@ Notes that decide whether these actually work: | `ai-model` | `gpt-4.1-mini` | Model ID, exactly as `ai-base-url` expects it. | | `ai-base-url` | `https://api.openai.com/v1` | OpenAI-compatible inference endpoint. | | `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | -| `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. | +| `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. Gemini 3.x has no `none` — use `minimal` or above. | +| `ai-temperature` | `0.1` | Sampling temperature. Gemini 3.x wants `1.0`; lowering it there causes looping per Google's docs. | +| `ai-max-tokens` | `4096` | Output cap per request. Raise alongside `ai-reasoning-effort`. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** diff --git a/action.yml b/action.yml index 0967522..ac50d05 100644 --- a/action.yml +++ b/action.yml @@ -46,6 +46,9 @@ inputs: ai-temperature: description: 'Sampling temperature. 0.1 suits OpenAI-style models for a deterministic verdict. Gemini 3.x recommends its default of 1.0 and warns that lowering it can cause looping or degraded reasoning — set 1.0 for those models.' default: '0.1' + ai-max-tokens: + description: 'Output token cap per request. The verdict itself is a few hundred tokens; the headroom exists so a reasoning model thought trace cannot squeeze out the JSON. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' + default: '4096' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -77,6 +80,7 @@ runs: INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} INPUT_AI-TEMPERATURE: ${{ inputs.ai-temperature }} + INPUT_AI-MAX-TOKENS: ${{ inputs.ai-max-tokens }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 8175c77..a76c944 100644 --- a/src/config.py +++ b/src/config.py @@ -37,6 +37,10 @@ # lowering temperature can cause looping or degraded reasoning, so that # provider needs this raised. DEFAULT_AI_TEMPERATURE = 0.1 +# A verdict payload is a few hundred tokens; the rest of this budget exists so +# a reasoning model's thought trace cannot squeeze the JSON out. max_tokens is +# a cap, not a reservation, so a larger value costs nothing when unused. +DEFAULT_AI_MAX_TOKENS = 4096 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -207,6 +211,7 @@ class Config: ai_api_key: str = "" ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT ai_temperature: float = DEFAULT_AI_TEMPERATURE + ai_max_tokens: int = DEFAULT_AI_MAX_TOKENS def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -343,6 +348,16 @@ def parse_config() -> Config: f"falling back to {DEFAULT_AI_TEMPERATURE}." ) ai_temperature = DEFAULT_AI_TEMPERATURE + + max_tokens_raw = _env("AI-MAX-TOKENS", str(DEFAULT_AI_MAX_TOKENS)) + try: + ai_max_tokens = int(max_tokens_raw) + except ValueError: + print( + f"::warning::Invalid ai-max-tokens '{max_tokens_raw}' — " + f"falling back to {DEFAULT_AI_MAX_TOKENS}." + ) + ai_max_tokens = DEFAULT_AI_MAX_TOKENS if ai_enabled and not ai_api_key: print( "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " @@ -380,5 +395,6 @@ def parse_config() -> Config: ai_api_key=ai_api_key, ai_reasoning_effort=ai_reasoning_effort, ai_temperature=ai_temperature, + ai_max_tokens=ai_max_tokens, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index 9f2818f..6c50180 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -32,6 +32,7 @@ from unidiff.errors import UnidiffParseError from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL +from src.config import DEFAULT_AI_MAX_TOKENS as _DEFAULT_AI_MAX_TOKENS from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT from src.config import DEFAULT_AI_TEMPERATURE as _DEFAULT_AI_TEMPERATURE from src.diff_utils import is_trivial_diff @@ -830,6 +831,7 @@ def _call_ai_for_batch( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, + max_tokens: int = _DEFAULT_AI_MAX_TOKENS, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -847,7 +849,7 @@ def _call_ai_for_batch( try: raw = _call_ai_provider( model, system_prompt, user_prompt, token, base_url, - reasoning_effort, temperature, + reasoning_effort, temperature, max_tokens, ) return raw, None except Exception as exc: @@ -875,6 +877,7 @@ def _call_ai_provider( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, + max_tokens: int = _DEFAULT_AI_MAX_TOKENS, ) -> str: """Call an OpenAI-compatible inference endpoint, return raw response text. @@ -901,12 +904,12 @@ def _call_ai_provider( extra["reasoning_effort"] = reasoning_effort try: - response = _create_completion(client, model, messages, extra, temperature) + response = _create_completion(client, model, messages, extra, temperature, max_tokens) except Exception as exc: if not extra or not _is_unsupported_reasoning_effort_error(exc): raise _REASONING_EFFORT_UNSUPPORTED.add((base_url, model)) - response = _create_completion(client, model, messages, {}, temperature) + response = _create_completion(client, model, messages, {}, temperature, max_tokens) if not response.choices: return "" @@ -919,13 +922,14 @@ def _create_completion( messages: list[object], extra: dict[str, object], temperature: float = _DEFAULT_AI_TEMPERATURE, + max_tokens: int = _DEFAULT_AI_MAX_TOKENS, ) -> object: """Issue the chat-completion request, with ``extra`` merged into kwargs.""" return client.chat.completions.create( model=model, messages=messages, temperature=temperature, - max_tokens=2048, + max_tokens=max_tokens, response_format=ResponseFormatJSONSchema( type="json_schema", json_schema=JSONSchema( @@ -987,6 +991,7 @@ def run_layer3( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, + max_tokens: int = _DEFAULT_AI_MAX_TOKENS, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1099,7 +1104,7 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, - base_url, reasoning_effort, temperature, + base_url, reasoning_effort, temperature, max_tokens, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index 0f5f525..b73f04d 100644 --- a/src/main.py +++ b/src/main.py @@ -146,6 +146,7 @@ def run_pipeline(config: Config) -> Report: base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, + max_tokens=config.ai_max_tokens, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 6724f94..b0a4318 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -775,6 +775,40 @@ def test_defaults_to_deterministic_temperature(self, mock_openai: MagicMock): assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 0.1 +class TestMaxTokens: + @patch("src.layer3_ai.OpenAI") + def test_forwards_max_tokens(self, mock_openai: MagicMock): + """Headroom for a thought trace is configurable, not hardcoded.""" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + max_tokens=16384, + ) + + assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 16384 + + @patch("src.layer3_ai.OpenAI") + def test_default_leaves_room_for_thinking(self, mock_openai: MagicMock): + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [] + mock_openai.return_value = mock_client + + _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 4096 + + class TestCallAiProvider: @patch("src.layer3_ai.OpenAI") def test_uses_configured_base_url(self, mock_openai: MagicMock): diff --git a/tests/test_main.py b/tests/test_main.py index 41a0430..6dba444 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -246,6 +246,7 @@ def test_full_pipeline_classifies_files_for_l3( base_url="https://api.openai.com/v1", reasoning_effort="none", temperature=0.1, + max_tokens=4096, confidence_threshold=0.7, unmeasurable_files=set(), ) From 1c815c86f3fe4ca9c18c6848d4d85cae2a7e9688 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 00:55:31 +0200 Subject: [PATCH 12/16] fix(layer3): raise output cap to 8192 and warn when it truncates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 4096 was picked without checking how thought tokens interact with the output cap. They are charged against it: when thinking exhausts the budget the API returns finish_reason="length" with empty content, which _parse_ai_response reads as SKIP at confidence 0.0 — indistinguishable from a model that had nothing to say. 8192 is the reported floor that avoids this with thinking enabled. The cap stays well below the 65536 Gemini 3.x allows on purpose: thought tokens are billed as output and burn free-tier tokens/minute, so an unbounded cap turns a runaway thought trace into quota exhaustion. The cap is not a reservation, so headroom is free when unused — the trade is between bounded worst-case spend and silent truncation, and 8192 buys both. Also emit a ::warning:: naming the cap and the two levers when a request truncates before producing a verdict, so the degradation is visible. 436 tests pass, ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 2 +- README.md | 4 ++-- action.yml | 4 ++-- src/config.py | 11 ++++++--- src/layer3_ai.py | 15 ++++++++++++- tests/test_layer3_ai.py | 50 ++++++++++++++++++++++++++++++++++++++++- tests/test_main.py | 2 +- 7 files changed, 77 insertions(+), 11 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index 37c372b..669a658 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -100,7 +100,7 @@ def run_local(repo: str, pr_number: int) -> None: ai_api_key=_resolve_ai_api_key(), ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"), ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), - ai_max_tokens=int(os.environ.get("AI_MAX_TOKENS", "4096")), + ai_max_tokens=int(os.environ.get("AI_MAX_TOKENS", "8192")), ) session = create_session(token) diff --git a/README.md b/README.md index f59830f..9d6098b 100644 --- a/README.md +++ b/README.md @@ -176,7 +176,7 @@ Notes that decide whether these actually work: - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. - **Model IDs go stale — verify before trusting one.** Gemini 2.5 was shut down ahead of its documented October 2026 date, and a retired ID returns a generic `404 models/ is not found for API version v1beta`, which reads like a URL bug. List what your key can actually reach: `curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY"`. - **Gemini 3.x cannot disable thinking — and you should not want to.** `minimal` is the floor and only "matches the no-thinking setting for most queries". Published intelligence scores for these models are *reasoning-enabled*: `gemini-3.1-flash-lite` scores 25 on the Artificial Analysis Intelligence Index versus 15 for `gpt-4.1-mini`, the baseline this prompt was tuned on — but that figure does not describe a minimal-thinking run, and Google recommends thinking for code reasoning specifically. Layer 3 is code judgment, so prefer `low` or higher and give it room via `ai-max-tokens` rather than flooring the effort. -- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-tokens` defaults to `4096`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. +- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-tokens` defaults to `8192`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. - **Gemini 3.x wants `ai-temperature: '1.0'`.** Google documents its default as strongly recommended and warns that lowering temperature can cause looping or degraded reasoning. The action default of `0.1` suits OpenAI-style models but is actively wrong here. - **Superseded guidance, kept for context: `gemini-2.5-flash` over `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. - **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. @@ -200,7 +200,7 @@ Notes that decide whether these actually work: | `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | | `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. Gemini 3.x has no `none` — use `minimal` or above. | | `ai-temperature` | `0.1` | Sampling temperature. Gemini 3.x wants `1.0`; lowering it there causes looping per Google's docs. | -| `ai-max-tokens` | `4096` | Output cap per request. Raise alongside `ai-reasoning-effort`. | +| `ai-max-tokens` | `8192` | Output cap per request. Raise alongside `ai-reasoning-effort`. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** diff --git a/action.yml b/action.yml index ac50d05..8a2f979 100644 --- a/action.yml +++ b/action.yml @@ -47,8 +47,8 @@ inputs: description: 'Sampling temperature. 0.1 suits OpenAI-style models for a deterministic verdict. Gemini 3.x recommends its default of 1.0 and warns that lowering it can cause looping or degraded reasoning — set 1.0 for those models.' default: '0.1' ai-max-tokens: - description: 'Output token cap per request. The verdict itself is a few hundred tokens; the headroom exists so a reasoning model thought trace cannot squeeze out the JSON. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' - default: '4096' + description: 'Output token cap per request. Thought tokens are charged against it, and exhausting it returns finish_reason=length with empty content, which is reported as a warning. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' + default: '8192' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' diff --git a/src/config.py b/src/config.py index a76c944..a69c5b9 100644 --- a/src/config.py +++ b/src/config.py @@ -38,9 +38,14 @@ # provider needs this raised. DEFAULT_AI_TEMPERATURE = 0.1 # A verdict payload is a few hundred tokens; the rest of this budget exists so -# a reasoning model's thought trace cannot squeeze the JSON out. max_tokens is -# a cap, not a reservation, so a larger value costs nothing when unused. -DEFAULT_AI_MAX_TOKENS = 4096 +# a reasoning model's thought trace cannot squeeze the JSON out. Thought tokens +# are charged against the output cap, and when they exhaust it the API returns +# finish_reason="length" with empty content — which parses as SKIP at +# confidence 0.0. 8192 is the community-reported floor that avoids that with +# thinking enabled; the cap is not a reservation, so unused headroom is free. +# It stays well under the 65536 Gemini 3.x allows, because thought tokens are +# billed as output and burn free-tier tokens/minute. +DEFAULT_AI_MAX_TOKENS = 8192 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") diff --git a/src/layer3_ai.py b/src/layer3_ai.py index 6c50180..778c823 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -913,7 +913,20 @@ def _call_ai_provider( if not response.choices: return "" - return response.choices[0].message.content or "" + + choice = response.choices[0] + content = choice.message.content or "" + # A reasoning model can spend the whole output cap on its thought trace and + # return finish_reason="length" with empty content. That parses as SKIP at + # confidence 0.0, which is indistinguishable from a model that simply had + # nothing to say — so name the cause instead of degrading quietly. + if getattr(choice, "finish_reason", None) == "length" and not content.strip(): + print( + f"::warning::{model} hit the {max_tokens}-token output cap before " + f"emitting a verdict — most likely the thought trace consumed it. " + f"Raise ai-max-tokens or lower ai-reasoning-effort." + ) + return content def _create_completion( diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index b0a4318..b20b5d7 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -775,6 +775,54 @@ def test_defaults_to_deterministic_temperature(self, mock_openai: MagicMock): assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 0.1 +class TestTruncationWarning: + """A thought trace that eats the output cap must not degrade silently.""" + + @patch("src.layer3_ai.OpenAI") + def test_warns_when_cap_consumed_before_verdict( + self, mock_openai: MagicMock, capsys, + ): + choice = MagicMock() + choice.finish_reason = "length" + choice.message.content = "" + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [choice] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gemini-3.1-flash-lite", + system_prompt="system", + user_prompt="user", + token="sk-fake", + max_tokens=8192, + ) + + assert result == "" + out = capsys.readouterr().out + assert "::warning::" in out + assert "8192-token output cap" in out + assert "ai-max-tokens" in out + + @patch("src.layer3_ai.OpenAI") + def test_no_warning_on_a_normal_verdict(self, mock_openai: MagicMock, capsys): + choice = MagicMock() + choice.finish_reason = "stop" + choice.message.content = '{"verdict": "pass"}' + mock_client = MagicMock() + mock_client.chat.completions.create.return_value.choices = [choice] + mock_openai.return_value = mock_client + + result = _call_ai_provider( + model="gpt-4.1-mini", + system_prompt="system", + user_prompt="user", + token="sk-fake", + ) + + assert result == '{"verdict": "pass"}' + assert "::warning::" not in capsys.readouterr().out + + class TestMaxTokens: @patch("src.layer3_ai.OpenAI") def test_forwards_max_tokens(self, mock_openai: MagicMock): @@ -806,7 +854,7 @@ def test_default_leaves_room_for_thinking(self, mock_openai: MagicMock): token="sk-fake", ) - assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 4096 + assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 8192 class TestCallAiProvider: diff --git a/tests/test_main.py b/tests/test_main.py index 6dba444..465c662 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -246,7 +246,7 @@ def test_full_pipeline_classifies_files_for_l3( base_url="https://api.openai.com/v1", reasoning_effort="none", temperature=0.1, - max_tokens=4096, + max_tokens=8192, confidence_threshold=0.7, unmeasurable_files=set(), ) From d43d603e21f847c7400d5ad8ad622f77d3e9929d Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 01:01:46 +0200 Subject: [PATCH 13/16] ci(dogfood): add dogfood-ai label to exercise Layer 3's AI path Two runs in a row reported "All files resolved by deterministic shortcuts": at coverage-threshold 100 with a full coverage report, Gate 3 (>= threshold) or Gate 5 (below, tests exist) decides every file, so the AI phase is structurally unreachable and the dogfood cannot validate the provider migration it was pointed at. Withhold the coverage report when a PR carries the `dogfood-ai` label, so files fall through to the AI gates (6-8). `labeled` joins the trigger types so adding the label starts a run without a push. Unlabelled PRs keep the strict coverage gate unchanged. Also cover the config fallbacks that the gate itself flagged: config.py sat at 79% because the invalid-ai-temperature and invalid-ai-max-tokens except branches had no tests. 439 tests pass, ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dogfood.yml | 10 +++++++-- tests/test_config.py | 39 +++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index e1cdfe0..f2f74eb 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -1,7 +1,8 @@ name: Test Guard (Dogfood) on: pull_request: - types: [opened, synchronize] + # `labeled` lets the dogfood-ai label below start a run without a push. + types: [opened, synchronize, labeled] # Cancel superseded runs: only the newest push per PR branch keeps running. concurrency: @@ -34,7 +35,12 @@ jobs: - name: Run Test Guard on itself uses: ./ with: - coverage-file: coverage.xml + # At threshold 100 with a full coverage report, Gates 3 and 5 resolve + # every file deterministically and the AI phase never runs. Add the + # `dogfood-ai` label to a PR to withhold coverage for one run, so + # files fall through to the AI gates (6-8) and Layer 3 is exercised + # end to end. Without the label this stays the strict coverage gate. + coverage-file: ${{ contains(github.event.pull_request.labels.*.name, 'dogfood-ai') && '' || 'coverage.xml' }} coverage-threshold: '100' # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and # now 404s. flash-lite defaults to minimal thinking, which suits the diff --git a/tests/test_config.py b/tests/test_config.py index bdcb756..d944236 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -68,6 +68,45 @@ def test_custom_endpoint_and_key(self, monkeypatch): assert cfg.ai_model == "openai/gpt-4.1-mini" assert cfg.ai_enabled is True + def test_invalid_temperature_falls_back(self, monkeypatch, capsys): + """A bad temperature must not abort the run — warn and use the default.""" + monkeypatch.setenv("INPUT_AI-TEMPERATURE", "not-a-number") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_temperature == 0.1 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-temperature" in out + + def test_invalid_max_tokens_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-MAX-TOKENS", "lots") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_max_tokens == 8192 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-max-tokens" in out + + def test_valid_temperature_and_max_tokens_are_used(self, monkeypatch): + monkeypatch.setenv("INPUT_AI-TEMPERATURE", "1.0") + monkeypatch.setenv("INPUT_AI-MAX-TOKENS", "16384") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_temperature == 1.0 + assert cfg.ai_max_tokens == 16384 + def test_missing_api_key_disables_ai(self, monkeypatch, capsys): """No key means no AI phase: GITHUB_TOKEN is not a provider credential. From 83f544d2ec3794489605f6f3ff8ed1f139dfea45 Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 01:05:01 +0200 Subject: [PATCH 14/16] ci(dogfood): fix label gate that a falsy ternary made inert The conditional coverage-file could never withhold coverage: ${{ contains(...) && '' || 'coverage.xml' }} In GitHub expressions the empty string is falsy, so when the condition is true the && yields '' and the || short-circuits straight to 'coverage.xml'. A ternary cannot produce an empty value. The labeled run fired correctly; the expression was incapable of acting on it. Gate two whole steps on the label instead, so the AI-path step omits coverage-file entirely. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/dogfood.yml | 43 ++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index f2f74eb..44ffef8 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -32,15 +32,46 @@ jobs: pip install -r requirements.txt pytest tests/ --cov=src --cov-report=xml + # Two steps rather than one with a conditional coverage-file: in GitHub + # expressions `cond && '' || 'x'` always yields 'x', because the empty + # string is falsy and short-circuits to the fallback. A ternary cannot + # produce an empty value, so gate whole steps instead. - name: Run Test Guard on itself + if: "!contains(github.event.pull_request.labels.*.name, 'dogfood-ai')" + uses: ./ + with: + coverage-file: coverage.xml + coverage-threshold: '100' + # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and + # now 404s. flash-lite defaults to minimal thinking, which suits the + # 2048-token output cap. + ai-model: 'gemini-3.1-flash-lite' + # Trailing slash is required — without it the compat layer 404s. + ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/' + # Layer 3 needs a provider key. Absent the secret, the gate degrades + # to Layer 1 + Layer 2 with a warning instead of failing. + ai-api-key: ${{ secrets.GEMINI_API_KEY }} + # Gemini 3.x has no "off" for thinking. Its published intelligence + # scores are reasoning-enabled, and Google recommends thinking for + # code reasoning, so don't floor this at minimal — the 4096-token + # default output cap leaves room for a short thought trace. + ai-reasoning-effort: 'low' + # Gemini 3.x warns that lowering temperature causes looping and + # degraded reasoning, so use its recommended default instead of 0.1. + ai-temperature: '1.0' + # Repo-specific: benchmarks/ is a standalone perf harness, not product + # code under test. Kept OUT of the shipped defaults on purpose — a + # test-adequacy gate should not silently skip a code tree for every + # user. Unioned onto the defaults, so no need to re-list them. + extra-exclude-patterns: 'benchmarks/**' + + # `dogfood-ai` label: withhold the coverage report so files fall through + # to Layer 3's AI gates (6-8) instead of being resolved by coverage + # shortcuts, which is the only way this workflow exercises the model. + - name: Run Test Guard on itself (AI path, no coverage) + if: contains(github.event.pull_request.labels.*.name, 'dogfood-ai') uses: ./ with: - # At threshold 100 with a full coverage report, Gates 3 and 5 resolve - # every file deterministically and the AI phase never runs. Add the - # `dogfood-ai` label to a PR to withhold coverage for one run, so - # files fall through to the AI gates (6-8) and Layer 3 is exercised - # end to end. Without the label this stays the strict coverage gate. - coverage-file: ${{ contains(github.event.pull_request.labels.*.name, 'dogfood-ai') && '' || 'coverage.xml' }} coverage-threshold: '100' # Gemini 2.5 was shut down ahead of its documented Oct 2026 date and # now 404s. flash-lite defaults to minimal thinking, which suits the From ebf640577ae6b388998ee7559c3f10d00f7dfa1f Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 01:22:07 +0200 Subject: [PATCH 15/16] feat(layer3): make the prompt budget configurable via ai-input-token-limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _INPUT_TOKEN_LIMIT was not dead code — it derives the user-prompt budget that drives batching and decides how much diff evidence reaches the model — but its value of 8000 was sized for the retired GitHub Models 8K input cap. At that budget a large test diff is shed, and since the prompt tells the model to treat omitted code as untested, the result is a false warning: exactly what happened to src/layer3_ai.py, whose test diff is 22,952 chars against a 6,120-token (~18k char) budget. - new ai-input-token-limit input, default 8000, threaded through run_layer3 and _build_ai_prompt via _user_prompt_budget() - dogfood and the local harness raise it to 32000 for Gemini's 1M context - the default stays conservative because providers meter tokens per minute; Groq's free tier allows only 8K/min Raising the token budget alone was not enough. A second, independent truncation clips each diff at 10k characters first, so the per-diff cap is now derived from the budget (half of it, 10k floor) and the two scale together. An explicitly passed cap still wins, which the tests caught: without that, the 413 retry's deliberate 3000-char shrink would have been inflated back up and the retry defeated. 443 tests pass, ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 5 +++- .github/workflows/dogfood.yml | 8 +++++++ README.md | 23 +++++++++++++++++++ action.yml | 4 ++++ src/config.py | 20 ++++++++++++++++ src/layer3_ai.py | 43 ++++++++++++++++++++++++++++++----- src/main.py | 1 + tests/test_config.py | 23 +++++++++++++++++++ tests/test_layer3_ai.py | 34 +++++++++++++++++++++++++++ tests/test_main.py | 1 + 10 files changed, 155 insertions(+), 7 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index 669a658..b18e0c5 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -15,7 +15,8 @@ GEMINI_API_KEY=... python local_pr_test.py Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key, -AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_TOKENS. +AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_TOKENS, +AI_INPUT_TOKEN_LIMIT. Defaults target gemini-3.1-flash-lite at temperature 1.0 with low thinking, per Google's guidance for Gemini 3.x. Without a key the AI phase is skipped. """ @@ -101,6 +102,7 @@ def run_local(repo: str, pr_number: int) -> None: ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"), ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), ai_max_tokens=int(os.environ.get("AI_MAX_TOKENS", "8192")), + ai_input_token_limit=int(os.environ.get("AI_INPUT_TOKEN_LIMIT", "32000")), ) session = create_session(token) @@ -171,6 +173,7 @@ def run_local(repo: str, pr_number: int) -> None: reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, max_tokens=config.ai_max_tokens, + input_token_limit=config.ai_input_token_limit, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 44ffef8..6328468 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -59,6 +59,10 @@ jobs: # Gemini 3.x warns that lowering temperature causes looping and # degraded reasoning, so use its recommended default instead of 0.1. ai-temperature: '1.0' + # 8000 was sized for the retired GitHub Models 8K input cap. + # Gemini has a 1M context, and at 8000 a large test diff is shed, + # which makes the model warn about code it was never shown. + ai-input-token-limit: '32000' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every @@ -90,6 +94,10 @@ jobs: # Gemini 3.x warns that lowering temperature causes looping and # degraded reasoning, so use its recommended default instead of 0.1. ai-temperature: '1.0' + # 8000 was sized for the retired GitHub Models 8K input cap. + # Gemini has a 1M context, and at 8000 a large test diff is shed, + # which makes the model warn about code it was never shown. + ai-input-token-limit: '32000' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every diff --git a/README.md b/README.md index 9d6098b..c8784da 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,28 @@ jobs: > No special GitHub permission is needed for AI analysis — the old `models: read` scope only ever gated GitHub Models and is now inert. What Layer 3 needs is `ai-api-key`. Set `ai-enabled: 'false'` to run Layers 1–2 only. +### Provider settings at a glance + +Every AI input in one place. Copy the column for your provider. + +| Setting | OpenAI (default) | Google Gemini | Azure AI Foundry | Groq (free) | +|:--------|:-----------------|:--------------|:-----------------|:------------| +| `ai-base-url` | `https://api.openai.com/v1` | `https://generativelanguage.googleapis.com/v1beta/openai/` | `https://.services.ai.azure.com/openai/v1` | `https://api.groq.com/openai/v1` | +| `ai-model` | `gpt-4.1-mini` | `gemini-3.1-flash-lite` | your deployment name | `openai/gpt-oss-120b` | +| `ai-api-key` | `OPENAI_API_KEY` | `GEMINI_API_KEY` | Foundry key | `GROQ_API_KEY` | +| `ai-reasoning-effort` | `none` (stripped automatically) | `low` | depends on model | `low` | +| `ai-temperature` | `0.1` | **`1.0`** | `0.1` | `0.1` | +| `ai-max-tokens` | `8192` | `8192` | `8192` | `8192` | +| `ai-input-token-limit` | `8000` | `32000` | `32000` | **`8000`** | +| Strict `json_schema` | yes | yes | yes | gpt-oss only | + +Why the non-obvious values: + +- **Gemini `ai-temperature: '1.0'`** — Google documents its default as strongly recommended and warns that lowering it causes looping or degraded reasoning. The `0.1` default suits OpenAI-style models and is actively wrong here. +- **Gemini `ai-input-token-limit: '32000'`** — a 1M-token context window makes the `8000` default needlessly tight, and a shed test diff produces false warnings (see below). +- **Groq stays at `8000`** — its free tier allows only 8K tokens *per minute*, so a larger prompt trips 429s, and Layer 3 has no 429 backoff. +- **OpenAI `ai-reasoning-effort`** — `gpt-4.1` rejects the parameter with a 400; Layer 3 detects that, retries once without it, and remembers the endpoint/model pair. + ### Pointing at another provider `ai-base-url` takes any OpenAI-compatible `/v1` endpoint. Use the model ID *that provider* expects. @@ -201,6 +223,7 @@ Notes that decide whether these actually work: | `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. Gemini 3.x has no `none` — use `minimal` or above. | | `ai-temperature` | `0.1` | Sampling temperature. Gemini 3.x wants `1.0`; lowering it there causes looping per Google's docs. | | `ai-max-tokens` | `8192` | Output cap per request. Raise alongside `ai-reasoning-effort`. | +| `ai-input-token-limit` | `8000` | Prompt budget: drives batching and how much diff evidence survives. Raise on large-context providers. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** diff --git a/action.yml b/action.yml index 8a2f979..47eb39c 100644 --- a/action.yml +++ b/action.yml @@ -49,6 +49,9 @@ inputs: ai-max-tokens: description: 'Output token cap per request. Thought tokens are charged against it, and exhausting it returns finish_reason=length with empty content, which is reported as a warning. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' default: '8192' + ai-input-token-limit: + description: 'Budget for the prompt Layer 3 sends, which drives batching and how much diff evidence survives. The default 8000 was sized for the retired GitHub Models 8K input cap; raise it (e.g. 32000) on a large-context provider so big test diffs are not shed, at the cost of more tokens per minute. Also scales the per-diff character cap.' + default: '8000' ai-confidence-threshold: description: 'Minimum AI confidence to enforce fail verdict (0.0-1.0). Below this, fail becomes warning.' default: '0.7' @@ -81,6 +84,7 @@ runs: INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} INPUT_AI-TEMPERATURE: ${{ inputs.ai-temperature }} INPUT_AI-MAX-TOKENS: ${{ inputs.ai-max-tokens }} + INPUT_AI-INPUT-TOKEN-LIMIT: ${{ inputs.ai-input-token-limit }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index a69c5b9..4fe96dd 100644 --- a/src/config.py +++ b/src/config.py @@ -46,6 +46,14 @@ # It stays well under the 65536 Gemini 3.x allows, because thought tokens are # billed as output and burn free-tier tokens/minute. DEFAULT_AI_MAX_TOKENS = 8192 +# Client-side budget for the prompt we send, which drives batching and decides +# how much diff evidence survives. 8000 was sized for GitHub Models' retired 8K +# input cap, and it is now the binding constraint on evidence quality: a test +# diff larger than the budget gets shed, and the prompt instructs the model to +# treat omitted code as untested, producing false warnings. It stays +# conservative by default because providers meter tokens per minute — Groq's +# free tier allows only 8K/min — so raise it deliberately per provider. +DEFAULT_AI_INPUT_TOKEN_LIMIT = 8000 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -217,6 +225,7 @@ class Config: ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT ai_temperature: float = DEFAULT_AI_TEMPERATURE ai_max_tokens: int = DEFAULT_AI_MAX_TOKENS + ai_input_token_limit: int = DEFAULT_AI_INPUT_TOKEN_LIMIT def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -363,6 +372,16 @@ def parse_config() -> Config: f"falling back to {DEFAULT_AI_MAX_TOKENS}." ) ai_max_tokens = DEFAULT_AI_MAX_TOKENS + + input_limit_raw = _env("AI-INPUT-TOKEN-LIMIT", str(DEFAULT_AI_INPUT_TOKEN_LIMIT)) + try: + ai_input_token_limit = int(input_limit_raw) + except ValueError: + print( + f"::warning::Invalid ai-input-token-limit '{input_limit_raw}' — " + f"falling back to {DEFAULT_AI_INPUT_TOKEN_LIMIT}." + ) + ai_input_token_limit = DEFAULT_AI_INPUT_TOKEN_LIMIT if ai_enabled and not ai_api_key: print( "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " @@ -401,5 +420,6 @@ def parse_config() -> Config: ai_reasoning_effort=ai_reasoning_effort, ai_temperature=ai_temperature, ai_max_tokens=ai_max_tokens, + ai_input_token_limit=ai_input_token_limit, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index 778c823..f066f5f 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -32,6 +32,7 @@ from unidiff.errors import UnidiffParseError from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL +from src.config import DEFAULT_AI_INPUT_TOKEN_LIMIT as _DEFAULT_AI_INPUT_TOKEN_LIMIT from src.config import DEFAULT_AI_MAX_TOKENS as _DEFAULT_AI_MAX_TOKENS from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT from src.config import DEFAULT_AI_TEMPERATURE as _DEFAULT_AI_TEMPERATURE @@ -241,7 +242,11 @@ def _build_ai_prompt( coverage_details: dict[str, float] | None, coverage_threshold: float, matched_tests: dict[str, str | list[str] | None], - max_diff_chars: int = 10_000, + # Both resolved at call time rather than as def-time defaults: max_diff_chars + # is derived from token_budget, and the budget constant is defined further + # down this module, so a def-time default would NameError. + max_diff_chars: int | None = None, + token_budget: int | None = None, ) -> str: """Build the structured Markdown user prompt for the AI batch. @@ -257,6 +262,17 @@ def _build_ai_prompt( if not files_for_ai: return "" + if token_budget is None: + token_budget = _user_prompt_budget() + # Two independent truncations guard the prompt: this per-diff character cap + # and the whole-diff shedding below. Raising only the token budget changes + # nothing while the char cap still clips every large diff, so derive the cap + # from the budget — half of it, leaving room for other files in the batch, + # and never below the historical 10k floor. An explicit value always wins: + # the 413 retry passes a deliberately smaller cap to shrink the request. + if max_diff_chars is None: + max_diff_chars = max(10_000, token_budget * _CHARS_PER_TOKEN // 2) + # Build a reverse map: test file → list of source files it is matched to. # Used to annotate each test diff in the prompt so the AI knows which # source files a given test is authoritative for. @@ -327,7 +343,7 @@ def assemble(entries: list[dict[str, object]]) -> str: # input cap even when the aggregate test payload is large. The evidence # banner (recomputed in assemble) tells the model what was dropped. kept = list(test_entries) - while kept and _estimate_tokens(assemble(kept)) > _USER_PROMPT_TOKEN_BUDGET: + while kept and _estimate_tokens(assemble(kept)) > token_budget: victim = max(kept, key=lambda e: (bool(e["is_candidate"]), int(e["cost"]))) kept.remove(victim) return assemble(kept) @@ -618,12 +634,21 @@ def _sanitize_diff( _REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() _CHARS_PER_TOKEN = 3 -_INPUT_TOKEN_LIMIT = 8000 +_INPUT_TOKEN_LIMIT = _DEFAULT_AI_INPUT_TOKEN_LIMIT _SYSTEM_OVERHEAD_TOKENS = 800 # system prompt + JSON schema overhead _SAFETY_FACTOR = 0.85 -_USER_PROMPT_TOKEN_BUDGET = int( - (_INPUT_TOKEN_LIMIT - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR -) # = 6120 tokens + + +def _user_prompt_budget(input_token_limit: int = _INPUT_TOKEN_LIMIT) -> int: + """Tokens available for the user prompt at a given input budget. + + Subtracts the system prompt and schema, then applies a safety factor + because token counts here are estimated from characters, not measured. + """ + return int((input_token_limit - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR) + + +_USER_PROMPT_TOKEN_BUDGET = _user_prompt_budget() # = 6120 tokens at 8000 _FILE_ENTRY_OVERHEAD_TOKENS = 25 # markdown headers, code fences per file _BATCH_OVERHEAD_TOKENS = 30 # section headers per batch prompt _RETRY_MAX_DIFF_CHARS = 3000 # tighter truncation on 413 retry @@ -832,6 +857,7 @@ def _call_ai_for_batch( reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, max_tokens: int = _DEFAULT_AI_MAX_TOKENS, + input_token_limit: int = _DEFAULT_AI_INPUT_TOKEN_LIMIT, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -842,9 +868,11 @@ def _call_ai_for_batch( batch_test_diffs = _filter_test_diffs_for_batch( batch_files, test_diffs, matched_tests, ) + token_budget = _user_prompt_budget(input_token_limit) user_prompt = _build_ai_prompt( batch_files, source_diffs, batch_test_diffs, coverage_details, coverage_threshold, matched_tests, + token_budget=token_budget, ) try: raw = _call_ai_provider( @@ -858,6 +886,7 @@ def _call_ai_for_batch( batch_files, source_diffs, batch_test_diffs, coverage_details, coverage_threshold, matched_tests, max_diff_chars=_RETRY_MAX_DIFF_CHARS, + token_budget=token_budget, ) try: raw = _call_ai_provider( @@ -1005,6 +1034,7 @@ def run_layer3( reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, max_tokens: int = _DEFAULT_AI_MAX_TOKENS, + input_token_limit: int = _DEFAULT_AI_INPUT_TOKEN_LIMIT, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1118,6 +1148,7 @@ def run_layer3( coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, base_url, reasoning_effort, temperature, max_tokens, + input_token_limit, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index b73f04d..e74e3ed 100644 --- a/src/main.py +++ b/src/main.py @@ -147,6 +147,7 @@ def run_pipeline(config: Config) -> Report: reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, max_tokens=config.ai_max_tokens, + input_token_limit=config.ai_input_token_limit, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. diff --git a/tests/test_config.py b/tests/test_config.py index d944236..7939fe7 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -82,6 +82,29 @@ def test_invalid_temperature_falls_back(self, monkeypatch, capsys): assert "::warning::" in out assert "ai-temperature" in out + def test_input_token_limit_is_configurable(self, monkeypatch): + """Raising the prompt budget is how large test diffs stop being shed.""" + monkeypatch.setenv("INPUT_AI-INPUT-TOKEN-LIMIT", "32000") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + assert parse_config().ai_input_token_limit == 32000 + + def test_invalid_input_token_limit_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-INPUT-TOKEN-LIMIT", "huge") + monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") + monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") + monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + + cfg = parse_config() + assert cfg.ai_input_token_limit == 8000 + out = capsys.readouterr().out + assert "::warning::" in out + assert "ai-input-token-limit" in out + def test_invalid_max_tokens_falls_back(self, monkeypatch, capsys): monkeypatch.setenv("INPUT_AI-MAX-TOKENS", "lots") monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index b20b5d7..743865c 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -775,6 +775,40 @@ def test_defaults_to_deterministic_temperature(self, mock_openai: MagicMock): assert mock_client.chat.completions.create.call_args.kwargs["temperature"] == 0.1 +class TestInputTokenBudget: + """The input budget decides how much test evidence survives into the prompt. + + Shed test diffs make the prompt's evidence-completeness rule kick in, and + the model correctly returns a warning for code it cannot see tested — so a + budget that is too small manufactures false warnings. + """ + + def test_budget_scales_with_the_limit(self): + assert layer3_ai._user_prompt_budget(8000) == 6120 + assert layer3_ai._user_prompt_budget(32000) == 26520 + + def test_raised_limit_keeps_a_large_test_diff(self): + big_test_diff = "+ assert something\n" * 1200 # ~22k chars, ~7.6k tokens + + shed = layer3_ai._build_ai_prompt( + ["src/thing.py"], + {"src/thing.py": "+ def thing(): pass"}, + {"tests/test_thing.py": big_test_diff}, + None, 80, {"src/thing.py": "tests/test_thing.py"}, + token_budget=layer3_ai._user_prompt_budget(8000), + ) + kept = layer3_ai._build_ai_prompt( + ["src/thing.py"], + {"src/thing.py": "+ def thing(): pass"}, + {"tests/test_thing.py": big_test_diff}, + None, 80, {"src/thing.py": "tests/test_thing.py"}, + token_budget=layer3_ai._user_prompt_budget(32000), + ) + + # The raised budget must retain strictly more evidence. + assert len(kept) > len(shed) + + class TestTruncationWarning: """A thought trace that eats the output cap must not degrade silently.""" diff --git a/tests/test_main.py b/tests/test_main.py index 465c662..a550676 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -247,6 +247,7 @@ def test_full_pipeline_classifies_files_for_l3( reasoning_effort="none", temperature=0.1, max_tokens=8192, + input_token_limit=8000, confidence_threshold=0.7, unmeasurable_files=set(), ) From 0fe29ea1aab60a02b95859107b62309f9a0906ab Mon Sep 17 00:00:00 2001 From: domenico Date: Sat, 1 Aug 2026 01:25:17 +0200 Subject: [PATCH 16/16] refactor: rename the token inputs to say which direction they govern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ai-max-tokens capped output while ai-input-token-limit bounded the prompt, so the pair read as if one were the input and the other the maximum. That ambiguity had already cost real work: the output cap got raised twice while the input budget was the actual cause of the false warning on src/layer3_ai.py. ai-max-tokens -> ai-max-output-tokens ai-input-token-limit -> ai-max-input-tokens Also renamed through the config fields, module constants and function parameters. Both inputs are new on this branch and unreleased, so nothing depended on the old names. The OpenAI SDK parameter is still literally max_tokens, so the call site reads max_tokens=max_output_tokens and the tests that assert on the outgoing kwarg still check "max_tokens" — only our own names changed. 443 tests pass, ruff clean on src and tests. Co-Authored-By: Claude Opus 5 (1M context) --- .dev/local_pr_test.py | 12 +++++----- .github/workflows/dogfood.yml | 4 ++-- README.md | 16 ++++++------- action.yml | 8 +++---- src/config.py | 32 +++++++++++++------------- src/layer3_ai.py | 42 ++++++++++++++++++----------------- src/main.py | 4 ++-- tests/test_config.py | 28 +++++++++++------------ tests/test_layer3_ai.py | 14 ++++++------ tests/test_main.py | 4 ++-- 10 files changed, 83 insertions(+), 81 deletions(-) diff --git a/.dev/local_pr_test.py b/.dev/local_pr_test.py index b18e0c5..5b3c24f 100644 --- a/.dev/local_pr_test.py +++ b/.dev/local_pr_test.py @@ -15,8 +15,8 @@ GEMINI_API_KEY=... python local_pr_test.py Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key, -AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_TOKENS, -AI_INPUT_TOKEN_LIMIT. +AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_OUTPUT_TOKENS, +AI_MAX_INPUT_TOKENS. Defaults target gemini-3.1-flash-lite at temperature 1.0 with low thinking, per Google's guidance for Gemini 3.x. Without a key the AI phase is skipped. """ @@ -101,8 +101,8 @@ def run_local(repo: str, pr_number: int) -> None: ai_api_key=_resolve_ai_api_key(), ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"), ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")), - ai_max_tokens=int(os.environ.get("AI_MAX_TOKENS", "8192")), - ai_input_token_limit=int(os.environ.get("AI_INPUT_TOKEN_LIMIT", "32000")), + ai_max_output_tokens=int(os.environ.get("AI_MAX_OUTPUT_TOKENS", "8192")), + ai_max_input_tokens=int(os.environ.get("AI_MAX_INPUT_TOKENS", "32000")), ) session = create_session(token) @@ -172,8 +172,8 @@ def run_local(repo: str, pr_number: int) -> None: base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, - max_tokens=config.ai_max_tokens, - input_token_limit=config.ai_input_token_limit, + max_output_tokens=config.ai_max_output_tokens, + max_input_tokens=config.ai_max_input_tokens, confidence_threshold=config.ai_confidence_threshold, ) report.layers.append(l3) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 6328468..fe251b9 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -62,7 +62,7 @@ jobs: # 8000 was sized for the retired GitHub Models 8K input cap. # Gemini has a 1M context, and at 8000 a large test diff is shed, # which makes the model warn about code it was never shown. - ai-input-token-limit: '32000' + ai-max-input-tokens: '32000' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every @@ -97,7 +97,7 @@ jobs: # 8000 was sized for the retired GitHub Models 8K input cap. # Gemini has a 1M context, and at 8000 a large test diff is shed, # which makes the model warn about code it was never shown. - ai-input-token-limit: '32000' + ai-max-input-tokens: '32000' # Repo-specific: benchmarks/ is a standalone perf harness, not product # code under test. Kept OUT of the shipped defaults on purpose — a # test-adequacy gate should not silently skip a code tree for every diff --git a/README.md b/README.md index c8784da..2bfa587 100644 --- a/README.md +++ b/README.md @@ -156,14 +156,14 @@ Every AI input in one place. Copy the column for your provider. | `ai-api-key` | `OPENAI_API_KEY` | `GEMINI_API_KEY` | Foundry key | `GROQ_API_KEY` | | `ai-reasoning-effort` | `none` (stripped automatically) | `low` | depends on model | `low` | | `ai-temperature` | `0.1` | **`1.0`** | `0.1` | `0.1` | -| `ai-max-tokens` | `8192` | `8192` | `8192` | `8192` | -| `ai-input-token-limit` | `8000` | `32000` | `32000` | **`8000`** | +| `ai-max-output-tokens` | `8192` | `8192` | `8192` | `8192` | +| `ai-max-input-tokens` | `8000` | `32000` | `32000` | **`8000`** | | Strict `json_schema` | yes | yes | yes | gpt-oss only | Why the non-obvious values: - **Gemini `ai-temperature: '1.0'`** — Google documents its default as strongly recommended and warns that lowering it causes looping or degraded reasoning. The `0.1` default suits OpenAI-style models and is actively wrong here. -- **Gemini `ai-input-token-limit: '32000'`** — a 1M-token context window makes the `8000` default needlessly tight, and a shed test diff produces false warnings (see below). +- **Gemini `ai-max-input-tokens: '32000'`** — a 1M-token context window makes the `8000` default needlessly tight, and a shed test diff produces false warnings (see below). - **Groq stays at `8000`** — its free tier allows only 8K tokens *per minute*, so a larger prompt trips 429s, and Layer 3 has no 429 backoff. - **OpenAI `ai-reasoning-effort`** — `gpt-4.1` rejects the parameter with a 400; Layer 3 detects that, retries once without it, and remembers the endpoint/model pair. @@ -197,11 +197,11 @@ Notes that decide whether these actually work: - **On Groq, only `openai/gpt-oss-20b` and `openai/gpt-oss-120b` support `strict: true`.** Other Groq models — including `moonshotai/kimi-k2-instruct-0905` — offer best-effort JSON or tool-use only, and will fail Layer 3's strict call. - **Groq's 8K tokens/minute free cap is tight for this workload.** Layer 3 budgets up to 8K input tokens per request, so one full-size batch can consume the entire per-minute allowance and the next batch 429s. Layer 3 has no 429 backoff, so a large PR degrades to L1+L2. Fine for small PRs; unreliable for big ones. - **Model IDs go stale — verify before trusting one.** Gemini 2.5 was shut down ahead of its documented October 2026 date, and a retired ID returns a generic `404 models/ is not found for API version v1beta`, which reads like a URL bug. List what your key can actually reach: `curl "https://generativelanguage.googleapis.com/v1beta/models?key=$GEMINI_API_KEY"`. -- **Gemini 3.x cannot disable thinking — and you should not want to.** `minimal` is the floor and only "matches the no-thinking setting for most queries". Published intelligence scores for these models are *reasoning-enabled*: `gemini-3.1-flash-lite` scores 25 on the Artificial Analysis Intelligence Index versus 15 for `gpt-4.1-mini`, the baseline this prompt was tuned on — but that figure does not describe a minimal-thinking run, and Google recommends thinking for code reasoning specifically. Layer 3 is code judgment, so prefer `low` or higher and give it room via `ai-max-tokens` rather than flooring the effort. -- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-tokens` defaults to `8192`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. +- **Gemini 3.x cannot disable thinking — and you should not want to.** `minimal` is the floor and only "matches the no-thinking setting for most queries". Published intelligence scores for these models are *reasoning-enabled*: `gemini-3.1-flash-lite` scores 25 on the Artificial Analysis Intelligence Index versus 15 for `gpt-4.1-mini`, the baseline this prompt was tuned on — but that figure does not describe a minimal-thinking run, and Google recommends thinking for code reasoning specifically. Layer 3 is code judgment, so prefer `low` or higher and give it room via `ai-max-output-tokens` rather than flooring the effort. +- **Budget output tokens for the thought trace.** The verdict JSON is a few hundred tokens, but thought tokens count as output and truncated JSON parses as SKIP at confidence 0.0. `ai-max-output-tokens` defaults to `8192`; raise it if you raise `ai-reasoning-effort`. Gemini 3.x permits up to 65,536 output tokens, so the model is not the constraint — free-tier tokens/minute is. - **Gemini 3.x wants `ai-temperature: '1.0'`.** Google documents its default as strongly recommended and warns that lowering temperature can cause looping or degraded reasoning. The action default of `0.1` suits OpenAI-style models but is actively wrong here. - **Superseded guidance, kept for context: `gemini-2.5-flash` over `flash-lite`.** Layer 3 is a judgment task — map changed behaviours to test assertions across four dimensions, then calibrate confidence to evidence quality. On Artificial Analysis, Gemini 2.5 Flash without reasoning scores 14 on the Intelligence Index against 15 for `gpt-4.1-mini`, the model this action was tuned on, so it is close to parity. Flash-Lite without reasoning scores 7. The risk with a weaker model is not missed problems — those fail open to L1+L2 — but *confidently wrong* FAILs: `ai-confidence-threshold` only softens a FAIL to a WARNING when the model reports low confidence, and weak models tend to report high confidence regardless. Weaker models also more often omit a file from the response, which discards the whole batch (`_validate_batch_verdicts`). If you do run Flash-Lite, raise `ai-confidence-threshold` to around `0.8`. -- **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. +- **Thinking is disabled by default.** Layer 3 sends `reasoning_effort: none`, which switches thinking off on Gemini 2.5 models and Groq's gpt-oss. This matters because Layer 3 caps `max_output_tokens` at 2048 and thought tokens count as output, so an unbounded thought trace can truncate the JSON — which parses as SKIP at confidence 0.0. Both `gemini-2.5-flash` and `gemini-2.5-flash-lite` are safe with this default. Raise it via `ai-reasoning-effort` if you want the model to think; note that thinking cannot be disabled at all on `gemini-2.5-pro` or the Gemini 3.x models. - **Providers that reject the parameter are handled automatically.** OpenAI's `gpt-4.1` family returns a 400 for `reasoning_effort`. Layer 3 detects that specific rejection, retries once without the parameter, and remembers the endpoint/model pair — so the default OpenAI setup costs one extra request per run, not per batch. - **Free tiers generally train on submitted prompts.** Layer 3 sends source and test diffs. Check the provider's data-use terms before pointing this at a private repository. @@ -222,8 +222,8 @@ Notes that decide whether these actually work: | `ai-api-key` | _(empty)_ | API key for `ai-base-url`; pass it as a secret. Layer 3 is skipped when empty. | | `ai-reasoning-effort` | `none` | Thinking budget for reasoning models: `none`, `minimal`, `low`, `medium`, `high`. Empty omits the parameter. Gemini 3.x has no `none` — use `minimal` or above. | | `ai-temperature` | `0.1` | Sampling temperature. Gemini 3.x wants `1.0`; lowering it there causes looping per Google's docs. | -| `ai-max-tokens` | `8192` | Output cap per request. Raise alongside `ai-reasoning-effort`. | -| `ai-input-token-limit` | `8000` | Prompt budget: drives batching and how much diff evidence survives. Raise on large-context providers. | +| `ai-max-output-tokens` | `8192` | Output cap per request. Raise alongside `ai-reasoning-effort`. | +| `ai-max-input-tokens` | `8000` | Prompt budget: drives batching and how much diff evidence survives. Raise on large-context providers. | | `ai-confidence-threshold` | `0.7` | AI FAIL verdicts below this confidence become WARNING. Float, `0.0`–`1.0`; other values fail the run. | **Default exclude patterns:** diff --git a/action.yml b/action.yml index 47eb39c..3ec7bfb 100644 --- a/action.yml +++ b/action.yml @@ -46,10 +46,10 @@ inputs: ai-temperature: description: 'Sampling temperature. 0.1 suits OpenAI-style models for a deterministic verdict. Gemini 3.x recommends its default of 1.0 and warns that lowering it can cause looping or degraded reasoning — set 1.0 for those models.' default: '0.1' - ai-max-tokens: + ai-max-output-tokens: description: 'Output token cap per request. Thought tokens are charged against it, and exhausting it returns finish_reason=length with empty content, which is reported as a warning. Raise it if you raise ai-reasoning-effort. Gemini 3.x permits up to 65536.' default: '8192' - ai-input-token-limit: + ai-max-input-tokens: description: 'Budget for the prompt Layer 3 sends, which drives batching and how much diff evidence survives. The default 8000 was sized for the retired GitHub Models 8K input cap; raise it (e.g. 32000) on a large-context provider so big test diffs are not shed, at the cost of more tokens per minute. Also scales the per-diff character cap.' default: '8000' ai-confidence-threshold: @@ -83,8 +83,8 @@ runs: INPUT_AI-API-KEY: ${{ inputs.ai-api-key }} INPUT_AI-REASONING-EFFORT: ${{ inputs.ai-reasoning-effort }} INPUT_AI-TEMPERATURE: ${{ inputs.ai-temperature }} - INPUT_AI-MAX-TOKENS: ${{ inputs.ai-max-tokens }} - INPUT_AI-INPUT-TOKEN-LIMIT: ${{ inputs.ai-input-token-limit }} + INPUT_AI-MAX-OUTPUT-TOKENS: ${{ inputs.ai-max-output-tokens }} + INPUT_AI-MAX-INPUT-TOKENS: ${{ inputs.ai-max-input-tokens }} INPUT_AI-CONFIDENCE-THRESHOLD: ${{ inputs.ai-confidence-threshold }} PYTHONPATH: ${{ github.action_path }} run: python ${{ github.action_path }}/src/main.py diff --git a/src/config.py b/src/config.py index 4fe96dd..74507a7 100644 --- a/src/config.py +++ b/src/config.py @@ -45,7 +45,7 @@ # thinking enabled; the cap is not a reservation, so unused headroom is free. # It stays well under the 65536 Gemini 3.x allows, because thought tokens are # billed as output and burn free-tier tokens/minute. -DEFAULT_AI_MAX_TOKENS = 8192 +DEFAULT_AI_MAX_OUTPUT_TOKENS = 8192 # Client-side budget for the prompt we send, which drives batching and decides # how much diff evidence survives. 8000 was sized for GitHub Models' retired 8K # input cap, and it is now the binding constraint on evidence quality: a test @@ -53,7 +53,7 @@ # treat omitted code as untested, producing false warnings. It stays # conservative by default because providers meter tokens per minute — Groq's # free tier allows only 8K/min — so raise it deliberately per provider. -DEFAULT_AI_INPUT_TOKEN_LIMIT = 8000 +DEFAULT_AI_MAX_INPUT_TOKENS = 8000 _DEFAULT_AI_CONFIDENCE_THRESHOLD = 0.7 _DEFAULT_AI_ENABLED_VALUES = ("true", "1", "yes") @@ -224,8 +224,8 @@ class Config: ai_api_key: str = "" ai_reasoning_effort: str = DEFAULT_AI_REASONING_EFFORT ai_temperature: float = DEFAULT_AI_TEMPERATURE - ai_max_tokens: int = DEFAULT_AI_MAX_TOKENS - ai_input_token_limit: int = DEFAULT_AI_INPUT_TOKEN_LIMIT + ai_max_output_tokens: int = DEFAULT_AI_MAX_OUTPUT_TOKENS + ai_max_input_tokens: int = DEFAULT_AI_MAX_INPUT_TOKENS def _parse_custom_test_patterns(raw: str) -> dict[str, dict[str, str]]: @@ -363,25 +363,25 @@ def parse_config() -> Config: ) ai_temperature = DEFAULT_AI_TEMPERATURE - max_tokens_raw = _env("AI-MAX-TOKENS", str(DEFAULT_AI_MAX_TOKENS)) + max_output_tokens_raw = _env("AI-MAX-OUTPUT-TOKENS", str(DEFAULT_AI_MAX_OUTPUT_TOKENS)) try: - ai_max_tokens = int(max_tokens_raw) + ai_max_output_tokens = int(max_output_tokens_raw) except ValueError: print( - f"::warning::Invalid ai-max-tokens '{max_tokens_raw}' — " - f"falling back to {DEFAULT_AI_MAX_TOKENS}." + f"::warning::Invalid ai-max-output-tokens '{max_output_tokens_raw}' — " + f"falling back to {DEFAULT_AI_MAX_OUTPUT_TOKENS}." ) - ai_max_tokens = DEFAULT_AI_MAX_TOKENS + ai_max_output_tokens = DEFAULT_AI_MAX_OUTPUT_TOKENS - input_limit_raw = _env("AI-INPUT-TOKEN-LIMIT", str(DEFAULT_AI_INPUT_TOKEN_LIMIT)) + input_limit_raw = _env("AI-MAX-INPUT-TOKENS", str(DEFAULT_AI_MAX_INPUT_TOKENS)) try: - ai_input_token_limit = int(input_limit_raw) + ai_max_input_tokens = int(input_limit_raw) except ValueError: print( - f"::warning::Invalid ai-input-token-limit '{input_limit_raw}' — " - f"falling back to {DEFAULT_AI_INPUT_TOKEN_LIMIT}." + f"::warning::Invalid ai-max-input-tokens '{input_limit_raw}' — " + f"falling back to {DEFAULT_AI_MAX_INPUT_TOKENS}." ) - ai_input_token_limit = DEFAULT_AI_INPUT_TOKEN_LIMIT + ai_max_input_tokens = DEFAULT_AI_MAX_INPUT_TOKENS if ai_enabled and not ai_api_key: print( "::warning::ai-enabled is true but ai-api-key is empty — Layer 3 " @@ -419,7 +419,7 @@ def parse_config() -> Config: ai_api_key=ai_api_key, ai_reasoning_effort=ai_reasoning_effort, ai_temperature=ai_temperature, - ai_max_tokens=ai_max_tokens, - ai_input_token_limit=ai_input_token_limit, + ai_max_output_tokens=ai_max_output_tokens, + ai_max_input_tokens=ai_max_input_tokens, ai_confidence_threshold=ai_confidence_threshold, ) diff --git a/src/layer3_ai.py b/src/layer3_ai.py index f066f5f..5a052e5 100644 --- a/src/layer3_ai.py +++ b/src/layer3_ai.py @@ -32,8 +32,8 @@ from unidiff.errors import UnidiffParseError from src.config import DEFAULT_AI_BASE_URL as _DEFAULT_AI_BASE_URL -from src.config import DEFAULT_AI_INPUT_TOKEN_LIMIT as _DEFAULT_AI_INPUT_TOKEN_LIMIT -from src.config import DEFAULT_AI_MAX_TOKENS as _DEFAULT_AI_MAX_TOKENS +from src.config import DEFAULT_AI_MAX_INPUT_TOKENS as _DEFAULT_AI_MAX_INPUT_TOKENS +from src.config import DEFAULT_AI_MAX_OUTPUT_TOKENS as _DEFAULT_AI_MAX_OUTPUT_TOKENS from src.config import DEFAULT_AI_REASONING_EFFORT as _DEFAULT_AI_REASONING_EFFORT from src.config import DEFAULT_AI_TEMPERATURE as _DEFAULT_AI_TEMPERATURE from src.diff_utils import is_trivial_diff @@ -634,18 +634,18 @@ def _sanitize_diff( _REASONING_EFFORT_UNSUPPORTED: set[tuple[str, str]] = set() _CHARS_PER_TOKEN = 3 -_INPUT_TOKEN_LIMIT = _DEFAULT_AI_INPUT_TOKEN_LIMIT +_MAX_INPUT_TOKENS = _DEFAULT_AI_MAX_INPUT_TOKENS _SYSTEM_OVERHEAD_TOKENS = 800 # system prompt + JSON schema overhead _SAFETY_FACTOR = 0.85 -def _user_prompt_budget(input_token_limit: int = _INPUT_TOKEN_LIMIT) -> int: +def _user_prompt_budget(max_input_tokens: int = _MAX_INPUT_TOKENS) -> int: """Tokens available for the user prompt at a given input budget. Subtracts the system prompt and schema, then applies a safety factor because token counts here are estimated from characters, not measured. """ - return int((input_token_limit - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR) + return int((max_input_tokens - _SYSTEM_OVERHEAD_TOKENS) * _SAFETY_FACTOR) _USER_PROMPT_TOKEN_BUDGET = _user_prompt_budget() # = 6120 tokens at 8000 @@ -856,8 +856,8 @@ def _call_ai_for_batch( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, - max_tokens: int = _DEFAULT_AI_MAX_TOKENS, - input_token_limit: int = _DEFAULT_AI_INPUT_TOKEN_LIMIT, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, + max_input_tokens: int = _DEFAULT_AI_MAX_INPUT_TOKENS, ) -> tuple[str | None, Exception | None]: """Call the AI for a single batch with one model. @@ -868,7 +868,7 @@ def _call_ai_for_batch( batch_test_diffs = _filter_test_diffs_for_batch( batch_files, test_diffs, matched_tests, ) - token_budget = _user_prompt_budget(input_token_limit) + token_budget = _user_prompt_budget(max_input_tokens) user_prompt = _build_ai_prompt( batch_files, source_diffs, batch_test_diffs, coverage_details, coverage_threshold, matched_tests, @@ -877,7 +877,7 @@ def _call_ai_for_batch( try: raw = _call_ai_provider( model, system_prompt, user_prompt, token, base_url, - reasoning_effort, temperature, max_tokens, + reasoning_effort, temperature, max_output_tokens, ) return raw, None except Exception as exc: @@ -906,7 +906,7 @@ def _call_ai_provider( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, - max_tokens: int = _DEFAULT_AI_MAX_TOKENS, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, ) -> str: """Call an OpenAI-compatible inference endpoint, return raw response text. @@ -933,12 +933,14 @@ def _call_ai_provider( extra["reasoning_effort"] = reasoning_effort try: - response = _create_completion(client, model, messages, extra, temperature, max_tokens) + response = _create_completion( + client, model, messages, extra, temperature, max_output_tokens, + ) except Exception as exc: if not extra or not _is_unsupported_reasoning_effort_error(exc): raise _REASONING_EFFORT_UNSUPPORTED.add((base_url, model)) - response = _create_completion(client, model, messages, {}, temperature, max_tokens) + response = _create_completion(client, model, messages, {}, temperature, max_output_tokens) if not response.choices: return "" @@ -951,9 +953,9 @@ def _call_ai_provider( # nothing to say — so name the cause instead of degrading quietly. if getattr(choice, "finish_reason", None) == "length" and not content.strip(): print( - f"::warning::{model} hit the {max_tokens}-token output cap before " + f"::warning::{model} hit the {max_output_tokens}-token output cap before " f"emitting a verdict — most likely the thought trace consumed it. " - f"Raise ai-max-tokens or lower ai-reasoning-effort." + f"Raise ai-max-output-tokens or lower ai-reasoning-effort." ) return content @@ -964,14 +966,14 @@ def _create_completion( messages: list[object], extra: dict[str, object], temperature: float = _DEFAULT_AI_TEMPERATURE, - max_tokens: int = _DEFAULT_AI_MAX_TOKENS, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, ) -> object: """Issue the chat-completion request, with ``extra`` merged into kwargs.""" return client.chat.completions.create( model=model, messages=messages, temperature=temperature, - max_tokens=max_tokens, + max_tokens=max_output_tokens, response_format=ResponseFormatJSONSchema( type="json_schema", json_schema=JSONSchema( @@ -1033,8 +1035,8 @@ def run_layer3( base_url: str = _DEFAULT_AI_BASE_URL, reasoning_effort: str = _DEFAULT_AI_REASONING_EFFORT, temperature: float = _DEFAULT_AI_TEMPERATURE, - max_tokens: int = _DEFAULT_AI_MAX_TOKENS, - input_token_limit: int = _DEFAULT_AI_INPUT_TOKEN_LIMIT, + max_output_tokens: int = _DEFAULT_AI_MAX_OUTPUT_TOKENS, + max_input_tokens: int = _DEFAULT_AI_MAX_INPUT_TOKENS, ) -> LayerResult: """Run the full Layer 3 evaluation pipeline. @@ -1147,8 +1149,8 @@ def run_layer3( batch, source_diffs, test_diffs, coverage_details, coverage_threshold, l2_matched_tests, models[current_model_idx], system_prompt, token, - base_url, reasoning_effort, temperature, max_tokens, - input_token_limit, + base_url, reasoning_effort, temperature, max_output_tokens, + max_input_tokens, ) if raw is not None: _, ai_confidence, batch_verdicts = _parse_ai_response(raw) diff --git a/src/main.py b/src/main.py index e74e3ed..ca1b59a 100644 --- a/src/main.py +++ b/src/main.py @@ -146,8 +146,8 @@ def run_pipeline(config: Config) -> Report: base_url=config.ai_base_url, reasoning_effort=config.ai_reasoning_effort, temperature=config.ai_temperature, - max_tokens=config.ai_max_tokens, - input_token_limit=config.ai_input_token_limit, + max_output_tokens=config.ai_max_output_tokens, + max_input_tokens=config.ai_max_input_tokens, confidence_threshold=config.ai_confidence_threshold, # L1 proved these are instrumented but contributed no executable changed # lines, so Gate 2 skips them instead of Gate 4 failing them at 0%. diff --git a/tests/test_config.py b/tests/test_config.py index 7939fe7..f2c6117 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -82,45 +82,45 @@ def test_invalid_temperature_falls_back(self, monkeypatch, capsys): assert "::warning::" in out assert "ai-temperature" in out - def test_input_token_limit_is_configurable(self, monkeypatch): + def test_max_input_tokens_is_configurable(self, monkeypatch): """Raising the prompt budget is how large test diffs stop being shed.""" - monkeypatch.setenv("INPUT_AI-INPUT-TOKEN-LIMIT", "32000") + monkeypatch.setenv("INPUT_AI-MAX-INPUT-TOKENS", "32000") monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") - assert parse_config().ai_input_token_limit == 32000 + assert parse_config().ai_max_input_tokens == 32000 - def test_invalid_input_token_limit_falls_back(self, monkeypatch, capsys): - monkeypatch.setenv("INPUT_AI-INPUT-TOKEN-LIMIT", "huge") + def test_invalid_max_input_tokens_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-MAX-INPUT-TOKENS", "huge") monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") cfg = parse_config() - assert cfg.ai_input_token_limit == 8000 + assert cfg.ai_max_input_tokens == 8000 out = capsys.readouterr().out assert "::warning::" in out - assert "ai-input-token-limit" in out + assert "ai-max-input-tokens" in out - def test_invalid_max_tokens_falls_back(self, monkeypatch, capsys): - monkeypatch.setenv("INPUT_AI-MAX-TOKENS", "lots") + def test_invalid_max_output_tokens_falls_back(self, monkeypatch, capsys): + monkeypatch.setenv("INPUT_AI-MAX-OUTPUT-TOKENS", "lots") monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") cfg = parse_config() - assert cfg.ai_max_tokens == 8192 + assert cfg.ai_max_output_tokens == 8192 out = capsys.readouterr().out assert "::warning::" in out - assert "ai-max-tokens" in out + assert "ai-max-output-tokens" in out - def test_valid_temperature_and_max_tokens_are_used(self, monkeypatch): + def test_valid_temperature_and_max_output_tokens_are_used(self, monkeypatch): monkeypatch.setenv("INPUT_AI-TEMPERATURE", "1.0") - monkeypatch.setenv("INPUT_AI-MAX-TOKENS", "16384") + monkeypatch.setenv("INPUT_AI-MAX-OUTPUT-TOKENS", "16384") monkeypatch.setenv("INPUT_AI-API-KEY", "sk-fake") monkeypatch.setenv("GITHUB_TOKEN", "ghp_fake123") monkeypatch.setenv("GITHUB_REPOSITORY", "owner/repo") @@ -128,7 +128,7 @@ def test_valid_temperature_and_max_tokens_are_used(self, monkeypatch): cfg = parse_config() assert cfg.ai_temperature == 1.0 - assert cfg.ai_max_tokens == 16384 + assert cfg.ai_max_output_tokens == 16384 def test_missing_api_key_disables_ai(self, monkeypatch, capsys): """No key means no AI phase: GITHUB_TOKEN is not a provider credential. diff --git a/tests/test_layer3_ai.py b/tests/test_layer3_ai.py index 743865c..ec41182 100644 --- a/tests/test_layer3_ai.py +++ b/tests/test_layer3_ai.py @@ -828,14 +828,14 @@ def test_warns_when_cap_consumed_before_verdict( system_prompt="system", user_prompt="user", token="sk-fake", - max_tokens=8192, + max_output_tokens=8192, ) assert result == "" out = capsys.readouterr().out assert "::warning::" in out assert "8192-token output cap" in out - assert "ai-max-tokens" in out + assert "ai-max-output-tokens" in out @patch("src.layer3_ai.OpenAI") def test_no_warning_on_a_normal_verdict(self, mock_openai: MagicMock, capsys): @@ -859,7 +859,7 @@ def test_no_warning_on_a_normal_verdict(self, mock_openai: MagicMock, capsys): class TestMaxTokens: @patch("src.layer3_ai.OpenAI") - def test_forwards_max_tokens(self, mock_openai: MagicMock): + def test_forwards_max_output_tokens(self, mock_openai: MagicMock): """Headroom for a thought trace is configurable, not hardcoded.""" mock_client = MagicMock() mock_client.chat.completions.create.return_value.choices = [] @@ -870,7 +870,7 @@ def test_forwards_max_tokens(self, mock_openai: MagicMock): system_prompt="system", user_prompt="user", token="sk-fake", - max_tokens=16384, + max_output_tokens=16384, ) assert mock_client.chat.completions.create.call_args.kwargs["max_tokens"] == 16384 @@ -2201,8 +2201,8 @@ def test_413_no_fallback_for_custom_model(self, mock_call: MagicMock): class TestTokenBudgetConstants: """Pin the token budget constants to their corrected values.""" - def test_input_token_limit(self): - assert layer3_ai._INPUT_TOKEN_LIMIT == 8000 + def test_max_input_tokens(self): + assert layer3_ai._MAX_INPUT_TOKENS == 8000 def test_chars_per_token(self): assert layer3_ai._CHARS_PER_TOKEN == 3 @@ -2219,7 +2219,7 @@ def test_user_prompt_token_budget(self): def test_budget_leaves_headroom_for_system_prompt(self): total = layer3_ai._SYSTEM_OVERHEAD_TOKENS + layer3_ai._USER_PROMPT_TOKEN_BUDGET - assert total < layer3_ai._INPUT_TOKEN_LIMIT + assert total < layer3_ai._MAX_INPUT_TOKENS # --------------------------------------------------------------------------- diff --git a/tests/test_main.py b/tests/test_main.py index a550676..2618ffd 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -246,8 +246,8 @@ def test_full_pipeline_classifies_files_for_l3( base_url="https://api.openai.com/v1", reasoning_effort="none", temperature=0.1, - max_tokens=8192, - input_token_limit=8000, + max_output_tokens=8192, + max_input_tokens=8000, confidence_threshold=0.7, unmeasurable_files=set(), )