Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
1a020c9
fix(layer3): replace retired GitHub Models with any OpenAI-compatible…
Ostico Jul 31, 2026
e80b8e5
docs: document free-tier providers and the strict json_schema constraint
Ostico Jul 31, 2026
fcdb461
docs: correct Groq strict json_schema support and free-tier viability
Ostico Jul 31, 2026
a9c4aa8
feat(layer3): add ai-reasoning-effort, defaulting to none
Ostico Jul 31, 2026
5ccadda
docs: recommend gemini-2.5-flash over flash-lite for Layer 3
Ostico Jul 31, 2026
b773c56
ci(dogfood): run Layer 3 on gemini-2.5-flash with thinking off
Ostico Jul 31, 2026
dd6e393
fix(action): remove ${{ }} expression from an input description
Ostico Jul 31, 2026
a39f931
fix(dev): point the local PR harness at the configured AI provider
Ostico Jul 31, 2026
92d26d6
fix(dev): bootstrap sys.path so the harness runs standalone
Ostico Jul 31, 2026
b8d6cab
fix(layer3): move to live Gemini 3 model ids, add ai-temperature
Ostico Jul 31, 2026
fd5bc85
feat(layer3): add ai-max-tokens, stop flooring Gemini 3 thinking at m…
Ostico Jul 31, 2026
1c815c8
fix(layer3): raise output cap to 8192 and warn when it truncates
Ostico Jul 31, 2026
d43d603
ci(dogfood): add dogfood-ai label to exercise Layer 3's AI path
Ostico Jul 31, 2026
83f544d
ci(dogfood): fix label gate that a falsy ternary made inert
Ostico Jul 31, 2026
ebf6405
feat(layer3): make the prompt budget configurable via ai-input-token-…
Ostico Jul 31, 2026
0fe29ea
refactor: rename the token inputs to say which direction they govern
Ostico Jul 31, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 55 additions & 4 deletions .dev/local_pr_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,32 @@
GITHUB_TOKEN=ghp_xxx python local_pr_test.py # explicit token

Token resolution: GITHUB_TOKEN env var → `gh auth token` CLI fallback.

Layer 3 runs with no coverage report (coverage_files=[]), so files reach the
AI gates instead of being resolved by coverage shortcuts — this is the way to
exercise the AI path end to end. It needs a provider key:

GEMINI_API_KEY=... python local_pr_test.py

Provider overrides: AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY for the key,
AI_MODEL, AI_BASE_URL, AI_REASONING_EFFORT, AI_TEMPERATURE, AI_MAX_OUTPUT_TOKENS,
AI_MAX_INPUT_TOKENS.
Defaults target gemini-3.1-flash-lite at temperature 1.0 with low thinking,
per Google's guidance for Gemini 3.x. Without a key the AI phase is skipped.
"""
from __future__ import annotations

import os
import subprocess
import sys
from pathlib import Path

# Running this file directly puts .dev/ on sys.path, not the repo root, so the
# src.* imports below fail. In CI action.yml sets PYTHONPATH; here we bootstrap
# it so `python .dev/local_pr_test.py` works from anywhere.
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

from src.config import Config, _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS
from src.config import _DEFAULT_EXCLUDE, _DEFAULT_TEST_PATTERNS, Config
from src.github_api import create_session
from src.github_client import format_report
from src.layer1_coverage import run_layer1
Expand All @@ -28,7 +47,6 @@


def _resolve_token() -> str:
import os
token = os.environ.get("GITHUB_TOKEN")
if token:
return token
Expand All @@ -41,6 +59,25 @@ def _resolve_token() -> str:
sys.exit(1)


def _resolve_ai_api_key() -> str:
"""Provider key for Layer 3, from the first env var that is set.

Deliberately not GITHUB_TOKEN: GitHub Models was retired on 2026-07-30 and
a GitHub token authenticates no inference endpoint. Without a key the AI
phase is skipped and only the shortcut gates run.
"""
for name in ("AI_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY"):
value = os.environ.get(name, "").strip()
if value:
return value
print(
"WARNING: no AI_API_KEY / GEMINI_API_KEY / OPENAI_API_KEY set — "
"Layer 3 will run shortcut gates only, no AI call.",
file=sys.stderr,
)
return ""


def run_local(repo: str, pr_number: int) -> None:
token = _resolve_token()

Expand All @@ -55,8 +92,17 @@ def run_local(repo: str, pr_number: int) -> None:
test_patterns=_DEFAULT_TEST_PATTERNS,
exclude_patterns=exclude_patterns,
ai_enabled=True,
ai_model="openai/gpt-5-mini",
ai_model=os.environ.get("AI_MODEL", "gemini-3.1-flash-lite"),
ai_confidence_threshold=0.7,
ai_base_url=os.environ.get(
"AI_BASE_URL",
"https://generativelanguage.googleapis.com/v1beta/openai/",
),
ai_api_key=_resolve_ai_api_key(),
ai_reasoning_effort=os.environ.get("AI_REASONING_EFFORT", "low"),
ai_temperature=float(os.environ.get("AI_TEMPERATURE", "1.0")),
ai_max_output_tokens=int(os.environ.get("AI_MAX_OUTPUT_TOKENS", "8192")),
ai_max_input_tokens=int(os.environ.get("AI_MAX_INPUT_TOKENS", "32000")),
)

session = create_session(token)
Expand Down Expand Up @@ -122,7 +168,12 @@ def run_local(repo: str, pr_number: int) -> None:
coverage_details=l1.coverage_details,
coverage_threshold=config.coverage_threshold,
model=config.ai_model,
token=config.github_token,
token=config.ai_api_key,
base_url=config.ai_base_url,
reasoning_effort=config.ai_reasoning_effort,
temperature=config.ai_temperature,
max_output_tokens=config.ai_max_output_tokens,
max_input_tokens=config.ai_max_input_tokens,
confidence_threshold=config.ai_confidence_threshold,
)
report.layers.append(l3)
Expand Down
67 changes: 64 additions & 3 deletions .github/workflows/dogfood.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
name: Test Guard (Dogfood)
on:
pull_request:
types: [opened, synchronize]
# `labeled` lets the dogfood-ai label below start a run without a push.
types: [opened, synchronize, labeled]

# Cancel superseded runs: only the newest push per PR branch keeps running.
concurrency:
Expand All @@ -12,7 +13,7 @@ permissions:
contents: read
pull-requests: write
statuses: write
models: read
# No `models: read` — it only ever gated GitHub Models, retired 2026-07-30.

jobs:
test-guard:
Expand All @@ -31,12 +32,72 @@ jobs:
pip install -r requirements.txt
pytest tests/ --cov=src --cov-report=xml

# Two steps rather than one with a conditional coverage-file: in GitHub
# expressions `cond && '' || 'x'` always yields 'x', because the empty
# string is falsy and short-circuits to the fallback. A ternary cannot
# produce an empty value, so gate whole steps instead.
- name: Run Test Guard on itself
if: "!contains(github.event.pull_request.labels.*.name, 'dogfood-ai')"
uses: ./
with:
coverage-file: coverage.xml
coverage-threshold: '100'
ai-model: 'openai/gpt-4.1-mini'
# Gemini 2.5 was shut down ahead of its documented Oct 2026 date and
# now 404s. flash-lite defaults to minimal thinking, which suits the
# 2048-token output cap.
ai-model: 'gemini-3.1-flash-lite'
# Trailing slash is required — without it the compat layer 404s.
ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
# Layer 3 needs a provider key. Absent the secret, the gate degrades
# to Layer 1 + Layer 2 with a warning instead of failing.
ai-api-key: ${{ secrets.GEMINI_API_KEY }}
# Gemini 3.x has no "off" for thinking. Its published intelligence
# scores are reasoning-enabled, and Google recommends thinking for
# code reasoning, so don't floor this at minimal — the 4096-token
# default output cap leaves room for a short thought trace.
ai-reasoning-effort: 'low'
# Gemini 3.x warns that lowering temperature causes looping and
# degraded reasoning, so use its recommended default instead of 0.1.
ai-temperature: '1.0'
# 8000 was sized for the retired GitHub Models 8K input cap.
# Gemini has a 1M context, and at 8000 a large test diff is shed,
# which makes the model warn about code it was never shown.
ai-max-input-tokens: '32000'
# Repo-specific: benchmarks/ is a standalone perf harness, not product
# code under test. Kept OUT of the shipped defaults on purpose — a
# test-adequacy gate should not silently skip a code tree for every
# user. Unioned onto the defaults, so no need to re-list them.
extra-exclude-patterns: 'benchmarks/**'

# `dogfood-ai` label: withhold the coverage report so files fall through
# to Layer 3's AI gates (6-8) instead of being resolved by coverage
# shortcuts, which is the only way this workflow exercises the model.
- name: Run Test Guard on itself (AI path, no coverage)
if: contains(github.event.pull_request.labels.*.name, 'dogfood-ai')
uses: ./
with:
coverage-threshold: '100'
# Gemini 2.5 was shut down ahead of its documented Oct 2026 date and
# now 404s. flash-lite defaults to minimal thinking, which suits the
# 2048-token output cap.
ai-model: 'gemini-3.1-flash-lite'
# Trailing slash is required — without it the compat layer 404s.
ai-base-url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
# Layer 3 needs a provider key. Absent the secret, the gate degrades
# to Layer 1 + Layer 2 with a warning instead of failing.
ai-api-key: ${{ secrets.GEMINI_API_KEY }}
# Gemini 3.x has no "off" for thinking. Its published intelligence
# scores are reasoning-enabled, and Google recommends thinking for
# code reasoning, so don't floor this at minimal — the 4096-token
# default output cap leaves room for a short thought trace.
ai-reasoning-effort: 'low'
# Gemini 3.x warns that lowering temperature causes looping and
# degraded reasoning, so use its recommended default instead of 0.1.
ai-temperature: '1.0'
# 8000 was sized for the retired GitHub Models 8K input cap.
# Gemini has a 1M context, and at 8000 a large test diff is shed,
# which makes the model warn about code it was never shown.
ai-max-input-tokens: '32000'
# Repo-specific: benchmarks/ is a standalone perf harness, not product
# code under test. Kept OUT of the shipped defaults on purpose — a
# test-adequacy gate should not silently skip a code tree for every
Expand Down
Loading
Loading