From d46480ab159416e68fda4afc961ee17b40dedfe5 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:31:33 -0700 Subject: [PATCH 01/64] feat: per-severity warning and outofscope caps in the quality gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #13, task T4. apply_quality_gate gains the keyword params max_warning_findings and max_outofscope_findings. The error cap's logic moves into a private _apply_severity_cap helper that all three caps share: the surviving findings of one severity are ranked by finding_rank_key, so ties break on content, and the excess is marked with " cap exceeded (max N)" rather than removed. The error cap's reason text and its env/default resolution are unchanged. None (the default) means unlimited and reads no environment variable; 0 drops every finding of that severity. spec is never capped and never counts toward a cap. The docstring records that under PRXREF_FAIL_ON=any a cap of 0 can turn exit 1 into exit 0: a cap narrows the gate, never widens it. Config keys and orchestrator wiring are left to later seats. With both params None the gate's output is identical to 0.14.0: a 20,000-case randomized probe against the base function matched on every case, and a warning cap of 0 as control differed on 9,071 of them. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/quality.py | 68 ++++++--- tests/test_severity_caps.py | 265 ++++++++++++++++++++++++++++++++++++ 2 files changed, 313 insertions(+), 20 deletions(-) create mode 100644 tests/test_severity_caps.py diff --git a/src/prxref/quality.py b/src/prxref/quality.py index 506e7420..f84c6729 100644 --- a/src/prxref/quality.py +++ b/src/prxref/quality.py @@ -1544,6 +1544,24 @@ def _resolve_max_errors(explicit: int | None) -> int: return DEFAULT_MAX_ERRORS +def _apply_severity_cap(staged: list[Finding], severity: str, cap: int) -> None: + active_indices: list[int] = [ + i for i, f in enumerate(staged) + if f.drop_reason is None and f.severity == severity + ] + + if len(active_indices) > cap: + ranked = sorted( + active_indices, + key=lambda idx: finding_rank_key(staged[idx]), + ) + for dropped_idx in ranked[cap:]: + staged[dropped_idx] = replace( + staged[dropped_idx], + drop_reason=f"{severity} cap exceeded (max {cap})", + ) + + def _hedge_span(text: str) -> str | None: for _name, pattern in HEDGE_RULES: m = pattern.search(text) @@ -1631,8 +1649,10 @@ def apply_quality_gate( *, confidence_floor: float | None = None, max_errors: int | None = None, + max_warning_findings: int | None = None, + max_outofscope_findings: int | None = None, ) -> list[Finding]: - """Filter findings through vocabulary, confidence, and per-review error caps. + """Filter findings through vocabulary, confidence, and per-severity caps. Order: 1. Severity vocabulary: non-empty lowercase must be in @@ -1640,10 +1660,28 @@ def apply_quality_gate( invalid severities are dropped. 2. Confidence floor: drop findings below the threshold (default 0.6). 3. Error cap: among surviving errors, keep the top N ranked by - :func:`finding_rank_key` and drop the rest, so ties are broken by - content rather than by arrival order. ``spec`` findings never count - toward the cap: a spec-heavy review is neither crowded out by it nor - crowding it out. + :func:`finding_rank_key` and drop the rest + (``error cap exceeded (max N)``), so ties are broken by content + rather than by arrival order. ``max_errors`` falls back to + ``PRXREF_MAX_ERROR_FINDINGS`` and then to :data:`DEFAULT_MAX_ERRORS`. + 4. Warning and outofscope caps: ``max_warning_findings`` and + ``max_outofscope_findings`` apply the same ranking to the surviving + findings of their own severity, dropping the excess as + ``warning cap exceeded (max N)`` and + ``outofscope cap exceeded (max N)``. ``None`` (the default) means + unlimited and reads no environment variable, so a call that omits + both is identical to one without these caps; ``0`` drops every + finding of that severity. + + Each cap counts only its own severity, and only findings that are still + active after steps 1 and 2. ``spec`` findings are never capped and never + count toward a cap: a spec-heavy review is neither crowded out by one + nor crowding one out. A capped finding is kept with its ``drop_reason`` + set, never removed. + + A cap narrows ``PRXREF_FAIL_ON`` and never widens it: under ``any``, a + cap of ``0`` removes that severity from the active findings, so a run + whose only findings were of that severity exits 0 instead of 1. The returned list is sorted by :func:`finding_sort_key`. """ @@ -1677,21 +1715,11 @@ def apply_quality_gate( staged.append(norm) - active_error_indices: list[int] = [ - i for i, f in enumerate(staged) - if f.drop_reason is None and f.severity == "error" - ] - - if len(active_error_indices) > cap: - ranked = sorted( - active_error_indices, - key=lambda idx: finding_rank_key(staged[idx]), - ) - for dropped_idx in ranked[cap:]: - staged[dropped_idx] = replace( - staged[dropped_idx], - drop_reason=f"error cap exceeded (max {cap})", - ) + _apply_severity_cap(staged, "error", cap) + if max_warning_findings is not None: + _apply_severity_cap(staged, "warning", max_warning_findings) + if max_outofscope_findings is not None: + _apply_severity_cap(staged, "outofscope", max_outofscope_findings) return sorted(staged, key=finding_sort_key) diff --git a/tests/test_severity_caps.py b/tests/test_severity_caps.py new file mode 100644 index 00000000..4d31dfc5 --- /dev/null +++ b/tests/test_severity_caps.py @@ -0,0 +1,265 @@ +"""Per-severity caps in ``apply_quality_gate`` (issue #13, T4).""" +from __future__ import annotations + +import itertools + +import pytest + +from prxref.cli import _fail_on_exit +from prxref.quality import active, apply_quality_gate +from prxref.triage import Finding + + +def _f(**kwargs) -> Finding: + defaults = { + "file": "src/app.py", + "line": 10, + "severity": "warning", + "confidence": 0.8, + "title": "Possible bug", + "body": "Details about the finding.", + } + defaults.update(kwargs) + return Finding(**defaults) + + +def _tied(severity: str) -> list[Finding]: + return [ + _f(file="src/a.py", line=1, severity=severity, confidence=0.9, title="alpha"), + _f(file="src/a.py", line=2, severity=severity, confidence=0.9, title="bravo"), + _f(file="src/a.py", line=3, severity=severity, confidence=0.9, title="charlie"), + ] + + +def _mixed() -> list[Finding]: + return [ + _f(file="src/a.py", line=1, severity="error", confidence=0.9, title="e1"), + _f(file="src/a.py", line=2, severity="error", confidence=0.7, title="e2"), + _f(file="src/a.py", line=3, severity="error", confidence=0.8, title="e3"), + _f(file="src/b.py", line=1, severity="warning", confidence=0.9, title="w1"), + _f(file="src/b.py", line=2, severity="Warning", confidence=0.7, title="w2"), + _f(file="src/b.py", line=3, severity="warning", confidence=0.4, title="w3"), + _f(file="src/c.py", line=1, severity="outofscope", confidence=0.9, title="o1"), + _f(file="src/c.py", line=2, severity="outofscope", confidence=0.8, title="o2"), + _f(file="src/d.py", line=1, severity="spec", confidence=0.9, title="s1"), + _f(file="src/d.py", line=2, severity="bogus", confidence=0.9, title="x1"), + _f( + file="src/e.py", line=1, severity="warning", confidence=0.9, + title="pre", drop_reason="hedged: \"if\"", + ), + ] + + +def _cap_kwargs(severity: str, cap: int | None) -> dict[str, int | None]: + return { + "warning": {"max_warning_findings": cap}, + "outofscope": {"max_outofscope_findings": cap}, + }[severity] + + +def _by_title(findings: list[Finding]) -> dict[str, Finding]: + return {f.title: f for f in findings} + + +CAPPED = ("warning", "outofscope") + + +class TestCapsHonoured: + """Each new cap keeps the top N of its severity, ranked by content.""" + + @pytest.mark.parametrize("severity", CAPPED) + def test_survivors_do_not_depend_on_arrival_order(self, severity): + seen = set() + for perm in itertools.permutations(_tied(severity)): + staged = apply_quality_gate( + list(perm), confidence_floor=0.6, **_cap_kwargs(severity, 2) + ) + seen.add(frozenset(f.title for f in staged if f.drop_reason is None)) + assert seen == {frozenset({"alpha", "bravo"})} + + @pytest.mark.parametrize("severity", CAPPED) + def test_excess_is_marked_not_removed(self, severity): + staged = apply_quality_gate( + _tied(severity), confidence_floor=0.6, **_cap_kwargs(severity, 2) + ) + assert len(staged) == 3 + assert _by_title(staged)["charlie"].drop_reason == ( + f"{severity} cap exceeded (max 2)" + ) + + @pytest.mark.parametrize("severity", CAPPED) + def test_confidence_decides_before_content(self, severity): + findings = [ + _f(file="src/a.py", line=1, severity=severity, confidence=0.7, title="aaa"), + _f(file="src/z.py", line=9, severity=severity, confidence=0.95, title="zzz"), + ] + staged = _by_title( + apply_quality_gate( + findings, confidence_floor=0.6, **_cap_kwargs(severity, 1) + ) + ) + assert staged["zzz"].drop_reason is None + assert staged["aaa"].drop_reason == f"{severity} cap exceeded (max 1)" + + def test_a_cap_counts_only_its_own_severity(self): + staged = _by_title( + apply_quality_gate( + _mixed(), + confidence_floor=0.6, + max_errors=10, + max_warning_findings=1, + max_outofscope_findings=2, + ) + ) + assert [t for t in ("e1", "e2", "e3") if staged[t].drop_reason] == [] + assert staged["w1"].drop_reason is None + assert staged["w2"].drop_reason == "warning cap exceeded (max 1)" + assert staged["o1"].drop_reason is None + assert staged["o2"].drop_reason is None + + def test_earlier_drops_keep_their_reason_and_take_no_slot(self): + staged = _by_title( + apply_quality_gate( + _mixed(), confidence_floor=0.6, max_warning_findings=2, + ) + ) + assert staged["w3"].drop_reason == "confidence 0.40 below floor 0.60" + assert staged["pre"].drop_reason == 'hedged: "if"' + assert staged["w1"].drop_reason is None + assert staged["w2"].drop_reason is None + + def test_a_case_normalized_severity_counts_toward_its_cap(self): + staged = _by_title( + apply_quality_gate( + _mixed(), confidence_floor=0.6, max_warning_findings=1, + ) + ) + assert staged["w2"].severity == "warning" + assert staged["w2"].drop_reason == "warning cap exceeded (max 1)" + + +class TestNoneAndZero: + @pytest.mark.parametrize("severity", CAPPED) + def test_none_is_unlimited(self, severity): + many = [ + _f(file="src/a.py", line=i, severity=severity, confidence=0.9, title=f"t{i}") + for i in range(1, 51) + ] + staged = apply_quality_gate( + many, confidence_floor=0.6, **_cap_kwargs(severity, None) + ) + assert len(active(staged)) == 50 + + @pytest.mark.parametrize("severity", CAPPED) + def test_none_reads_no_environment_variable(self, severity, monkeypatch): + monkeypatch.setenv("PRXREF_MAX_WARNING_FINDINGS", "0") + monkeypatch.setenv("PRXREF_MAX_OUTOFSCOPE_FINDINGS", "0") + staged = apply_quality_gate(_tied(severity), confidence_floor=0.6) + assert len(active(staged)) == 3 + + @pytest.mark.parametrize("severity", CAPPED) + def test_zero_drops_every_finding_of_that_severity(self, severity): + staged = apply_quality_gate( + _mixed(), confidence_floor=0.6, max_errors=10, + **_cap_kwargs(severity, 0), + ) + survivors = active(staged) + assert [f.title for f in survivors if f.severity == severity] == [] + capped = [ + f for f in staged if f.drop_reason == f"{severity} cap exceeded (max 0)" + ] + assert capped and all(f.severity == severity for f in capped) + other = {f.severity for f in survivors} + assert other == {"error", "warning", "outofscope", "spec"} - {severity} + + +class TestErrorCapUnchanged: + def test_error_reason_text_is_unchanged_beside_the_new_caps(self): + staged = _by_title( + apply_quality_gate( + _mixed(), + confidence_floor=0.6, + max_errors=1, + max_warning_findings=5, + max_outofscope_findings=5, + ) + ) + assert staged["e1"].drop_reason is None + assert staged["e3"].drop_reason == "error cap exceeded (max 1)" + assert staged["e2"].drop_reason == "error cap exceeded (max 1)" + + def test_the_error_cap_ignores_the_new_caps(self): + base = apply_quality_gate(_mixed(), confidence_floor=0.6, max_errors=2) + capped = apply_quality_gate( + _mixed(), confidence_floor=0.6, max_errors=2, + max_warning_findings=0, max_outofscope_findings=0, + ) + errors = [(f.title, f.drop_reason) for f in base if f.severity == "error"] + assert errors == [ + (f.title, f.drop_reason) for f in capped if f.severity == "error" + ] + + +class TestSpecNeverCapped: + def test_spec_survives_every_cap_at_zero(self): + specs = [ + _f(file="src/s.py", line=i, severity="spec", confidence=0.9, title=f"s{i}") + for i in range(1, 21) + ] + staged = apply_quality_gate( + specs, + confidence_floor=0.6, + max_errors=0, + max_warning_findings=0, + max_outofscope_findings=0, + ) + assert len(active(staged)) == 20 + + def test_spec_takes_no_slot_from_a_capped_severity(self): + findings = [ + _f(file="src/a.py", line=1, severity="spec", confidence=0.99, title="s"), + _f(file="src/a.py", line=2, severity="warning", confidence=0.7, title="w"), + _f(file="src/a.py", line=3, severity="outofscope", confidence=0.7, title="o"), + ] + staged = apply_quality_gate( + findings, + confidence_floor=0.6, + max_warning_findings=1, + max_outofscope_findings=1, + ) + assert len(active(staged)) == 3 + + +class TestDefaultsAreIdentical: + def test_omitting_the_caps_equals_passing_none(self): + omitted = apply_quality_gate(_mixed(), confidence_floor=0.6, max_errors=2) + explicit = apply_quality_gate( + _mixed(), + confidence_floor=0.6, + max_errors=2, + max_warning_findings=None, + max_outofscope_findings=None, + ) + assert omitted == explicit + assert repr(omitted) == repr(explicit) + + def test_defaults_leave_every_warning_and_outofscope_active(self): + staged = _by_title(apply_quality_gate(_mixed(), confidence_floor=0.6)) + for title in ("w1", "w2", "o1", "o2"): + assert staged[title].drop_reason is None + + +class TestFailOnInteraction: + """Documented: under ``PRXREF_FAIL_ON=any`` a cap of 0 can turn 1 into 0.""" + + def test_a_zero_cap_narrows_fail_on_any(self): + warnings = _tied("warning") + uncapped = apply_quality_gate(warnings, confidence_floor=0.6) + capped = apply_quality_gate( + warnings, confidence_floor=0.6, max_warning_findings=0 + ) + assert _fail_on_exit({"findings_active": active(uncapped)}, "any")[0] == 1 + assert _fail_on_exit({"findings_active": active(capped)}, "any") == (0, None) + + def test_the_docstring_names_the_interaction(self): + assert "PRXREF_FAIL_ON" in (apply_quality_gate.__doc__ or "") From cb32673e3f792dfe09452db9571395207b4ca172 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:32:10 -0700 Subject: [PATCH 02/64] feat(quality): title similarity scorer for reworded duplicates (#10 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a dedicated title tokenizer and a Jaccard scorer so the sweep dedup can recognise a reworded restatement of a chunk finding on the same line. Issue #10 measured 14 chunk + sweep pairs that the exact (file, normalized title) key misses. - _title_tokens: [a-z0-9]+ words of normalize_title, at least three characters, minus function words and generic remedy verbs/hedges. It is kept apart from the thread-dedup _tokens, whose stopwords drop "null" and "string" and whose 4-char floor drops "key"; with those tokens the issue's P1 pair ties the closest different-problem pair. - title_similarity(a, b) -> (jaccard, shared). - titles_similar(a, b, threshold): jaccard >= threshold AND at least TITLE_MIN_SHARED_TOKENS (3) shared tokens, so a short title such as "SQL injection" never merges into a longer one that contains it. Pure addition next to normalize_title. apply_sweep_dedup is untouched; the next seat wires the similarity tier into it behind PRXREF_DEDUP_SIMILARITY. tests/test_title_similarity.py covers the issue's three reworded pairs (similar at 0.5), six different-problem same-line pairs and three short-title cases (not similar), the token floor on its own, threshold wiring, symmetry and determinism. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/quality.py | 76 +++++++++++++++ tests/test_title_similarity.py | 169 +++++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 tests/test_title_similarity.py diff --git a/src/prxref/quality.py b/src/prxref/quality.py index 506e7420..c7a74f1f 100644 --- a/src/prxref/quality.py +++ b/src/prxref/quality.py @@ -1012,6 +1012,82 @@ def normalize_title(title: str) -> str: return " ".join(trimmed.split()) +_TITLE_WORD_RE = re.compile(r"[a-z0-9]+") + +_TITLE_TOKEN_MIN_LEN = 3 + +_TITLE_STOPWORDS: frozenset[str] = frozenset({ + "the", "and", "but", "for", "nor", "with", "without", "within", "into", + "onto", "from", "via", "per", "instead", "rather", "than", "then", + "that", "this", "these", "those", "its", "their", "when", "where", + "which", "while", "can", "could", "should", "would", "must", "may", + "might", "will", "are", "was", "were", "been", "being", "has", "have", + "had", "does", "did", + "use", "uses", "using", "replace", "add", "restore", "missing", + "remove", "avoid", "ensure", "consider", "prefer", "fix", "make", + "potential", "possible", "possibly", "likely", +}) + +TITLE_MIN_SHARED_TOKENS: int = 3 +"""Fewest distinct title tokens two findings must share to count as similar. + +Jaccard alone over-merges short titles: "SQL injection" against "SQL +injection in search query" scores exactly 0.5 while naming only two +words. Requiring three shared tokens keeps such a pair apart and still +admits every reworded duplicate measured for issue #10, which share +four or five. +""" + + +def _title_tokens(title: str) -> set[str]: + """Content words of a finding title, for reworded-duplicate scoring. + + The ASCII ``[a-z0-9]+`` words of :func:`normalize_title`, at least + three characters long, minus English function words and the generic + remedy verbs and hedges (``use``, ``replace``, ``add``, ``restore``, + ``missing``, ``potential``) that reworded titles swap freely. Hyphens + and other punctuation split words, so ``string-literal`` and + ``string literal`` yield the same tokens. Deliberately separate from + :func:`_tokens`: its stopword list drops ``null`` and ``string`` and + its four-character floor drops ``key``, which are exactly the words + that tell two same-line titles apart. + """ + return { + word + for word in _TITLE_WORD_RE.findall(normalize_title(title)) + if len(word) >= _TITLE_TOKEN_MIN_LEN and word not in _TITLE_STOPWORDS + } + + +def title_similarity(a: str, b: str) -> tuple[float, int]: + """Score how closely two finding titles restate one problem. + + Returns ``(jaccard, shared)``: the Jaccard index of the two titles' + :func:`_title_tokens` sets and the number of distinct tokens they + share. Two titles with no tokens between them score ``(0.0, 0)``. + Symmetric in its arguments and a pure function of the two strings. + """ + tokens_a = _title_tokens(a) + tokens_b = _title_tokens(b) + shared = len(tokens_a & tokens_b) + union = len(tokens_a | tokens_b) + if union == 0: + return 0.0, 0 + return shared / union, shared + + +def titles_similar(a: str, b: str, threshold: float) -> bool: + """Return True when two finding titles are reworded restatements. + + Similar means :func:`title_similarity` reaches ``threshold`` on the + Jaccard index AND the titles share at least + :data:`TITLE_MIN_SHARED_TOKENS` tokens. The token floor is what keeps + a short title from merging into any longer one that contains it. + """ + jaccard, shared = title_similarity(a, b) + return jaccard >= threshold and shared >= TITLE_MIN_SHARED_TOKENS + + def finding_sort_key(finding: Finding) -> tuple[str, int, str]: """Content-derived ordering key for a finding: ``(file, line, title)``. diff --git a/tests/test_title_similarity.py b/tests/test_title_similarity.py new file mode 100644 index 00000000..7280d277 --- /dev/null +++ b/tests/test_title_similarity.py @@ -0,0 +1,169 @@ +"""Tests for the reworded-duplicate title scorer in prxref.quality (issue #10). + +P1-P3 are the three reworded same-line pairs quoted in issue #10. N1 and N2 +are the closest different-problem negatives from the issue map; N3-N6 are +further same-line different-problem pairs written for this suite. S1 and S2 +are the short-title cases that an overlap coefficient over-merges. +""" +from __future__ import annotations + +import pytest + +from prxref.quality import ( + TITLE_MIN_SHARED_TOKENS, + _title_tokens, + title_similarity, + titles_similar, +) + +THRESHOLD = 0.5 + +P1_A = "Replace manual null check with Optional in resolve()" +P1_B = "Use Optional instead of manual null check in adapter lookup" +P2_A = "Restore admin-only scope on destructive migration admin endpoints" +P2_B = "Restore admin-only scope on migration admin controller" +P3_A = "Use a logging-key constant instead of a string literal key" +P3_B = "Use a logging-key constant instead of a string-literal key" + +REWORDED = { + "P1": (P1_A, P1_B), + "P2": (P2_A, P2_B), + "P3": (P3_A, P3_B), +} + +DIFFERENT_PROBLEM = { + "N1": (P1_A, "Missing null check on resolve() return value"), + "N2": (P2_B, "Add rate limiting to migration admin controller"), + "N3": ( + "Unbounded retry loop in payment webhook handler", + "Payment webhook handler logs the raw card payload", + ), + "N4": ( + "Race condition on cache entry expiry in session store", + "Session store cache key includes the user email", + ), + "N5": ( + "Missing await on async save in order repository", + "Order repository save swallows persistence exceptions", + ), + "N6": ( + "Off-by-one in pagination offset for user list endpoint", + "User list endpoint returns soft-deleted users", + ), +} + +SHORT_TITLE = { + "S1-chunk": ("Missing null check", P1_A), + "S1-sweep": ("Missing null check", P1_B), + "S2": ("SQL injection", "SQL injection in search query"), +} + +ALL_PAIRS = {**REWORDED, **DIFFERENT_PROBLEM, **SHORT_TITLE} + + +class TestReworded: + @pytest.mark.parametrize("name", sorted(REWORDED)) + def test_reworded_pair_is_similar(self, name): + a, b = REWORDED[name] + assert titles_similar(a, b, THRESHOLD) is True + + @pytest.mark.parametrize( + ("name", "expected"), + [("P1", (4 / 7, 4)), ("P2", (4 / 7, 4)), ("P3", (1.0, 5))], + ) + def test_reworded_pair_scores(self, name, expected): + assert title_similarity(*REWORDED[name]) == expected + + def test_identical_titles_score_one(self): + assert title_similarity(P1_A, P1_A) == (1.0, 5) + + +class TestNotSimilar: + @pytest.mark.parametrize("name", sorted(DIFFERENT_PROBLEM)) + def test_different_problem_pair_is_not_similar(self, name): + a, b = DIFFERENT_PROBLEM[name] + assert titles_similar(a, b, THRESHOLD) is False + + @pytest.mark.parametrize("name", sorted(DIFFERENT_PROBLEM)) + def test_different_problem_pair_is_kept_by_the_threshold(self, name): + jaccard, shared = title_similarity(*DIFFERENT_PROBLEM[name]) + assert shared >= TITLE_MIN_SHARED_TOKENS + assert jaccard < THRESHOLD + + @pytest.mark.parametrize("name", sorted(SHORT_TITLE)) + def test_short_title_is_not_similar(self, name): + a, b = SHORT_TITLE[name] + assert titles_similar(a, b, THRESHOLD) is False + + def test_short_title_is_kept_by_the_token_floor_alone(self): + a, b = SHORT_TITLE["S2"] + assert title_similarity(a, b) == (0.5, 2) + assert titles_similar(a, b, THRESHOLD) is False + assert titles_similar(a, b, 0.1) is False + + def test_scorer_separates_every_reworded_pair_from_every_negative(self): + reworded = [title_similarity(*pair)[0] for pair in REWORDED.values()] + negatives = [title_similarity(*pair)[0] for pair in DIFFERENT_PROBLEM.values()] + assert min(reworded) > max(negatives) + + def test_p1_outscores_n2(self): + assert title_similarity(*REWORDED["P1"])[0] > title_similarity(*DIFFERENT_PROBLEM["N2"])[0] + + +class TestThreshold: + def test_threshold_is_honoured(self): + assert titles_similar(P1_A, P1_B, 0.5) is True + assert titles_similar(P1_A, P1_B, 0.6) is False + + def test_lower_threshold_admits_a_pair_with_enough_shared_tokens(self): + a, b = DIFFERENT_PROBLEM["N1"] + assert title_similarity(a, b) == (3 / 7, 3) + assert titles_similar(a, b, 0.4) is True + + def test_min_shared_tokens_is_three(self): + assert TITLE_MIN_SHARED_TOKENS == 3 + + +class TestTitleTokens: + def test_keeps_words_the_thread_tokenizer_drops(self): + assert _title_tokens(P3_A) == {"logging", "key", "constant", "string", "literal"} + assert {"null", "check"} <= _title_tokens(P1_A) + + def test_drops_function_words_generic_verbs_and_short_words(self): + assert _title_tokens("Use the id of a DB instead") == set() + assert _title_tokens("Replace or restore the missing lock") == {"lock"} + + def test_hyphen_and_space_tokenize_alike(self): + assert _title_tokens("string-literal key") == _title_tokens("string literal key") + + def test_case_quotes_and_backticks_are_ignored(self): + assert _title_tokens("`Resolve()` returns \"NULL\"") == _title_tokens("resolve() returns null") + + def test_word_order_does_not_matter(self): + assert title_similarity("manual null check", "check null manual") == (1.0, 3) + + +class TestEdges: + @pytest.mark.parametrize( + ("a", "b"), + [("", ""), ("!!!", "..."), ("", P1_A), ("Use the DB", "Add an id")], + ) + def test_titles_without_shared_tokens_score_zero(self, a, b): + assert title_similarity(a, b) == (0.0, 0) + assert titles_similar(a, b, 0.0) is False + + +class TestSymmetryAndDeterminism: + @pytest.mark.parametrize("name", sorted(ALL_PAIRS)) + def test_symmetric(self, name): + a, b = ALL_PAIRS[name] + assert title_similarity(a, b) == title_similarity(b, a) + assert titles_similar(a, b, THRESHOLD) == titles_similar(b, a, THRESHOLD) + + @pytest.mark.parametrize("name", sorted(ALL_PAIRS)) + def test_deterministic(self, name): + a, b = ALL_PAIRS[name] + first = title_similarity(a, b) + assert all(title_similarity(a, b) == first for _ in range(5)) + verdict = titles_similar(a, b, THRESHOLD) + assert all(titles_similar(a, b, THRESHOLD) is verdict for _ in range(5)) From 793432d8316a5fe552f84a13232e46da8e2554b0 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:32:25 -0700 Subject: [PATCH 03/64] feat(rules): match_globs, the applies_to path matcher (#12 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add rules.match_globs(path, patterns) for scoped review rules. Each pattern is an fnmatchcase glob over the whole POSIX path, as PRXREF_SIZE_IGNORE_GLOBS is: case-sensitive, and * crosses /. A leading ! negates. A path is selected when any positive pattern matches and no negation does, in any order (map-12 Design 1 rejects gitignore's last-match-wins). An empty list, or negations only, select nothing. Stdlib fnmatch needs a / wherever a pattern says **/, so **/*.java misses a root-level Foo.java and the issue's own !**/src/test/** misses src/test/A.java. match_globs also tries every copy of the pattern with some of its leading or after-slash **/ removed; a run such as **/**/ counts as one. is_size_ignored is untouched and keeps plain fnmatch (decisions #12), and a test pins that. Tests: tests/test_scoped_rules_globs.py, 125 table-driven cases. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/rules.py | 71 +++++++++++- tests/test_scoped_rules_globs.py | 185 +++++++++++++++++++++++++++++++ 2 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 tests/test_scoped_rules_globs.py diff --git a/src/prxref/rules.py b/src/prxref/rules.py index f706ca72..12a7b419 100644 --- a/src/prxref/rules.py +++ b/src/prxref/rules.py @@ -34,12 +34,14 @@ import codecs import errno +import fnmatch import hashlib +import itertools import logging import os import re import stat -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from .llm import ConfigError @@ -302,3 +304,70 @@ def load_review_rules(path: str | None, *, max_chars: int, source: str) -> Revie if not capped.text and not severity_map: logger.warning("%s: rules file %r is empty; no rules injected", source, path) return ReviewRules(path=path, body=capped, severity_map=severity_map, ignored_keys=ignored) + + +_ANY_DIRS = "**/" + + +def match_globs(path: str, patterns: Sequence[str]) -> bool: + """True when ``patterns``, a scoped rules file's ``applies_to`` list, selects ``path``. + + ``path`` is a diff path: POSIX, relative to the repository root. Each + pattern is matched against the whole path with + :func:`fnmatch.fnmatchcase`, as ``PRXREF_SIZE_IGNORE_GLOBS`` is, so the + match is case-sensitive on every host and ``*`` crosses ``/``. A pattern + that starts with ``!`` negates the glob after it. + + The path is selected when at least one positive pattern matches it and no + negation does. Order does not matter: a negation vetoes the path wherever + it sits in the list, and a positive pattern after it cannot bring the path + back, unlike the last-match-wins rule of ``.gitignore``. An empty list, or + a list of negations only, selects nothing. A leading ``!`` always + negates, so a pattern for a path that itself starts with ``!`` starts + with ``?`` instead. + + One addition to plain ``fnmatch``: a ``**/`` that starts the pattern or + follows a ``/`` also matches zero directories, so ``**/*.java`` selects a + root-level ``Foo.java``, ``!**/src/test/**`` vetoes ``src/test/A.java``, + and ``src/**/*.java`` selects ``src/Foo.java``. Stdlib ``fnmatch`` needs a + ``/`` there, and ``PRXREF_SIZE_IGNORE_GLOBS`` keeps that stricter match. + A pattern matches when it, or any copy of it with some of those ``**/`` + removed, matches under ``fnmatchcase``; a run such as ``**/**/`` counts as + one. Each of ``k`` such ``**/`` doubles the copies tried, so the cost is + ``2**k`` ``fnmatchcase`` calls at worst. + """ + selected = False + for pattern in patterns: + if pattern.startswith("!"): + if _glob_matches(path, pattern[1:]): + return False + elif not selected: + selected = _glob_matches(path, pattern) + return selected + + +def _glob_matches(path: str, pattern: str) -> bool: + """True when ``path`` matches ``pattern`` or a copy of it with some ``**/`` removed.""" + head, *tail = _split_any_dirs(pattern) + for kept in itertools.product((_ANY_DIRS, ""), repeat=len(tail)): + variant = head + "".join(sep + piece for sep, piece in zip(kept, tail, strict=True)) + if fnmatch.fnmatchcase(path, variant): + return True + return False + + +def _split_any_dirs(pattern: str) -> list[str]: + """Split ``pattern`` at each ``**/`` that starts it or follows a ``/``, a run counting as one.""" + pieces: list[str] = [] + start = index = 0 + while index < len(pattern): + if pattern.startswith(_ANY_DIRS, index) and (index == 0 or pattern[index - 1] == "/"): + pieces.append(pattern[start:index]) + index += len(_ANY_DIRS) + while pattern.startswith(_ANY_DIRS, index): + index += len(_ANY_DIRS) + start = index + else: + index += 1 + pieces.append(pattern[start:]) + return pieces diff --git a/tests/test_scoped_rules_globs.py b/tests/test_scoped_rules_globs.py new file mode 100644 index 00000000..70c41f05 --- /dev/null +++ b/tests/test_scoped_rules_globs.py @@ -0,0 +1,185 @@ +"""Issue #12 T1: ``match_globs``, the path matcher behind ``applies_to``. + +Pinned here, as one table of ``(path, patterns, selected)`` rows: + +- stdlib ``fnmatchcase`` semantics, shared with ``PRXREF_SIZE_IGNORE_GLOBS``: + case-sensitive, ``*`` crosses ``/``, the whole path must match; +- the zero-directory ``**/`` fold: a ``**/`` that starts the pattern or + follows a ``/`` also matches no directory at all, so a root-level file + matches ``**/*.java`` and ``src/Foo.java`` matches ``src/**/*.java``; +- negation: a ``!`` pattern vetoes the path wherever it sits in the list, and + a later positive pattern cannot bring the path back; +- an empty list, and a list of negations only, select nothing; +- the issue's own ``["**/*.java", "!**/src/test/**"]`` example. + +The fold is for ``applies_to`` only (decisions #12): the size advisory's +``is_size_ignored`` keeps plain ``fnmatchcase``, and the last class pins that. +""" +from __future__ import annotations + +import fnmatch + +import pytest + +from prxref.rules import match_globs +from prxref.triage import is_size_ignored + +ISSUE_EXAMPLE = ("**/*.java", "!**/src/test/**") + +ROOT_LEVEL = [ + ("Foo.java", ["**/*.java"], True), + ("pom.xml", ["**/*.java"], False), + ("Foo.java", ["**/**/*.java"], True), + ("Foo.java", ["**/**/**/*.java"], True), + ("README.md", ["**"], True), + ("README.md", ["**/*"], True), + ("Chart.yaml", ["**/Chart.yaml"], True), + ("xChart.yaml", ["**/Chart.yaml"], False), +] + +NESTED = [ + ("src/main/java/Foo.java", ["**/*.java"], True), + ("a/b/c/d/e/Foo.java", ["**/*.java"], True), + ("src/Foo.java", ["src/*.java"], True), + ("src/deep/Foo.java", ["src/*.java"], True), + ("lib/Foo.java", ["src/*.java"], False), + ("docs/guide.md", ["docs/**"], True), + ("docs/a/b/guide.md", ["docs/**"], True), + ("docs", ["docs/**"], False), + ("helm/charts/app/Chart.yaml", ["**/Chart.yaml"], True), + ("src/Foo.java", ["Foo.java"], False), +] + +MIDDLE_DOUBLESTAR = [ + ("src/Foo.java", ["src/**/*.java"], True), + ("src/a/b/Foo.java", ["src/**/*.java"], True), + ("lib/src/Foo.java", ["src/**/*.java"], False), + ("charts/templates/deploy.yaml", ["charts/**/templates/*.yaml"], True), + ("charts/app/templates/deploy.yaml", ["charts/**/templates/*.yaml"], True), + ("charts/app/values.yaml", ["charts/**/templates/*.yaml"], False), + ("src/Foo.java", ["**/src/**/*.java"], True), + ("mod/src/Foo.java", ["**/src/**/*.java"], True), + ("mod/src/a/b/Foo.java", ["**/src/**/*.java"], True), + ("mod/lib/Foo.java", ["**/src/**/*.java"], False), + ("a/b", ["a/**/**/b"], True), + ("a/x/y/b", ["a/**/**/b"], True), + ("a/b", ["a**/b"], True), + ("ab", ["a**/b"], False), + ("xb", ["x**/b"], False), + ("src/Foo.java", ["src/F**/oo.java"], False), +] + +NEGATION = [ + ("src/test/A.java", ["**/*.java", "!**/src/test/**", "**/src/test/**/*.java"], False), + ("src/test/A.java", ["**/src/test/**/*.java", "!**/src/test/**"], False), + ("src/main/A.java", ["!**/src/test/**"], False), + ("src/main/A.java", ["!**/src/test/**", "!**/*.md"], False), + ("src/main/A.java", ["**/*.java", "!**/*.md"], True), + ("README.md", ["**/*.java", "!**/*.md"], False), + ("gen/A.java", ["**/*.java", "!gen/**", "!**/*Test.java"], False), + ("src/ATest.java", ["**/*.java", "!gen/**", "!**/*Test.java"], False), + ("src/A.java", ["**/*.java", "!gen/**", "!**/*Test.java"], True), + ("a!b.txt", ["a!b.txt"], True), + ("!notes.md", ["?notes.md"], True), + ("!notes.md", ["!notes.md"], False), +] + +EMPTY = [ + ("Foo.java", [], False), + ("Foo.java", (), False), + ("README.md", [], False), + ("src/deep/path/x.py", [], False), +] + +LETTER_CASE = [ + ("Foo.JAVA", ["**/*.java"], False), + ("Foo.java", ["**/*.JAVA"], False), + ("SRC/Foo.java", ["src/**/*.java"], False), + ("src/Foo.java", ["SRC/**"], False), + ("README.md", ["readme.md"], False), + ("readme.md", ["readme.md"], True), + ("Src/Test/A.java", list(ISSUE_EXAMPLE), True), + ("src/Test/A.java", ["**/*.java", "!**/src/test/**"], True), +] + +ISSUE_PATHS = [ + ("Foo.java", True), + ("src/main/java/com/acme/Foo.java", True), + ("service/src/main/java/Foo.java", True), + ("src/test/java/FooTest.java", False), + ("service/src/test/java/FooTest.java", False), + ("src/test/Foo.java", False), + ("src/main/resources/application.yaml", False), + ("helm/values.yaml", False), + ("src/testing/Foo.java", True), + ("mysrc/test/Foo.java", True), +] + + +def _rows(label, rows): + return [pytest.param(path, patterns, selected, id=f"{label}:{path}:{','.join(patterns)}") + for path, patterns, selected in rows] + + +TABLE = ( + _rows("root", ROOT_LEVEL) + + _rows("nested", NESTED) + + _rows("middle", MIDDLE_DOUBLESTAR) + + _rows("negation", NEGATION) + + _rows("empty", EMPTY) + + _rows("case", LETTER_CASE) +) + + +@pytest.mark.parametrize(("path", "patterns", "selected"), TABLE) +def test_match_globs_table(path, patterns, selected): + assert match_globs(path, patterns) is selected + + +@pytest.mark.parametrize(("path", "selected"), ISSUE_PATHS, ids=[p for p, _ in ISSUE_PATHS]) +class TestIssueExample: + """The issue's ``applies_to: ["**/*.java", "!**/src/test/**"]``, in both orders.""" + + def test_as_written(self, path, selected): + assert match_globs(path, list(ISSUE_EXAMPLE)) is selected + + def test_reversed(self, path, selected): + assert match_globs(path, list(reversed(ISSUE_EXAMPLE))) is selected + + def test_as_tuple(self, path, selected): + assert match_globs(path, ISSUE_EXAMPLE) is selected + + +@pytest.mark.parametrize(("path", "patterns", "_selected"), _rows("order", NEGATION + LETTER_CASE)) +def test_order_never_changes_the_result(path, patterns, _selected): + forward = match_globs(path, list(patterns)) + assert match_globs(path, list(reversed(patterns))) is forward + + +@pytest.mark.parametrize(("path", "pattern"), [ + ("Foo.java", "**/*.java"), + ("src/test/A.java", "**/src/test/**"), + ("src/Foo.java", "src/**/*.java"), + ("charts/templates/deploy.yaml", "charts/**/templates/*.yaml"), +]) +class TestZeroDirectoryFold: + """Each fold row is a path stdlib ``fnmatchcase`` misses and ``match_globs`` selects.""" + + def test_stdlib_fnmatch_misses_it(self, path, pattern): + assert fnmatch.fnmatchcase(path, pattern) is False + + def test_match_globs_selects_it(self, path, pattern): + assert match_globs(path, [pattern]) is True + + def test_a_negation_vetoes_it(self, path, pattern): + assert match_globs(path, ["**", f"!{pattern}"]) is False + + def test_size_advisory_keeps_plain_fnmatch(self, path, pattern): + assert is_size_ignored(path, ignore_globs=[pattern]) is False + + +def test_patterns_are_not_consumed(): + patterns = ["**/*.java", "!**/src/test/**"] + assert match_globs("Foo.java", patterns) is True + assert match_globs("src/test/A.java", patterns) is False + assert patterns == ["**/*.java", "!**/src/test/**"] From 0e51db25e566642206a3124c237d2d652f752ec3 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:32:34 -0700 Subject: [PATCH 04/64] refactor(forges): share the diff renderer and generalize GitHub paging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #15, tasks T1 and T2: the groundwork for rebuilding a too-large GitHub diff from the /pulls/{n}/files listing. T1. _render_diff_entries moves out of forges/gitlab.py into a new shared module, forges/_diff_render.py, as render_diff_entries(entries). The body is unchanged apart from the parameter name, and the docstring now describes the GitLab-shaped entry keys instead of GitLab itself. gitlab.py imports it under its old private name, so every existing gitlab._render_diff_entries caller and test passes unchanged. T2. github.ForgeImpl._iter_comment_pages becomes _iter_pages(ref, url, headers, *, what, extra_params=None), shaped like GitLab's walker: what names the listing in every FeedReadError, and extra_params merges over per_page/page. The three comment readers pass what="comment feed", so their transport and HTTP error text is byte-identical. Two tails follow GitLab's noun-free wording: "not a list of comments" is now "not a list", and the budget error counts "entries" rather than "comments". No test or doc pinned either tail. Page size stays _PAGE_SIZE (100) and the budget stays _MAX_PAGES (50). The /files fallback and the 406 detection (T3/T4) are not built here. Tests: tests/test_forge_diff_render.py (23 tests) pins every renderer header shape byte for byte, the gitlab alias identity, and _iter_pages reading to a short last page, sending per_page=100 plus extra_params, naming what on each failure, and raising when the page budget runs out. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/forges/_diff_render.py | 60 ++++++ src/prxref/forges/github.py | 48 +++-- src/prxref/forges/gitlab.py | 52 +----- tests/test_forge_diff_render.py | 299 ++++++++++++++++++++++++++++++ 4 files changed, 392 insertions(+), 67 deletions(-) create mode 100644 src/prxref/forges/_diff_render.py create mode 100644 tests/test_forge_diff_render.py diff --git a/src/prxref/forges/_diff_render.py b/src/prxref/forges/_diff_render.py new file mode 100644 index 00000000..e0a003e4 --- /dev/null +++ b/src/prxref/forges/_diff_render.py @@ -0,0 +1,60 @@ +"""Render per-file diff entries as one git-style unified diff. + +Shared by the adapters whose forge serves a merge request, a pull request or a +compare as a list of per-file entries rather than as raw diff text. +""" +from __future__ import annotations + + +def render_diff_entries(entries: list[dict]) -> str: + """Render per-file diff entries as one git-style unified diff. + + Each entry is GitLab-shaped: ``old_path`` and ``new_path`` name the file on + either side, the ``new_file``, ``deleted_file`` and ``renamed_file`` flags + pick the header form, and ``diff`` holds the file's hunks without their + headers. This rebuilds each file's ``diff --git`` header from those keys, + then appends the hunks, so the parser downstream sees the same text shape + a raw-diff endpoint returns. An entry with no ``diff`` (missing, ``None`` + or empty) is rendered as a header-only file. An empty list renders as + ``""``. + """ + diff_parts: list[str] = [] + for d in entries: + old_path = d.get("old_path") or "" + new_path = d.get("new_path") or "" + new_file = d.get("new_file", False) + deleted_file = d.get("deleted_file", False) + renamed_file = d.get("renamed_file", False) + raw_diff = d.get("diff") or "" + + header_lines = [f"diff --git a/{old_path} b/{new_path}"] + if new_file: + header_lines.append("new file mode 100644") + header_lines.append("--- /dev/null") + header_lines.append(f"+++ b/{new_path}") + elif deleted_file: + header_lines.append("deleted file mode 100644") + header_lines.append(f"--- a/{old_path}") + header_lines.append("+++ /dev/null") + elif renamed_file: + header_lines.append(f"rename from {old_path}") + header_lines.append(f"rename to {new_path}") + header_lines.append(f"--- a/{old_path}") + header_lines.append(f"+++ b/{new_path}") + else: + header_lines.append(f"--- a/{old_path}") + header_lines.append(f"+++ b/{new_path}") + + file_unified = "\n".join(header_lines) + if raw_diff: + if not raw_diff.startswith("\n"): + file_unified += "\n" + file_unified += raw_diff + if not file_unified.endswith("\n"): + file_unified += "\n" + else: + file_unified += "\n" + + diff_parts.append(file_unified) + + return "".join(diff_parts) diff --git a/src/prxref/forges/github.py b/src/prxref/forges/github.py index 98f6b5be..bc9a4570 100644 --- a/src/prxref/forges/github.py +++ b/src/prxref/forges/github.py @@ -199,14 +199,24 @@ def get_compare_diff(self, ref: PRRef, *, base_sha: str, head_sha: str) -> str: resp.raise_for_status() return resp.text - def _iter_comment_pages( - self, ref: PRRef, url: str, headers: dict[str, str] + def _iter_pages( + self, + ref: PRRef, + url: str, + headers: dict[str, str], + *, + what: str, + extra_params: dict[str, str] | None = None, ) -> Iterator[list[dict]]: - """Yield a comment listing one page at a time, oldest page first. + """Yield a paginated GitHub collection one page at a time, oldest page first. + + Every request carries ``per_page`` and ``page``; ``extra_params`` is + merged over them. ``what`` names the collection in every error, so a + failed read says which listing it was reading. A page at a time rather than one flat list, so a caller hunting for a - single comment stops at the page it appears on instead of paying for - the whole feed. Any read that does not reach the end — transport + single entry stops at the page it appears on instead of paying for + the whole listing. Any read that does not reach the end — transport failure, non-OK status, unparseable body, or the page budget running out — raises ``FeedReadError`` instead of returning short, so no caller can mistake "I stopped early" for "that was all". @@ -217,34 +227,40 @@ def _iter_comment_pages( """ where = f"{ref.owner}/{ref.repo}#{ref.number}" for page_number in range(1, _MAX_PAGES + 1): + params: dict[str, int | str] = { + "per_page": _PAGE_SIZE, + "page": page_number, + } + if extra_params: + params.update(extra_params) try: resp = self.session.get( url, headers=headers, - params={"per_page": _PAGE_SIZE, "page": page_number}, + params=params, timeout=_REQUEST_TIMEOUT, ) except requests.RequestException as e: raise FeedReadError( - f"comment feed for {where} could not be read at page " + f"{what} for {where} could not be read at page " f"{page_number}: {e}" ) from e if not resp.ok: raise FeedReadError( - f"comment feed for {where} returned HTTP " + f"{what} for {where} returned HTTP " f"{resp.status_code} at page {page_number}" ) try: items = resp.json() except ValueError as e: raise FeedReadError( - f"comment feed for {where} returned an unreadable body at " + f"{what} for {where} returned an unreadable body at " f"page {page_number}: {e}" ) from e if not isinstance(items, list): raise FeedReadError( - f"comment feed for {where} returned " - f"{type(items).__name__}, not a list of comments" + f"{what} for {where} returned " + f"{type(items).__name__}, not a list" ) yield [item for item in items if isinstance(item, dict)] @@ -253,8 +269,8 @@ def _iter_comment_pages( return raise FeedReadError( - f"comment feed for {where} outran the {_MAX_PAGES}-page budget " - f"({_MAX_PAGES * _PAGE_SIZE} comments) without reaching the end" + f"{what} for {where} outran the {_MAX_PAGES}-page budget " + f"({_MAX_PAGES * _PAGE_SIZE} entries) without reaching the end" ) def post_summary(self, ref: PRRef, body: str) -> None: @@ -270,7 +286,7 @@ def post_summary(self, ref: PRRef, body: str) -> None: body = with_summary_marker(body) existing_comment_id: int | None = None - for comments in self._iter_comment_pages(ref, list_url, headers): + for comments in self._iter_pages(ref, list_url, headers, what="comment feed"): for c in comments: if SUMMARY_MARKER in (c.get("body") or ""): existing_comment_id = c.get("id") @@ -350,7 +366,7 @@ def list_threads(self, ref: PRRef) -> list[Thread]: threads: list[Thread] = [] try: - for data in self._iter_comment_pages(ref, url, headers): + for data in self._iter_pages(ref, url, headers, what="comment feed"): for item in data: path = item.get("path") line = item.get("line") or item.get("original_line") or item.get("position") @@ -439,7 +455,7 @@ def prune_inline_comments(self, ref: PRRef) -> int: headers = self._headers(ref.host) removed = 0 try: - for comments in self._iter_comment_pages(ref, list_url, headers): + for comments in self._iter_pages(ref, list_url, headers, what="comment feed"): for comment in comments: if ATTRIBUTION_MARKER not in (comment.get("body") or ""): continue diff --git a/src/prxref/forges/gitlab.py b/src/prxref/forges/gitlab.py index ace14622..08e6620d 100644 --- a/src/prxref/forges/gitlab.py +++ b/src/prxref/forges/gitlab.py @@ -10,6 +10,7 @@ import requests from requests.adapters import HTTPAdapter +from prxref.forges._diff_render import render_diff_entries as _render_diff_entries from prxref.forges.base import ( ATTRIBUTION_MARKER, SUMMARY_MARKER, @@ -106,57 +107,6 @@ def _make_retry_session() -> requests.Session: _DEFAULT_SESSION = _make_retry_session() -def _render_diff_entries(diffs: list[dict]) -> str: - """Render GitLab's structured diff entries as one git-style unified diff. - - GitLab serves no raw diff for a merge request or a compare, only a list of - per-file entries whose ``diff`` holds the hunks without their headers. This - rebuilds each file's ``diff --git`` header from the entry's flags, then - appends the hunks, so the parser downstream sees the same text shape every - other forge returns. - """ - diff_parts: list[str] = [] - for d in diffs: - old_path = d.get("old_path") or "" - new_path = d.get("new_path") or "" - new_file = d.get("new_file", False) - deleted_file = d.get("deleted_file", False) - renamed_file = d.get("renamed_file", False) - raw_diff = d.get("diff") or "" - - header_lines = [f"diff --git a/{old_path} b/{new_path}"] - if new_file: - header_lines.append("new file mode 100644") - header_lines.append("--- /dev/null") - header_lines.append(f"+++ b/{new_path}") - elif deleted_file: - header_lines.append("deleted file mode 100644") - header_lines.append(f"--- a/{old_path}") - header_lines.append("+++ /dev/null") - elif renamed_file: - header_lines.append(f"rename from {old_path}") - header_lines.append(f"rename to {new_path}") - header_lines.append(f"--- a/{old_path}") - header_lines.append(f"+++ b/{new_path}") - else: - header_lines.append(f"--- a/{old_path}") - header_lines.append(f"+++ b/{new_path}") - - file_unified = "\n".join(header_lines) - if raw_diff: - if not raw_diff.startswith("\n"): - file_unified += "\n" - file_unified += raw_diff - if not file_unified.endswith("\n"): - file_unified += "\n" - else: - file_unified += "\n" - - diff_parts.append(file_unified) - - return "".join(diff_parts) - - class ForgeImpl: """GitLab Forge adapter.""" diff --git a/tests/test_forge_diff_render.py b/tests/test_forge_diff_render.py new file mode 100644 index 00000000..7cad450a --- /dev/null +++ b/tests/test_forge_diff_render.py @@ -0,0 +1,299 @@ +"""Tests for the shared diff renderer and GitHub's generalized page walker. + +``render_diff_entries`` moved out of the GitLab adapter so GitHub can rebuild +a diff from its ``/pulls/{n}/files`` listing with the same header logic. +``github.ForgeImpl._iter_pages`` is the comment-feed walker generalized to +any listing, named by ``what`` in every error it raises. +""" +from __future__ import annotations + +import json +from unittest.mock import MagicMock + +import pytest +import requests + +from prxref.forges import _diff_render, github, gitlab +from prxref.forges._diff_render import render_diff_entries +from prxref.forges.base import FeedReadError +from prxref.forges.github import ForgeImpl + +PAGE_SIZE = 100 +URL = "https://api.github.com/repos/acme/api/pulls/42/files" +HEADERS = {"Accept": "application/vnd.github+json"} + + +def _mock_response(status_code=200, json_data=None, text=""): + resp = MagicMock(spec=requests.Response) + resp.status_code = status_code + resp.ok = 200 <= status_code < 300 + resp.headers = {} + if json_data is not None: + resp.json.return_value = json_data + resp.text = json.dumps(json_data) + else: + resp.text = text + resp.json.side_effect = ValueError("No JSON") + return resp + + +def _page(count, start=0): + return _mock_response(json_data=[{"id": start + i} for i in range(count)]) + + +def _ref(): + ref = ForgeImpl.parse_pr_url("https://github.com/acme/api/pull/42") + assert ref is not None + return ref + + +def _walk(session, what="PR files listing", extra_params=None): + return list( + ForgeImpl(session=session)._iter_pages( + _ref(), URL, HEADERS, what=what, extra_params=extra_params + ) + ) + + +# --- render_diff_entries ------------------------------------------------------ + + +def test_render_new_file(): + entry = {"old_path": "src/new.py", "new_path": "src/new.py", "new_file": True, "diff": "@@ -0,0 +1 @@\n+x\n"} + assert render_diff_entries([entry]) == ( + "diff --git a/src/new.py b/src/new.py\n" + "new file mode 100644\n" + "--- /dev/null\n" + "+++ b/src/new.py\n" + "@@ -0,0 +1 @@\n+x\n" + ) + + +def test_render_deleted_file(): + entry = {"old_path": "src/old.py", "new_path": "src/old.py", "deleted_file": True, "diff": "@@ -1 +0,0 @@\n-y\n"} + assert render_diff_entries([entry]) == ( + "diff --git a/src/old.py b/src/old.py\n" + "deleted file mode 100644\n" + "--- a/src/old.py\n" + "+++ /dev/null\n" + "@@ -1 +0,0 @@\n-y\n" + ) + + +def test_render_renamed_file_with_hunks(): + entry = { + "old_path": "docs/a.md", + "new_path": "docs/b.md", + "renamed_file": True, + "diff": "@@ -1 +1 @@\n-old\n+new\n", + } + assert render_diff_entries([entry]) == ( + "diff --git a/docs/a.md b/docs/b.md\n" + "rename from docs/a.md\n" + "rename to docs/b.md\n" + "--- a/docs/a.md\n" + "+++ b/docs/b.md\n" + "@@ -1 +1 @@\n-old\n+new\n" + ) + + +def test_render_modified_file(): + entry = {"old_path": "src/a.py", "new_path": "src/a.py", "diff": "@@ -1 +1 @@\n-a\n+b\n"} + assert render_diff_entries([entry]) == ( + "diff --git a/src/a.py b/src/a.py\n" + "--- a/src/a.py\n" + "+++ b/src/a.py\n" + "@@ -1 +1 @@\n-a\n+b\n" + ) + + +@pytest.mark.parametrize( + "entry", + [ + {"old_path": "img/logo.png", "new_path": "img/logo.png"}, + {"old_path": "img/logo.png", "new_path": "img/logo.png", "diff": None}, + {"old_path": "img/logo.png", "new_path": "img/logo.png", "diff": ""}, + ], + ids=["missing", "none", "empty"], +) +def test_render_entry_without_a_diff_is_header_only(entry): + assert render_diff_entries([entry]) == ( + "diff --git a/img/logo.png b/img/logo.png\n" + "--- a/img/logo.png\n" + "+++ b/img/logo.png\n" + ) + + +def test_render_new_file_without_a_diff_keeps_its_new_file_header(): + entry = {"old_path": "bin/tool", "new_path": "bin/tool", "new_file": True} + assert render_diff_entries([entry]) == ( + "diff --git a/bin/tool b/bin/tool\n" + "new file mode 100644\n" + "--- /dev/null\n" + "+++ b/bin/tool\n" + ) + + +def test_render_normalizes_the_newlines_around_a_hunk_body(): + unterminated = {"old_path": "a", "new_path": "a", "diff": "@@ -1 +1 @@\n-a\n+b"} + led_by_newline = {"old_path": "b", "new_path": "b", "diff": "\n@@ -1 +1 @@\n-a\n+b\n"} + assert render_diff_entries([unterminated, led_by_newline]) == ( + "diff --git a/a b/a\n--- a/a\n+++ b/a\n@@ -1 +1 @@\n-a\n+b\n" + "diff --git a/b b/b\n--- a/b\n+++ b/b\n@@ -1 +1 @@\n-a\n+b\n" + ) + + +def test_render_concatenates_entries_in_order_and_empty_renders_empty(): + first = {"old_path": "a", "new_path": "a", "diff": "@@ -1 +1 @@\n-a\n+b\n"} + second = {"old_path": "b", "new_path": "b"} + assert render_diff_entries([first, second]) == ( + render_diff_entries([first]) + render_diff_entries([second]) + ) + assert render_diff_entries([]) == "" + + +def test_gitlab_alias_is_the_shared_renderer(): + assert gitlab._render_diff_entries is render_diff_entries + assert gitlab._render_diff_entries is _diff_render.render_diff_entries + + +# --- github._iter_pages ------------------------------------------------------- + + +def test_iter_pages_reads_to_a_short_last_page(): + session = MagicMock(spec=requests.Session) + session.get.side_effect = [_page(PAGE_SIZE), _page(30, start=PAGE_SIZE)] + + pages = _walk(session) + + assert [len(p) for p in pages] == [PAGE_SIZE, 30] + assert [item["id"] for p in pages for item in p] == list(range(PAGE_SIZE + 30)) + assert session.get.call_count == 2 + assert [c.kwargs["params"]["page"] for c in session.get.call_args_list] == [1, 2] + assert all(c.args[0] == URL for c in session.get.call_args_list) + + +def test_iter_pages_stops_after_one_short_page(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _page(0) + + assert _walk(session) == [[]] + assert session.get.call_count == 1 + + +def test_iter_pages_sends_the_page_size_the_headers_and_extra_params(): + session = MagicMock(spec=requests.Session) + session.get.side_effect = [_page(PAGE_SIZE), _page(1)] + + _walk(session, extra_params={"state": "all"}) + + sent = [c.kwargs["params"] for c in session.get.call_args_list] + assert sent == [ + {"per_page": PAGE_SIZE, "page": 1, "state": "all"}, + {"per_page": PAGE_SIZE, "page": 2, "state": "all"}, + ] + assert github._PAGE_SIZE == PAGE_SIZE + assert all(c.kwargs["headers"] == HEADERS for c in session.get.call_args_list) + assert all(c.kwargs["timeout"] == github._REQUEST_TIMEOUT for c in session.get.call_args_list) + + +def test_iter_pages_without_extra_params_sends_only_paging(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _page(3) + + _walk(session) + + assert session.get.call_args.kwargs["params"] == {"per_page": PAGE_SIZE, "page": 1} + + +def test_iter_pages_skips_non_dict_items_but_counts_them_toward_a_full_page(): + session = MagicMock(spec=requests.Session) + full = [{"id": i} for i in range(PAGE_SIZE - 1)] + ["not a dict"] + session.get.side_effect = [_mock_response(json_data=full), _page(0)] + + pages = _walk(session) + + assert [len(p) for p in pages] == [PAGE_SIZE - 1, 0] + assert session.get.call_count == 2 + + +def test_iter_pages_raises_naming_what_on_a_non_ok_page(): + session = MagicMock(spec=requests.Session) + session.get.side_effect = [_page(PAGE_SIZE), _mock_response(500, json_data={"message": "boom"})] + + with pytest.raises(FeedReadError) as excinfo: + _walk(session, what="PR files listing") + + message = str(excinfo.value) + assert message == "PR files listing for acme/api#42 returned HTTP 500 at page 2" + assert "comment feed" not in message + + +def test_iter_pages_raises_naming_what_on_a_transport_failure(): + session = MagicMock(spec=requests.Session) + session.get.side_effect = requests.ConnectionError("down") + + with pytest.raises(FeedReadError, match=r"^PR files listing for acme/api#42 could not be read at page 1: down$"): + _walk(session, what="PR files listing") + + +def test_iter_pages_raises_naming_what_on_an_unreadable_body(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _mock_response(text="") + + with pytest.raises(FeedReadError, match=r"^PR files listing for acme/api#42 returned an unreadable body at page 1"): + _walk(session, what="PR files listing") + + +def test_iter_pages_raises_naming_what_on_a_body_that_is_not_a_list(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _mock_response(json_data={"message": "Not Found"}) + + with pytest.raises(FeedReadError, match=r"^PR files listing for acme/api#42 returned dict, not a list$"): + _walk(session, what="PR files listing") + + +def test_iter_pages_raises_when_the_page_budget_runs_out(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _page(PAGE_SIZE) + + with pytest.raises(FeedReadError, match="page budget") as excinfo: + _walk(session, what="PR files listing") + + assert session.get.call_count == github._MAX_PAGES == 50 + assert str(excinfo.value) == ( + "PR files listing for acme/api#42 outran the 50-page budget " + "(5000 entries) without reaching the end" + ) + + +def test_iter_pages_keeps_the_comment_feed_wording_for_comment_callers(): + session = MagicMock(spec=requests.Session) + session.get.return_value = _mock_response(403, json_data={"message": "rate limited"}) + + with pytest.raises(FeedReadError) as excinfo: + ForgeImpl(session=session).post_summary(_ref(), "body") + + assert str(excinfo.value) == "comment feed for acme/api#42 returned HTTP 403 at page 1" + session.post.assert_not_called() + + +def test_iter_pages_is_what_every_comment_reader_walks(): + for method in ("post_summary", "list_threads", "prune_inline_comments"): + session = MagicMock(spec=requests.Session) + session.get.side_effect = requests.ConnectionError("down") + forge = ForgeImpl(session=session) + seen: list[str] = [] + real = forge._iter_pages + + def spy(*args, _real=real, _seen=seen, **kwargs): + _seen.append(kwargs["what"]) + return _real(*args, **kwargs) + + forge._iter_pages = spy + args = (_ref(), "body") if method == "post_summary" else (_ref(),) + try: + getattr(forge, method)(*args) + except FeedReadError: + pass + assert seen == ["comment feed"], method From 791f638f0540fba49e2fd901528a481ff1dbc96a Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:33:10 -0700 Subject: [PATCH 05/64] feat: prompt-template override loader, validation and record (#11 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/prxref/prompt_templates.py, the loader behind PRXREF_PROMPTS_DIR / --prompts-dir. load_prompt_templates(path, *, source) reads worker.md, systemic.md and summary.md from an operator directory (any subset), falls back to the packaged templates via importlib.resources exactly as reviewer.load_prompt reads them, and returns a frozen PromptTemplates with the effective text of all three plus record(): {"dir", "templates": {name: {path, sha256, chars}}} for each overridden file, sha256 over the raw bytes. Validation is fail-fast, because a review template without the "## Review Context" marker raises in every chunk's render, is recorded as a crashed worker, and silently zeroes the review with exit 0. worker.md and systemic.md must keep the marker and, below it, every placeholder the packaged template has below it, minus the optional feature slots {scope_example} and {rule_example}; the required sets are computed from the packaged files at load, so a slot #13 adds becomes required automatically. summary.md requires only {findings}. A ConfigError starting with the source refuses a URL, a missing or non-directory path, a directory symlinked out of the working directory, a template symlinked out of the prompts directory, a non-regular file, a template over 256 KiB (never truncated), invalid UTF-8 and NUL bytes. Unknown placeholders, a known placeholder above the marker, a second marker, unrecognised files and an empty directory warn and load. Nothing calls the module yet: the renderer, orchestrator and CLI wiring are later tasks, so every existing run is byte-identical. Tests in tests/test_prompt_templates.py (81 cases). 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/prompt_templates.py | 338 +++++++++++++++++++++++++ tests/test_prompt_templates.py | 439 +++++++++++++++++++++++++++++++++ 2 files changed, 777 insertions(+) create mode 100644 src/prxref/prompt_templates.py create mode 100644 tests/test_prompt_templates.py diff --git a/src/prxref/prompt_templates.py b/src/prxref/prompt_templates.py new file mode 100644 index 00000000..76c84d6b --- /dev/null +++ b/src/prxref/prompt_templates.py @@ -0,0 +1,338 @@ +"""Operator prompt-template overrides: load, validate and fingerprint a prompts directory. + +``PRXREF_PROMPTS_DIR`` (or ``--prompts-dir DIR``) names a directory that may +hold any of ``worker.md``, ``systemic.md`` and ``summary.md``. Each file found +there replaces the packaged template of the same name for the whole run; a +file that is absent falls back to the packaged copy, read from +``prxref/prompts`` exactly as :func:`prxref.reviewer.load_prompt` reads it. +The judge prompt is not overridable, and no other name is read. + +The directory is loaded once, before any network call, and every failure is a +:class:`~prxref.llm.ConfigError` whose message starts with the ``source`` that +supplied the path, so the CLI exits 2 naming the env var or flag. Validation +is mandatory, not advisory: a review template without the +:data:`CONTEXT_MARKER` line raises inside every review unit's render, which +the orchestrator records as a crashed chunk, so an unvalidated template would +silently zero the review and still exit 0. + +What an override must contain is derived from the packaged templates at load, +never from a hand-kept list, so a slot added to a packaged template becomes +required of every override automatically: + +- ``worker.md`` and ``systemic.md`` must contain :data:`CONTEXT_MARKER` and, + after it, every placeholder the packaged template has after it, except the + feature slots in :data:`OPTIONAL_PLACEHOLDERS`. Only the text after the + marker is filled, so a placeholder above it does not count. +- ``summary.md`` must contain ``{findings}``; the other summary slots may be + dropped. + +Refused outright: a URL instead of a local path, a path that does not exist +or is not a directory, a directory that symlinks out of the working +directory, a template that symlinks out of the prompts directory, a template +that is not a regular file, one over :data:`MAX_TEMPLATE_BYTES` (never +truncated, because a cut template loses its placeholders), invalid UTF-8 and +NUL bytes. Logged as a WARNING and loaded anyway: a placeholder that is not a +slot of that template (a likely typo, which renders literally), a known +placeholder above the marker, a second marker, an unrecognised file in the +directory (dotfiles are skipped), and a directory holding none of the three +templates. + +Whoever controls the directory controls the reviewer's whole policy, so the +path must come from the operator's configuration and never from the pull +request under review; a PR that commits a prompts directory into its own +checkout rewrites its own review. This loader reads only a configured local +path, never through a forge. +""" +from __future__ import annotations + +import codecs +import errno +import hashlib +import logging +import os +import re +import stat +from dataclasses import dataclass +from importlib import resources + +from .llm import ConfigError +from .reviewer import _CONTEXT_MARKER +from .text_inputs import check_readable_path, confine_to_cwd, decode_text + +logger = logging.getLogger(__name__) + +CONTEXT_MARKER = _CONTEXT_MARKER +TEMPLATE_NAMES: tuple[str, ...] = ("worker", "systemic", "summary") +REVIEW_TEMPLATES: frozenset[str] = frozenset({"worker", "systemic"}) +OPTIONAL_PLACEHOLDERS: frozenset[str] = frozenset({"scope_example", "rule_example"}) +SUMMARY_REQUIRED: frozenset[str] = frozenset({"findings"}) +MAX_TEMPLATE_BYTES = 256 * 1024 +PLACEHOLDER_RE = re.compile(r"\{([A-Za-z_]\w*)\}") + +_URL_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://") + + +@dataclass(frozen=True) +class TemplateFile: + """One override read from the prompts directory. + + ``name`` is the template (``"worker"``, ``"systemic"`` or ``"summary"``); + ``path`` is the file's path as configured (the directory joined with the + file name, not resolved); ``text`` is the decoded text (a BOM dropped, + CRLF and CR folded to LF); ``sha256`` is the lowercase hex digest of the + raw file bytes, so it equals ``shasum -a 256 FILE``; ``chars`` is + ``len(text)``. + """ + + name: str + path: str + text: str + sha256: str + chars: int + + def record(self) -> dict[str, object]: + """Return the run-record fingerprint ``{path, sha256, chars}``, never the text.""" + return {"path": self.path, "sha256": self.sha256, "chars": self.chars} + + +@dataclass(frozen=True) +class PromptTemplates: + """The effective templates of one run: overrides where present, packaged text elsewhere. + + ``worker``, ``systemic`` and ``summary`` hold the text every render of + that template uses. ``dir`` is the directory as configured. ``overrides`` + holds one :class:`TemplateFile` per overridden template, in + :data:`TEMPLATE_NAMES` order, and is empty when the directory held none. + """ + + dir: str + worker: str + systemic: str + summary: str + overrides: tuple[TemplateFile, ...] = () + + @property + def overridden(self) -> tuple[str, ...]: + """The names of the overridden templates, in :data:`TEMPLATE_NAMES` order.""" + return tuple(f.name for f in self.overrides) + + def text(self, name: str) -> str: + """The effective text of template ``name``; any other name raises ``ValueError``.""" + _require_name(name) + return getattr(self, name) + + def override(self, name: str) -> str: + """The override text of template ``name``, or ``""`` when it is packaged. + + ``""`` is the renderers' "use the packaged template" value, so a caller + can pass this straight through and leave a non-overridden template on + the packaged path. Any other name raises ``ValueError``. + """ + _require_name(name) + for f in self.overrides: + if f.name == name: + return f.text + return "" + + def record(self) -> dict[str, object]: + """Return the run-record view, JSON-native values only and never template text. + + ``{"dir": , "templates": {: {"path", + "sha256", "chars"}}}``, with one entry per overridden template in + :data:`TEMPLATE_NAMES` order; a template left packaged has no entry. + """ + return {"dir": self.dir, "templates": {f.name: f.record() for f in self.overrides}} + + +def placeholders(text: str) -> frozenset[str]: + """Every ``{identifier}`` placeholder name in ``text``. + + Uses the brace-identifier shape :func:`prxref.reviewer.fill_template` + fills, so JSON braces and ``{ spaced }`` text are not placeholders. + """ + return frozenset(PLACEHOLDER_RE.findall(text)) + + +def required_placeholders(name: str) -> frozenset[str]: + """The placeholders an override of template ``name`` must contain. + + For ``worker`` and ``systemic``: every placeholder after + :data:`CONTEXT_MARKER` in the packaged template, minus + :data:`OPTIONAL_PLACEHOLDERS`, computed from the packaged file on every + call. For ``summary``: :data:`SUMMARY_REQUIRED`. Any other name raises + ``ValueError``. + """ + _require_name(name) + if name not in REVIEW_TEMPLATES: + return SUMMARY_REQUIRED + _, _, tail = _packaged_text(name).partition(CONTEXT_MARKER) + return placeholders(tail) - OPTIONAL_PLACEHOLDERS + + +def load_prompt_templates(path: str | os.PathLike[str] | None, *, source: str) -> PromptTemplates | None: + """Load and validate the prompt-template overrides in directory ``path``. + + ``None``, an empty or a whitespace-only path means "no overrides" and + returns ``None``. ``source`` names the input that supplied the path + (``PRXREF_PROMPTS_DIR`` or ``--prompts-dir``); every failure listed in + the module docstring is a :class:`~prxref.llm.ConfigError` whose message + starts with it, and every warning names it too. The directory is read + once: only the exact file names ``worker.md``, ``systemic.md`` and + ``summary.md`` are loaded, and a template not present falls back to its + packaged text. Makes no network call. + """ + if path is None: + return None + raw = os.fspath(path) + if not raw.strip(): + return None + if _URL_RE.match(raw.strip()): + raise ConfigError(f"{source}: prompts directory must be a local path, not a URL: {raw!r}") + root = _open_root(raw, source) + try: + entries = set(os.listdir(raw)) + except OSError as exc: + raise ConfigError(f"{source}: cannot list prompts directory {raw!r}: {_reason(exc)}") from exc + files = [f"{name}.md" for name in TEMPLATE_NAMES] + unknown = sorted(e for e in entries if not e.startswith(".") and e not in files) + if unknown: + logger.warning( + "%s: prompts directory %r: ignoring unrecognised %s; only %s are read", + source, raw, ", ".join(unknown), ", ".join(files), + ) + overrides = tuple( + _load_one(name, os.path.join(raw, f"{name}.md"), root, source) + for name in TEMPLATE_NAMES + if f"{name}.md" in entries + ) + if not overrides: + logger.warning( + "%s: prompts directory %r holds none of %s; the packaged templates are used", + source, raw, ", ".join(files), + ) + effective = {name: _packaged_text(name) for name in TEMPLATE_NAMES} + effective.update({f.name: f.text for f in overrides}) + return PromptTemplates(dir=raw, overrides=overrides, **effective) + + +def _open_root(raw: str, source: str) -> str: + try: + st = os.stat(raw) + except FileNotFoundError as exc: + raise ConfigError(f"{source}: prompts directory {raw!r} does not exist") from exc + except OSError as exc: + raise ConfigError(f"{source}: cannot read prompts directory {raw!r}: {_reason(exc)}") from exc + if not stat.S_ISDIR(st.st_mode): + raise ConfigError(f"{source}: prompts directory {raw!r} is not a directory") + try: + return confine_to_cwd(raw) + except PermissionError as exc: + raise ConfigError(f"{source}: prompts directory {raw!r} {_reason(exc)}") from exc + + +def _load_one(name: str, path: str, root: str, source: str) -> TemplateFile: + if not _is_within(os.path.realpath(path), root): + raise ConfigError(f"{source}: prompt template {path!r} resolves outside the prompts directory") + try: + resolved = check_readable_path(path, confine=True) + with open(resolved, "rb") as fh: + st = os.fstat(fh.fileno()) + if not stat.S_ISREG(st.st_mode): + raise OSError(errno.EINVAL, "not a regular file", path) + if st.st_size > MAX_TEMPLATE_BYTES: + raise _too_large(source, path, st.st_size) + data = fh.read(MAX_TEMPLATE_BYTES + 1) + except ConfigError: + raise + except (OSError, ValueError) as exc: + raise ConfigError(f"{source}: cannot read prompt template {path!r}: {_reason(exc)}") from exc + if len(data) > MAX_TEMPLATE_BYTES: + raise _too_large(source, path, len(data)) + try: + text = decode_text(data) + except UnicodeDecodeError as exc: + offset = exc.start + (len(codecs.BOM_UTF8) if data.startswith(codecs.BOM_UTF8) else 0) + raise ConfigError( + f"{source}: prompt template {path!r} is not UTF-8 text ({exc.reason} at byte {offset})" + ) from exc + if "\x00" in text: + raise ConfigError(f"{source}: prompt template {path!r} contains NUL bytes; expected Markdown text") + _validate(name, text, path, source) + return TemplateFile(name=name, path=path, text=text, sha256=hashlib.sha256(data).hexdigest(), chars=len(text)) + + +def _validate(name: str, text: str, path: str, source: str) -> None: + packaged = _packaged_text(name) + known = placeholders(packaged) + found = placeholders(text) + if name in REVIEW_TEMPLATES: + known |= OPTIONAL_PLACEHOLDERS + head, marker, tail = text.partition(CONTEXT_MARKER) + if not marker: + raise ConfigError( + f"{source}: prompt template {path!r} is missing the {CONTEXT_MARKER!r} marker, which " + "splits the system prompt (above it) from the user prompt (below it)" + ) + missing = required_placeholders(name) - placeholders(tail) + if missing: + raise ConfigError( + f"{source}: prompt template {path!r} is missing required placeholder(s) " + f"{_braced(missing)} after the {CONTEXT_MARKER!r} marker" + ) + if CONTEXT_MARKER in tail: + logger.warning( + "%s: prompt template %r has more than one %r marker; only the first splits the " + "prompt, and the rest reach the user prompt as text", + source, path, CONTEXT_MARKER, + ) + above = placeholders(head) & known + if above: + logger.warning( + "%s: prompt template %r has %s above the %r marker; only text below it is filled, " + "so these reach the system prompt literally", + source, path, _braced(above), CONTEXT_MARKER, + ) + else: + missing = SUMMARY_REQUIRED - found + if missing: + raise ConfigError( + f"{source}: prompt template {path!r} is missing the required {_braced(missing)} placeholder" + ) + unknown = found - known + if unknown: + logger.warning( + "%s: prompt template %r has unknown placeholder(s) %s, which render literally " + "(a typo?); %s.md fills %s", + source, path, _braced(unknown), name, _braced(known), + ) + + +def _packaged_text(name: str) -> str: + return resources.files("prxref").joinpath("prompts").joinpath(f"{name}.md").read_text(encoding="utf-8") + + +def _require_name(name: str) -> None: + if name not in TEMPLATE_NAMES: + raise ValueError(f"template name must be one of {', '.join(TEMPLATE_NAMES)}, got {name!r}") + + +def _too_large(source: str, path: str, size: int) -> ConfigError: + return ConfigError( + f"{source}: prompt template {path!r} is {size} bytes, over the {MAX_TEMPLATE_BYTES}-byte " + "(256 KiB) limit; templates are never truncated" + ) + + +def _braced(names: frozenset[str]) -> str: + return ", ".join(f"{{{n}}}" for n in sorted(names)) + + +def _reason(exc: BaseException) -> str: + return getattr(exc, "strerror", None) or str(exc) + + +def _is_within(path: str, root: str) -> bool: + try: + return os.path.commonpath([path, root]) == root + except ValueError: + return False diff --git a/tests/test_prompt_templates.py b/tests/test_prompt_templates.py new file mode 100644 index 00000000..0a69a07a --- /dev/null +++ b/tests/test_prompt_templates.py @@ -0,0 +1,439 @@ +"""Prompt-template overrides (#11 T1): the loader, validation and run record.""" +from __future__ import annotations + +import dataclasses +import hashlib +import json +import logging +import os +import re +from pathlib import Path + +import pytest + +from prxref import prompt_templates as pt +from prxref.llm import ConfigError +from prxref.prompt_templates import ( + CONTEXT_MARKER, + MAX_TEMPLATE_BYTES, + OPTIONAL_PLACEHOLDERS, + TEMPLATE_NAMES, + PromptTemplates, + load_prompt_templates, + placeholders, + required_placeholders, +) +from prxref.reviewer import _CONTEXT_MARKER, load_prompt + +SOURCES = ("PRXREF_PROMPTS_DIR", "--prompts-dir") +_SLOT = re.compile(r"\{([A-Za-z_]\w*)\}") + + +def _packaged(name: str) -> str: + return load_prompt(f"{name}.md") + + +def _override(name: str) -> str: + return "TEAM OVERRIDE: style-guide findings are welcome.\n\n" + _packaged(name) + + +def _prompts(work: Path, files: dict[str, str | bytes], dirname: str = "prompts") -> Path: + d = work / dirname + d.mkdir() + for fname, content in files.items(): + data = content if isinstance(content, bytes) else content.encode("utf-8") + (d / fname).write_bytes(data) + return d + + +def _warnings(caplog) -> list[str]: + return [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + + +@pytest.fixture +def work(tmp_path, monkeypatch) -> Path: + root = tmp_path / "work" + root.mkdir() + monkeypatch.chdir(root) + return root + + +REQUIRED_CASES = [(n, p) for n in ("worker", "systemic") for p in sorted(required_placeholders(n))] + + +class TestPackagedContract: + @pytest.mark.parametrize("name", TEMPLATE_NAMES) + def test_the_packaged_fallback_is_what_reviewer_load_prompt_reads(self, name): + assert pt._packaged_text(name) == load_prompt(f"{name}.md") + + def test_the_marker_is_the_one_the_renderer_splits_on(self): + assert CONTEXT_MARKER == _CONTEXT_MARKER == "## Review Context" + + @pytest.mark.parametrize("name", ["worker", "systemic"]) + def test_review_requirements_are_every_packaged_slot_below_the_marker_bar_the_optional_ones(self, name): + _, marker, tail = _packaged(name).partition(_CONTEXT_MARKER) + assert marker + expected = set(_SLOT.findall(tail)) - OPTIONAL_PLACEHOLDERS + assert required_placeholders(name) == expected + assert {"pr_title", "ticket_context", "spec_digest"} <= expected + assert "scope_example" not in expected + + def test_worker_requires_diff_and_systemic_requires_digest(self): + assert "diff" in required_placeholders("worker") + assert "digest" in required_placeholders("systemic") + + def test_summary_requires_only_findings(self): + assert required_placeholders("summary") == frozenset({"findings"}) + assert "findings" in placeholders(_packaged("summary")) + + def test_a_slot_added_to_a_packaged_template_is_required_of_every_override(self, work, monkeypatch): + original = pt._packaged_text + + def grown(name): + text = original(name) + return text + "\n{new_slot}\n" if name == "worker" else text + + d = _prompts(work, {"worker.md": _packaged("worker")}) + monkeypatch.setattr(pt, "_packaged_text", grown) + assert "new_slot" in required_placeholders("worker") + with pytest.raises(ConfigError, match=r"\{new_slot\}"): + load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + + def test_the_packaged_templates_copied_verbatim_load_without_warnings(self, work, caplog): + d = _prompts(work, {f"{n}.md": _packaged(n) for n in TEMPLATE_NAMES}) + with caplog.at_level(logging.INFO, logger="prxref"): + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + assert loaded.overridden == TEMPLATE_NAMES + for name in TEMPLATE_NAMES: + assert loaded.text(name) == _packaged(name) + + +class TestOverride: + def test_a_full_override_replaces_all_three_templates(self, work, caplog): + files = {f"{n}.md": _override(n) for n in TEMPLATE_NAMES} + d = _prompts(work, files) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert _warnings(caplog) == [] + assert isinstance(loaded, PromptTemplates) + assert (loaded.worker, loaded.systemic, loaded.summary) == tuple(_override(n) for n in TEMPLATE_NAMES) + assert loaded.overridden == ("worker", "systemic", "summary") + for name in TEMPLATE_NAMES: + assert loaded.override(name) == _override(name) + assert list(loaded.record()["templates"]) == ["worker", "systemic", "summary"] + + def test_a_partial_override_falls_back_to_the_packaged_templates(self, work): + d = _prompts(work, {"worker.md": _override("worker")}) + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert loaded.worker == _override("worker") + assert loaded.systemic == _packaged("systemic") + assert loaded.summary == _packaged("summary") + assert loaded.overridden == ("worker",) + assert loaded.override("worker") == _override("worker") + assert loaded.override("systemic") == "" + assert loaded.override("summary") == "" + assert list(loaded.record()["templates"]) == ["worker"] + + def test_a_summary_holding_only_findings_loads_silently(self, work, caplog): + d = _prompts(work, {"summary.md": "Review notes:\n\n{findings}\n"}) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert _warnings(caplog) == [] + assert loaded.summary == "Review notes:\n\n{findings}\n" + + @pytest.mark.parametrize("path", [None, "", " "]) + def test_no_path_means_no_overrides(self, path): + assert load_prompt_templates(path, source="PRXREF_PROMPTS_DIR") is None + + def test_the_record_is_json_native_and_holds_the_dir_as_configured(self, work): + raw = b"Summary:\n{findings}\n" + _prompts(work, {"summary.md": raw}) + loaded = load_prompt_templates("prompts", source="--prompts-dir") + record = loaded.record() + assert json.loads(json.dumps(record)) == record == { + "dir": "prompts", + "templates": { + "summary": { + "path": os.path.join("prompts", "summary.md"), + "sha256": hashlib.sha256(raw).hexdigest(), + "chars": len("Summary:\n{findings}\n"), + }, + }, + } + + def test_sha256_covers_the_raw_bytes_while_chars_counts_the_decoded_text(self, work): + raw = b"\xef\xbb\xbf" + _override("worker").replace("\n", "\r\n").encode("utf-8") + d = _prompts(work, {"worker.md": raw}) + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + entry = loaded.record()["templates"]["worker"] + assert entry["sha256"] == hashlib.sha256(raw).hexdigest() + assert entry["sha256"] != hashlib.sha256(loaded.worker.encode("utf-8")).hexdigest() + assert loaded.worker == _override("worker") + assert entry["chars"] == len(_override("worker")) + + def test_a_path_object_is_recorded_as_a_string(self, work): + d = _prompts(work, {"summary.md": "{findings}"}) + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert loaded.dir == os.fspath(d) + assert loaded.record()["templates"]["summary"]["path"] == os.path.join(os.fspath(d), "summary.md") + + def test_a_symlink_inside_the_directory_loads_and_hashes_its_target(self, work): + d = _prompts(work, {}) + (d / "variants").mkdir() + target = d / "variants" / "worker-v2.md" + target.write_bytes(_override("worker").encode("utf-8")) + (d / "worker.md").symlink_to(Path("variants") / "worker-v2.md") + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + assert loaded.worker == _override("worker") + assert loaded.record()["templates"]["worker"]["sha256"] == hashlib.sha256(target.read_bytes()).hexdigest() + + def test_an_absolute_directory_outside_the_working_directory_is_the_operators_choice(self, work, tmp_path): + d = _prompts(tmp_path, {"summary.md": "{findings}"}, dirname="ci-prompts") + loaded = load_prompt_templates(str(d), source="PRXREF_PROMPTS_DIR") + assert loaded.overridden == ("summary",) + + def test_the_result_is_immutable(self, work): + d = _prompts(work, {"worker.md": _override("worker")}) + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + with pytest.raises(dataclasses.FrozenInstanceError): + loaded.worker = "x" + with pytest.raises(dataclasses.FrozenInstanceError): + loaded.overrides[0].sha256 = "0" + assert isinstance(loaded.overrides, tuple) + + @pytest.mark.parametrize("method", ["text", "override"]) + def test_an_unknown_template_name_is_a_value_error(self, work, method): + d = _prompts(work, {"summary.md": "{findings}"}) + loaded = load_prompt_templates(d, source="PRXREF_PROMPTS_DIR") + with pytest.raises(ValueError, match="judge"): + getattr(loaded, method)("judge") + + +def _missing_dir(work: Path, tmp_path: Path) -> str: + return "no-such-prompts" + + +def _not_a_directory(work: Path, tmp_path: Path) -> str: + (work / "prompts.md").write_text("x") + return "prompts.md" + + +def _url(work: Path, tmp_path: Path) -> str: + return "https://example.com/prompts" + + +def _worker_without_marker(work: Path, tmp_path: Path) -> str: + _prompts(work, {"worker.md": _packaged("worker").replace(_CONTEXT_MARKER, "## Context")}) + return "prompts" + + +def _systemic_without_marker(work: Path, tmp_path: Path) -> str: + _prompts(work, {"systemic.md": _packaged("systemic").replace(_CONTEXT_MARKER, "## Context")}) + return "prompts" + + +def _summary_without_findings(work: Path, tmp_path: Path) -> str: + _prompts(work, {"summary.md": _packaged("summary").replace("{findings}", "")}) + return "prompts" + + +def _required_slot_only_above_the_marker(work: Path, tmp_path: Path) -> str: + head, marker, tail = _packaged("worker").partition(_CONTEXT_MARKER) + _prompts(work, {"worker.md": head + "{diff}\n" + marker + tail.replace("{diff}", "")}) + return "prompts" + + +def _over_the_size_cap(work: Path, tmp_path: Path) -> str: + data = _packaged("worker").encode("utf-8") + _prompts(work, {"worker.md": data + b"x" * (MAX_TEMPLATE_BYTES + 1 - len(data))}) + return "prompts" + + +def _not_utf8(work: Path, tmp_path: Path) -> str: + _prompts(work, {"summary.md": b"{findings}\n\xff\xfe latin-1 \xe9\n"}) + return "prompts" + + +def _nul_bytes(work: Path, tmp_path: Path) -> str: + _prompts(work, {"summary.md": b"{findings}\x00"}) + return "prompts" + + +def _symlink_out_of_the_directory(work: Path, tmp_path: Path) -> str: + (work / "elsewhere.md").write_text(_packaged("worker")) + d = _prompts(work, {}) + (d / "worker.md").symlink_to(Path("..") / "elsewhere.md") + return "prompts" + + +def _symlink_out_of_the_working_directory(work: Path, tmp_path: Path) -> str: + outside = tmp_path / "secret.md" + outside.write_text("{findings}") + d = _prompts(work, {}) + (d / "summary.md").symlink_to(outside) + return "prompts" + + +def _directory_symlink_out_of_the_working_directory(work: Path, tmp_path: Path) -> str: + real = _prompts(tmp_path, {"summary.md": "{findings}"}, dirname="outside-prompts") + (work / "prompts").symlink_to(real) + return "prompts" + + +def _template_is_a_directory(work: Path, tmp_path: Path) -> str: + d = _prompts(work, {}) + (d / "worker.md").mkdir() + return "prompts" + + +INVALID_CASES = [ + (_missing_dir, "does not exist"), + (_not_a_directory, "is not a directory"), + (_url, "not a URL"), + (_worker_without_marker, "missing the '## Review Context' marker"), + (_systemic_without_marker, "missing the '## Review Context' marker"), + (_summary_without_findings, "missing the required {findings} placeholder"), + (_required_slot_only_above_the_marker, "missing required placeholder(s) {diff} after"), + (_over_the_size_cap, f"{MAX_TEMPLATE_BYTES + 1} bytes, over the {MAX_TEMPLATE_BYTES}-byte (256 KiB) limit"), + (_not_utf8, "is not UTF-8 text"), + (_nul_bytes, "contains NUL bytes"), + (_symlink_out_of_the_directory, "resolves outside the prompts directory"), + (_symlink_out_of_the_working_directory, "resolves outside the prompts directory"), + (_directory_symlink_out_of_the_working_directory, "resolves outside the working directory"), + (_template_is_a_directory, "cannot read prompt template"), +] + + +class TestInvalid: + @pytest.mark.parametrize("source", SOURCES) + @pytest.mark.parametrize(("build", "expected"), INVALID_CASES, ids=[c[0].__name__[1:] for c in INVALID_CASES]) + def test_each_invalid_case_is_a_config_error_naming_the_source(self, work, tmp_path, build, expected, source): + path = build(work, tmp_path) + with pytest.raises(ConfigError) as excinfo: + load_prompt_templates(path, source=source) + message = str(excinfo.value) + assert message.startswith(f"{source}: ") + assert expected in message + + @pytest.mark.parametrize(("name", "slot"), REQUIRED_CASES, ids=[f"{n}-{p}" for n, p in REQUIRED_CASES]) + def test_dropping_any_required_placeholder_is_a_config_error(self, work, name, slot): + _prompts(work, {f"{name}.md": _packaged(name).replace(f"{{{slot}}}", "")}) + with pytest.raises(ConfigError, match=re.escape(f"{{{slot}}}")) as excinfo: + load_prompt_templates("prompts", source="--prompts-dir") + assert str(excinfo.value).startswith("--prompts-dir: ") + assert os.path.join("prompts", f"{name}.md") in str(excinfo.value) + + def test_every_missing_placeholder_is_named_at_once(self, work): + text = _packaged("systemic").replace("{digest}", "").replace("{pr_title}", "") + _prompts(work, {"systemic.md": text}) + with pytest.raises(ConfigError, match=re.escape("{digest}, {pr_title}")): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + + def test_a_template_of_exactly_256_kib_loads_and_is_not_truncated(self, work): + data = _packaged("worker").encode("utf-8") + data += b"x" * (MAX_TEMPLATE_BYTES - len(data)) + _prompts(work, {"worker.md": data}) + loaded = load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + assert loaded.worker.encode("utf-8") == data + assert loaded.record()["templates"]["worker"]["chars"] == len(data.decode("utf-8")) + + def test_the_utf8_error_reports_the_byte_offset(self, work): + _prompts(work, {"summary.md": b"{findings}\n\xff"}) + with pytest.raises(ConfigError, match="at byte 11"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + + def test_a_failing_template_refuses_the_whole_directory(self, work): + _prompts(work, {"worker.md": _override("worker"), "summary.md": "no slot here"}) + with pytest.raises(ConfigError, match="summary.md"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + + +class TestWarnings: + def test_a_typo_placeholder_warns_and_still_loads(self, work, caplog): + text = _packaged("worker").replace("{pr_title}", "{pr_title} {pr_titel}") + _prompts(work, {"worker.md": text}) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates("prompts", source="--prompts-dir") + assert loaded.worker == text + (message,) = _warnings(caplog) + assert message.startswith("--prompts-dir: ") + assert "unknown placeholder(s) {pr_titel}" in message + assert "worker.md" in message + + def test_a_placeholder_foreign_to_the_summary_warns(self, work, caplog): + _prompts(work, {"summary.md": "{findings}\n{scope_example}\n{diff}"}) + with caplog.at_level(logging.WARNING, logger="prxref"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + (message,) = _warnings(caplog) + assert "unknown placeholder(s) {diff}, {scope_example}" in message + + def test_an_unknown_file_warns_and_dotfiles_are_skipped(self, work, caplog): + _prompts(work, { + "worker.md": _override("worker"), + "notes.md": "scratch", + "Summary.md": "{findings}", + ".DS_Store": b"\x00", + ".gitkeep": b"", + }) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + (message,) = _warnings(caplog) + assert message.startswith("PRXREF_PROMPTS_DIR: ") + assert "ignoring unrecognised Summary.md, notes.md" in message + assert ".DS_Store" not in message and ".gitkeep" not in message + assert loaded.overridden == ("worker",) + assert loaded.summary == _packaged("summary") + + @pytest.mark.parametrize("files", [{}, {".gitkeep": b""}], ids=["empty", "dotfiles-only"]) + def test_a_directory_without_templates_warns_and_uses_the_packaged_ones(self, work, caplog, files): + _prompts(work, files) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + (message,) = _warnings(caplog) + assert message.startswith("PRXREF_PROMPTS_DIR: ") + assert "holds none of worker.md, systemic.md, summary.md" in message + assert loaded.record() == {"dir": "prompts", "templates": {}} + assert loaded.overrides == () + for name in TEMPLATE_NAMES: + assert loaded.text(name) == _packaged(name) + assert loaded.override(name) == "" + + def test_a_known_placeholder_above_the_marker_warns(self, work, caplog): + head, marker, tail = _packaged("systemic").partition(_CONTEXT_MARKER) + _prompts(work, {"systemic.md": head + "Title: {pr_title}\n\n" + marker + tail}) + with caplog.at_level(logging.WARNING, logger="prxref"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + (message,) = _warnings(caplog) + assert "{pr_title} above the '## Review Context' marker" in message + + def test_a_second_marker_warns(self, work, caplog): + _prompts(work, {"worker.md": _packaged("worker") + "\n" + _CONTEXT_MARKER + "\n"}) + with caplog.at_level(logging.WARNING, logger="prxref"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + (message,) = _warnings(caplog) + assert "more than one '## Review Context' marker" in message + + +class TestOptionalSlots: + @pytest.mark.parametrize("name", ["worker", "systemic"]) + def test_dropping_scope_example_is_allowed_silently(self, work, caplog, name): + assert "scope_example" in placeholders(_packaged(name)) + text = _packaged(name).replace("{scope_example}", "") + _prompts(work, {f"{name}.md": text}) + with caplog.at_level(logging.WARNING, logger="prxref"): + loaded = load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + assert _warnings(caplog) == [] + assert loaded.text(name) == text + + @pytest.mark.parametrize("name", ["worker", "systemic"]) + def test_the_rule_example_feature_slot_is_known_but_not_required(self, work, caplog, name): + assert "rule_example" not in required_placeholders(name) + text = _packaged(name) + "\n{rule_example}\n" + _prompts(work, {f"{name}.md": text}) + with caplog.at_level(logging.WARNING, logger="prxref"): + load_prompt_templates("prompts", source="PRXREF_PROMPTS_DIR") + assert _warnings(caplog) == [] + + def test_the_optional_set_is_exactly_the_feature_slots(self): + assert OPTIONAL_PLACEHOLDERS == frozenset({"scope_example", "rule_example"}) From 0bf23de20f89bfd61fb51c649038a0cf70803b88 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:34:50 -0700 Subject: [PATCH 06/64] feat(config): land the seven 0.15.0 configuration keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds every 0.15.0 key in one place (decisions D2), so the feature seats read them from the merged tree rather than racing on the config tables: - PRXREF_DEDUP_SIMILARITY (#10): float, None = off, _Range(0.0, 1.0) - PRXREF_PROMPTS_DIR (#11): path string, None = packaged templates - PRXREF_SCOPED_RULES (#12): list key, [] = off - PRXREF_SCOPED_RULES_MAX_CHARS (#12): int, 24000, _Range(0) - PRXREF_GROUP_FINDINGS (#13): bool, False - PRXREF_MAX_WARNING_FINDINGS (#13): int, None = unlimited, >= 0 - PRXREF_MAX_OUTOFSCOPE_FINDINGS (#13): int, None = unlimited, >= 0 Each key is on all four surfaces (the config tables, the config.py docstring, .env.example and docs/env-vars.md), marked with the feature it switches and "0.15.0". The docs state the shipped behaviour. The minor-cap entry says outofscope is a severity, not ticket scope out. The _Range and _check_ranges docstrings name the new None-means-off keys and the second bounded quantity. The env-vars counts were recomputed from _DEFAULTS: 62 keys, 63 accepted names, 44 in LLM / Pipeline. The "only configuration that touches the filtering" sentence now names the four new filtering levers. No wiring: cli.py and orchestrator.py are untouched and no flag is added, so every run is unchanged until the feature seats land. tests/test_config.py gains 139 tests: default, table membership, coercion, range, None/empty unset and override precedence for each key. A bad value exits 2 naming its variable through `prxref review`. Every surface has exactly one entry per key that names 0.15.0, and the outofscope entries state the ticket-scope distinction. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- .env.example | 48 ++++++ docs/env-vars.md | 15 +- src/prxref/config.py | 98 +++++++++-- tests/test_config.py | 393 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 539 insertions(+), 15 deletions(-) diff --git a/.env.example b/.env.example index ccfcc3cb..9474f8a6 100644 --- a/.env.example +++ b/.env.example @@ -83,6 +83,31 @@ PRXREF_CONFIDENCE_FLOOR=0.6 # (Legacy alias: PRXREF_MAX_ERRORS.) PRXREF_MAX_ERROR_FINDINGS=10 +# Per-severity caps (0.15.0): max warning-severity findings reported per +# review, the excess dropped lowest-confidence-first. Must be >= 0; 0 caps +# every warning. Empty (the default) = unlimited. +# PRXREF_MAX_WARNING_FINDINGS= + +# Per-severity caps (0.15.0): the same cap for the minor tier, findings of +# severity "outofscope". That is a severity, NOT ticket scope "out" from +# PRXREF_TICKET_CONTEXT_FILE. "spec" findings are never capped. Must be >= 0; +# empty (the default) = unlimited. +# PRXREF_MAX_OUTOFSCOPE_FINDINGS= + +# Finding grouping (0.15.0): set to exactly 1 to fold findings that break the +# same rule in the same file into one comment listing the other locations +# ("Also at: ..."); without a rule, the normalized title groups them. Runs +# before the caps, so PRXREF_MAX_ERROR_FINDINGS counts groups. Default off: +# prompts and output are unchanged. +# PRXREF_GROUP_FINDINGS=0 + +# Reworded-duplicate dedup (0.15.0): two findings on the same file and line +# whose titles score at or above this Jaccard similarity (and share at least +# 3 title words) are one finding; a chunk copy beats a sweep copy of equal or +# lower severity. Greater than 0 and at most 1.0. Empty (the default) = off: +# only the exact-title dedup runs. Not yet tuned on real review data. +# PRXREF_DEDUP_SIMILARITY= + # Max diff chunks reviewed per PR (default 8). Must be > 0. PRXREF_MAX_CHUNKS=8 @@ -204,6 +229,29 @@ PRXREF_MAX_CHUNKS=8 # a longer body is truncated with a warning. Must be > 0. # PRXREF_REVIEW_RULES_MAX_CHARS=12000 +# Path-scoped review rules (0.15.0): rules files and directories (*.md, one +# level deep) whose `applies_to:` front-matter globs pick the chunks each file +# reaches; the whole-PR sweep gets the union. Added to PRXREF_REVIEW_RULES, +# never replacing it. A file without applies_to reaches every unit; a URL, +# `applies_to: []` or a malformed file is a configuration error (exit 2). +# Comma- or whitespace-separated; repeatable `--scoped-rules PATH` flags +# replace this list. Empty (the default) = off. Read from a trusted checkout. +# PRXREF_SCOPED_RULES= + +# Path-scoped review rules (0.15.0): characters of scoped-rules text one +# review unit (a chunk or the sweep) receives; the file that overflows is +# truncated with a warning. Must be > 0. +# PRXREF_SCOPED_RULES_MAX_CHARS=24000 + +# Prompt template overrides (0.15.0): directory of replacement worker.md, +# systemic.md and summary.md; an absent file keeps the packaged template. Each +# one is validated before any network call, and a bad one is a configuration +# error (exit 2). `--prompts-dir DIR` wins for one run; `prxref prompts export +# DIR` writes the packaged templates to start from. Read it from a trusted +# checkout: a PR that commits a template rewrites its own review. Empty (the +# default) = the packaged templates. +# PRXREF_PROMPTS_DIR= + # Plain-text or Markdown file holding the ticket this PR implements. When set, # every finding is marked in, out of, or of unknown ticket scope. An empty # file means "this PR has no ticket". A missing or non-UTF-8 file is a diff --git a/docs/env-vars.md b/docs/env-vars.md index d585f416..e82e0f53 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -23,6 +23,10 @@ Configuration is loaded from built-in defaults, overridden by environment variab | `PRXREF_LLM_CLI_CONCURRENCY` | `2` | `claude-cli` / `kiro-cli` only: how many CLI processes one client may run at once. Must be **greater than 0**. Each call is a full CLI process, and subscription limits are per account. | | `PRXREF_CONFIDENCE_FLOOR` | `0.6` | Minimum confidence score. Must be **within `[0.0, 1.0]` inclusive** — it is a probability everywhere in the pipeline. Findings below the floor are dropped. | | `PRXREF_MAX_ERROR_FINDINGS` | `10` | Maximum number of error-severity findings reported per review. Excess errors are dropped lowest-confidence-first. Must be **>= 0**; `0` is legal and caps every error. (Legacy alias: `PRXREF_MAX_ERRORS`.) | +| `PRXREF_MAX_WARNING_FINDINGS` | *(empty — unlimited)* | **Per-severity caps (0.15.0).** Maximum number of warning-severity findings reported per review. Excess warnings are dropped lowest-confidence-first, with `drop_reason` `warning cap exceeded (max N)`. Must be **>= 0**; `0` is legal and caps every warning, distinct from unset, which caps nothing. | +| `PRXREF_MAX_OUTOFSCOPE_FINDINGS` | *(empty — unlimited)* | **Per-severity caps (0.15.0).** The same cap for the minor tier: findings of severity `outofscope`, dropped past the cap with `outofscope cap exceeded (max N)`. `outofscope` is a finding **severity**; it is **not** ticket scope `out` from `PRXREF_TICKET_CONTEXT_FILE`, so a finding marked out of the ticket's scope counts against its own severity's cap, not this one. `spec` findings are never capped. Must be **>= 0**; unset caps nothing. | +| `PRXREF_GROUP_FINDINGS` | `False` | **Finding grouping (0.15.0).** Set to the literal `1` to fold chunk findings that break the same rule in the same file into one comment: the model is asked to name the rule it applied, and findings without a rule are grouped by normalized title. The comment sits at the group's first line with its highest severity and confidence and lists the other locations as `Also at: …`; the other members are dropped with `grouped into :`. Grouping runs before the caps, so `PRXREF_MAX_ERROR_FINDINGS` counts groups, not lines. Whole-PR sweep findings are never grouped. Off by default, which keeps the prompts and the output unchanged. | +| `PRXREF_DEDUP_SIMILARITY` | *(empty — off)* | **Reworded-duplicate dedup (0.15.0).** Title similarity at or above which two findings in the same file and on the same line are treated as one: Jaccard over a dedicated title tokenizer, and the titles must also share at least 3 tokens. A chunk copy is kept over a sweep copy of equal or lower severity; within one side the more severe, then the more confident, finding is kept. The dropped one reads `duplicate of finding (reworded, similarity S)`. Must be **greater than 0 and at most 1.0**. Unset leaves only the exact-title dedup running, so output is unchanged; the threshold is not yet tuned on real review data. No CLI flag. | | `PRXREF_MAX_CHUNKS` | `8` | Maximum number of diff chunks reviewed per PR. Must be **greater than 0**. Overridable per run with `--max-chunks`. | | `PRXREF_CHUNK_TOKEN_BUDGET` | `25000` | Approximate token budget per diff chunk. Must be **greater than 0**. Lowering it splits a PR into more, smaller chunks: more LLM calls, but less diff per call. Chunk count is still capped by `PRXREF_MAX_CHUNKS`, and overflow past that cap lands in the smallest existing chunk rather than opening a new one. | | `PRXREF_CHUNK_MAX_FILES` | `5` | Maximum files placed in one review chunk. Must be **greater than 0**. Chunks stay under the cap while any chunk has room; once `PRXREF_MAX_CHUNKS` is reached and every chunk is full, overflow files join the smallest chunk past the cap rather than being dropped from review. | @@ -45,6 +49,9 @@ Configuration is loaded from built-in defaults, overridden by environment variab | `PRXREF_SPEC_DIGEST_TOKENS` | `3000` | Token budget for the final spec-constraint digest injected into worker prompts (estimated at 4 characters per token, like the systemic digest). Must be **greater than 0**. | | `PRXREF_REVIEW_RULES` | *(empty — off)* | Path to a team review-rules file (Markdown, with optional front matter carrying a `severity:` map) added to every review prompt. A missing, unreadable or malformed file raises `ConfigError` naming its source and `prxref review` exits `2`. `--rules-file PATH` wins for one run, and `--rules-file ""` turns the file off. Read it from a trusted checkout: in CI the workspace is usually the PR's own code, so a rules file inside it lets the PR rewrite its own review rules. See [docs/review-rules.md](review-rules.md). | | `PRXREF_REVIEW_RULES_MAX_CHARS` | `12000` | Characters of the rules body (after the front matter) kept in the prompt; a longer body is truncated with a warning. Must be **greater than 0**. | +| `PRXREF_SCOPED_RULES` | *(empty — off)* | **Path-scoped review rules (0.15.0).** Rules files and directories (each directory scanned one level deep for `*.md`, in name order, at most 50 files) whose front-matter `applies_to:` globs (alias `applyTo`) decide which review units get each file: a chunk gets every file matching one of its paths, and the whole-PR sweep gets the union. Globs match like `PRXREF_SIZE_IGNORE_GLOBS` (case-sensitive `fnmatch`, `*` crosses `/`), plus a `!` pattern that excludes, and a `**/` that also matches zero directories (for this key only). A file without `applies_to` reaches every unit; a URL, a path escaping the working directory, `applies_to: []`, a leading `/`, or two files mapping one word to different severity tiers raises `ConfigError` and `prxref review` exits `2`. Each file is capped at `PRXREF_REVIEW_RULES_MAX_CHARS`. Added to `PRXREF_REVIEW_RULES`, never replacing it. Comma- or whitespace-separated; the repeatable `--scoped-rules PATH` flag replaces this list for one run and can name a path containing a space. Read it from a trusted checkout, as with `PRXREF_REVIEW_RULES`. Unset leaves the prompts unchanged. | +| `PRXREF_SCOPED_RULES_MAX_CHARS` | `24000` | **Path-scoped review rules (0.15.0).** Characters of scoped-rules text one review unit (a chunk or the sweep) receives: whole files in name order, the first that does not fit is truncated with an omission marker, and one warning per run names this variable. Must be **greater than 0**. | +| `PRXREF_PROMPTS_DIR` | *(empty — off)* | **Prompt template overrides (0.15.0).** Directory holding replacement `worker.md`, `systemic.md` and `summary.md` prompt templates; a file that is absent keeps the packaged one. Every template is validated before any network call: `worker.md` and `systemic.md` must keep the `## Review Context` marker and every packaged placeholder below it (the feature slots `{scope_example}` and `{rule_example}` are optional), and `summary.md` needs only `{findings}`. A missing directory, a failed check, or a template over 256 KiB raises `ConfigError` and `prxref review` exits `2`; an empty directory or an unknown placeholder only warns. The run record stamps each template's source and SHA-256. `--prompts-dir DIR` wins for one run, and `prxref prompts export DIR [--force]` writes the packaged templates to start from. The judge prompt of `prxref eval` cannot be overridden. Read it from a trusted checkout: a PR that commits a template rewrites its own review. Unset uses the packaged templates, byte for byte. | | `PRXREF_TICKET_CONTEXT_FILE` | *(empty — off)* | Plain-text or Markdown file holding the ticket this PR implements. When set, every finding is marked in, out of, or of unknown ticket scope. An empty or whitespace-only file means "this PR has no ticket". A missing or non-UTF-8 file raises `ConfigError` and `prxref review` exits `2`. The webhook daemon ignores it (and says so once). `--context-file PATH` wins for one run, and `--context-file ""` turns it off. | | `PRXREF_TICKET_CONTEXT_MAX_CHARS` | `6000` | Characters of ticket text kept in the prompt; longer text is truncated with a visible marker. Must be **greater than 0**. | @@ -148,15 +155,15 @@ Neither knob affects the exit code — `PRXREF_FAIL_ON` is the only one that can ## Quality Passes and Drop Reasons -The two knobs above are the only configuration that touches the filtering. The eleven deterministic passes themselves, the team severity map and spec grounding that run before them, the release-shaped-PR check, and every `drop_reason` string they emit are documented in one place: **[docs/quality.md](quality.md)**. Everything on that page other than the confidence floor and the error cap is a correctness check against the diff itself, not a noise lever, and has no environment variable. +The two knobs above, plus the four opt-in levers added in 0.15.0 (`PRXREF_MAX_WARNING_FINDINGS`, `PRXREF_MAX_OUTOFSCOPE_FINDINGS`, `PRXREF_GROUP_FINDINGS` and `PRXREF_DEDUP_SIMILARITY`, all off by default), are the only configuration that touches the filtering. The eleven deterministic passes themselves, the team severity map and spec grounding that run before them, the release-shaped-PR check, and every `drop_reason` string they emit are documented in one place: **[docs/quality.md](quality.md)**. Everything on that page other than the confidence floor, the per-severity caps, finding grouping and reworded-duplicate dedup is a correctness check against the diff itself, not a noise lever, and has no environment variable. ## Environment Cross-Check & Defaults -The tables above define all **55** configuration keys in `src/prxref/config.py` (`_DEFAULTS`), and every one of them appears in `.env.example`: +The tables above define all **62** configuration keys in `src/prxref/config.py` (`_DEFAULTS`), and every one of them appears in `.env.example`: -- **LLM / Pipeline (37):** `PRXREF_LLM_BACKEND`, `PRXREF_LLM_BASE_URL`, `PRXREF_LLM_API_KEY`, `PRXREF_LLM_MODELS`, `PRXREF_LLM_REASONING_EFFORT`, `PRXREF_LLM_MAX_TOKENS`, `PRXREF_LLM_TIMEOUT`, `PRXREF_LLM_TEMPERATURE`, `PRXREF_LLM_SEED`, `PRXREF_LLM_CLI_PATH`, `PRXREF_LLM_CLI_CONCURRENCY`, `PRXREF_CONFIDENCE_FLOOR`, `PRXREF_MAX_ERROR_FINDINGS`, `PRXREF_MAX_CHUNKS`, `PRXREF_CHUNK_TOKEN_BUDGET`, `PRXREF_CHUNK_MAX_FILES`, `PRXREF_CHUNK_CONTEXT_LINES`, `PRXREF_MAX_WORKERS`, `PRXREF_MAX_INLINE_COMMENTS`, `PRXREF_FAIL_ON`, `PRXREF_DRY_RUN`, `PRXREF_TRACE_FILE`, `PRXREF_TRACE_DIR`, `PRXREF_POST_MODE`, `PRXREF_POST_VERDICT`, `PRXREF_PRICE_TABLE`, `PRXREF_POST_COST`, `PRXREF_SIZE_WARN_LINES`, `PRXREF_SIZE_WARN_FILES`, `PRXREF_SIZE_IGNORE_GLOBS`, `PRXREF_SPEC_SOURCES`, `PRXREF_SPEC_MAX_CHARS`, `PRXREF_SPEC_DIGEST_TOKENS`, `PRXREF_REVIEW_RULES`, `PRXREF_REVIEW_RULES_MAX_CHARS`, `PRXREF_TICKET_CONTEXT_FILE`, `PRXREF_TICKET_CONTEXT_MAX_CHARS` +- **LLM / Pipeline (44):** `PRXREF_LLM_BACKEND`, `PRXREF_LLM_BASE_URL`, `PRXREF_LLM_API_KEY`, `PRXREF_LLM_MODELS`, `PRXREF_LLM_REASONING_EFFORT`, `PRXREF_LLM_MAX_TOKENS`, `PRXREF_LLM_TIMEOUT`, `PRXREF_LLM_TEMPERATURE`, `PRXREF_LLM_SEED`, `PRXREF_LLM_CLI_PATH`, `PRXREF_LLM_CLI_CONCURRENCY`, `PRXREF_CONFIDENCE_FLOOR`, `PRXREF_MAX_ERROR_FINDINGS`, `PRXREF_MAX_WARNING_FINDINGS`, `PRXREF_MAX_OUTOFSCOPE_FINDINGS`, `PRXREF_GROUP_FINDINGS`, `PRXREF_DEDUP_SIMILARITY`, `PRXREF_MAX_CHUNKS`, `PRXREF_CHUNK_TOKEN_BUDGET`, `PRXREF_CHUNK_MAX_FILES`, `PRXREF_CHUNK_CONTEXT_LINES`, `PRXREF_MAX_WORKERS`, `PRXREF_MAX_INLINE_COMMENTS`, `PRXREF_FAIL_ON`, `PRXREF_DRY_RUN`, `PRXREF_TRACE_FILE`, `PRXREF_TRACE_DIR`, `PRXREF_POST_MODE`, `PRXREF_POST_VERDICT`, `PRXREF_PRICE_TABLE`, `PRXREF_POST_COST`, `PRXREF_SIZE_WARN_LINES`, `PRXREF_SIZE_WARN_FILES`, `PRXREF_SIZE_IGNORE_GLOBS`, `PRXREF_SPEC_SOURCES`, `PRXREF_SPEC_MAX_CHARS`, `PRXREF_SPEC_DIGEST_TOKENS`, `PRXREF_REVIEW_RULES`, `PRXREF_REVIEW_RULES_MAX_CHARS`, `PRXREF_SCOPED_RULES`, `PRXREF_SCOPED_RULES_MAX_CHARS`, `PRXREF_PROMPTS_DIR`, `PRXREF_TICKET_CONTEXT_FILE`, `PRXREF_TICKET_CONTEXT_MAX_CHARS` - **Per-Forge Auth (10):** `PRXREF_BITBUCKET_TOKEN`, `PRXREF_BITBUCKET_USER`, `PRXREF_BITBUCKET_APP_PASSWORD`, `PRXREF_BITBUCKET_SERVER_TOKEN`, `PRXREF_BITBUCKET_SERVER_USER`, `PRXREF_BITBUCKET_SERVER_PASSWORD`, `PRXREF_GITHUB_TOKEN`, `PRXREF_GITHUB_ENTERPRISE_TOKEN`, `PRXREF_GITLAB_TOKEN`, `PRXREF_AZURE_DEVOPS_TOKEN` - **Spec Sources / Jira (3):** `PRXREF_JIRA_BASE_URL`, `PRXREF_JIRA_EMAIL`, `PRXREF_JIRA_API_TOKEN` - **Webhooks (5):** `PRXREF_BITBUCKET_WEBHOOK_SECRET`, `PRXREF_GITHUB_WEBHOOK_SECRET`, `PRXREF_GITLAB_WEBHOOK_SECRET`, `PRXREF_AZURE_DEVOPS_WEBHOOK_SECRET`, `PRXREF_ALLOW_UNSIGNED` -*(55 configuration keys, plus one deprecated alias — `PRXREF_MAX_ERRORS` for `PRXREF_MAX_ERROR_FINDINGS` — for 56 accepted variable names.)* +*(62 configuration keys, plus one deprecated alias — `PRXREF_MAX_ERRORS` for `PRXREF_MAX_ERROR_FINDINGS` — for 63 accepted variable names.)* diff --git a/src/prxref/config.py b/src/prxref/config.py index 9ced00e6..efb944ed 100644 --- a/src/prxref/config.py +++ b/src/prxref/config.py @@ -49,6 +49,34 @@ PRXREF_MAX_ERROR_FINDINGS Max error-severity findings reported per review; >= 0, where 0 caps every error (legacy alias: PRXREF_MAX_ERRORS) + PRXREF_MAX_WARNING_FINDINGS Per-severity caps (0.15.0): max + warning-severity findings reported per + review, the excess dropped + lowest-confidence-first; >= 0, where 0 caps + every warning. Unset (default) = unlimited + PRXREF_MAX_OUTOFSCOPE_FINDINGS + Per-severity caps (0.15.0): the same cap for + the minor tier, ``outofscope``-severity + findings. ``outofscope`` is a severity, not + ticket scope ``out`` (see + PRXREF_TICKET_CONTEXT_FILE); ``spec`` is + never capped. >= 0; unset (default) = + unlimited + PRXREF_GROUP_FINDINGS Finding grouping (0.15.0): literal "1" folds + chunk findings that break the same rule in + the same file (by normalized title when the + model names no rule) into one comment that + lists the other locations; runs before the + caps, so they count groups. Default off: + prompts and output unchanged + PRXREF_DEDUP_SIMILARITY Reworded-duplicate dedup (0.15.0): title + similarity (Jaccard over title tokens, at + least 3 shared) at or above which two + findings on the same file and line are one; + a chunk copy beats a sweep copy of equal or + lower severity. Greater than 0 and at most + 1.0; unset (default) = off, only the + exact-title dedup runs PRXREF_MAX_CHUNKS Max diff chunks reviewed per PR; positive int (default 8) PRXREF_CHUNK_TOKEN_BUDGET Approximate token budget per diff chunk; @@ -171,6 +199,31 @@ front matter) kept in the prompt; longer is truncated with a warning; positive int (default 12000) + PRXREF_SCOPED_RULES Path-scoped review rules (0.15.0): rules + files and directories (``*.md`` one level + deep) whose ``applies_to:`` front-matter + globs pick the chunks each file reaches; the + sweep gets the union. Added to + PRXREF_REVIEW_RULES, never replacing it. A + file without ``applies_to`` reaches every + unit; a URL, ``applies_to: []`` or a + malformed file is a configuration error. + The repeatable ``--scoped-rules PATH`` flag + replaces the list. Empty (the default) = off + PRXREF_SCOPED_RULES_MAX_CHARS Path-scoped review rules (0.15.0): + characters of scoped-rules text one review + unit receives; the file that overflows is + truncated with a warning; positive int + (default 24000) + PRXREF_PROMPTS_DIR Prompt template overrides (0.15.0): + directory of replacement ``worker.md``, + ``systemic.md`` and ``summary.md``; an + absent file keeps the packaged one. Each + template is validated before any network + call (marker, placeholders, 256 KiB), and a + failure is a configuration error. + ``--prompts-dir DIR`` wins. Unset (the + default) = the packaged templates PRXREF_TICKET_CONTEXT_FILE Path to a text file holding the ticket this PR implements; each finding is then marked in, out of, or of unknown ticket scope. An @@ -234,10 +287,10 @@ PRXREF_ALLOW_UNSIGNED literal "1" accepts unsigned webhooks (default off; insecure) -List-valued keys (PRXREF_LLM_MODELS, PRXREF_SPEC_SOURCES and -PRXREF_SIZE_IGNORE_GLOBS) split on any run of commas and/or whitespace, so no -item can contain either; a glob that must match a literal space writes it as -``?``. +List-valued keys (PRXREF_LLM_MODELS, PRXREF_SPEC_SOURCES, +PRXREF_SIZE_IGNORE_GLOBS and PRXREF_SCOPED_RULES) split on any run of commas +and/or whitespace, so no item can contain either; a glob that must match a +literal space writes it as ``?``. Precedence: built-in defaults < environment < ``overrides`` kwargs. An error names the source that actually supplied the offending value — the @@ -292,6 +345,10 @@ "llm_cli_concurrency": 2, "confidence_floor": DEFAULT_CONFIDENCE_FLOOR, "max_error_findings": DEFAULT_MAX_ERRORS, + "max_warning_findings": None, + "max_outofscope_findings": None, + "group_findings": False, + "dedup_similarity": None, "max_chunks": 8, "chunk_token_budget": DEFAULT_TOKEN_BUDGET, "chunk_max_files": DEFAULT_MAX_FILES_PER_CHUNK, @@ -323,6 +380,9 @@ "spec_digest_tokens": 3000, "review_rules": "", "review_rules_max_chars": 12000, + "scoped_rules": [], + "scoped_rules_max_chars": 24000, + "prompts_dir": None, "ticket_context_file": "", "ticket_context_max_chars": 6000, "jira_base_url": "", @@ -352,10 +412,15 @@ "spec_max_chars", "spec_digest_tokens", "llm_cli_concurrency", "review_rules_max_chars", "ticket_context_max_chars", "size_warn_lines", "size_warn_files", + "max_warning_findings", "max_outofscope_findings", "scoped_rules_max_chars", +}) +_FLOAT_KEYS = frozenset({"confidence_floor", "llm_timeout", "dedup_similarity"}) +_BOOL_KEYS = frozenset({ + "allow_unsigned", "dry_run", "post_verdict", "post_cost", "group_findings", +}) +_LIST_KEYS = frozenset({ + "llm_models", "spec_sources", "size_ignore_globs", "scoped_rules", }) -_FLOAT_KEYS = frozenset({"confidence_floor", "llm_timeout"}) -_BOOL_KEYS = frozenset({"allow_unsigned", "dry_run", "post_verdict", "post_cost"}) -_LIST_KEYS = frozenset({"llm_models", "spec_sources", "size_ignore_globs"}) # An enum-valued key has no numeric interval to check, so its legal vocabulary # is declared here instead and enforced on the same pass as the ranges. A @@ -380,15 +445,18 @@ class _Range(NamedTuple): machine-specific, and an invented limit would be worse than none. Only a semantically bounded quantity gets a real ``high``: today that is the confidence floor, which is a 0-1 probability everywhere in - ``triage.Finding`` and in the prompts. + ``triage.Finding`` and in the prompts, and the dedup similarity, a 0-1 + Jaccard score whose low end is open because 0 would merge every pair of + findings on a line. ``low_inclusive`` distinguishes "must be positive" from "must not be negative". Zero is meaningless for a token budget (it asks the model for an empty completion), for a timeout (every request fails instantly), for a worker count (``ThreadPoolExecutor`` rejects it) and for a chunk count (``build_chunks`` raises on the overflow branch). Zero IS meaningful for the - error cap, where it means "report no errors", for the context-line count, - where it means "emit the changed lines only", for the sampling seed, + error, warning and outofscope caps, where it means "report none of that + severity", for the context-line count, where it means "emit the changed + lines only", for the sampling seed, where 0 is a perfectly valid seed, and for the PR-size thresholds, where 0 flags any change at all. """ @@ -437,7 +505,11 @@ def describe(self) -> str: "ticket_context_max_chars": _Range(0), "size_warn_lines": _Range(0, low_inclusive=True), "size_warn_files": _Range(0, low_inclusive=True), + "max_warning_findings": _Range(0, low_inclusive=True), + "max_outofscope_findings": _Range(0, low_inclusive=True), + "scoped_rules_max_chars": _Range(0), "confidence_floor": _Range(0.0, 1.0, low_inclusive=True), + "dedup_similarity": _Range(0.0, 1.0), } _LEGACY_ENV_ALIASES: dict[str, str] = { @@ -498,7 +570,11 @@ def _check_ranges(cfg: dict[str, object], sources: dict[str, str]) -> None: "not configured" is not a violation. The PR-size thresholds (``size_warn_lines``, ``size_warn_files``) are the second such class: ``None`` means the advisory is off, because 0 is a legal threshold and - cannot double as "unset". Every value that is not ``None`` — + cannot double as "unset". The 0.15.0 keys add two more: the warning and + outofscope caps (``max_warning_findings``, ``max_outofscope_findings``), + where ``None`` means unlimited because 0 caps every finding of that + severity, and ``dedup_similarity``, where ``None`` means the + reworded-duplicate pass is off. Every value that is not ``None`` — including one smuggled in through an override — is still checked. """ for key, rng in sorted(_RANGES.items()): diff --git a/tests/test_config.py b/tests/test_config.py index 2905714c..2937e02b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ """Tests for prxref.config: env loading, type coercion, overrides, and forge factory.""" import os +import re import pytest import requests @@ -11,6 +12,7 @@ from prxref.llm import ConfigError from prxref.quality import DEFAULT_MAX_ERRORS from tests.conftest import clear_prxref_env, prxref_env_names +from tests.test_docs_consistency import SURFACES def _make_ref(forge: str) -> PRRef: @@ -1465,3 +1467,394 @@ def test_the_factory_owns_the_backend_vocabulary(self): def test_a_mixed_case_backend_loads(self, monkeypatch): monkeypatch.setenv("PRXREF_LLM_BACKEND", "Claude-CLI") assert load_config()["llm_backend"] == "Claude-CLI" + + +_KEYS_0_15 = [ + ("dedup_similarity", "PRXREF_DEDUP_SIMILARITY"), + ("prompts_dir", "PRXREF_PROMPTS_DIR"), + ("scoped_rules", "PRXREF_SCOPED_RULES"), + ("scoped_rules_max_chars", "PRXREF_SCOPED_RULES_MAX_CHARS"), + ("group_findings", "PRXREF_GROUP_FINDINGS"), + ("max_warning_findings", "PRXREF_MAX_WARNING_FINDINGS"), + ("max_outofscope_findings", "PRXREF_MAX_OUTOFSCOPE_FINDINGS"), +] + + +class TestDedupSimilarity: + """PRXREF_DEDUP_SIMILARITY (#10): a 0-1 Jaccard threshold, off when unset. + + ``None`` is the declared unset, like the size thresholds, so the + reworded-duplicate pass runs only when an operator asks for it. The low + bound is open: 0 would merge every pair of findings on a line. + """ + + def test_default_is_none_meaning_off(self): + assert config._DEFAULTS["dedup_similarity"] is None + assert load_config()["dedup_similarity"] is None + + def test_declared_as_a_float_key_with_an_open_low_bound(self): + assert "dedup_similarity" in config._FLOAT_KEYS + assert "dedup_similarity" not in config._INT_KEYS + assert config._RANGES["dedup_similarity"] == config._Range(0.0, 1.0) + assert config._RANGES["dedup_similarity"].low_inclusive is False + + @pytest.mark.parametrize("raw,expected", [ + ("0.5", 0.5), ("1", 1.0), ("1.0", 1.0), (" 0.75 ", 0.75), ("0.01", 0.01), + ]) + def test_env_coerces_to_a_float(self, monkeypatch, raw, expected): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", raw) + value = load_config()["dedup_similarity"] + assert value == expected + assert isinstance(value, float) + + @pytest.mark.parametrize("raw", ["", " "]) + def test_empty_or_whitespace_reads_as_unset(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", raw) + assert load_config()["dedup_similarity"] is None + + @pytest.mark.parametrize("raw", ["0", "0.0", "-0.1", "1.01", "2", "nan", "inf"]) + def test_out_of_band_value_rejected(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", raw) + with pytest.raises(ConfigError, match=r"^PRXREF_DEDUP_SIMILARITY: "): + load_config() + + @pytest.mark.parametrize("raw", ["high", "50%", "0,5"]) + def test_malformed_value_names_the_variable(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", raw) + with pytest.raises(ConfigError, match=r"^PRXREF_DEDUP_SIMILARITY: "): + load_config() + + def test_message_states_the_bound_it_broke(self, monkeypatch): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", "0") + with pytest.raises(ConfigError) as exc: + load_config() + assert "greater than 0.0" in str(exc.value) + assert "at most 1.0" in str(exc.value) + + @pytest.mark.parametrize("value", [0, 0.0, 1.5, -1, float("nan"), True]) + def test_an_override_cannot_smuggle_an_out_of_band_value(self, value): + with pytest.raises(ConfigError, match=r"^dedup_similarity: ") as exc: + load_config(dedup_similarity=value) + assert "PRXREF_DEDUP_SIMILARITY" not in str(exc.value) + + def test_an_override_wins_over_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", "0.9") + assert load_config(dedup_similarity=0.5)["dedup_similarity"] == 0.5 + + +class TestPromptsDir: + """PRXREF_PROMPTS_DIR (#11): a path string, ``None`` when unset. + + The directory is only read by the prompt-template loader, so + ``load_config`` stays I/O-free for it and passes the value through. + """ + + def test_default_is_none(self): + assert config._DEFAULTS["prompts_dir"] is None + assert load_config()["prompts_dir"] is None + + def test_in_no_coercion_table(self): + assert "prompts_dir" not in ( + config._INT_KEYS | config._FLOAT_KEYS | config._BOOL_KEYS | config._LIST_KEYS + ) + assert "prompts_dir" not in config._RANGES + assert "prompts_dir" not in config._CHOICE_KEYS + + @pytest.mark.parametrize("raw", [".prxref/prompts", "team prompts/v2", "~/prompts"]) + def test_env_value_passes_through_verbatim(self, monkeypatch, raw): + """Not a list key: a path containing a space survives whole.""" + monkeypatch.setenv("PRXREF_PROMPTS_DIR", raw) + assert load_config()["prompts_dir"] == raw + + @pytest.mark.parametrize("raw", ["", " "]) + def test_empty_or_whitespace_reads_as_unset(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_PROMPTS_DIR", raw) + assert load_config()["prompts_dir"] is None + + def test_a_missing_dir_is_not_read_at_load_time(self, monkeypatch, tmp_path): + monkeypatch.setenv("PRXREF_PROMPTS_DIR", str(tmp_path / "absent")) + assert load_config()["prompts_dir"].endswith("absent") + + def test_an_empty_string_override_arrives_as_the_empty_string(self, monkeypatch): + """Only ``None`` overrides are ignored, so a ``--prompts-dir ""`` style + override reaches the caller as ``""``: consumers treat both ``None`` and + ``""`` as off.""" + monkeypatch.setenv("PRXREF_PROMPTS_DIR", ".prxref/prompts") + assert load_config(prompts_dir="")["prompts_dir"] == "" + + def test_an_override_wins_over_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_PROMPTS_DIR", "env-dir") + assert load_config(prompts_dir="flag-dir")["prompts_dir"] == "flag-dir" + + +class TestScopedRules: + """PRXREF_SCOPED_RULES (#12): the fourth list key, empty by default.""" + + def test_default_is_an_empty_list(self): + assert config._DEFAULTS["scoped_rules"] == [] + assert load_config()["scoped_rules"] == [] + + def test_declared_as_a_list_key(self): + assert "scoped_rules" in config._LIST_KEYS + assert "scoped_rules" not in config._RANGES + assert "scoped_rules" not in config._CHOICE_KEYS + + @pytest.mark.parametrize("raw,expected", [ + (".prxref/rules", [".prxref/rules"]), + ("rules/java.md,rules/helm.md", ["rules/java.md", "rules/helm.md"]), + ("rules/java.md rules/helm.md", ["rules/java.md", "rules/helm.md"]), + (" rules/java.md ,\n .prxref/scoped\t", ["rules/java.md", ".prxref/scoped"]), + ]) + def test_splits_on_commas_and_whitespace(self, monkeypatch, raw, expected): + monkeypatch.setenv("PRXREF_SCOPED_RULES", raw) + assert load_config()["scoped_rules"] == expected + + def test_whitespace_only_reads_as_unset(self, monkeypatch): + monkeypatch.setenv("PRXREF_SCOPED_RULES", " \t ") + assert load_config()["scoped_rules"] == [] + + @pytest.mark.parametrize("value", [[""], ["team rules/java.md"]]) + def test_an_override_list_is_taken_as_given(self, value): + """No split and no blank-dropping on the override path: a flag can name + a path with a space, and the loader drops blank entries itself.""" + assert load_config(scoped_rules=value)["scoped_rules"] == value + + def test_an_override_replaces_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_SCOPED_RULES", "a.md,b.md") + assert load_config(scoped_rules=["c.md"])["scoped_rules"] == ["c.md"] + + +class TestScopedRulesMaxChars: + """PRXREF_SCOPED_RULES_MAX_CHARS (#12): the per-unit total, int > 0.""" + + def test_default(self): + assert config._DEFAULTS["scoped_rules_max_chars"] == 24000 + value = load_config()["scoped_rules_max_chars"] + assert value == 24000 + assert isinstance(value, int) + + def test_declared_in_the_int_and_range_tables(self): + assert "scoped_rules_max_chars" in config._INT_KEYS + assert config._RANGES["scoped_rules_max_chars"] == config._Range(0) + + @pytest.mark.parametrize("raw,expected", [(" 7 ", 7), ("1", 1), ("10000000", 10_000_000)]) + def test_env_coerces_to_an_int_with_no_invented_ceiling( + self, monkeypatch, raw, expected + ): + monkeypatch.setenv("PRXREF_SCOPED_RULES_MAX_CHARS", raw) + value = load_config()["scoped_rules_max_chars"] + assert value == expected + assert isinstance(value, int) + + def test_whitespace_only_reads_as_unset(self, monkeypatch): + monkeypatch.setenv("PRXREF_SCOPED_RULES_MAX_CHARS", " ") + assert load_config()["scoped_rules_max_chars"] == 24000 + + @pytest.mark.parametrize("raw", ["lots", "2.5", "0", "-1"]) + def test_malformed_or_non_positive_value_names_the_variable(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_SCOPED_RULES_MAX_CHARS", raw) + with pytest.raises(ConfigError, match=r"^PRXREF_SCOPED_RULES_MAX_CHARS: "): + load_config() + + def test_an_override_is_range_checked_and_named_as_itself(self): + with pytest.raises(ConfigError, match=r"^scoped_rules_max_chars: ") as exc: + load_config(scoped_rules_max_chars=0) + assert "PRXREF_SCOPED_RULES_MAX_CHARS" not in str(exc.value) + + def test_an_override_wins_over_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_SCOPED_RULES_MAX_CHARS", "3") + assert load_config(scoped_rules_max_chars=5)["scoped_rules_max_chars"] == 5 + + +class TestGroupFindings: + """PRXREF_GROUP_FINDINGS (#13): explicit opt-in, parsed by ``_truthy``.""" + + def test_defaults_to_off(self): + assert config._DEFAULTS["group_findings"] is False + assert load_config()["group_findings"] is False + + def test_literal_one_enables_it(self, monkeypatch): + monkeypatch.setenv("PRXREF_GROUP_FINDINGS", "1") + assert load_config()["group_findings"] is True + + @pytest.mark.parametrize("raw", ["true", "True", "yes", "on", "0", "y"]) + def test_only_the_literal_one_enables_it(self, monkeypatch, raw): + monkeypatch.setenv("PRXREF_GROUP_FINDINGS", raw) + assert load_config()["group_findings"] is False + + def test_whitespace_only_reads_as_unset(self, monkeypatch): + monkeypatch.setenv("PRXREF_GROUP_FINDINGS", " ") + assert load_config()["group_findings"] is False + + def test_it_is_a_bool_key_not_a_numeric_one(self): + assert "group_findings" in config._BOOL_KEYS + assert "group_findings" not in config._INT_KEYS | config._FLOAT_KEYS + assert "group_findings" not in config._RANGES + + def test_an_override_wins_over_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_GROUP_FINDINGS", "1") + assert load_config(group_findings=False)["group_findings"] is False + + +_SEVERITY_CAPS = [ + ("max_warning_findings", "PRXREF_MAX_WARNING_FINDINGS"), + ("max_outofscope_findings", "PRXREF_MAX_OUTOFSCOPE_FINDINGS"), +] + + +class TestPerSeverityCaps: + """PRXREF_MAX_WARNING_FINDINGS / _OUTOFSCOPE_FINDINGS (#13). + + ``None`` means unlimited, in the key's own type, because 0 is a legal cap + that drops every finding of that severity. So the low bound is inclusive + and ``_check_ranges`` skips the key only while it is unset. + """ + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + def test_default_is_none_meaning_unlimited(self, key, env): + assert config._DEFAULTS[key] is None + assert load_config()[key] is None + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + def test_declared_in_the_int_and_range_tables(self, key, env): + assert key in config._INT_KEYS + assert config._RANGES[key] == config._Range(0, low_inclusive=True) + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + @pytest.mark.parametrize( + "raw,expected", [("0", 0), ("3", 3), (" 12 ", 12), ("100000", 100_000)] + ) + def test_env_coerces_to_an_int(self, monkeypatch, key, env, raw, expected): + monkeypatch.setenv(env, raw) + value = load_config()[key] + assert value == expected + assert isinstance(value, int) + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + def test_zero_is_distinct_from_unset(self, monkeypatch, key, env): + monkeypatch.setenv(env, "0") + assert load_config()[key] == 0 + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + @pytest.mark.parametrize("raw", ["", " "]) + def test_empty_or_whitespace_reads_as_unset(self, monkeypatch, key, env, raw): + monkeypatch.setenv(env, raw) + assert load_config()[key] is None + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + @pytest.mark.parametrize("raw", ["-1", "-50"]) + def test_negative_value_rejected(self, monkeypatch, key, env, raw): + monkeypatch.setenv(env, raw) + with pytest.raises(ConfigError, match=rf"^{env}: "): + load_config() + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + @pytest.mark.parametrize("raw", ["many", "1.5", "off"]) + def test_malformed_value_names_the_variable(self, monkeypatch, key, env, raw): + monkeypatch.setenv(env, raw) + with pytest.raises(ConfigError, match=rf"^{env}: "): + load_config() + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + def test_an_override_cannot_smuggle_a_negative_cap(self, key, env): + with pytest.raises(ConfigError, match=rf"^{key}: ") as exc: + load_config(**{key: -1}) + assert env not in str(exc.value) + + @pytest.mark.parametrize("key,env", _SEVERITY_CAPS) + def test_an_override_wins_over_the_environment(self, monkeypatch, key, env): + monkeypatch.setenv(env, "10") + assert load_config(**{key: 0})[key] == 0 + + def test_the_caps_are_independent_of_each_other_and_of_the_error_cap( + self, monkeypatch + ): + monkeypatch.setenv("PRXREF_MAX_WARNING_FINDINGS", "2") + cfg = load_config() + assert cfg["max_warning_findings"] == 2 + assert cfg["max_outofscope_findings"] is None + assert cfg["max_error_findings"] == DEFAULT_MAX_ERRORS + + +class TestKeys015ReachTheEntryPoint: + """Observed through ``prxref review``: a bad 0.15 value is exit 2 naming + its variable, before the review runs; a good one gets past config.""" + + @pytest.mark.parametrize("env,raw", [ + ("PRXREF_DEDUP_SIMILARITY", "0"), + ("PRXREF_SCOPED_RULES_MAX_CHARS", "0"), + ("PRXREF_MAX_WARNING_FINDINGS", "-1"), + ("PRXREF_MAX_OUTOFSCOPE_FINDINGS", "few"), + ]) + def test_a_bad_value_exits_2_before_the_review_runs( + self, monkeypatch, capsys, env, raw + ): + calls = [] + monkeypatch.setattr(cli, "_run_review", lambda *a, **k: calls.append(a)) + monkeypatch.setenv(env, raw) + + rc = cli.main(["review", "--pr-url", "https://github.com/org/repo/pull/7"]) + + assert rc == 2 + _, err = capsys.readouterr() + assert f"configuration error: {env}: " in err + assert calls == [] + + def test_good_values_reach_the_review(self, monkeypatch): + calls = [] + + def _record(*args, **kwargs): + calls.append(args) + raise RuntimeError("stop after config") + + monkeypatch.setattr(cli, "_run_review", _record) + monkeypatch.setenv("PRXREF_DEDUP_SIMILARITY", "0.5") + monkeypatch.setenv("PRXREF_SCOPED_RULES_MAX_CHARS", "100") + monkeypatch.setenv("PRXREF_MAX_WARNING_FINDINGS", "0") + monkeypatch.setenv("PRXREF_MAX_OUTOFSCOPE_FINDINGS", "3") + monkeypatch.setenv("PRXREF_GROUP_FINDINGS", "1") + + cli.main(["review", "--pr-url", "https://github.com/org/repo/pull/7"]) + + assert len(calls) == 1 + + +def _doc_entry(surface: str, env: str) -> str: + """The one entry that documents ``env`` on ``surface``, whitespace-folded.""" + text = SURFACES[surface] + if surface == "docs/env-vars.md": + entries = [ln for ln in text.splitlines() if ln.startswith(f"| `{env}` |")] + elif surface == ".env.example": + entries = [p for p in text.split("\n\n") if re.search(rf"^# {env}=", p, re.M)] + else: + entries = [ + m.group(0) + for m in re.finditer( + rf"^ {env}\b.*?(?=^ PRXREF_|^\S|\n\n)", text, re.M | re.S + ) + ] + assert len(entries) == 1, f"{surface}: expected one entry for {env}, got {len(entries)}" + return " ".join(entries[0].split()) + + +class TestKeys015AreDocumented: + """The four-surface rule, tightened for the 0.15 keys. + + ``test_docs_consistency`` checks a SUBSTRING, so ``PRXREF_SCOPED_RULES`` + would pass on the strength of ``PRXREF_SCOPED_RULES_MAX_CHARS`` alone. + """ + + @pytest.mark.parametrize("key,env", _KEYS_0_15) + def test_the_env_name_is_derived_for_the_suite_wide_clear(self, key, env): + assert config._ENV_PREFIX + key.upper() == env + assert env in prxref_env_names() + + @pytest.mark.parametrize("surface", sorted(SURFACES)) + @pytest.mark.parametrize("key,env", _KEYS_0_15) + def test_each_surface_has_one_entry_that_names_the_release(self, key, env, surface): + assert "0.15.0" in _doc_entry(surface, env) + + @pytest.mark.parametrize("surface", sorted(SURFACES)) + def test_the_outofscope_cap_says_it_is_not_ticket_scope_out(self, surface): + entry = _doc_entry(surface, "PRXREF_MAX_OUTOFSCOPE_FINDINGS") + assert re.search(r"\bnot\W+ticket scope\W+out\b", entry, re.I), entry From 3adc5c1d910da3bd7ea7a62e5fddfd67e17dae78 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:36:07 -0700 Subject: [PATCH 07/64] feat(eval): judge prompt, response parser and per-ref cap (#14 T5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the pure half of the eval judge: prompts/judge.md, a single-shot prompt that grades each human finding full, partial or none against the AI findings in the same files, and src/prxref/judge.py. - assign_refs numbers a record's post-gate findings A1..An, skipping dropped rows. - build_judge_prompt fills the template through reviewer.fill_template and loads it through reviewer.load_prompt. split_judge_prompt cuts it at "## Case" into the (system, user) pair LLMClient.invoke takes. - parse_judge_response parses with parser.loads_lenient and returns exactly one Grade per human finding, in label order. Code enforces the scoring rules: a credited ai_ref must exist and be in the human finding's file, and one AI finding credits at most 2 human findings. For the cap, full outranks partial and then label order decides; the excess is graded none and logged. An unknown human_id is dropped with a warning, and a missing one is graded none. Malformed output raises JudgeParseError. - judge_cache_key is the sha256 of canonical JSON over the template sha256, the model, and the labels and AI findings as the prompt shows them. - JUDGE_PROMPT_VERSION = 1. The LLM client, cache storage and judge cost are left to T6. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/judge.py | 378 +++++++++++++++++++++++++++++++++ src/prxref/prompts/judge.md | 42 ++++ tests/test_judge.py | 401 ++++++++++++++++++++++++++++++++++++ 3 files changed, 821 insertions(+) create mode 100644 src/prxref/judge.py create mode 100644 src/prxref/prompts/judge.md create mode 100644 tests/test_judge.py diff --git a/src/prxref/judge.py b/src/prxref/judge.py new file mode 100644 index 00000000..cf834bda --- /dev/null +++ b/src/prxref/judge.py @@ -0,0 +1,378 @@ +"""The LLM judge of ``prxref eval score``: prompt, response parser, grade cap, cache key. + +A case whose human labels carry no ``must_match`` predicate is scored by one +single-shot judge call. This module holds the pure half of that call; the +client, the cache storage and the cost live with the caller. + +1. :func:`assign_refs` numbers one run record's post-gate findings ``A1``, + ``A2``, ... in record order, skipping any row with a ``drop_reason``. +2. :func:`build_judge_prompt` fills ``prompts/judge.md`` with the case's human + findings and the AI findings in the same files; :func:`split_judge_prompt` + cuts it into the ``system`` / ``user`` pair ``LLMClient.invoke`` takes. +3. :func:`parse_judge_response` turns the judge's JSON into exactly one + :class:`Grade` per human finding, in label order. Code, not the judge, + enforces the scoring rules: a credited ``ai_ref`` must exist and sit in the + human finding's file, and one AI finding credits at most + :data:`MAX_CREDITS_PER_AI_FINDING` human findings. A grade those rules + override becomes ``none`` with a :attr:`Grade.reason` and a logged warning. + Output that is not the promised shape raises :class:`JudgeParseError`, + which the caller records as a judge error, never as a score of 0. +4. :func:`judge_cache_key` addresses a cached grading by the template sha256, + the judge model, and the labels and AI findings exactly as the prompt shows + them. + +Cases and findings are read by duck typing: a mapping key or an attribute of +the same name, so a run record's JSON rows, :class:`prxref.triage.Finding` +objects and the eval case loader's records all work. A human label supplies +``id``, ``file``, ``line``, ``severity`` and optional ``text``; a case supplies +its labels as ``expected``. Human severity is passed through as given. +""" +from __future__ import annotations + +import hashlib +import json +import logging +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from . import reviewer +from .parser import loads_lenient + +logger = logging.getLogger("prxref") + +JUDGE_PROMPT_NAME = "judge" +JUDGE_PROMPT_VERSION = 1 +JUDGE_SPLIT_MARKER = "## Case" + +GRADE_FULL = "full" +GRADE_PARTIAL = "partial" +GRADE_NONE = "none" +GRADES = (GRADE_FULL, GRADE_PARTIAL, GRADE_NONE) + +MAX_CREDITS_PER_AI_FINDING = 2 +AI_REF_PREFIX = "A" + +REASON_MISSING = "missing" +REASON_UNKNOWN_REF = "unknown_ref" +REASON_OTHER_FILE = "other_file" +REASON_OVER_CAP = "over_cap" + +_GRADE_RANK = {GRADE_FULL: 0, GRADE_PARTIAL: 1} + + +class JudgeParseError(ValueError): + """The judge's response is not a ``{"grades": [...]}`` object of well-formed rows. + + Raised for unparseable JSON, a non-object top level, a missing or + non-list ``grades``, a row that is not an object, a row without a string + ``human_id``, and a ``grade`` outside :data:`GRADES`. The caller counts + the case as a judge error; it is never scored as 0. + """ + + +@dataclass(frozen=True) +class AIFinding: + """One post-gate AI finding as the judge sees it, under its ``ref``.""" + + ref: str + file: str + line: int | None + severity: str + title: str + body: str + + +@dataclass(frozen=True) +class Grade: + """The judge's verdict on one human finding, after the code-enforced rules. + + ``grade`` is one of :data:`GRADES`. ``ai_ref`` is the crediting AI + finding's ref for ``full`` / ``partial`` and ``None`` for ``none``. + ``reason`` is ``None`` when the judge's own grade stands, otherwise why + the code made it ``none``: :data:`REASON_MISSING` (the judge skipped the + finding), :data:`REASON_UNKNOWN_REF` (no such ref, or none cited), + :data:`REASON_OTHER_FILE` (the ref is in another file) or + :data:`REASON_OVER_CAP` (the ref already credits + :data:`MAX_CREDITS_PER_AI_FINDING` findings). + """ + + human_id: str + grade: str + ai_ref: str | None + reason: str | None = None + + +def _get(obj: Any, key: str) -> Any: + if isinstance(obj, Mapping): + return obj.get(key) + return getattr(obj, key, None) + + +def _text(value: Any) -> str: + return value if isinstance(value, str) else "" + + +def _line(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _ident(value: Any) -> str | None: + if isinstance(value, bool): + return None + if isinstance(value, int): + return str(value) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def _label_rows(case: Any) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for label in _get(case, "expected") or (): + human_id = _ident(_get(label, "id")) + if human_id is None: + raise ValueError(f"human finding without an id: {label!r}") + if human_id in seen: + raise ValueError(f"duplicate human finding id {human_id!r}") + seen.add(human_id) + rows.append({ + "id": human_id, + "file": _text(_get(label, "file")), + "line": _line(_get(label, "line")), + "severity": _text(_get(label, "severity")), + "text": _text(_get(label, "text")), + }) + return rows + + +def _ai_row(finding: Any) -> dict[str, Any]: + return { + "ref": _text(_get(finding, "ref")), + "file": _text(_get(finding, "file")), + "line": _line(_get(finding, "line")), + "severity": _text(_get(finding, "severity")), + "title": _text(_get(finding, "title")), + "body": _text(_get(finding, "body")), + } + + +def _judge_inputs(case: Any, ai_findings: Iterable[Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + labels = _label_rows(case) + files = {row["file"] for row in labels} + visible = [row for row in map(_ai_row, ai_findings) if row["file"] in files] + return labels, visible + + +def assign_refs(findings: Iterable[Any]) -> list[AIFinding]: + """Number one run record's post-gate findings ``A1``, ``A2``, ... in input order. + + Takes the record's ``findings`` rows (mappings) or finding objects. A row + whose ``drop_reason`` is set was gated out and is skipped without using a + number, because eval scores post-gate output only. A grouped finding that + covers several locations should arrive as one row per location, so each + location is its own ref under the per-ref cap. + """ + kept = [f for f in findings if not _get(f, "drop_reason")] + return [ + AIFinding( + ref=f"{AI_REF_PREFIX}{index}", + file=_text(_get(f, "file")), + line=_line(_get(f, "line")), + severity=_text(_get(f, "severity")), + title=_text(_get(f, "title")), + body=_text(_get(f, "body")), + ) + for index, f in enumerate(kept, start=1) + ] + + +def human_files(case: Any) -> dict[str, str]: + """Map each of the case's human finding ids to its file, in label order. + + This is the ``human_ids`` argument of :func:`parse_judge_response`. + Raises ``ValueError`` for a label without an id or a repeated id. + """ + return {row["id"]: row["file"] for row in _label_rows(case)} + + +def ai_ref_files(ai_findings: Iterable[Any]) -> dict[str, str]: + """Map each AI finding's ref to its file; the ``ai_refs`` of :func:`parse_judge_response`.""" + return {row["ref"]: row["file"] for row in map(_ai_row, ai_findings) if row["ref"]} + + +def load_judge_template() -> str: + """The packaged ``prompts/judge.md`` text, read through :func:`prxref.reviewer.load_prompt`.""" + return reviewer.load_prompt(JUDGE_PROMPT_NAME) + + +def judge_prompt_sha(template: str | None = None) -> str: + """The sha256 hex digest of the judge template (the packaged one by default).""" + text = load_judge_template() if template is None else template + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _render_rows(rows: Sequence[Mapping[str, Any]]) -> str: + return json.dumps(list(rows), indent=2, ensure_ascii=False) + + +def build_judge_prompt(case: Any, ai_findings: Sequence[Any]) -> str: + """Fill ``prompts/judge.md`` for one case. + + The ``{human_findings}`` slot gets the case's labels (``id``, ``file``, + ``line``, ``severity``, ``text``) and the ``{ai_findings}`` slot gets the + ref'd AI findings (``ref``, ``file``, ``line``, ``severity``, ``title``, + ``body``) whose file holds at least one label, both as JSON arrays in + input order. An AI finding in any other file could never be credited, so + the judge does not see it. The template is filled in one pass by + :func:`prxref.reviewer.fill_template`, so a finding that quotes a + placeholder renders literally. + """ + labels, visible = _judge_inputs(case, ai_findings) + return reviewer.fill_template(load_judge_template(), { + "human_findings": _render_rows(labels), + "ai_findings": _render_rows(visible), + }) + + +def split_judge_prompt(prompt: str) -> tuple[str, str]: + """Cut a filled judge prompt at :data:`JUDGE_SPLIT_MARKER` into ``(system, user)``. + + The system half is the fixed grading instructions; the user half starts + at the marker and carries the case data and the output format. Raises + ``ValueError`` when the marker is absent. + """ + head, marker, tail = prompt.partition(JUDGE_SPLIT_MARKER) + if not marker: + raise ValueError(f"judge prompt is missing the {JUDGE_SPLIT_MARKER!r} split marker") + return head.strip(), (marker + tail).strip() + + +def judge_cache_key(prompt_sha: str, model: str, case: Any, ai_findings: Sequence[Any]) -> str: + """The sha256 hex digest that addresses one case's cached judge grading. + + Hashes canonical JSON (sorted keys, no whitespace) of the template + sha256, the judge model, and the case's labels and AI findings projected + exactly as :func:`build_judge_prompt` renders them. A mapping's key order + never changes the key; a change the judge would see (a label's text, a + finding's body, the finding order that decides its refs) always does, + and a field the prompt omits (confidence, category, acceptance) never does. + """ + labels, visible = _judge_inputs(case, ai_findings) + payload = { + "prompt_sha256": prompt_sha, + "model": model, + "labels": labels, + "ai_findings": visible, + } + blob = json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest() + + +def _read_rows(text: str) -> list[Any]: + if not isinstance(text, str) or not text.strip(): + raise JudgeParseError("judge response is empty") + try: + data = loads_lenient(text) + except json.JSONDecodeError as exc: + raise JudgeParseError(f"judge response is not JSON: {exc}") from exc + if not isinstance(data, dict): + raise JudgeParseError(f"judge response is a JSON {type(data).__name__}, not an object") + rows = data.get("grades") + if not isinstance(rows, list): + raise JudgeParseError("judge response has no 'grades' list") + return rows + + +def _answers(rows: list[Any], human_ids: Mapping[str, str]) -> dict[str, tuple[str, str | None]]: + answers: dict[str, tuple[str, str | None]] = {} + for index, row in enumerate(rows): + if not isinstance(row, dict): + raise JudgeParseError(f"grades[{index}] is not an object") + human_id = _ident(row.get("human_id")) + if human_id is None: + raise JudgeParseError(f"grades[{index}] has no human_id") + grade = row.get("grade") + grade = grade.strip().lower() if isinstance(grade, str) else grade + if grade not in GRADES: + raise JudgeParseError(f"grades[{index}] has grade {row.get('grade')!r}, not one of {', '.join(GRADES)}") + if human_id not in human_ids: + logger.warning("judge graded unknown human finding %r; dropped", human_id) + continue + if human_id in answers: + logger.warning("judge graded human finding %r more than once; kept the first", human_id) + continue + answers[human_id] = (grade, _ident(row.get("ai_ref"))) + return answers + + +def _validated(human_id: str, file: str, answer: tuple[str, str | None] | None, ai_refs: Mapping[str, str]) -> Grade: + if answer is None: + logger.warning("judge returned no grade for human finding %r; graded none", human_id) + return Grade(human_id, GRADE_NONE, None, REASON_MISSING) + grade, ref = answer + if grade == GRADE_NONE: + return Grade(human_id, GRADE_NONE, None) + if ref is None or ref not in ai_refs: + logger.warning("judge credited human finding %r to unknown AI finding %r; graded none", human_id, ref) + return Grade(human_id, GRADE_NONE, None, REASON_UNKNOWN_REF) + if ai_refs[ref] != file: + logger.warning( + "judge credited human finding %r (%s) to AI finding %r in another file (%s); graded none", + human_id, file, ref, ai_refs[ref], + ) + return Grade(human_id, GRADE_NONE, None, REASON_OTHER_FILE) + return Grade(human_id, grade, ref) + + +def _capped(grades: list[Grade]) -> list[Grade]: + by_ref: dict[str, list[int]] = {} + for index, grade in enumerate(grades): + if grade.ai_ref is not None: + by_ref.setdefault(grade.ai_ref, []).append(index) + capped = list(grades) + for ref, indexes in by_ref.items(): + if len(indexes) <= MAX_CREDITS_PER_AI_FINDING: + continue + ranked = sorted(indexes, key=lambda i: (_GRADE_RANK[grades[i].grade], i)) + kept = sorted(ranked[:MAX_CREDITS_PER_AI_FINDING]) + dropped = sorted(ranked[MAX_CREDITS_PER_AI_FINDING:]) + for i in dropped: + capped[i] = Grade(grades[i].human_id, GRADE_NONE, None, REASON_OVER_CAP) + logger.warning( + "judge credited AI finding %r to %d human findings; kept %s, graded none %s (cap %d)", + ref, len(indexes), + ", ".join(grades[i].human_id for i in kept), + ", ".join(grades[i].human_id for i in dropped), + MAX_CREDITS_PER_AI_FINDING, + ) + return capped + + +def parse_judge_response(text: str, human_ids: Mapping[str, str], ai_refs: Mapping[str, str]) -> list[Grade]: + """Parse the judge's reply into one :class:`Grade` per human finding, in ``human_ids`` order. + + ``human_ids`` maps each human finding id to its file (:func:`human_files`) + and ``ai_refs`` maps each AI ref to its file (:func:`ai_ref_files`). The + reply is parsed with :func:`prxref.parser.loads_lenient`; ``grade`` is + matched after trimming and lowercasing, and an integer id is read as its + decimal string. + + - An id not in ``human_ids`` is dropped with a warning; a repeated id + keeps its first row, with a warning. + - An id the reply omits is graded ``none`` (:data:`REASON_MISSING`). + - A ``full`` / ``partial`` grade whose ``ai_ref`` is null or not in + ``ai_refs`` becomes ``none`` (:data:`REASON_UNKNOWN_REF`); one whose ref + sits in a different file than the human finding becomes ``none`` + (:data:`REASON_OTHER_FILE`). A ``none`` grade never keeps a ref. + - One ref credits at most :data:`MAX_CREDITS_PER_AI_FINDING` human + findings. Deterministically, ``full`` outranks ``partial`` and then + ``human_ids`` order decides; each excess grade becomes ``none`` + (:data:`REASON_OVER_CAP`) and the drop is logged as a warning. + + Raises :class:`JudgeParseError` when the reply is not the promised shape. + """ + answers = _answers(_read_rows(text), human_ids) + graded = [_validated(hid, file, answers.get(hid), ai_refs) for hid, file in human_ids.items()] + return _capped(graded) diff --git a/src/prxref/prompts/judge.md b/src/prxref/prompts/judge.md new file mode 100644 index 00000000..eae869b6 --- /dev/null +++ b/src/prxref/prompts/judge.md @@ -0,0 +1,42 @@ +You grade an automated code reviewer against human reviewers. You see the findings human reviewers left on one pull request and the findings the automated reviewer (the AI) produced in the same files. You do not see the code or the diff. For each human finding, decide whether one AI finding raises the same problem. + +## Grades + +- `full` — one AI finding identifies the same defect or concern as the human finding, in the same file, even when it is worded differently or proposes a different fix. +- `partial` — one AI finding points at the same code and the same underlying concern but misses its substance: it names a symptom without the cause, makes a weaker or vaguer claim, or covers only part of a multi-part human finding. +- `none` — no AI finding raises this concern. An AI finding on the same line about a different problem is `none`. + +## Rules + +- Grade every human finding exactly once, by its `id`. Do not invent ids. +- `ai_ref` is the `ref` of the one AI finding that earns the grade, and that AI finding must be in the SAME `file` as the human finding. Cite the single best match. For `none`, `ai_ref` is `null`. +- One AI finding may credit at most two human findings. When one seems to cover more, credit the two it covers best and grade the others on their own evidence. +- Judge the substance, not the wording and not the severity: a different severity never lowers a grade. +- A vague AI finding ("consider adding error handling") does not match a specific human finding unless it names the same defect. +- Line numbers are a hint, not a requirement: a match may sit a few lines away from the human finding. + +## Case + +### Human findings + +{human_findings} + +### AI findings + +{ai_findings} + +## Output Format + +Return exactly one JSON object, no prose, no fences: + +```json +{ + "grades": [ + {"human_id": "H1", "grade": "full", "ai_ref": "A2"}, + {"human_id": "H2", "grade": "partial", "ai_ref": "A1"}, + {"human_id": "H3", "grade": "none", "ai_ref": null} + ] +} +``` + +`grade` is one of `full`, `partial` or `none`. Emit one entry per human finding listed above. diff --git a/tests/test_judge.py b/tests/test_judge.py new file mode 100644 index 00000000..da7286e9 --- /dev/null +++ b/tests/test_judge.py @@ -0,0 +1,401 @@ +"""Issue #14 T5: the judge prompt, its response parser, the per-ref cap and the cache key.""" +from __future__ import annotations + +import hashlib +import json +import logging +import re + +import pytest + +from prxref import judge, reviewer +from prxref.judge import ( + JUDGE_PROMPT_VERSION, + JUDGE_SPLIT_MARKER, + MAX_CREDITS_PER_AI_FINDING, + REASON_MISSING, + REASON_OTHER_FILE, + REASON_OVER_CAP, + REASON_UNKNOWN_REF, + AIFinding, + Grade, + JudgeParseError, + ai_ref_files, + assign_refs, + build_judge_prompt, + human_files, + judge_cache_key, + judge_prompt_sha, + parse_judge_response, + split_judge_prompt, +) +from prxref.triage import Finding + +_PLACEHOLDER = re.compile(r"\{([a-z_]+)\}") + +CASE = { + "id": "case-1", + "expected": [ + {"id": "H1", "file": "src/a.py", "line": 10, "severity": "error", "text": "Divides by zero when size is 0."}, + {"id": "H2", "file": "src/a.py", "line": 12, "severity": "warning", "text": "Leaks the file handle."}, + {"id": "H3", "file": "src/a.py", "line": 30, "severity": "minor", "text": "Misleading name."}, + {"id": "H4", "file": "src/b.py", "line": 5, "severity": "error", "text": "Token logged in clear."}, + ], +} + +RECORD_FINDINGS = [ + {"file": "src/a.py", "line": 10, "severity": "error", "confidence": 0.9, "title": "Divide by zero", + "body": "size may be 0.", "drop_reason": None}, + {"file": "src/a.py", "line": 11, "severity": "warning", "confidence": 0.4, "title": "Low confidence", + "body": "gated out.", "drop_reason": "confidence"}, + {"file": "src/b.py", "line": 5, "severity": "error", "confidence": 0.8, "title": "Secret in log", + "body": "token is logged.", "drop_reason": None}, + {"file": "src/c.py", "line": 1, "severity": "warning", "confidence": 0.7, "title": "Unlabelled file", + "body": "no human finding here.", "drop_reason": None}, +] + +AI = assign_refs(RECORD_FINDINGS) +HUMANS = human_files(CASE) +REFS = ai_ref_files(AI) + + +def _reply(*rows: dict) -> str: + return json.dumps({"grades": list(rows)}) + + +def _row(human_id, grade, ai_ref=None) -> dict: + return {"human_id": human_id, "grade": grade, "ai_ref": ai_ref} + + +def _by_id(grades: list[Grade]) -> dict[str, Grade]: + return {g.human_id: g for g in grades} + + +class TestRefsAndMaps: + def test_refs_number_post_gate_findings_only_in_record_order(self): + assert [(f.ref, f.file, f.title) for f in AI] == [ + ("A1", "src/a.py", "Divide by zero"), + ("A2", "src/b.py", "Secret in log"), + ("A3", "src/c.py", "Unlabelled file"), + ] + + def test_finding_objects_are_accepted(self): + found = assign_refs([ + Finding("src/a.py", 3, "error", 0.9, "T", "B"), + Finding("src/a.py", 4, "error", 0.9, "Gone", "B", drop_reason="dedup"), + ]) + assert found == [AIFinding(ref="A1", file="src/a.py", line=3, severity="error", title="T", body="B")] + + def test_human_files_keep_label_order(self): + assert list(HUMANS.items()) == [ + ("H1", "src/a.py"), ("H2", "src/a.py"), ("H3", "src/a.py"), ("H4", "src/b.py"), + ] + + def test_ai_ref_files(self): + assert REFS == {"A1": "src/a.py", "A2": "src/b.py", "A3": "src/c.py"} + + def test_a_repeated_human_id_is_refused(self): + case = {"expected": [{"id": "H1", "file": "a"}, {"id": "H1", "file": "b"}]} + with pytest.raises(ValueError, match="duplicate human finding id 'H1'"): + human_files(case) + + def test_a_label_without_an_id_is_refused(self): + with pytest.raises(ValueError, match="without an id"): + human_files({"expected": [{"file": "a"}]}) + + +class TestPrompt: + def test_version_is_one(self): + assert JUDGE_PROMPT_VERSION == 1 + + def test_the_template_has_exactly_the_two_data_slots(self): + template = reviewer.load_prompt("judge") + assert sorted(set(_PLACEHOLDER.findall(template))) == ["ai_findings", "human_findings"] + + def test_the_prompt_fills_every_placeholder(self): + prompt = build_judge_prompt(CASE, AI) + assert _PLACEHOLDER.findall(prompt) == [] + + def test_the_prompt_shows_labels_and_same_file_ai_findings(self): + prompt = build_judge_prompt(CASE, AI) + for text in ("H1", "H4", "Divides by zero when size is 0.", "Misleading name.", '"minor"'): + assert text in prompt + assert '"ref": "A1"' in prompt + assert '"ref": "A2"' in prompt + assert "A3" not in prompt + assert "Unlabelled file" not in prompt + assert "gated out." not in prompt + + def test_the_rendered_rows_carry_the_documented_fields(self): + _system, user = split_judge_prompt(build_judge_prompt(CASE, AI)) + humans_block = user.split("### Human findings", 1)[1].split("### AI findings", 1)[0] + ai_block = user.split("### AI findings", 1)[1].split("## Output Format", 1)[0] + humans = json.loads(humans_block) + ais = json.loads(ai_block) + assert humans[0] == { + "id": "H1", "file": "src/a.py", "line": 10, "severity": "error", + "text": "Divides by zero when size is 0.", + } + assert [row["ref"] for row in ais] == ["A1", "A2"] + assert set(ais[0]) == {"ref", "file", "line", "severity", "title", "body"} + + def test_a_label_without_text_renders_an_empty_text(self): + case = {"expected": [{"id": "S1", "file": "src/a.py", "line": 2, "severity": "spec"}]} + prompt = build_judge_prompt(case, []) + assert '"text": ""' in prompt + assert _PLACEHOLDER.findall(prompt) == [] + + def test_a_value_quoting_a_placeholder_renders_literally(self): + case = {"expected": [{"id": "H1", "file": "f", "line": 1, "severity": "error", "text": "see {ai_findings}"}]} + prompt = build_judge_prompt(case, []) + assert "see {ai_findings}" in prompt + + def test_the_split_puts_instructions_in_system_and_the_case_in_user(self): + system, user = split_judge_prompt(build_judge_prompt(CASE, AI)) + assert JUDGE_SPLIT_MARKER not in system + assert "## Grades" in system + assert "Divides by zero" not in system + assert user.startswith(JUDGE_SPLIT_MARKER) + assert "Divides by zero when size is 0." in user + assert '"grades"' in user + + def test_the_split_refuses_a_prompt_without_the_marker(self): + with pytest.raises(ValueError, match="split marker"): + split_judge_prompt("no marker here") + + def test_the_template_is_read_through_reviewer_load_prompt(self, monkeypatch): + seen = [] + + def fake_load(name): + seen.append(name) + return "H={human_findings}\n## Case\nA={ai_findings}" + + monkeypatch.setattr(reviewer, "load_prompt", fake_load) + prompt = build_judge_prompt(CASE, AI) + assert seen == ["judge"] + assert prompt.startswith('H=[\n {\n "id": "H1"') + + def test_the_template_sha_is_sha256_of_the_packaged_text(self): + text = reviewer.load_prompt("judge") + assert judge_prompt_sha() == hashlib.sha256(text.encode("utf-8")).hexdigest() + assert judge_prompt_sha("x") == hashlib.sha256(b"x").hexdigest() + + def test_the_prompt_name_is_not_an_overridable_review_prompt(self): + assert judge.JUDGE_PROMPT_NAME not in {"worker", "systemic", "summary"} + + +class TestMalformed: + @pytest.mark.parametrize( + "text", + [ + "", + " ", + "Sorry, I cannot grade these findings.", + "[1, 2, 3]", + '"grades"', + '{"verdicts": []}', + '{"grades": {"human_id": "H1"}}', + '{"grades": ["H1 full A1"]}', + '{"grades": [{"grade": "full", "ai_ref": "A1"}]}', + '{"grades": [{"human_id": null, "grade": "full", "ai_ref": "A1"}]}', + '{"grades": [{"human_id": "H1", "ai_ref": "A1"}]}', + '{"grades": [{"human_id": "H1", "grade": "match", "ai_ref": "A1"}]}', + '{"grades": [{"human_id": "H1", "grade": 1, "ai_ref": "A1"}]}', + ], + ) + def test_malformed_output_raises(self, text): + with pytest.raises(JudgeParseError): + parse_judge_response(text, HUMANS, REFS) + + def test_the_error_is_a_value_error(self): + assert issubclass(JudgeParseError, ValueError) + + def test_a_malformed_row_raises_even_for_an_unknown_id(self): + with pytest.raises(JudgeParseError, match="grade"): + parse_judge_response(_reply(_row("H9", "maybe")), HUMANS, REFS) + + def test_fenced_json_with_prose_still_parses(self): + text = "Here you go:\n```json\n" + _reply(_row("H1", "full", "A1")) + "\n```\n" + grades = parse_judge_response(text, HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "full", "A1") + + def test_grade_case_and_whitespace_are_tolerated(self): + grades = parse_judge_response(_reply(_row("H1", " Full ", "A1")), HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "full", "A1") + + +class TestIds: + def test_every_human_finding_gets_one_grade_in_label_order(self): + reply = _reply(_row("H4", "partial", "A2"), _row("H2", "none"), _row("H1", "full", "A1"), _row("H3", "none")) + grades = parse_judge_response(reply, HUMANS, REFS) + assert grades == [ + Grade("H1", "full", "A1"), + Grade("H2", "none", None), + Grade("H3", "none", None), + Grade("H4", "partial", "A2"), + ] + + def test_an_unknown_human_id_is_dropped_with_a_warning(self, caplog): + reply = _reply(_row("H1", "full", "A1"), _row("H99", "full", "A1")) + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(reply, HUMANS, REFS) + assert "H99" not in _by_id(grades) + assert [g.human_id for g in grades] == ["H1", "H2", "H3", "H4"] + assert "unknown human finding 'H99'" in caplog.text + + def test_a_missing_human_id_is_graded_none(self, caplog): + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(_reply(_row("H1", "full", "A1")), HUMANS, REFS) + by_id = _by_id(grades) + assert by_id["H1"] == Grade("H1", "full", "A1") + for human_id in ("H2", "H3", "H4"): + assert by_id[human_id] == Grade(human_id, "none", None, REASON_MISSING) + assert "no grade for human finding 'H2'" in caplog.text + + def test_an_empty_grades_list_grades_everything_none(self): + grades = parse_judge_response('{"grades": []}', HUMANS, REFS) + assert {g.grade for g in grades} == {"none"} + assert {g.reason for g in grades} == {REASON_MISSING} + + def test_a_repeated_human_id_keeps_the_first_row(self, caplog): + reply = _reply(_row("H1", "none"), _row("H1", "full", "A1")) + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(reply, HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "none", None) + assert "more than once" in caplog.text + + def test_integer_ids_read_as_their_decimal_string(self): + humans = {"7": "src/a.py"} + grades = parse_judge_response('{"grades": [{"human_id": 7, "grade": "full", "ai_ref": "A1"}]}', humans, REFS) + assert grades == [Grade("7", "full", "A1")] + + +class TestRefValidation: + def test_an_ai_ref_in_another_file_is_rejected(self, caplog): + reply = _reply(_row("H1", "full", "A2"), _row("H4", "full", "A2")) + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(reply, HUMANS, REFS) + by_id = _by_id(grades) + assert by_id["H1"] == Grade("H1", "none", None, REASON_OTHER_FILE) + assert by_id["H4"] == Grade("H4", "full", "A2") + assert "in another file" in caplog.text + + def test_an_unknown_ai_ref_is_rejected(self, caplog): + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(_reply(_row("H1", "partial", "A9")), HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "none", None, REASON_UNKNOWN_REF) + assert "unknown AI finding 'A9'" in caplog.text + + @pytest.mark.parametrize("ref", [None, "", " ", ["A1"]]) + def test_credit_without_a_usable_ref_is_rejected(self, ref): + grades = parse_judge_response(_reply(_row("H1", "full", ref)), HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "none", None, REASON_UNKNOWN_REF) + + def test_a_none_grade_never_keeps_a_ref(self): + grades = parse_judge_response(_reply(_row("H1", "none", "A1")), HUMANS, REFS) + assert _by_id(grades)["H1"] == Grade("H1", "none", None) + + +class TestCap: + def test_the_cap_is_two(self): + assert MAX_CREDITS_PER_AI_FINDING == 2 + + def test_two_credits_on_one_ref_both_stand(self): + reply = _reply(_row("H1", "full", "A1"), _row("H2", "partial", "A1")) + by_id = _by_id(parse_judge_response(reply, HUMANS, REFS)) + assert by_id["H1"] == Grade("H1", "full", "A1") + assert by_id["H2"] == Grade("H2", "partial", "A1") + + def test_over_cap_drops_the_excess_and_logs_it(self, caplog): + reply = _reply(_row("H1", "full", "A1"), _row("H2", "full", "A1"), _row("H3", "full", "A1")) + with caplog.at_level(logging.WARNING, logger="prxref"): + grades = parse_judge_response(reply, HUMANS, REFS) + assert grades[:3] == [ + Grade("H1", "full", "A1"), + Grade("H2", "full", "A1"), + Grade("H3", "none", None, REASON_OVER_CAP), + ] + assert "AI finding 'A1' to 3 human findings; kept H1, H2, graded none H3 (cap 2)" in caplog.text + + def test_full_outranks_partial_under_the_cap(self): + reply = _reply(_row("H1", "partial", "A1"), _row("H2", "full", "A1"), _row("H3", "full", "A1")) + grades = parse_judge_response(reply, HUMANS, REFS) + assert grades[:3] == [ + Grade("H1", "none", None, REASON_OVER_CAP), + Grade("H2", "full", "A1"), + Grade("H3", "full", "A1"), + ] + + def test_the_drop_does_not_depend_on_reply_order(self): + rows = [_row("H1", "full", "A1"), _row("H2", "partial", "A1"), _row("H3", "partial", "A1")] + forward = parse_judge_response(_reply(*rows), HUMANS, REFS) + backward = parse_judge_response(_reply(*reversed(rows)), HUMANS, REFS) + assert forward == backward + assert _by_id(forward)["H3"] == Grade("H3", "none", None, REASON_OVER_CAP) + assert _by_id(forward)["H2"] == Grade("H2", "partial", "A1") + + def test_rejected_refs_do_not_count_toward_the_cap(self): + humans = {"H1": "src/a.py", "H2": "src/a.py", "H3": "src/b.py"} + reply = _reply(_row("H1", "full", "A1"), _row("H2", "full", "A1"), _row("H3", "full", "A1")) + grades = parse_judge_response(reply, humans, REFS) + assert grades == [ + Grade("H1", "full", "A1"), + Grade("H2", "full", "A1"), + Grade("H3", "none", None, REASON_OTHER_FILE), + ] + + +class TestCacheKey: + SHA = "0" * 64 + + def test_the_key_is_a_sha256_hex_digest(self): + key = judge_cache_key(self.SHA, "judge-model", CASE, AI) + assert re.fullmatch(r"[0-9a-f]{64}", key) + + def test_the_key_is_stable_across_mapping_key_order(self): + shuffled_case = {"expected": [dict(reversed(list(label.items()))) for label in CASE["expected"]]} + as_rows = [dict(reversed(list(vars(f).items()))) for f in AI] + assert judge_cache_key(self.SHA, "m", CASE, AI) == judge_cache_key(self.SHA, "m", shuffled_case, as_rows) + + def test_the_key_is_repeatable(self): + assert judge_cache_key(self.SHA, "m", CASE, AI) == judge_cache_key(self.SHA, "m", CASE, list(AI)) + + @pytest.mark.parametrize( + "change", + ["prompt_sha", "model", "label_text", "ai_body", "ai_order"], + ) + def test_anything_the_judge_sees_changes_the_key(self, change): + sha, model, case, ai = self.SHA, "m", CASE, list(AI) + if change == "prompt_sha": + sha = "1" * 64 + elif change == "model": + model = "other" + elif change == "label_text": + case = {"expected": [{**CASE["expected"][0], "text": "reworded"}, *CASE["expected"][1:]]} + elif change == "ai_body": + ai = [AIFinding(**{**vars(ai[0]), "body": "changed"}), *ai[1:]] + else: + ai = assign_refs([RECORD_FINDINGS[2], RECORD_FINDINGS[0]]) + assert judge_cache_key(sha, model, case, ai) != judge_cache_key(self.SHA, "m", CASE, AI) + + def test_fields_the_prompt_omits_do_not_change_the_key(self): + case = {"expected": [{**label, "category": "security", "accepted": True} for label in CASE["expected"]]} + rows = [{**vars(f), "confidence": 0.1} for f in AI] + assert judge_cache_key(self.SHA, "m", case, rows) == judge_cache_key(self.SHA, "m", CASE, AI) + + def test_ai_findings_in_unlabelled_files_do_not_change_the_key(self): + assert judge_cache_key(self.SHA, "m", CASE, AI[:2]) == judge_cache_key(self.SHA, "m", CASE, AI) + + +class TestRoundTrip: + def test_a_stub_judge_reply_grades_the_case(self): + prompt = build_judge_prompt(CASE, AI) + assert '"ref": "A1"' in prompt + reply = "```json\n" + _reply(_row("H1", "full", "A1"), _row("H4", "partial", "A2")) + "\n```" + grades = parse_judge_response(reply, human_files(CASE), ai_ref_files(AI)) + assert [(g.human_id, g.grade, g.ai_ref) for g in grades] == [ + ("H1", "full", "A1"), + ("H2", "none", None), + ("H3", "none", None), + ("H4", "partial", "A2"), + ] From 823cbbd54480333b27a2fb339286274decfc6ce7 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:37:54 -0700 Subject: [PATCH 08/64] feat: eval case loader and validation (#14 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/prxref/eval_cases.py, the --cases input of the coming `prxref eval` subcommand. It loads a cases.json file ({version: 1, cases: [...]}) or a directory of case-*/ dirs (the tests/evals layout, with line_hint mapped to line and source to category) into frozen EvalCase and ExpectedFinding records. Every case is validated up front, and each problem is a ConfigError reading "--cases: case '': : ", so the CLI can exit 2 before any review runs. The checks: a safe, unique case id (it becomes a path segment of the run output), no unknown fields, the replay rules of review (a pr_url with no diff_file needs both SHAs, full hex, different, stored lowercased), a pr_url a forge recognises, context_file and spec paths that exist, label shape (int line >= 1, severity in error|warning|minor|spec|outofscope stored as given, a must_match regex that compiles), and, whenever the case has a diff, the anchor check ported from tests/evals: every label sits on a line the diff adds. check_anchors is public so a pinned-range case can be checked once its diff is fetched. Human severity also accepts spec and outofscope, because the repo's own tests/evals dataset labels its planted violations spec; the map's error|warning|minor alone would refuse that dataset. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/eval_cases.py | 486 +++++++++++++++++++++++++++++++++++++ tests/test_eval_cases.py | 509 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 995 insertions(+) create mode 100644 src/prxref/eval_cases.py create mode 100644 tests/test_eval_cases.py diff --git a/src/prxref/eval_cases.py b/src/prxref/eval_cases.py new file mode 100644 index 00000000..2de836d9 --- /dev/null +++ b/src/prxref/eval_cases.py @@ -0,0 +1,486 @@ +"""Eval cases: load and validate the ``--cases`` input of ``prxref eval`` (issue #14). + +``--cases`` names either a ``cases.json`` file or a directory of ``case-*/`` +directories, the layout of ``tests/evals/``. Both forms load into the same +frozen :class:`EvalCase` records holding their :class:`ExpectedFinding` +labels, so the run and score steps never see which form a dataset used. + +A ``cases.json`` file is ``{"version": 1, "cases": [...]}``. Each case is an +object with a required ``id`` and ``expected`` and the optional ``pr_url``, +``base_sha``, ``head_sha``, ``diff_file``, ``context_file`` and ``spec`` (one +string or a list of them). ``expected`` is a list, possibly empty, of human +findings, each with ``id``, ``file``, ``line`` and ``severity`` and the +optional ``category``, ``accepted``, ``text`` and ``must_match``. An optional +field set to ``null`` counts as absent. A relative ``diff_file``, +``context_file`` or ``spec`` path is read relative to the directory holding +the ``cases.json`` file; a ``spec`` entry that is an ``http(s)`` URL is kept +as given. + +A ``case-*/`` directory supplies ``diff.patch`` (required) as its +``diff_file``, ``expected.json`` (required: the ``expected`` list above) and, +when present, ``ticket.md`` as its ``context_file`` and ``docs/`` as its one +``spec`` source. Its id is the directory name. ``expected.json`` spells +``line`` as ``line_hint`` and ``category`` as ``source``, and every message +about it uses those spellings. + +A case replays either a local diff (``diff_file``) or a pinned commit range of +a pull request (``pr_url`` with both ``base_sha`` and ``head_sha``), and may +give ``pr_url`` beside ``diff_file`` as ``prxref review`` allows. The SHA +rules are those of the replay flags: full 40- or 64-character hex, stored +lowercased, naming two different commits. + +Human severity is one of :data:`HUMAN_SEVERITIES` and is stored as given. +``minor`` is the labeller's lowest tier, which a scorer maps to ``warning`` +only when it compares severities; ``spec`` and ``outofscope`` are prxref's +own tiers, which the directory dataset labels with. ``must_match`` is a plain +substring, or a regular expression when prefixed :data:`MUST_MATCH_REGEX_PREFIX`. + +Every case is validated when the dataset loads, before any review runs, so a +bad dataset is a configuration error: each problem raises +:class:`~prxref.llm.ConfigError` whose message is +``: case '': : ``, naming ``--cases``, the case +and the field, and the CLI exits 2. A case whose id is itself the problem is +named by its position, ``cases[]``; a problem with the file as a whole +names the path instead of a case. When a case has a diff file, every label +must anchor on a line that diff adds (:func:`check_anchors`); a case pinned +to a pull request's range has no diff until it runs, so its labels are +checked for shape only. +""" +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from prxref.forges.base import detect_forge +from prxref.llm import ConfigError +from prxref.triage import FileDiff, parse_unified_diff + +CASES_VERSION = 1 +HUMAN_SEVERITIES = ("error", "warning", "minor", "spec", "outofscope") +MUST_MATCH_REGEX_PREFIX = "re:" + +_CASE_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +_FULL_SHA_RE = re.compile(r"[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?") +_TOP_KEYS = ("version", "cases") +_CASE_KEYS = ( + "id", "pr_url", "base_sha", "head_sha", "diff_file", "context_file", "spec", "expected", +) +_EXPECTED_KEYS = ( + "id", "file", "line", "severity", "category", "accepted", "text", "must_match", +) +_JSON_SPELLING: dict[str, str] = {} +_DIRECTORY_SPELLING = {"line": "line_hint", "category": "source"} + + +@dataclass(frozen=True) +class ExpectedFinding: + """One human-labelled finding that a review of its case should report. + + ``line`` is the 1-based line in the new (post-image) file. ``severity`` + is one of :data:`HUMAN_SEVERITIES`, exactly as labelled. ``category`` is + the labeller's free-form class (the directory form's ``source``, + ``spec`` or ``generic``), ``accepted`` whether the author acted on the + comment, and ``text`` the reviewer's words; each is ``None`` when not + labelled. ``must_match`` is the deterministic acceptance predicate, or + ``None`` when the label is left to a judge. + """ + + id: str + file: str + line: int + severity: str + category: str | None = None + accepted: bool | None = None + text: str | None = None + must_match: str | None = None + + +@dataclass(frozen=True) +class EvalCase: + """One validated eval case: what to replay and the findings it should yield. + + ``id`` is a single safe path segment, unique within its dataset. + ``diff_file`` and ``context_file`` are paths ready to open, already + joined onto the ``cases.json`` directory or the case directory; ``spec`` + holds the spec sources in order, each such a path or an ``http(s)`` URL, + and is empty when the case sets none. ``base_sha`` and ``head_sha`` are + both full lowercased SHAs, and ``pr_url`` is set, or both are ``None``. + ``expected`` keeps the labels in file order and may be empty. + """ + + id: str + expected: tuple[ExpectedFinding, ...] + pr_url: str | None = None + base_sha: str | None = None + head_sha: str | None = None + diff_file: str | None = None + context_file: str | None = None + spec: tuple[str, ...] = () + + +def load_cases(path: str | os.PathLike[str], *, source: str = "--cases") -> list[EvalCase]: + """Load and validate every eval case under ``path``, in dataset order. + + A directory is read as ``case-*/`` directories, sorted by name; anything + else is read as a ``cases.json`` file. Nothing is returned until every + case has passed, so a dataset is used whole or not at all. Each problem + raises :class:`~prxref.llm.ConfigError` naming ``source`` (the flag that + supplied ``path``), the case and the field. + """ + root = Path(path) + if root.is_dir(): + return _load_directory(root, source) + return _load_cases_json(root, source) + + +def check_anchors(case: EvalCase, diff_text: str, *, source: str = "--cases") -> None: + """Refuse a label of ``case`` that does not anchor on a line ``diff_text`` adds. + + Every expected finding's ``file`` must be a file the diff touches and its + ``line`` one of that file's added lines. The loader runs this check on + every case with a diff file; a caller holding the diff of a pinned range + can run it on that diff. A failure raises + :class:`~prxref.llm.ConfigError` naming ``source``, the case and the field. + """ + files = parse_unified_diff(diff_text) + _check_anchors(case.expected, files, f"case {case.id!r}", "expected", _JSON_SPELLING, source) + + +def _load_cases_json(root: Path, source: str) -> list[EvalCase]: + data = _read_json(root, source) + where = repr(str(root)) + if not isinstance(data, dict): + raise ConfigError( + f'{source}: {where}: must be an object like {{"version": 1, "cases": [...]}}, ' + f"got {_kind(data)}" + ) + unknown = sorted(set(data) - set(_TOP_KEYS)) + if unknown: + raise ConfigError( + f"{source}: {where}: {', '.join(unknown)}: unknown field; " + f"expected {' and '.join(_TOP_KEYS)}" + ) + version = data.get("version") + if type(version) is not int or version != CASES_VERSION: + raise ConfigError(f"{source}: {where}: version: must be {CASES_VERSION}, got {_kind(version)}") + entries = data.get("cases") + if not isinstance(entries, list) or not entries: + raise ConfigError(f"{source}: {where}: cases: must be a non-empty array, got {_kind(entries)}") + cases: list[EvalCase] = [] + first_index: dict[str, int] = {} + for index, entry in enumerate(entries): + case = _case_from_json(entry, index, root.parent, source) + if case.id in first_index: + raise ConfigError( + f"{source}: case {case.id!r}: id: duplicate of cases[{first_index[case.id]}] " + f"(cases[{index}])" + ) + first_index[case.id] = index + cases.append(case) + return cases + + +def _case_from_json(entry: Any, index: int, base: Path, source: str) -> EvalCase: + if not isinstance(entry, dict): + raise ConfigError(f"{source}: cases[{index}]: must be an object, got {_kind(entry)}") + case_id = entry.get("id") + if not isinstance(case_id, str) or not _CASE_ID_RE.fullmatch(case_id): + raise ConfigError( + f"{source}: cases[{index}]: id: must be a name of letters, digits, '.', '_' " + f"and '-' that starts with a letter or digit, got {_kind(case_id)}" + ) + where = f"case {case_id!r}" + unknown = sorted(set(entry) - set(_CASE_KEYS)) + if unknown: + raise ConfigError( + f"{source}: {where}: {', '.join(unknown)}: unknown field; " + f"allowed: {', '.join(_CASE_KEYS)}" + ) + pr_url = _optional_text(entry, "pr_url", where, source) + base_sha = _optional_text(entry, "base_sha", where, source) + head_sha = _optional_text(entry, "head_sha", where, source) + diff_file = _optional_text(entry, "diff_file", where, source) + context_file = _optional_text(entry, "context_file", where, source) + base_sha, head_sha = _check_replay(pr_url, base_sha, head_sha, diff_file, where, source) + if pr_url is not None and detect_forge(pr_url) is None: + raise ConfigError( + f"{source}: {where}: pr_url: not a pull request URL of a supported forge, " + f"got {pr_url!r}" + ) + if context_file is not None: + context_file = _join(base, context_file) + if not Path(context_file).is_file(): + raise ConfigError(f"{source}: {where}: context_file: no such file {context_file!r}") + spec = _spec_sources(entry.get("spec"), base, where, source) + if "expected" not in entry: + raise ConfigError(f"{source}: {where}: expected: required (an array of findings, possibly empty)") + expected = _expected_findings(entry["expected"], where, "expected", _JSON_SPELLING, source) + if diff_file is not None: + diff_file = _join(base, diff_file) + files = _read_diff(diff_file, where, "diff_file", source) + _check_anchors(expected, files, where, "expected", _JSON_SPELLING, source) + return EvalCase( + id=case_id, expected=expected, pr_url=pr_url, base_sha=base_sha, head_sha=head_sha, + diff_file=diff_file, context_file=context_file, spec=spec, + ) + + +def _check_replay( + pr_url: str | None, + base_sha: str | None, + head_sha: str | None, + diff_file: str | None, + where: str, + source: str, +) -> tuple[str | None, str | None]: + if pr_url is None and diff_file is None: + raise ConfigError( + f"{source}: {where}: pr_url/diff_file: one is required (diff_file replays a " + "local diff; pr_url with base_sha and head_sha replays a pinned range)" + ) + if (base_sha is None) != (head_sha is None): + missing, given = ("head_sha", "base_sha") if head_sha is None else ("base_sha", "head_sha") + raise ConfigError(f"{source}: {where}: {missing}: required together with {given}") + if base_sha is None or head_sha is None: + if diff_file is None: + raise ConfigError( + f"{source}: {where}: base_sha/head_sha: required with pr_url when there is " + "no diff_file, so the replay is pinned to a commit range" + ) + return None, None + for field, value in (("base_sha", base_sha), ("head_sha", head_sha)): + if not _FULL_SHA_RE.fullmatch(value): + raise ConfigError( + f"{source}: {where}: {field}: must be a full 40- or 64-character hex " + f"commit SHA, got {value!r}" + ) + base_sha, head_sha = base_sha.lower(), head_sha.lower() + if base_sha == head_sha: + raise ConfigError(f"{source}: {where}: base_sha/head_sha: must name two different commits") + if pr_url is None: + raise ConfigError( + f"{source}: {where}: pr_url: required with base_sha/head_sha (the range is " + "resolved in that pull request's repository)" + ) + return base_sha, head_sha + + +def _spec_sources(raw: Any, base: Path, where: str, source: str) -> tuple[str, ...]: + if raw is None: + return () + if isinstance(raw, str): + items: list[Any] = [raw] + labels = ["spec"] + elif isinstance(raw, list): + items = raw + labels = [f"spec[{i}]" for i in range(len(raw))] + else: + raise ConfigError( + f"{source}: {where}: spec: must be a string or an array of strings, got {_kind(raw)}" + ) + out: list[str] = [] + for item, label in zip(items, labels, strict=True): + if not isinstance(item, str) or not item.strip(): + raise ConfigError(f"{source}: {where}: {label}: must be a non-empty string, got {_kind(item)}") + parsed = urlparse(item) + if parsed.scheme in ("http", "https") and parsed.netloc: + out.append(item) + continue + resolved = _join(base, item) + if not Path(resolved).exists(): + raise ConfigError(f"{source}: {where}: {label}: no such file or directory {resolved!r}") + out.append(resolved) + return tuple(out) + + +def _load_directory(root: Path, source: str) -> list[EvalCase]: + try: + case_dirs = sorted(p for p in root.iterdir() if p.is_dir() and p.name.startswith("case-")) + except OSError as exc: + raise ConfigError(f"{source}: cannot read {str(root)!r}: {exc.strerror or exc}") from exc + if not case_dirs: + raise ConfigError( + f"{source}: {str(root)!r} has no case-*/ directories (pass a cases.json file, " + "or a directory of case-*/ directories)" + ) + return [_case_from_directory(case_dir, source) for case_dir in case_dirs] + + +def _case_from_directory(case_dir: Path, source: str) -> EvalCase: + case_id = case_dir.name + where = f"case {case_id!r}" + if not _CASE_ID_RE.fullmatch(case_id): + raise ConfigError( + f"{source}: {where}: id: the directory name must be letters, digits, '.', '_' " + "and '-' only" + ) + diff_file = str(case_dir / "diff.patch") + files = _read_diff(diff_file, where, "diff.patch", source) + expected_path = case_dir / "expected.json" + raw = _read_json(expected_path, f"{source}: {where}: expected.json") + expected = _expected_findings(raw, where, "expected.json", _DIRECTORY_SPELLING, source) + _check_anchors(expected, files, where, "expected.json", _DIRECTORY_SPELLING, source) + ticket = case_dir / "ticket.md" + docs = case_dir / "docs" + return EvalCase( + id=case_id, + expected=expected, + diff_file=diff_file, + context_file=str(ticket) if ticket.is_file() else None, + spec=(str(docs),) if docs.is_dir() else (), + ) + + +def _expected_findings( + raw: Any, where: str, prefix: str, spelling: dict[str, str], source: str +) -> tuple[ExpectedFinding, ...]: + if not isinstance(raw, list): + raise ConfigError(f"{source}: {where}: {prefix}: must be an array of findings, got {_kind(raw)}") + allowed = {spelling.get(key, key): key for key in _EXPECTED_KEYS} + findings: list[ExpectedFinding] = [] + seen: dict[str, int] = {} + for index, entry in enumerate(raw): + at = f"{prefix}[{index}]" + if not isinstance(entry, dict): + raise ConfigError(f"{source}: {where}: {at}: must be an object, got {_kind(entry)}") + unknown = sorted(set(entry) - set(allowed)) + if unknown: + raise ConfigError( + f"{source}: {where}: {at}.{unknown[0]}: unknown field; " + f"allowed: {', '.join(allowed)}" + ) + values = {key: entry.get(name) for name, key in allowed.items()} + problem = _finding_problem(entry, values, spelling) + if problem is None and values["id"] in seen: + problem = ("id", f"duplicate of {prefix}[{seen[values['id']]}]") + if problem is not None: + key, text = problem + raise ConfigError(f"{source}: {where}: {at}.{spelling.get(key, key)}: {text}") + seen[values["id"]] = index + findings.append(ExpectedFinding(**values)) + return tuple(findings) + + +def _finding_problem( + entry: dict, values: dict[str, Any], spelling: dict[str, str] +) -> tuple[str, str] | None: + for key in ("id", "file", "line", "severity"): + if spelling.get(key, key) not in entry: + return key, "required" + for key in ("id", "file"): + if not isinstance(values[key], str) or not values[key].strip(): + return key, f"must be a non-empty string, got {_kind(values[key])}" + line = values["line"] + if type(line) is not int or line < 1: + return "line", f"must be an integer >= 1, got {_kind(line)}" + severity = values["severity"] + if not isinstance(severity, str) or severity not in HUMAN_SEVERITIES: + return "severity", f"must be one of {', '.join(HUMAN_SEVERITIES)}, got {_kind(severity)}" + for key in ("category", "text", "must_match"): + value = values[key] + if value is not None and (not isinstance(value, str) or not value.strip()): + return key, f"must be a non-empty string or null, got {_kind(value)}" + accepted = values["accepted"] + if accepted is not None and not isinstance(accepted, bool): + return "accepted", f"must be true, false or null, got {_kind(accepted)}" + problem = _must_match_problem(values["must_match"]) + if problem: + return "must_match", problem + return None + + +def _must_match_problem(pattern: str | None) -> str: + if pattern is None or not pattern.startswith(MUST_MATCH_REGEX_PREFIX): + return "" + body = pattern[len(MUST_MATCH_REGEX_PREFIX):] + if not body: + return f"an empty regular expression after {MUST_MATCH_REGEX_PREFIX!r} matches everything" + try: + re.compile(body) + except re.error as exc: + return f"invalid regular expression after {MUST_MATCH_REGEX_PREFIX!r}: {exc}" + return "" + + +def _check_anchors( + expected: tuple[ExpectedFinding, ...], + files: list[FileDiff], + where: str, + prefix: str, + spelling: dict[str, str], + source: str, +) -> None: + by_path = {f.path: f for f in files} + for index, finding in enumerate(expected): + at = f"{prefix}[{index}]" + file_diff = by_path.get(finding.file) + if file_diff is None: + raise ConfigError( + f"{source}: {where}: {at}.file: {finding.file!r} is not a file the diff touches" + ) + if finding.line not in file_diff.added_lines: + raise ConfigError( + f"{source}: {where}: {at}.{spelling.get('line', 'line')}: line {finding.line} " + f"is not a line the diff adds to {finding.file!r}" + ) + + +def _read_diff(path: str, where: str, field: str, source: str) -> list[FileDiff]: + try: + text = Path(path).read_text(encoding="utf-8", errors="replace") + except OSError as exc: + raise ConfigError(f"{source}: {where}: {field}: cannot read {path!r}: {exc.strerror or exc}") from exc + files = parse_unified_diff(text) + if not files: + raise ConfigError(f"{source}: {where}: {field}: {path!r} holds no file diffs") + return files + + +def _read_json(path: Path, head: str) -> Any: + try: + text = path.read_text(encoding="utf-8") + except OSError as exc: + raise ConfigError(f"{head}: cannot read {str(path)!r}: {exc.strerror or exc}") from exc + except UnicodeDecodeError as exc: + raise ConfigError(f"{head}: {str(path)!r} is not valid UTF-8") from exc + try: + return json.loads(text) + except json.JSONDecodeError as exc: + raise ConfigError( + f"{head}: {str(path)!r} is not valid JSON " + f"({exc.msg} at line {exc.lineno} column {exc.colno})" + ) from exc + + +def _optional_text(entry: dict, key: str, where: str, source: str) -> str | None: + value = entry.get(key) + if value is None: + return None + if not isinstance(value, str) or not value.strip(): + raise ConfigError(f"{source}: {where}: {key}: must be a non-empty string or null, got {_kind(value)}") + return value + + +def _join(base: Path, value: str) -> str: + return str(base / value) + + +def _kind(value: object) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return f"the boolean {str(value).lower()}" + if isinstance(value, int | float): + return f"the number {value!r}" + if isinstance(value, str): + return f"the string {value!r}" + if isinstance(value, dict): + return "an object" + if isinstance(value, list): + return "an array" + return type(value).__name__ diff --git a/tests/test_eval_cases.py b/tests/test_eval_cases.py new file mode 100644 index 00000000..90e73afa --- /dev/null +++ b/tests/test_eval_cases.py @@ -0,0 +1,509 @@ +"""Eval case loading (issue #14, T1): ``prxref.eval_cases.load_cases``. + +``--cases`` is a ``cases.json`` file or a directory of ``case-*/`` dirs. Both +forms load into the same frozen records; the directory form maps +``line_hint`` onto ``line`` and ``source`` onto ``category``; the repo's own +``tests/evals`` dataset loads whole; and every bad case is a ``ConfigError`` +whose message names ``--cases``, the case and the field, so the CLI can exit 2. +All identities are placeholders. +""" +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest + +from prxref.eval_cases import ( + HUMAN_SEVERITIES, + EvalCase, + ExpectedFinding, + check_anchors, + load_cases, +) +from prxref.llm import ConfigError +from tests.evals.test_evals import CASE_DIRS, EVALS_DIR + +DIFF = ( + "diff --git a/src/app.py b/src/app.py\n" + "--- a/src/app.py\n" + "+++ b/src/app.py\n" + "@@ -1,2 +1,3 @@\n" + " import os\n" + "+import sys\n" + " print(os.name)\n" +) +OTHER_DIFF = DIFF.replace("src/app.py", "src/other.py") +PR_URL = "https://github.com/acme/widgets/pull/7" +BASE = "a" * 40 +HEAD = "b" * 40 +DROP = object() + + +def _finding(**over) -> dict: + entry = {"id": "H1", "file": "src/app.py", "line": 2, "severity": "error"} + entry.update(over) + return {key: value for key, value in entry.items() if value is not DROP} + + +def _case(**over) -> dict: + entry = {"id": "case-a", "diff_file": "change.diff", "expected": [_finding()]} + entry.update(over) + return {key: value for key, value in entry.items() if value is not DROP} + + +def _dataset(tmp_path: Path, cases, *, version=1) -> Path: + (tmp_path / "change.diff").write_text(DIFF, encoding="utf-8") + path = tmp_path / "cases.json" + path.write_text(json.dumps({"version": version, "cases": cases}), encoding="utf-8") + return path + + +def _refusal(path, **kwargs) -> str: + with pytest.raises(ConfigError) as exc: + load_cases(path, **kwargs) + return str(exc.value) + + +def _case_dir(root: Path, name: str, expected, *, diff: str = DIFF, ticket=True, docs=True) -> Path: + case_dir = root / name + case_dir.mkdir(parents=True) + (case_dir / "diff.patch").write_text(diff, encoding="utf-8") + (case_dir / "expected.json").write_text(json.dumps(expected), encoding="utf-8") + if ticket: + (case_dir / "ticket.md").write_text("## Summary\nplaceholder ticket\n", encoding="utf-8") + if docs: + (case_dir / "docs").mkdir() + (case_dir / "docs" / "rules.md").write_text("Imports MUST be sorted.\n", encoding="utf-8") + return case_dir + + +def _dir_finding(**over) -> dict: + entry = { + "id": "S1", "file": "src/app.py", "line_hint": 2, "severity": "spec", + "must_match": "sys", "source": "spec", + } + entry.update(over) + return {key: value for key, value in entry.items() if value is not DROP} + + +class TestCasesJson: + def test_a_diff_file_case_loads_with_paths_joined_onto_the_dataset_dir(self, tmp_path): + (tmp_path / "docs").mkdir() + (tmp_path / "ticket.md").write_text("placeholder\n", encoding="utf-8") + path = _dataset(tmp_path, [_case( + context_file="ticket.md", + spec=["docs", "https://docs.example.com/spec"], + expected=[_finding( + category="security", accepted=True, text="unused import", must_match="re:sys|unused", + )], + )]) + + (case,) = load_cases(path) + + assert case == EvalCase( + id="case-a", + expected=(ExpectedFinding( + id="H1", file="src/app.py", line=2, severity="error", category="security", + accepted=True, text="unused import", must_match="re:sys|unused", + ),), + diff_file=str(tmp_path / "change.diff"), + context_file=str(tmp_path / "ticket.md"), + spec=(str(tmp_path / "docs"), "https://docs.example.com/spec"), + ) + + def test_a_pinned_pr_case_loads_with_lowercased_shas_and_no_diff(self, tmp_path): + path = _dataset(tmp_path, [_case( + diff_file=DROP, pr_url=PR_URL, base_sha=BASE.upper(), head_sha=HEAD, + expected=[_finding(file="src/not-in-any-local-diff.py", line=40)], + )]) + + (case,) = load_cases(path) + + assert (case.pr_url, case.base_sha, case.head_sha, case.diff_file) == (PR_URL, BASE, HEAD, None) + assert case.expected[0].line == 40 + + def test_64_character_shas_are_accepted(self, tmp_path): + path = _dataset(tmp_path, [_case( + diff_file=DROP, pr_url=PR_URL, base_sha="c" * 64, head_sha="d" * 64, + )]) + + (case,) = load_cases(path) + + assert (case.base_sha, case.head_sha) == ("c" * 64, "d" * 64) + + def test_pr_url_beside_a_diff_file_needs_no_shas(self, tmp_path): + (case,) = load_cases(_dataset(tmp_path, [_case(pr_url=PR_URL)])) + + assert (case.pr_url, case.base_sha, case.head_sha) == (PR_URL, None, None) + assert case.diff_file == str(tmp_path / "change.diff") + + def test_null_optional_fields_count_as_absent(self, tmp_path): + path = _dataset(tmp_path, [_case( + pr_url=None, base_sha=None, head_sha=None, context_file=None, spec=None, + expected=[_finding(category=None, accepted=None, text=None, must_match=None)], + )]) + + (case,) = load_cases(path) + + assert case == EvalCase( + id="case-a", + expected=(ExpectedFinding(id="H1", file="src/app.py", line=2, severity="error"),), + diff_file=str(tmp_path / "change.diff"), + ) + + def test_a_single_spec_string_becomes_a_one_item_tuple(self, tmp_path): + (tmp_path / "docs").mkdir() + + (case,) = load_cases(_dataset(tmp_path, [_case(spec="docs")])) + + assert case.spec == (str(tmp_path / "docs"),) + + def test_an_absolute_path_is_kept_as_given(self, tmp_path): + outside = tmp_path / "elsewhere" + outside.mkdir() + (outside / "ticket.md").write_text("placeholder\n", encoding="utf-8") + dataset = tmp_path / "dataset" + dataset.mkdir() + + (case,) = load_cases(_dataset(dataset, [_case(context_file=str(outside / "ticket.md"))])) + + assert case.context_file == str(outside / "ticket.md") + + def test_a_relative_cases_path_keeps_the_case_paths_relative(self, tmp_path, monkeypatch): + _dataset(tmp_path, [_case()]) + monkeypatch.chdir(tmp_path) + + (case,) = load_cases("cases.json") + + assert case.diff_file == "change.diff" + + def test_every_human_severity_is_stored_as_given(self, tmp_path): + expected = [ + _finding(id=f"H{i}", severity=severity) for i, severity in enumerate(HUMAN_SEVERITIES) + ] + + (case,) = load_cases(_dataset(tmp_path, [_case(expected=expected)])) + + assert [f.severity for f in case.expected] == ["error", "warning", "minor", "spec", "outofscope"] + + def test_an_empty_expected_list_is_a_clean_case(self, tmp_path): + (case,) = load_cases(_dataset(tmp_path, [_case(expected=[])])) + + assert case.expected == () + + def test_cases_keep_file_order(self, tmp_path): + cases = [_case(id=name) for name in ("zeta", "alpha", "mid.1_x")] + + loaded = load_cases(_dataset(tmp_path, cases)) + + assert [case.id for case in loaded] == ["zeta", "alpha", "mid.1_x"] + + def test_the_records_are_frozen(self, tmp_path): + (case,) = load_cases(_dataset(tmp_path, [_case()])) + + with pytest.raises(dataclasses.FrozenInstanceError): + case.id = "other" + with pytest.raises(dataclasses.FrozenInstanceError): + case.expected[0].line = 3 + assert isinstance(case.expected, tuple) + assert isinstance(case.spec, tuple) + + +class TestDirectoryForm: + def test_case_dirs_load_sorted_with_fields_mapped(self, tmp_path): + _case_dir(tmp_path, "case-b", [_dir_finding()]) + _case_dir(tmp_path, "case-a", [_dir_finding(id="G1", severity="error", source="generic")], + ticket=False, docs=False) + (tmp_path / "notes").mkdir() + (tmp_path / "case-file.txt").write_text("not a case\n", encoding="utf-8") + + loaded = load_cases(tmp_path) + + assert loaded == [ + EvalCase( + id="case-a", + expected=(ExpectedFinding( + id="G1", file="src/app.py", line=2, severity="error", category="generic", + must_match="sys", + ),), + diff_file=str(tmp_path / "case-a" / "diff.patch"), + ), + EvalCase( + id="case-b", + expected=(ExpectedFinding( + id="S1", file="src/app.py", line=2, severity="spec", category="spec", + must_match="sys", + ),), + diff_file=str(tmp_path / "case-b" / "diff.patch"), + context_file=str(tmp_path / "case-b" / "ticket.md"), + spec=(str(tmp_path / "case-b" / "docs"),), + ), + ] + + def test_accepted_and_text_are_optional_in_expected_json(self, tmp_path): + _case_dir(tmp_path, "case-a", [_dir_finding(accepted=False, text="placeholder", must_match=DROP, source=DROP)]) + + (case,) = load_cases(str(tmp_path)) + + assert case.expected == (ExpectedFinding( + id="S1", file="src/app.py", line=2, severity="spec", accepted=False, text="placeholder", + ),) + + def test_the_repo_eval_dataset_loads_whole(self): + loaded = load_cases(EVALS_DIR) + + assert [case.id for case in loaded] == [p.name for p in CASE_DIRS] + assert len(loaded) >= 3 + for case, case_dir in zip(loaded, CASE_DIRS, strict=True): + labels = json.loads((case_dir / "expected.json").read_text(encoding="utf-8")) + assert case.diff_file == str(case_dir / "diff.patch") + assert case.context_file == str(case_dir / "ticket.md") + assert case.spec == (str(case_dir / "docs"),) + assert (case.pr_url, case.base_sha, case.head_sha) == (None, None, None) + assert [ + (f.id, f.file, f.line, f.severity, f.category, f.must_match) for f in case.expected + ] == [ + (e["id"], e["file"], e["line_hint"], e["severity"], e["source"], e["must_match"]) + for e in labels + ] + assert any(f.severity == "spec" for case in loaded for f in case.expected) + + +class TestCasesJsonRefusals: + @pytest.mark.parametrize( + ("case", "field"), + [ + (_case(diff_fle="change.diff"), "diff_fle"), + (_case(diff_file=DROP), "pr_url/diff_file"), + (_case(diff_file=DROP, pr_url=PR_URL), "base_sha/head_sha"), + (_case(diff_file=DROP, pr_url=PR_URL, base_sha=BASE), "head_sha"), + (_case(diff_file=DROP, pr_url=PR_URL, head_sha=HEAD), "base_sha"), + (_case(diff_file=DROP, pr_url=PR_URL, base_sha="abc1234", head_sha=HEAD), "base_sha"), + (_case(diff_file=DROP, pr_url=PR_URL, base_sha=BASE, head_sha="g" * 40), "head_sha"), + (_case(diff_file=DROP, pr_url=PR_URL, base_sha=BASE, head_sha=BASE.upper()), "base_sha/head_sha"), + (_case(base_sha=BASE, head_sha=HEAD), "pr_url"), + (_case(pr_url="https://example.com/acme/widgets"), "pr_url"), + (_case(pr_url=7), "pr_url"), + (_case(pr_url=" "), "pr_url"), + (_case(diff_file="missing.diff"), "diff_file"), + (_case(diff_file=["change.diff"]), "diff_file"), + (_case(context_file="missing.md"), "context_file"), + (_case(spec="missing-docs"), "spec"), + (_case(spec=3), "spec"), + (_case(spec=["missing-docs"]), "spec[0]"), + (_case(spec=["https://docs.example.com/spec", 4]), "spec[1]"), + (_case(expected=DROP), "expected"), + (_case(expected={"id": "H1"}), "expected"), + (_case(expected=["H1"]), "expected[0]"), + (_case(expected=[_finding(line_hint=2, line=DROP)]), "expected[0].line_hint"), + (_case(expected=[_finding(id=DROP)]), "expected[0].id"), + (_case(expected=[_finding(id="")]), "expected[0].id"), + (_case(expected=[_finding(), _finding()]), "expected[1].id"), + (_case(expected=[_finding(file=DROP)]), "expected[0].file"), + (_case(expected=[_finding(file=3)]), "expected[0].file"), + (_case(expected=[_finding(line=DROP)]), "expected[0].line"), + (_case(expected=[_finding(line=0)]), "expected[0].line"), + (_case(expected=[_finding(line="2")]), "expected[0].line"), + (_case(expected=[_finding(line=True)]), "expected[0].line"), + (_case(expected=[_finding(line=2.0)]), "expected[0].line"), + (_case(expected=[_finding(severity=DROP)]), "expected[0].severity"), + (_case(expected=[_finding(severity="critical")]), "expected[0].severity"), + (_case(expected=[_finding(severity="Error")]), "expected[0].severity"), + (_case(expected=[_finding(severity=None)]), "expected[0].severity"), + (_case(expected=[_finding(category=3)]), "expected[0].category"), + (_case(expected=[_finding(category="")]), "expected[0].category"), + (_case(expected=[_finding(accepted="yes")]), "expected[0].accepted"), + (_case(expected=[_finding(text=5)]), "expected[0].text"), + (_case(expected=[_finding(must_match="")]), "expected[0].must_match"), + (_case(expected=[_finding(must_match="re:(")]), "expected[0].must_match"), + (_case(expected=[_finding(must_match="re:")]), "expected[0].must_match"), + (_case(expected=[_finding(file="src/other.py")]), "expected[0].file"), + (_case(expected=[_finding(line=1)]), "expected[0].line"), + (_case(expected=[_finding(), _finding(id="H2", line=3)]), "expected[1].line"), + (_case(expected=[_finding(line=99)]), "expected[0].line"), + ], + ) + def test_a_bad_case_names_the_flag_the_case_and_the_field(self, tmp_path, case, field): + message = _refusal(_dataset(tmp_path, [case])) + + assert message.startswith(f"--cases: case 'case-a': {field}: "), message + + def test_an_empty_diff_file_is_refused(self, tmp_path): + (tmp_path / "empty.diff").write_text("", encoding="utf-8") + + message = _refusal(_dataset(tmp_path, [_case(diff_file="empty.diff")])) + + assert message.startswith("--cases: case 'case-a': diff_file: "), message + assert "holds no file diffs" in message + + def test_a_label_off_the_added_lines_says_which_line_and_file(self, tmp_path): + message = _refusal(_dataset(tmp_path, [_case(expected=[_finding(line=3)])])) + + assert message == ( + "--cases: case 'case-a': expected[0].line: line 3 is not a line the diff adds to 'src/app.py'" + ) + + def test_the_second_bad_case_is_named_after_a_good_one(self, tmp_path): + cases = [_case(), _case(id="case-b", expected=[_finding(severity="blocker")])] + + message = _refusal(_dataset(tmp_path, cases)) + + assert message.startswith("--cases: case 'case-b': expected[0].severity: must be one of "), message + assert "error, warning, minor, spec, outofscope" in message + + @pytest.mark.parametrize("bad_id", [DROP, "", "../escape", ".hidden", "a/b", "has space", 7, None]) + def test_a_bad_case_id_is_named_by_position(self, tmp_path, bad_id): + message = _refusal(_dataset(tmp_path, [_case(), _case(id=bad_id)])) + + assert message.startswith("--cases: cases[1]: id: "), message + + def test_a_case_that_is_not_an_object_is_named_by_position(self, tmp_path): + message = _refusal(_dataset(tmp_path, [_case(), "case-b"])) + + assert message == "--cases: cases[1]: must be an object, got the string 'case-b'" + + def test_a_duplicate_case_id_is_refused(self, tmp_path): + message = _refusal(_dataset(tmp_path, [_case(), _case(id="case-b"), _case()])) + + assert message == "--cases: case 'case-a': id: duplicate of cases[0] (cases[2])" + + def test_the_source_label_leads_every_message(self, tmp_path): + message = _refusal(_dataset(tmp_path, [_case(diff_file=DROP)]), source="--eval-cases") + + assert message.startswith("--eval-cases: case 'case-a': pr_url/diff_file: "), message + + +class TestCasesFileRefusals: + def test_a_missing_path_is_refused(self, tmp_path): + missing = tmp_path / "nope.json" + + message = _refusal(missing) + + assert message.startswith(f"--cases: cannot read {str(missing)!r}: "), message + + def test_invalid_json_is_refused_with_its_position(self, tmp_path): + path = tmp_path / "cases.json" + path.write_text('{"version": 1,\n "cases": [}', encoding="utf-8") + + message = _refusal(path) + + assert message.startswith(f"--cases: {str(path)!r} is not valid JSON ("), message + assert "line 2" in message + + def test_non_utf8_is_refused(self, tmp_path): + path = tmp_path / "cases.json" + path.write_bytes(b'{"version": 1, "cases": ["\xff"]}') + + assert _refusal(path) == f"--cases: {str(path)!r} is not valid UTF-8" + + @pytest.mark.parametrize( + ("document", "fragment"), + [ + ([], 'must be an object like {"version": 1, "cases": [...]}, got an array'), + ({"version": 1, "cases": [_case()], "extra": 1}, "extra: unknown field; expected version and cases"), + ({"cases": [_case()]}, "version: must be 1, got null"), + ({"version": 2, "cases": [_case()]}, "version: must be 1, got the number 2"), + ({"version": True, "cases": [_case()]}, "version: must be 1, got the boolean true"), + ({"version": 1.0, "cases": [_case()]}, "version: must be 1, got the number 1.0"), + ({"version": "1", "cases": [_case()]}, "version: must be 1, got the string '1'"), + ({"version": 1}, "cases: must be a non-empty array, got null"), + ({"version": 1, "cases": []}, "cases: must be a non-empty array, got an array"), + ({"version": 1, "cases": {"case-a": {}}}, "cases: must be a non-empty array, got an object"), + ], + ) + def test_a_bad_top_level_names_the_file_and_the_field(self, tmp_path, document, fragment): + (tmp_path / "change.diff").write_text(DIFF, encoding="utf-8") + path = tmp_path / "cases.json" + path.write_text(json.dumps(document), encoding="utf-8") + + assert _refusal(path) == f"--cases: {str(path)!r}: {fragment}" + + +class TestDirectoryRefusals: + def test_a_directory_without_case_dirs_is_refused(self, tmp_path): + (tmp_path / "notes").mkdir() + + message = _refusal(tmp_path) + + assert message.startswith(f"--cases: {str(tmp_path)!r} has no case-*/ directories"), message + + def test_a_missing_diff_patch_is_refused(self, tmp_path): + case_dir = _case_dir(tmp_path, "case-a", [_dir_finding()]) + (case_dir / "diff.patch").unlink() + + message = _refusal(tmp_path) + + assert message.startswith("--cases: case 'case-a': diff.patch: cannot read "), message + + def test_a_missing_expected_json_is_refused(self, tmp_path): + case_dir = _case_dir(tmp_path, "case-a", [_dir_finding()]) + (case_dir / "expected.json").unlink() + + message = _refusal(tmp_path) + + assert message.startswith("--cases: case 'case-a': expected.json: cannot read "), message + + def test_invalid_expected_json_is_refused(self, tmp_path): + case_dir = _case_dir(tmp_path, "case-a", [_dir_finding()]) + (case_dir / "expected.json").write_text("[{", encoding="utf-8") + + message = _refusal(tmp_path) + + assert message.startswith("--cases: case 'case-a': expected.json: "), message + assert "is not valid JSON" in message + + @pytest.mark.parametrize( + ("expected", "field"), + [ + ({"id": "S1"}, "expected.json"), + ([_dir_finding(line=2, line_hint=DROP)], "expected.json[0].line"), + ([_dir_finding(category="spec", source=DROP)], "expected.json[0].category"), + ([_dir_finding(line_hint=DROP)], "expected.json[0].line_hint"), + ([_dir_finding(line_hint=0)], "expected.json[0].line_hint"), + ([_dir_finding(source=3)], "expected.json[0].source"), + ([_dir_finding(must_match="re:[")], "expected.json[0].must_match"), + ([_dir_finding(), _dir_finding()], "expected.json[1].id"), + ([_dir_finding(line_hint=1)], "expected.json[0].line_hint"), + ([_dir_finding(file="src/other.py")], "expected.json[0].file"), + ], + ) + def test_a_bad_label_is_named_in_the_directory_spelling(self, tmp_path, expected, field): + _case_dir(tmp_path, "case-a", expected) + + message = _refusal(tmp_path) + + assert message.startswith(f"--cases: case 'case-a': {field}: "), message + + def test_a_directory_name_that_is_not_a_safe_id_is_refused(self, tmp_path): + _case_dir(tmp_path, "case-a b", [_dir_finding()]) + + message = _refusal(tmp_path) + + assert message.startswith("--cases: case 'case-a b': id: "), message + + +class TestCheckAnchors: + def test_a_pinned_case_can_be_checked_against_its_fetched_diff(self, tmp_path): + (case,) = load_cases(_dataset(tmp_path, [_case( + diff_file=DROP, pr_url=PR_URL, base_sha=BASE, head_sha=HEAD, + )])) + + check_anchors(case, DIFF) + + with pytest.raises(ConfigError) as exc: + check_anchors(case, OTHER_DIFF) + assert str(exc.value) == ( + "--cases: case 'case-a': expected[0].file: 'src/app.py' is not a file the diff touches" + ) + + def test_a_label_on_a_context_line_fails_the_check(self): + case = EvalCase( + id="case-a", + expected=(ExpectedFinding(id="H1", file="src/app.py", line=1, severity="minor"),), + ) + + with pytest.raises(ConfigError) as exc: + check_anchors(case, DIFF, source="--label") + assert str(exc.value).startswith("--label: case 'case-a': expected[0].line: line 1 "), str(exc.value) From ded136402739ebfc6d3dc24933d79966cc4d5ab8 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:38:58 -0700 Subject: [PATCH 09/64] feat: PRHistory and pure description pinning for replays (#16 T1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A replay keeps the PR's live title and description, and authors often edit the description after review to list the fixes, so a replay hands the reviewer the answers. This lands the pure pieces the later seats wire in, with no forge and no network: - forges/base.py: frozen DescriptionVersion(text, edited_at), TitleRename(previous_title, current_title, created_at) and PRHistory (created_at, description_versions, title_renames, first_review_at, head_committed_at, complete). Every datetime must be timezone-aware; a naive one raises ValueError naming the field. - Forge Protocol: the optional get_pr_history(ref, *, head_sha=None), resolved with getattr like get_compare_diff. No forge implements it yet (T2 GitHub, T7 Bitbucket Cloud). - forges/replay.py: PinnedMetadata(title, description, status), pin_pr_metadata (the version in force at the cutoff, ordered by edited_at; 0 edits is pinned; a deleted or unreached version is live; the title follows the description) and choose_cutoff (flag, then first-review, then head-commit, else None). 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/forges/base.py | 105 ++++++++ src/prxref/forges/replay.py | 89 ++++++- tests/test_issue_16_replay_description.py | 301 ++++++++++++++++++++++ 3 files changed, 494 insertions(+), 1 deletion(-) create mode 100644 tests/test_issue_16_replay_description.py diff --git a/src/prxref/forges/base.py b/src/prxref/forges/base.py index 34c76e87..8f9450b7 100644 --- a/src/prxref/forges/base.py +++ b/src/prxref/forges/base.py @@ -13,6 +13,7 @@ from collections.abc import Sequence from dataclasses import dataclass +from datetime import datetime from typing import Protocol @@ -63,6 +64,97 @@ class PRData: raw: dict # forge-native payload, for forge-specific needs +def _require_aware(value: datetime, name: str) -> None: + """Raise ``ValueError`` naming ``name`` unless ``value`` is a timezone-aware datetime. + + A naive datetime is refused rather than assumed to be UTC: comparing it + with an aware one either raises a ``TypeError`` that names nothing or, for + ``==``, is silently ``False``, and a cutoff an hour off picks the wrong + description version without any error at all. + """ + if not isinstance(value, datetime): + raise ValueError(f"{name} must be a datetime, got {type(value).__name__}") + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError(f"{name} must be timezone-aware, got the naive {value.isoformat()}") + + +@dataclass(frozen=True) +class DescriptionVersion: + """One version of a PR description: its full ``text`` from ``edited_at`` onwards. + + ``text`` is ``None`` when the forge no longer holds that version (GitHub + nulls the ``diff`` of a deleted edit), which pins nothing. + """ + + text: str | None + edited_at: datetime + + def __post_init__(self) -> None: + _require_aware(self.edited_at, "DescriptionVersion.edited_at") + + +@dataclass(frozen=True) +class TitleRename: + """One PR title rename: ``previous_title`` became ``current_title`` at ``created_at``.""" + + previous_title: str + current_title: str + created_at: datetime + + def __post_init__(self) -> None: + _require_aware(self.created_at, "TitleRename.created_at") + + +@dataclass(frozen=True) +class PRHistory: + """What a replay needs to pin a PR's title and description to a cutoff, from ``get_pr_history``. + + ``description_versions`` holds one entry per version of the description, + the original included and the live text last, in any order: consumers + order them by ``edited_at``, never by position. It is empty for a PR whose + description was never edited, whose live description is then the + original. ``title_renames`` holds every rename, in any order. + ``first_review_at`` is the earliest human review or comment by someone + other than the author (prxref's own posts excluded), and + ``head_committed_at`` the commit date of the + head the history was read for; either is ``None`` when unknown. + + ``complete`` is ``False`` when the description versions stop short of + the original. Every forge pages its history newest first, so the + versions held are then the newest ones, contiguous back from the live + text, and the older ones are missing. A forge that cannot read every + title rename returns ``complete=False`` with no description versions, + which pins nothing. + + Every datetime must be timezone-aware; a naive one raises ``ValueError``. + Both sequences are stored as tuples, so a history is immutable and hashable. + """ + + created_at: datetime + description_versions: tuple[DescriptionVersion, ...] = () + title_renames: tuple[TitleRename, ...] = () + first_review_at: datetime | None = None + head_committed_at: datetime | None = None + complete: bool = True + + def __post_init__(self) -> None: + _require_aware(self.created_at, "PRHistory.created_at") + if self.first_review_at is not None: + _require_aware(self.first_review_at, "PRHistory.first_review_at") + if self.head_committed_at is not None: + _require_aware(self.head_committed_at, "PRHistory.head_committed_at") + object.__setattr__(self, "description_versions", tuple(self.description_versions)) + object.__setattr__(self, "title_renames", tuple(self.title_renames)) + for version in self.description_versions: + if not isinstance(version, DescriptionVersion): + raise ValueError( + f"PRHistory.description_versions must hold DescriptionVersion, got {type(version).__name__}" + ) + for rename in self.title_renames: + if not isinstance(rename, TitleRename): + raise ValueError(f"PRHistory.title_renames must hold TitleRename, got {type(rename).__name__}") + + SUMMARY_MARKER = "" # The attribution prefix every posted comment carries (the full line is @@ -164,6 +256,19 @@ def get_compare_diff(self, ref: PRRef, *, base_sha: str, head_sha: str) -> str: """ ... + def get_pr_history(self, ref: PRRef, *, head_sha: str | None = None) -> PRHistory: + """Return the PR's description versions, title renames and replay cutoff inputs. + + Optional: callers resolve it with ``getattr(forge, "get_pr_history", None)``, + so a Forge without it is still valid, and not every forge implements it + (a replay then keeps the live title and description). ``head_sha`` is + the pinned head whose commit date becomes ``head_committed_at``; with + ``None`` it is the PR's current head. Raises on transport, HTTP or + authentication failure like ``get_diff``; a history that could not be + paged to the end is returned with ``complete=False``, never raised. + """ + ... + def detect_forge(url: str) -> PRRef | None: """Try each registered forge's URL parser in order. diff --git a/src/prxref/forges/replay.py b/src/prxref/forges/replay.py index 2f16974d..f87bf8fb 100644 --- a/src/prxref/forges/replay.py +++ b/src/prxref/forges/replay.py @@ -24,9 +24,11 @@ import email.utils import re from collections.abc import Sequence +from datetime import datetime from pathlib import Path +from typing import Literal -from .base import Forge, InlineComment, PRData, PRRef, Thread +from .base import Forge, InlineComment, PRData, PRHistory, PRRef, Thread, _require_aware NEVER_POSTS = "replay runs never write to a forge" @@ -253,3 +255,88 @@ def post_inline_comments(self, ref: PRRef, comments: Sequence[InlineComment]) -> def prune_inline_comments(self, *args: object, **kwargs: object) -> int: """Always raises ``RuntimeError``: a replay never writes to a forge.""" raise RuntimeError(NEVER_POSTS) + + +PinStatus = Literal["pinned", "live"] +CutoffSource = Literal["flag", "first-review", "head-commit"] + + +@dataclasses.dataclass(frozen=True) +class PinnedMetadata: + """The title and description a replay shows, and whether they were pinned. + + ``status`` is ``"pinned"`` when both are the ones in force at the cutoff, + and ``"live"`` when both are the PR's current ones because the history + could not answer. The title never has a status of its own: it is pinned + exactly when the description is. + """ + + title: str + description: str + status: PinStatus + + +def pin_pr_metadata( + history: PRHistory, *, live_title: str, live_description: str, cutoff: datetime, +) -> PinnedMetadata: + """Return the title and description in force at ``cutoff``, or the live ones. + + The description in force is the last version, ordered by ``edited_at`` + and never by position, whose ``edited_at`` is at or before the cutoff. + A complete history with no such version gives its oldest, which is the + original, so a cutoff before ``history.created_at`` clamps to the + original. With no versions at all, a complete history was never edited, + so ``live_description`` is the original and is ``pinned``. + The result is ``live``, with ``live_title`` and ``live_description``, + when the version in force has no text (deleted), or when an incomplete + history holds no version at or before the cutoff and so does not reach + it. When the description is pinned, the title is the ``previous_title`` + of the first rename, ordered by ``created_at``, made strictly after the + cutoff, else ``live_title``. + + Raises ``ValueError`` when ``cutoff`` is naive. + """ + _require_aware(cutoff, "cutoff") + live = PinnedMetadata(title=live_title, description=live_description, status="live") + versions = sorted(history.description_versions, key=lambda version: version.edited_at) + if versions: + reached = [version for version in versions if version.edited_at <= cutoff] + if reached: + in_force = reached[-1] + elif history.complete: + in_force = versions[0] + else: + return live + if in_force.text is None: + return live + description = in_force.text + elif history.complete: + description = live_description + else: + return live + renames = sorted(history.title_renames, key=lambda rename: rename.created_at) + title = next((rename.previous_title for rename in renames if rename.created_at > cutoff), live_title) + return PinnedMetadata(title=title, description=description, status="pinned") + + +def choose_cutoff(as_of: datetime | None, history: PRHistory | None) -> tuple[datetime, CutoffSource] | None: + """Return the replay cutoff and where it came from, or ``None`` when nothing gives one. + + The ladder is ``as_of`` (``"flag"``, from ``--as-of``), else the + history's ``first_review_at`` (``"first-review"``), else its + ``head_committed_at`` (``"head-commit"``). ``history`` is ``None`` for a + forge with no ``get_pr_history``, which leaves only the flag. The cutoff + is returned as given, unclamped; ``pin_pr_metadata`` clamps it. + + Raises ``ValueError`` when ``as_of`` is naive. + """ + if as_of is not None: + _require_aware(as_of, "as_of") + return as_of, "flag" + if history is None: + return None + if history.first_review_at is not None: + return history.first_review_at, "first-review" + if history.head_committed_at is not None: + return history.head_committed_at, "head-commit" + return None diff --git a/tests/test_issue_16_replay_description.py b/tests/test_issue_16_replay_description.py new file mode 100644 index 00000000..ee89cc90 --- /dev/null +++ b/tests/test_issue_16_replay_description.py @@ -0,0 +1,301 @@ +"""Issue #16 T1: pin a replay's PR title and description to a cutoff. + +A replay of a PR must not show the reviewer a description the author edited +after review to list the fixes. These are the pure pieces, with no forge and +no network: the ``PRHistory`` a forge's optional ``get_pr_history`` returns, +``pin_pr_metadata`` (the title and description in force at a cutoff, or the +live ones) and ``choose_cutoff`` (``--as-of``, else the first human review, +else the head commit's date). The CLI wiring, the stamp and the forge +implementations are later tasks and are tested there. +""" +from __future__ import annotations + +import dataclasses +import inspect +import itertools +import socket +from datetime import UTC, datetime, timedelta, timezone, tzinfo + +import pytest + +from prxref.forges.base import DescriptionVersion, Forge, PRHistory, TitleRename +from prxref.forges.replay import PinnedMetadata, choose_cutoff, pin_pr_metadata +from tests.test_orchestrator import FakeForge + +CREATED = datetime(2026, 3, 1, 9, 0, tzinfo=UTC) +EDIT_1 = CREATED + timedelta(days=1) +EDIT_2 = CREATED + timedelta(days=3) +REVIEW = CREATED + timedelta(days=2) +HEAD_COMMIT = CREATED + timedelta(hours=5) + +ORIGINAL = "Adds the parser." +MIDDLE = "Adds the parser and its tests." +LIVE = "Adds the parser.\n\n| Review comment | Fix |\n| rename data | done |" +LIVE_TITLE = "parser: add, address review" + +V0 = DescriptionVersion(text=ORIGINAL, edited_at=CREATED) +V1 = DescriptionVersion(text=MIDDLE, edited_at=EDIT_1) +V2 = DescriptionVersion(text=LIVE, edited_at=EDIT_2) +VERSIONS = (V0, V1, V2) + +RENAME_1 = TitleRename(previous_title="parser: add", current_title="parser: add it", created_at=EDIT_1) +RENAME_2 = TitleRename(previous_title="parser: add it", current_title=LIVE_TITLE, created_at=EDIT_2) +RENAMES = (RENAME_1, RENAME_2) + + +def _history(**overrides) -> PRHistory: + fields = {"created_at": CREATED, "description_versions": VERSIONS, "title_renames": RENAMES} + fields.update(overrides) + return PRHistory(**fields) + + +def _pin(history: PRHistory, cutoff: datetime) -> PinnedMetadata: + return pin_pr_metadata(history, live_title=LIVE_TITLE, live_description=LIVE, cutoff=cutoff) + + +LIVE_RESULT = PinnedMetadata(title=LIVE_TITLE, description=LIVE, status="live") + + +class TestProtocolDeclaration: + def test_the_protocol_declares_it_with_a_keyword_only_head_sha(self): + signature = inspect.signature(Forge.get_pr_history) + params = signature.parameters + assert list(params) == ["self", "ref", "head_sha"] + assert params["head_sha"].kind is inspect.Parameter.KEYWORD_ONLY + assert params["head_sha"].default is None + assert signature.return_annotation == "PRHistory" + + def test_the_docstring_says_it_is_optional_and_how_to_resolve_it(self): + doc = inspect.getdoc(Forge.get_pr_history) + assert 'getattr(forge, "get_pr_history", None)' in doc + assert "not every forge implements it" in doc + assert "complete=False" in doc + + def test_a_forge_without_it_resolves_to_none(self): + assert getattr(FakeForge(), "get_pr_history", None) is None + + +class TestPRHistory: + def test_defaults_are_an_unedited_complete_history(self): + history = PRHistory(created_at=CREATED) + assert history.description_versions == () + assert history.title_renames == () + assert history.first_review_at is None + assert history.head_committed_at is None + assert history.complete is True + + def test_lists_are_stored_as_tuples_so_a_history_is_hashable(self): + history = PRHistory(created_at=CREATED, description_versions=[V0, V1], title_renames=[RENAME_1]) + assert history.description_versions == (V0, V1) + assert history.title_renames == (RENAME_1,) + assert hash(history) == hash(PRHistory(created_at=CREATED, description_versions=(V0, V1), + title_renames=(RENAME_1,))) + + @pytest.mark.parametrize("instance, field", [ + (PRHistory(created_at=CREATED), "complete"), + (V0, "text"), + (RENAME_1, "previous_title"), + (PinnedMetadata(title="t", description="d", status="pinned"), "status"), + ]) + def test_every_type_is_frozen(self, instance, field): + with pytest.raises(dataclasses.FrozenInstanceError): + setattr(instance, field, None) + + def test_a_version_may_have_no_text(self): + assert DescriptionVersion(text=None, edited_at=EDIT_1).text is None + + @pytest.mark.parametrize("field", ["description_versions", "title_renames"]) + def test_a_foreign_element_is_refused_naming_the_field(self, field): + with pytest.raises(ValueError, match=f"PRHistory.{field} must hold"): + PRHistory(created_at=CREATED, **{field: [{"text": "x"}]}) + + +NAIVE = datetime(2026, 3, 1, 9, 0) + + +class TestNaiveDatetimesAreRefused: + @pytest.mark.parametrize("build, name", [ + (lambda: PRHistory(created_at=NAIVE), "PRHistory.created_at"), + (lambda: PRHistory(created_at=CREATED, first_review_at=NAIVE), "PRHistory.first_review_at"), + (lambda: PRHistory(created_at=CREATED, head_committed_at=NAIVE), "PRHistory.head_committed_at"), + (lambda: DescriptionVersion(text="x", edited_at=NAIVE), "DescriptionVersion.edited_at"), + (lambda: TitleRename(previous_title="a", current_title="b", created_at=NAIVE), "TitleRename.created_at"), + (lambda: _pin(_history(), NAIVE), "cutoff"), + (lambda: choose_cutoff(NAIVE, None), "as_of"), + ]) + def test_naive_raises_value_error_naming_the_field(self, build, name): + with pytest.raises(ValueError, match=rf"^{name} must be timezone-aware"): + build() + + def test_a_non_datetime_is_refused_too(self): + with pytest.raises(ValueError, match=r"^PRHistory.created_at must be a datetime, got str"): + PRHistory(created_at="2026-03-01T09:00:00Z") + + def test_a_tzinfo_with_no_offset_counts_as_naive(self): + class NoOffset(tzinfo): + def utcoffset(self, dt): + return None + + with pytest.raises(ValueError, match="must be timezone-aware"): + PRHistory(created_at=datetime(2026, 3, 1, tzinfo=NoOffset())) + + +class TestDescriptionInForce: + def test_edits_before_and_after_the_cutoff_give_the_pre_cutoff_text(self): + result = _pin(_history(), REVIEW) + assert result == PinnedMetadata(title="parser: add it", description=MIDDLE, status="pinned") + + @pytest.mark.parametrize("order", list(itertools.permutations(VERSIONS))) + def test_versions_are_ordered_by_edited_at_not_by_position(self, order): + assert _pin(_history(description_versions=order), REVIEW).description == MIDDLE + + def test_an_edit_exactly_at_the_cutoff_is_in_force(self): + assert _pin(_history(), EDIT_1).description == MIDDLE + + def test_a_cutoff_after_every_edit_gives_the_live_version_pinned(self): + result = _pin(_history(), EDIT_2 + timedelta(days=1)) + assert result == PinnedMetadata(title=LIVE_TITLE, description=LIVE, status="pinned") + + def test_a_cutoff_before_created_at_clamps_to_the_original(self): + result = _pin(_history(), CREATED - timedelta(days=30)) + assert result == PinnedMetadata(title="parser: add", description=ORIGINAL, status="pinned") + + def test_a_rename_stamped_before_created_at_still_leaves_the_original_title(self): + skewed = TitleRename(previous_title="parser: add", current_title=LIVE_TITLE, + created_at=CREATED - timedelta(seconds=10)) + result = _pin(_history(title_renames=(skewed,)), CREATED - timedelta(days=1)) + assert result == PinnedMetadata(title="parser: add", description=ORIGINAL, status="pinned") + + def test_a_complete_history_whose_oldest_version_is_after_the_cutoff_gives_the_oldest(self): + late = (DescriptionVersion(text=MIDDLE, edited_at=EDIT_1), V2) + result = _pin(_history(description_versions=late, title_renames=()), CREATED + timedelta(hours=1)) + assert result == PinnedMetadata(title=LIVE_TITLE, description=MIDDLE, status="pinned") + + +class TestZeroEdits: + def test_zero_edits_gives_the_live_body_and_counts_as_pinned(self): + result = _pin(_history(description_versions=(), title_renames=()), REVIEW) + assert result == PinnedMetadata(title=LIVE_TITLE, description=LIVE, status="pinned") + + def test_zero_edits_still_pins_the_title(self): + result = _pin(_history(description_versions=()), REVIEW) + assert result == PinnedMetadata(title="parser: add it", description=LIVE, status="pinned") + + def test_zero_edits_in_an_incomplete_history_is_live(self): + assert _pin(_history(description_versions=(), complete=False), REVIEW) == LIVE_RESULT + + +class TestDeletedVersion: + def test_a_deleted_version_in_force_gives_live(self): + deleted = (V0, DescriptionVersion(text=None, edited_at=EDIT_1), V2) + assert _pin(_history(description_versions=deleted), REVIEW) == LIVE_RESULT + + def test_a_deleted_version_not_in_force_does_not_matter(self): + deleted = (DescriptionVersion(text=None, edited_at=CREATED), V1, V2) + result = _pin(_history(description_versions=deleted), REVIEW) + assert result == PinnedMetadata(title="parser: add it", description=MIDDLE, status="pinned") + + +class TestIncompleteHistory: + def test_an_incomplete_history_that_does_not_reach_the_cutoff_gives_live(self): + newest_only = (V2,) + assert _pin(_history(description_versions=newest_only, complete=False), REVIEW) == LIVE_RESULT + + def test_an_incomplete_history_that_reaches_the_cutoff_is_pinned(self): + newest_two = (V2, V1) + result = _pin(_history(description_versions=newest_two, complete=False), REVIEW) + assert result == PinnedMetadata(title="parser: add it", description=MIDDLE, status="pinned") + + def test_an_incomplete_history_cannot_clamp_to_an_original_it_does_not_hold(self): + newest_two = (V2, V1) + assert _pin(_history(description_versions=newest_two, complete=False), CREATED - timedelta(days=1)) \ + == LIVE_RESULT + + +class TestTitle: + def test_the_title_is_the_previous_title_of_the_first_rename_after_the_cutoff(self): + assert _pin(_history(), REVIEW).title == "parser: add it" + + @pytest.mark.parametrize("order", [RENAMES, tuple(reversed(RENAMES))]) + def test_renames_are_ordered_by_created_at_not_by_position(self, order): + assert _pin(_history(title_renames=order), CREATED + timedelta(hours=1)).title == "parser: add" + + def test_no_rename_after_the_cutoff_gives_the_live_title(self): + assert _pin(_history(), EDIT_2 + timedelta(seconds=1)).title == LIVE_TITLE + + def test_a_rename_exactly_at_the_cutoff_is_not_after_it(self): + assert _pin(_history(), EDIT_1).title == "parser: add it" + assert _pin(_history(), EDIT_2).title == LIVE_TITLE + + def test_the_title_is_live_whenever_the_description_is(self): + deleted = (V0, DescriptionVersion(text=None, edited_at=EDIT_1), V2) + assert _pin(_history(description_versions=VERSIONS), REVIEW).title == "parser: add it" + result = _pin(_history(description_versions=deleted), REVIEW) + assert result.status == "live" + assert result.title == LIVE_TITLE + + +class TestTimezones: + def test_offsets_are_compared_as_instants_not_wall_clocks(self): + pacific = timezone(timedelta(hours=-8)) + india = timezone(timedelta(hours=5, minutes=30)) + created = datetime(2026, 3, 1, 0, 0, tzinfo=UTC) + early = DescriptionVersion(text="early", edited_at=datetime(2026, 3, 1, 20, 0, tzinfo=india)) + late = DescriptionVersion(text="late", edited_at=datetime(2026, 3, 1, 10, 0, tzinfo=pacific)) + history = PRHistory(created_at=created, description_versions=(late, early)) + cutoff = datetime(2026, 3, 1, 16, 0, tzinfo=UTC) + assert early.edited_at < cutoff < late.edited_at + assert late.edited_at.replace(tzinfo=None) < cutoff.replace(tzinfo=None) + result = pin_pr_metadata(history, live_title="t", live_description="late", cutoff=cutoff) + assert result == PinnedMetadata(title="t", description="early", status="pinned") + + def test_a_cutoff_in_another_offset_names_the_same_instant(self): + plus_two = timezone(timedelta(hours=2)) + assert _pin(_history(), EDIT_1.astimezone(plus_two)).description == MIDDLE + assert _pin(_history(), (EDIT_1 - timedelta(seconds=1)).astimezone(plus_two)).description == ORIGINAL + + +class TestChooseCutoff: + def test_the_flag_wins_over_everything(self): + history = _history(first_review_at=REVIEW, head_committed_at=HEAD_COMMIT) + as_of = EDIT_1 + timedelta(minutes=7) + assert choose_cutoff(as_of, history) == (as_of, "flag") + + def test_the_flag_needs_no_history(self): + assert choose_cutoff(REVIEW, None) == (REVIEW, "flag") + + def test_the_first_review_comes_next(self): + history = _history(first_review_at=REVIEW, head_committed_at=HEAD_COMMIT) + assert choose_cutoff(None, history) == (REVIEW, "first-review") + + def test_the_head_commit_comes_last(self): + assert choose_cutoff(None, _history(head_committed_at=HEAD_COMMIT)) == (HEAD_COMMIT, "head-commit") + + def test_nothing_available_gives_none(self): + assert choose_cutoff(None, _history()) is None + + def test_no_flag_and_no_history_gives_none(self): + assert choose_cutoff(None, None) is None + + def test_the_cutoff_is_returned_unclamped(self): + early = CREATED - timedelta(days=5) + assert choose_cutoff(early, _history()) == (early, "flag") + + def test_a_head_commit_before_the_pr_was_opened_pins_the_original(self): + commit = CREATED - timedelta(days=2) + history = _history(head_committed_at=commit) + cutoff, source = choose_cutoff(None, history) + assert (cutoff, source) == (commit, "head-commit") + assert _pin(history, cutoff) == PinnedMetadata(title="parser: add", description=ORIGINAL, status="pinned") + + +class TestPureLogic: + def test_pinning_and_choosing_never_touch_the_network(self, monkeypatch): + def refuse(*args, **kwargs): + raise AssertionError("network used") + + monkeypatch.setattr(socket, "socket", refuse) + monkeypatch.setattr(socket, "create_connection", refuse) + history = _history(first_review_at=REVIEW) + cutoff, _source = choose_cutoff(None, history) + assert _pin(history, cutoff).description == MIDDLE From ce7d7fd79ac6927a3bcacece28119f5f07f9e4bb Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:40:42 -0700 Subject: [PATCH 10/64] feat: parse and validate applies_to in rules front matter (#12 T2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add rules.parse_applies_to(text, *, source, path), which reads the fence that split_front_matter reads and returns a scoped rules file's path globs as a tuple in file order. It returns None when the key is absent, which means the file applies to every unit. The key is matched casefolded against APPLIES_TO_KEYS, so the applyTo alias works. It accepts three forms: a bare or quoted scalar split on commas, a one-line JSON-style flow list, and indented '- ' lines. Each of the following is a ConfigError of the form ': :: ': - an empty value ([], "", ~, or no entries), per decisions.md #12; - an empty entry, or an entry that is not a string; - a malformed flow list or quoted string; - a flow list or scalar that continues onto indented lines; - a block scalar; - a bare '!', or a glob with a leading '/'; - a list that has only negated globs; - a duplicate key. ReviewRules gains applies_to: tuple[str, ...] | None = None for the scoped loader (T3). prompt_block and record do not read it. split_front_matter keeps its 3-tuple contract and still lists the key in ignored_keys. The always-on PRXREF_REVIEW_RULES file does not parse or validate the key, so it keeps loading as before. Per map-12, the loader now names the key in a WARNING instead of the INFO line. Prompt blocks for a legacy file stay byte-identical, pinned against sha256 values measured at daec2d0. docs/review-rules.md is corrected to match. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- docs/review-rules.md | 4 +- src/prxref/rules.py | 223 +++++++++++++- tests/test_scoped_rules_frontmatter.py | 390 +++++++++++++++++++++++++ 3 files changed, 607 insertions(+), 10 deletions(-) create mode 100644 tests/test_scoped_rules_frontmatter.py diff --git a/docs/review-rules.md b/docs/review-rules.md index de417361..5449de2a 100644 --- a/docs/review-rules.md +++ b/docs/review-rules.md @@ -120,7 +120,9 @@ being dropped as `invalid severity: 'blocker'`. See - **Only `severity:` is read.** Every other top-level key (`name:`, `description:`, …) is ignored and named once in an INFO log line, and any indented lines under it are skipped. So a Claude-style skill file, - multi-line `description: |` included, works unmodified. + multi-line `description: |` included, works unmodified. An `applies_to:` + or `applyTo:` key is skipped the same way but named in a WARNING instead, + because this file reaches every unit whatever the key says. - **The map.** `severity:` has nothing after the colon, and each indented line under it is `: `. Either side may be quoted, and a word may contain spaces (`Must Fix`). Words are case-insensitive, and runs of diff --git a/src/prxref/rules.py b/src/prxref/rules.py index f706ca72..e9314749 100644 --- a/src/prxref/rules.py +++ b/src/prxref/rules.py @@ -15,11 +15,13 @@ The front matter is optional and only its ``severity:`` key is read: a block of indented ``: `` lines, where the tier is one of -:data:`MAPPABLE_SEVERITIES`. Every other key is ignored and reported at INFO, -so a Claude-style skill file (``name:``, ``description: |``) can be pointed -at unmodified. ``spec`` is never a legal target: it means "violates a quoted -spec constraint", and a team word mapped onto it would mint spec findings on -runs with no spec at all. +:data:`MAPPABLE_SEVERITIES`. An ``applies_to:`` key (alias ``applyTo``) +scopes a scoped rules file to paths (:func:`parse_applies_to`); the always-on +file does not read it and says so in a WARNING. Every other key is ignored +and reported at INFO, so a Claude-style skill file (``name:``, +``description: |``) can be pointed at unmodified. ``spec`` is never a legal +target: it means "violates a quoted spec constraint", and a team word mapped +onto it would mint spec findings on runs with no spec at all. The rules steer the model, and whoever controls the file controls that steering, so the path must never come from the pull request under review. @@ -35,6 +37,7 @@ import codecs import errno import hashlib +import json import logging import os import re @@ -91,13 +94,18 @@ class ReviewRules: fingerprinted by the raw file bytes (front matter included); ``severity_map`` maps a casefolded team word to one of :data:`MAPPABLE_SEVERITIES`, in file order; ``ignored_keys`` names the - other front-matter keys, which are not used. + other front-matter keys, which are not used. ``applies_to`` holds the + file's path globs in file order, as :func:`parse_applies_to` returns + them, or ``None`` when the file applies to every unit; + :func:`load_review_rules` always leaves it ``None``, and neither + :meth:`prompt_block` nor :meth:`record` reads it. """ path: str body: CappedText severity_map: Mapping[str, str] ignored_keys: tuple[str, ...] = () + applies_to: tuple[str, ...] | None = None def prompt_block(self, unit: str) -> str: """The system-prompt block for one review unit (``"worker"`` or ``"sweep"``). @@ -242,6 +250,191 @@ def fail(lineno: int, problem: str) -> ConfigError: return severity_map, tuple(ignored), "\n".join(lines[close + 1:]) +APPLIES_TO_KEYS: frozenset[str] = frozenset({"applies_to", "applyto"}) + +_ITEM_RE = re.compile(r"^[ \t]+-(?:[ \t]+(.*))?$") +_SINGLE_QUOTED_RE = re.compile(r"^'((?:[^']|'')*)'$") +_COLLECTION_RE = re.compile(r"^(?:[\[{]|-(?:[ \t]|$))|:(?:[ \t]|$)") +_YAML_NULL_RE = re.compile(r"^(?:~|null|Null|NULL)$") +_YAML_NON_STRING_RE = re.compile( + r"^(?:~|null|Null|NULL|true|True|TRUE|false|False|FALSE" + r"|[-+]?[0-9]+|0o[0-7]+|0x[0-9a-fA-F]+" + r"|[-+]?(?:\.[0-9]+|[0-9]+(?:\.[0-9]*)?)(?:[eE][-+]?[0-9]+)?" + r"|[-+]?\.(?:inf|Inf|INF)|\.(?:nan|NaN|NAN))$" +) + + +def _glob_scalar(value: str, name: str) -> str: + if value.startswith('"'): + try: + return json.loads(value) + except json.JSONDecodeError: + raise ValueError(f"'{name}' has a malformed quoted string, got {value!r}") from None + if value.startswith("'"): + quoted = _SINGLE_QUOTED_RE.match(value) + if quoted is None: + raise ValueError(f"'{name}' has a malformed quoted string, got {value!r}") + return quoted.group(1).replace("''", "'") + if _COLLECTION_RE.search(value) or _YAML_NON_STRING_RE.match(value): + raise ValueError( + f"'{name}' entries must be glob strings, got {value!r}; " + "quote a glob that YAML reads as another type" + ) + return value + + +def _inline_globs(value: str, name: str) -> list[str]: + if value.startswith("["): + try: + parsed = json.loads(value) + except json.JSONDecodeError: + raise ValueError( + f"'{name}' flow list must be JSON-style, with double-quoted globs such as " + f'["**/*.java"], got {value!r}' + ) from None + for entry in parsed: + if not isinstance(entry, str): + raise ValueError(f"'{name}' entries must be glob strings, got {json.dumps(entry)}") + return parsed + if _YAML_NULL_RE.match(value): + return [] + scalar = _glob_scalar(value, name) + return scalar.split(",") if scalar.strip() else [] + + +def _glob_problem(glob: str) -> str | None: + if not glob: + return "has an empty entry; every entry must be a glob" + pattern = glob[1:] if glob[0] == "!" else glob + if not pattern or pattern[0].isspace(): + return f"entry {glob!r} must put its glob right after '!', as in '!**/test/**'" + if pattern[0] == "/": + return ( + f"entry {glob!r} starts with '/', but diff paths are relative to the " + "repository root; drop the leading '/'" + ) + return None + + +def parse_applies_to(text: str, *, source: str, path: str) -> tuple[str, ...] | None: + """Return the path globs of ``text``'s ``applies_to`` front-matter key. + + ``text`` is decoded, newline-normalised file text, and the fence, the + comments and the line numbers are those of :func:`split_front_matter`, + which still lists the key in its ``ignored_keys``. The key is matched + casefolded against :data:`APPLIES_TO_KEYS`, so ``applies_to`` and its + alias ``applyTo`` (the ``.github/instructions`` spelling) both work. + ``None`` means the file has no such key, or no closed front matter, and + so applies to every unit; this never logs, since + :func:`split_front_matter` already warns about an unclosed fence. + + The value takes one of three forms, and the result is a tuple of globs, + each stripped, in file order: + + - a scalar, bare or quoted, split on commas: + ``applies_to: "**/*.ts, **/*.tsx"``; + - a one-line JSON-style flow list of double-quoted globs, never split: + ``applies_to: ["**/*.java", "!**/src/test/**"]``; + - nothing after the colon, then indented ``- `` lines, bare or + quoted, never split. + + A glob starting with ``!`` excludes paths. The globs are validated here + but not matched. Every problem is a :class:`~prxref.llm.ConfigError` of + the form ``": :: "``, where the line is + 1-based over the whole file: the entry's own line in a block list, the + key's line otherwise. The problems are a second ``applies_to`` or + ``applyTo`` key; an empty value (``[]``, ``""``, ``~`` or no entries); an + empty entry; an entry that is not a string (``[1]``, ``- 42``, + ``- true``, ``- a: b``, a nested list or mapping); a malformed flow list + or quoted string (single-quoted flow items included); a flow list that + does not close on its own line; an inline value followed by indented + lines; an indented line that is not ``- ``; a block scalar + (``|`` or ``>``); a ``!`` not followed directly by a glob; a glob + starting with ``/``, since diff paths are relative; and a list with + only ``!`` globs, which can match no path. + """ + lines = text.split("\n") + if not _FENCE_RE.match(lines[0]): + return None + close = next((k for k in range(1, len(lines)) if _FENCE_RE.match(lines[k])), None) + if close is None: + return None + + def fail(lineno: int, problem: str) -> ConfigError: + return ConfigError(f"{source}: {path}:{lineno}: {problem}") + + name = "" + key_line = 0 + inline = "" + in_key = False + entries: list[tuple[int, str]] = [] + for index in range(1, close): + lineno = index + 1 + raw = lines[index] + line = _COMMENT_RE.sub(r"\1", raw) + if not line.strip(): + continue + if line[0] not in " \t": + key = _KEY_RE.match(line) + in_key = key is not None and key.group(1).casefold() in APPLIES_TO_KEYS + if key is None or not in_key: + continue + if key_line: + raise fail(lineno, f"duplicate '{key.group(1)}' key ('{name}' is already set on line {key_line})") + name, key_line, inline = key.group(1), lineno, key.group(2).strip() + if inline[:1] in ("|", ">"): + raise fail( + lineno, + f"'{name}' cannot be a block scalar ('|' or '>'); give a glob, a " + "comma-separated string of globs, or a list", + ) + continue + if not in_key: + continue + if inline.startswith("["): + raise fail( + lineno, + f"'{name}' flow list must close on the line that opens it; write a long " + "list as indented '- ' lines", + ) + if inline: + raise fail(lineno, f"'{name}' has a value after the colon, so it cannot continue on an indented line") + item = _ITEM_RE.match(line) + if item is None: + raise fail(lineno, f"'{name}' list entries must be '- ' lines, got {raw.strip()!r}") + try: + entries.append((lineno, _glob_scalar((item.group(1) or "").strip(), name))) + except ValueError as exc: + raise fail(lineno, str(exc)) from None + if not key_line: + return None + if inline: + try: + entries = [(key_line, glob) for glob in _inline_globs(inline, name)] + except ValueError as exc: + raise fail(key_line, str(exc)) from None + if not entries: + raise fail( + key_line, + f"'{name}' is empty; list at least one glob, or omit the key to apply the " + "file to every unit", + ) + globs: list[str] = [] + for lineno, entry in entries: + glob = entry.strip() + problem = _glob_problem(glob) + if problem is not None: + raise fail(lineno, f"'{name}' {problem}") + globs.append(glob) + if all(glob.startswith("!") for glob in globs): + raise fail( + key_line, + f"'{name}' has only negated ('!') globs, so it matches no path; add a glob " + "the file applies to", + ) + return tuple(globs) + + def load_review_rules(path: str | None, *, max_chars: int, source: str) -> ReviewRules | None: """Load the team review-rules file at ``path``, capped at ``max_chars``. @@ -262,7 +455,11 @@ def load_review_rules(path: str | None, *, max_chars: int, source: str) -> Revie A truncated body is logged once as a WARNING naming ``PRXREF_REVIEW_RULES_MAX_CHARS``, an empty body with no map as a WARNING, and ignored front-matter keys at INFO; each still returns the - loaded rules, since the file was configured. + loaded rules, since the file was configured. An ``applies_to`` or + ``applyTo`` key is neither parsed nor validated here, stays in + ``ignored_keys`` and leaves ``applies_to`` ``None``; it is logged as a + WARNING instead of at INFO, because this file reaches every unit + whatever it says. """ if path is None or not path.strip(): return None @@ -291,8 +488,16 @@ def load_review_rules(path: str | None, *, max_chars: int, source: str) -> Revie raise ConfigError(f"{source}: rules file {path!r} contains NUL bytes; expected Markdown or plain text") severity_map, ignored, body = split_front_matter(text, source=source, path=path) capped = cap_text(body.strip(), max_chars, sha256=sha256) - if ignored: - logger.info("%s: ignoring front-matter keys other than 'severity': %s", source, ", ".join(ignored)) + scoping = [key for key in ignored if key.casefold() in APPLIES_TO_KEYS] + unused = [key for key in ignored if key not in scoping] + if unused: + logger.info("%s: ignoring front-matter keys other than 'severity': %s", source, ", ".join(unused)) + if scoping: + logger.warning( + "%s: rules file %r sets %s, which only a scoped rules file (PRXREF_SCOPED_RULES / " + "--scoped-rules) reads; this file still reaches every unit", + source, path, ", ".join(f"'{key}'" for key in scoping), + ) if capped.truncated: logger.warning( "%s: rules file %r has %d characters (after front matter); only the first %d " diff --git a/tests/test_scoped_rules_frontmatter.py b/tests/test_scoped_rules_frontmatter.py new file mode 100644 index 00000000..9a81b057 --- /dev/null +++ b/tests/test_scoped_rules_frontmatter.py @@ -0,0 +1,390 @@ +"""Issue #12 T2: the ``applies_to`` front-matter key of a scoped rules file. + +``parse_applies_to`` reads the fence ``split_front_matter`` reads and returns +the file's path globs in file order, or ``None`` when the file has no such key +and so applies to every unit. What is pinned here: + +- the three value forms (a scalar, a comma-separated scalar, a list, as a flow + list or as indented ``- `` lines) and the ``applyTo`` alias; +- ``None`` for an absent key, and a ConfigError, never ``()``, for an empty one; +- the exact ``: ::`` message of every rejected value; +- the new ``ReviewRules.applies_to`` field, which no prompt or record reads; +- that the always-on ``PRXREF_REVIEW_RULES`` file parses and prompts byte for + byte as it did before, and only warns about an ``applies_to`` it ignores. +""" +from __future__ import annotations + +import hashlib +import logging +from dataclasses import replace + +import pytest + +from prxref.llm import ConfigError +from prxref.rules import ( + APPLIES_TO_KEYS, + ReviewRules, + load_review_rules, + parse_applies_to, + split_front_matter, +) +from prxref.text_inputs import cap_text +from tests.test_issue_63_review_rules import BODY, MAP, SKILL + +SOURCE = "PRXREF_SCOPED_RULES" +PATH = "rules/java.md" +JAVA = "**/*.java" +NOT_TESTS = "!**/src/test/**" +UNITS = ("worker", "sweep") + +GOLDEN_SKILL_BLOCKS = { + "worker": ("492058d625ae4dc7ccadc61403aa903866ae06b7af22de39d98fe7ba2915efdc", 765), + "sweep": ("89e91832bc7eb5c6be983d8e0030184bfcfb4b0514ccd0970e8197f4460605c8", 968), +} +"""sha256 and length of the issue #63 SKILL file's prompt blocks, measured at +daec2d0, before any ``applies_to`` code existed.""" + + +def _fm(*lines: str, body: str = "# Java\n- rule\n") -> str: + return "---\n" + "".join(f"{line}\n" for line in lines) + "---\n" + body + + +def _parse(text: str): + return parse_applies_to(text, source=SOURCE, path=PATH) + + +def _error(text: str) -> str: + with pytest.raises(ConfigError) as exc: + _parse(text) + return str(exc.value) + + +def _empty(name: str = "applies_to") -> str: + return f"'{name}' is empty; list at least one glob, or omit the key to apply the file to every unit" + + +def _not_a_string(value: str) -> str: + return ( + f"'applies_to' entries must be glob strings, got {value!r}; " + "quote a glob that YAML reads as another type" + ) + + +def _not_a_json_string(shown: str) -> str: + return f"'applies_to' entries must be glob strings, got {shown}" + + +def _bad_flow(value: str) -> str: + return ( + "'applies_to' flow list must be JSON-style, with double-quoted globs such as " + f'["**/*.java"], got {value!r}' + ) + + +def _bad_quote(value: str) -> str: + return f"'applies_to' has a malformed quoted string, got {value!r}" + + +def _bad_negation(glob: str) -> str: + return f"'applies_to' entry {glob!r} must put its glob right after '!', as in '!**/test/**'" + + +def _leading_slash(glob: str) -> str: + return ( + f"'applies_to' entry {glob!r} starts with '/', but diff paths are relative to the " + "repository root; drop the leading '/'" + ) + + +EMPTY_ENTRY = "'applies_to' has an empty entry; every entry must be a glob" +ONLY_NEGATED = ( + "'applies_to' has only negated ('!') globs, so it matches no path; add a glob the file applies to" +) +BLOCK_SCALAR = ( + "'applies_to' cannot be a block scalar ('|' or '>'); give a glob, a comma-separated " + "string of globs, or a list" +) +FLOW_ACROSS_LINES = ( + "'applies_to' flow list must close on the line that opens it; write a long list as " + "indented '- ' lines" +) +SCALAR_ACROSS_LINES = "'applies_to' has a value after the colon, so it cannot continue on an indented line" + + +def _not_an_entry(raw: str) -> str: + return f"'applies_to' list entries must be '- ' lines, got {raw!r}" + + +ACCEPTS = [ + ("a bare scalar", _fm("applies_to: **/*.java"), (JAVA,)), + ("a double-quoted scalar", _fm('applies_to: "**/*.java"'), (JAVA,)), + ("a single-quoted scalar with an escaped quote", _fm("applies_to: 'docs/it''s/*.md'"), ("docs/it's/*.md",)), + ("a comma-separated bare scalar", _fm("applies_to: **/*.java, !**/src/test/**"), (JAVA, NOT_TESTS)), + ("a comma-separated quoted scalar", _fm('applies_to: "**/*.ts,**/*.tsx"'), ("**/*.ts", "**/*.tsx")), + ("the issue's flow list", _fm('applies_to: ["**/*.java", "!**/src/test/**"]'), (JAVA, NOT_TESTS)), + ("a flow entry keeps its comma", _fm('applies_to: ["src/a,b/**", "**/*.md"]'), ("src/a,b/**", "**/*.md")), + ("flow entries are stripped", _fm('applies_to: [ " **/*.java " ]'), (JAVA,)), + ( + "a block list, bare and quoted", + _fm("applies_to:", " - **/*.java", ' - "!**/src/test/**"', " - 'helm/**'"), + (JAVA, NOT_TESTS, "helm/**"), + ), + ("a block entry keeps its comma", _fm("applies_to:", ' - "a,b/*.md"'), ("a,b/*.md",)), + ( + "comments, blank lines and a tab in a block list", + _fm( + "applies_to: # java only", "", " # production code", "\t- **/*.java # all of it", + " - '!**/src/test/**'", + ), + (JAVA, NOT_TESTS), + ), + ("a comment after a flow list", _fm('applies_to: ["**/*.java"] # java'), (JAVA,)), + ("a negation may come first", _fm('applies_to: ["!**/gen/**", "**/*.py"]'), ("!**/gen/**", "**/*.py")), + ("a glob with an inner space", _fm('applies_to: "docs/team notes/*.md"'), ("docs/team notes/*.md",)), + ("a colon not followed by a space", _fm("applies_to: src/a:b/*.md"), ("src/a:b/*.md",)), + ("a number-like glob when quoted", _fm("applies_to:", ' - "2024"', " - '2025'"), ("2024", "2025")), + ("a repeated glob is kept in order", _fm("applies_to: a/**, b/**, a/**"), ("a/**", "b/**", "a/**")), + ( + "beside skill keys, a block scalar and a severity map", + _fm( + "name: java", "description: |", " applies_to: prose inside a block scalar", + "severity:", " blocker: error", "applies_to:", " - **/*.java", + ), + (JAVA,), + ), + ("tabs and trailing blanks on the fences", "--- \napplies_to: **/*.java\n---\t\nbody", (JAVA,)), +] + +ABSENT = [ + ("no front matter", "# Rules\n- be nice\n"), + ("front matter without the key", _fm("severity:", " blocker: error")), + ("the issue #63 skill file", SKILL), + ("the key only inside a block scalar", _fm("description: |", " applies_to: **/*.java")), + ("the key only in the body", _fm("name: x", body="applies_to: **/*.java\n")), + ("an unclosed fence", "---\napplies_to: **/*.java\n# body\n"), + ("a fence that is not the first line", "\n---\napplies_to: **/*.java\n---\n"), + ("an empty file", ""), + ("keys that only resemble it", _fm("applies_tox: **/*.java", "apply_to: **/*.java", "appliesTo: x/**")), +] + +REJECTS = [ + ("an empty flow list", _fm("applies_to: []"), 2, _empty()), + ("an empty flow list with a space", _fm("applies_to: [ ]"), 2, _empty()), + ("no value and no entries", _fm("applies_to:", "severity:", " blocker: error"), 2, _empty()), + ("no value at the end of the front matter", _fm("applies_to: # todo"), 2, _empty()), + ("an empty quoted scalar", _fm('applies_to: ""'), 2, _empty()), + ("a blank quoted scalar", _fm("applies_to: ' '"), 2, _empty()), + ("a YAML null", _fm("applies_to: ~"), 2, _empty()), + ("a YAML null word", _fm("applies_to: null"), 2, _empty()), + ("the alias, empty", _fm("applyTo: []"), 2, _empty("applyTo")), + ("line numbers count comments and blanks", _fm("# scope", "", "name: x", "applies_to: []"), 5, _empty()), + ("an empty entry in a comma list", _fm("applies_to: a/**,,b/**"), 2, EMPTY_ENTRY), + ("a trailing comma", _fm("applies_to: a/**,"), 2, EMPTY_ENTRY), + ("an empty flow entry", _fm('applies_to: ["a/**", " "]'), 2, EMPTY_ENTRY), + ("a bare dash", _fm("applies_to:", " - a/**", " -"), 4, EMPTY_ENTRY), + ("an empty quoted block entry", _fm("applies_to:", ' - ""'), 3, EMPTY_ENTRY), + ("a number in a flow list", _fm('applies_to: [1, "a/**"]'), 2, _not_a_json_string("1")), + ("a null in a flow list", _fm('applies_to: ["a/**", null]'), 2, _not_a_json_string("null")), + ("a boolean in a flow list", _fm("applies_to: [true]"), 2, _not_a_json_string("true")), + ("a list in a flow list", _fm('applies_to: [["a/**"]]'), 2, _not_a_json_string('["a/**"]')), + ("a mapping in a flow list", _fm('applies_to: [{"a": "b"}]'), 2, _not_a_json_string('{"a": "b"}')), + ("a number in a block list", _fm("applies_to:", " - 42"), 3, _not_a_string("42")), + ("a boolean in a block list", _fm("applies_to:", " - a/**", " - True"), 4, _not_a_string("True")), + ("a null in a block list", _fm("applies_to:", " - ~"), 3, _not_a_string("~")), + ("a float in a block list", _fm("applies_to:", " - 1.5e3"), 3, _not_a_string("1.5e3")), + ("a mapping in a block list", _fm("applies_to:", " - a: b"), 3, _not_a_string("a: b")), + ("a key with no value in a block list", _fm("applies_to:", " - src:"), 3, _not_a_string("src:")), + ("a flow list in a block list", _fm("applies_to:", " - [a, b]"), 3, _not_a_string("[a, b]")), + ("a flow mapping in a block list", _fm("applies_to:", " - {a: b}"), 3, _not_a_string("{a: b}")), + ("a nested block list", _fm("applies_to:", " - - a/**"), 3, _not_a_string("- a/**")), + ("a number scalar", _fm("applies_to: 42"), 2, _not_a_string("42")), + ("a flow mapping scalar", _fm("applies_to: {a: b}"), 2, _not_a_string("{a: b}")), + ("a single-quoted flow list", _fm("applies_to: ['**/*.java']"), 2, _bad_flow("['**/*.java']")), + ("a bare flow list", _fm("applies_to: [**/*.java]"), 2, _bad_flow("[**/*.java]")), + ("text after a flow list", _fm('applies_to: ["a/**"] b/**'), 2, _bad_flow('["a/**"] b/**')), + ("a trailing comma in a flow list", _fm('applies_to: ["a/**",]'), 2, _bad_flow('["a/**",]')), + ("a flow list that never closes", _fm('applies_to: ["a/**"'), 2, _bad_flow('["a/**"')), + ("a flow list across lines", _fm('applies_to: ["a/**",', ' "b/**"]'), 3, FLOW_ACROSS_LINES), + ("a scalar across lines", _fm("applies_to: a/**", " b/**"), 3, SCALAR_ACROSS_LINES), + ("a scalar followed by entries", _fm("applies_to: a/**", " - b/**"), 3, SCALAR_ACROSS_LINES), + ("an indented line that is not an entry", _fm("applies_to:", " a/**"), 3, _not_an_entry("a/**")), + ("a dash glued to its glob", _fm("applies_to:", " - a/**", " -b/**"), 4, _not_an_entry("-b/**")), + ("an unterminated double quote", _fm("applies_to:", ' - "a/**'), 3, _bad_quote('"a/**')), + ("an unterminated single quote", _fm("applies_to: 'a/**"), 2, _bad_quote("'a/**")), + ("text after a quoted scalar", _fm('applies_to: "a/**" b/**'), 2, _bad_quote('"a/**" b/**')), + ("a literal block scalar", _fm("applies_to: |", " a/**"), 2, BLOCK_SCALAR), + ("a folded block scalar", _fm("applies_to: >-"), 2, BLOCK_SCALAR), + ("a bare negation", _fm("applies_to: a/**, !"), 2, _bad_negation("!")), + ("a space after the negation", _fm('applies_to: ["a/**", "! b/**"]'), 2, _bad_negation("! b/**")), + ("a leading slash", _fm("applies_to: /src/**"), 2, _leading_slash("/src/**")), + ( + "a negated leading slash", _fm("applies_to:", " - src/**", " - '!/src/gen/**'"), 4, + _leading_slash("!/src/gen/**"), + ), + ("only negated globs", _fm('applies_to: ["!**/test/**", "!**/gen/**"]'), 2, ONLY_NEGATED), + ("only a negated block entry", _fm("applies_to:", "", " - '!**/test/**'"), 2, ONLY_NEGATED), + ( + "a second key", _fm("applies_to: a/**", "applies_to: b/**"), 3, + "duplicate 'applies_to' key ('applies_to' is already set on line 2)", + ), + ( + "the key and its alias", _fm("applyTo: a/**", "name: x", "applies_to: b/**"), 4, + "duplicate 'applies_to' key ('applyTo' is already set on line 2)", + ), +] + + +class TestParseAppliesTo: + @pytest.mark.parametrize(("text", "expected"), [c[1:] for c in ACCEPTS], ids=[c[0] for c in ACCEPTS]) + def test_accepted(self, text, expected): + assert _parse(text) == expected + + @pytest.mark.parametrize("text", [c[1] for c in ABSENT], ids=[c[0] for c in ABSENT]) + def test_absent_key_is_none(self, text): + assert _parse(text) is None + + @pytest.mark.parametrize(("text", "lineno", "problem"), [c[1:] for c in REJECTS], ids=[c[0] for c in REJECTS]) + def test_rejected_naming_source_path_and_line(self, text, lineno, problem): + assert _error(text) == f"{SOURCE}: {PATH}:{lineno}: {problem}" + + def test_the_result_is_a_tuple_in_file_order(self): + globs = _parse(_fm("applies_to:", " - z/**", " - a/**", " - m/**")) + assert type(globs) is tuple + assert globs == ("z/**", "a/**", "m/**") + + def test_absent_is_none_and_empty_is_an_error_never_an_empty_tuple(self): + assert _parse(_fm("name: x")) is None + for empty in ("applies_to: []", "applies_to:", 'applies_to: ""'): + with pytest.raises(ConfigError, match="is empty; list at least one glob"): + _parse(_fm(empty)) + + def test_parsing_never_logs_not_even_for_an_unclosed_fence(self, caplog): + with caplog.at_level(logging.DEBUG, logger="prxref"): + assert _parse("---\napplies_to: **/*.java\n") is None + assert _parse(_fm("applies_to: **/*.java")) == (JAVA,) + assert caplog.records == [] + + def test_the_source_and_path_are_quoted_as_given(self): + with pytest.raises(ConfigError) as exc: + parse_applies_to(_fm("applies_to: []"), source="--scoped-rules", path="./team rules/api.md") + assert str(exc.value) == f"--scoped-rules: ./team rules/api.md:2: {_empty()}" + + +FORMS = { + "scalar": ("{key}: **/*.java",), + "comma-separated": ("{key}: **/*.java, !**/src/test/**",), + "flow list": ('{key}: ["**/*.java", "!**/src/test/**"]',), + "block list": ("{key}:", " - **/*.java", " - '!**/src/test/**'"), +} + + +class TestAlias: + def test_the_keys_are_the_casefolded_spelling_and_its_alias(self): + assert APPLIES_TO_KEYS == frozenset({"applies_to", "applyto"}) + + @pytest.mark.parametrize("form", list(FORMS)) + @pytest.mark.parametrize("key", ["applyTo", "APPLIES_TO", "Applies_To", "applyto"]) + def test_every_spelling_reads_every_form_like_applies_to(self, key, form): + lines = FORMS[form] + assert _parse(_fm(*(line.format(key=key) for line in lines))) == _parse( + _fm(*(line.format(key="applies_to") for line in lines)) + ) + + def test_rejections_name_the_key_as_written(self): + assert _error(_fm("applyTo:", " - 42")) == f"{SOURCE}: {PATH}:3: " + _not_a_string("42").replace( + "'applies_to'", "'applyTo'" + ) + + +class TestReviewRulesField: + def _rules(self, **kwargs) -> ReviewRules: + return ReviewRules(path="r.md", body=cap_text(BODY, 12000, sha256="0" * 64), severity_map=MAP, **kwargs) + + def test_the_field_defaults_to_none(self): + assert self._rules().applies_to is None + + def test_no_prompt_block_or_record_reads_the_field(self): + plain = self._rules() + scoped = replace(plain, applies_to=(JAVA, NOT_TESTS)) + assert scoped.applies_to == (JAVA, NOT_TESTS) + for unit in UNITS: + assert scoped.prompt_block(unit) == plain.prompt_block(unit) + assert scoped.record() == plain.record() + assert "applies_to" not in plain.record() + + +def _write(tmp_path, text: str, name: str = "rules.md") -> str: + path = tmp_path / name + path.write_text(text, encoding="utf-8") + return str(path) + + +def _always_on(tmp_path, text: str, caplog, name: str = "rules.md"): + path = _write(tmp_path, text, name) + with caplog.at_level(logging.INFO, logger="prxref"): + rules = load_review_rules(path, max_chars=12000, source="--rules-file") + infos = [r.getMessage() for r in caplog.records if r.levelno == logging.INFO] + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + return path, rules, infos, warnings + + +def _scoping_warning(path: str, keys: str) -> str: + return ( + f"--rules-file: rules file {path!r} sets {keys}, which only a scoped rules file " + "(PRXREF_SCOPED_RULES / --scoped-rules) reads; this file still reaches every unit" + ) + + +SKILL_WITH_SCOPE = SKILL.replace("severity:\n", 'applies_to: ["**/*.java"]\nseverity:\n', 1) +LEGACY_INFO = "--rules-file: ignoring front-matter keys other than 'severity': name, description" + + +class TestAlwaysOnFileIsUnchanged: + def test_the_legacy_prompt_blocks_match_the_pre_change_golden_bytes(self, tmp_path, caplog): + _, rules, infos, warnings = _always_on(tmp_path, SKILL, caplog) + for unit in UNITS: + block = rules.prompt_block(unit) + assert (hashlib.sha256(block.encode("utf-8")).hexdigest(), len(block)) == GOLDEN_SKILL_BLOCKS[unit] + assert rules.applies_to is None + assert rules.ignored_keys == ("name", "description") + assert (infos, warnings) == ([LEGACY_INFO], []) + + def test_an_applies_to_key_changes_no_prompt_byte_and_is_not_applied(self, tmp_path, caplog): + path, rules, infos, warnings = _always_on(tmp_path, SKILL_WITH_SCOPE, caplog) + legacy = load_review_rules(_write(tmp_path, SKILL, "legacy.md"), max_chars=12000, source="--rules-file") + for unit in UNITS: + block = rules.prompt_block(unit) + assert block == legacy.prompt_block(unit) + assert (hashlib.sha256(block.encode("utf-8")).hexdigest(), len(block)) == GOLDEN_SKILL_BLOCKS[unit] + assert rules.applies_to is None + assert rules.ignored_keys == ("name", "description", "applies_to") + record, legacy_record = rules.record(), legacy.record() + assert record.pop("sha256") != legacy_record.pop("sha256") + record.pop("path") + legacy_record.pop("path") + assert record == legacy_record + assert infos == [LEGACY_INFO] + assert warnings == [_scoping_warning(path, "'applies_to'")] + + def test_split_front_matter_still_lists_the_key_as_ignored(self): + assert split_front_matter(SKILL_WITH_SCOPE, source="--rules-file", path="r.md") == ( + MAP, ("name", "description", "applies_to"), BODY + "\n", + ) + + def test_a_scope_only_file_warns_and_logs_no_info(self, tmp_path, caplog): + path, rules, infos, warnings = _always_on(tmp_path, _fm("applyTo: '**/*.ts'", body="- rule\n"), caplog) + assert rules.body.text == "- rule" + assert (infos, warnings) == ([], [_scoping_warning(path, "'applyTo'")]) + + @pytest.mark.parametrize("value", ["[]", "[1", "{a: b}", "|", "/abs/**"]) + def test_the_always_on_file_does_not_validate_the_key(self, tmp_path, caplog, value): + path, rules, _, warnings = _always_on(tmp_path, _fm(f"applies_to: {value}", body="- rule\n"), caplog) + assert (rules.body.text, rules.applies_to) == ("- rule", None) + assert warnings == [_scoping_warning(path, "'applies_to'")] + + def test_both_spellings_share_one_warning(self, tmp_path, caplog): + text = _fm("applies_to: a/**", "name: x", "applyTo: b/**", body="- rule\n") + path, rules, infos, warnings = _always_on(tmp_path, text, caplog) + assert rules.ignored_keys == ("applies_to", "name", "applyTo") + assert infos == ["--rules-file: ignoring front-matter keys other than 'severity': name"] + assert warnings == [_scoping_warning(path, "'applies_to', 'applyTo'")] From f7fc6fa7a5ab182b249a9c1dbd815ede3126e2a2 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:41:39 -0700 Subject: [PATCH 11/64] feat(eval): deterministic must_match matcher and score metrics (#14 T4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/prxref/eval_metrics.py, the pure layer behind `prxref eval score`: - match_case grades every human finding that carries must_match with the spec-grounded-review section 7.2 rule: an AI finding in the same file, within quality.DEFAULT_LINE_TOLERANCE (5) lines, whose title plus body passes the predicate (plain substring compared through quality.normalize_title, or a case-insensitive `re:` regex). Only rows whose drop_reason is null are credited. Credit is a maximum matching in which one AI finding credits at most 2 human findings; a grouped finding (#13) that lists extra `locations` is credited per location, each with its own cap. Ties go to the nearest line, then the most shared quality._tokens, then content order, so grades never depend on input order. No LLM call. - score_cases turns GradedCase inputs into JSON-safe metrics plus per-case rows: micro recall overall (the headline), by severity, by category and over accepted findings, with partial credit weighted 0.5; judge_error is counted apart and left out of every denominator; unmatched AI findings per PR; severity agreement with human minor read as warning for the comparison only; chunks_failed and elapsed_ms totals; review and judge cost, where a None cost is never summed (costs.valid_usd). Tests: tests/test_eval_metrics.py, synthetic grades and duck-typed fakes. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/eval_metrics.py | 553 +++++++++++++++++++++++++++++++++++++ tests/test_eval_metrics.py | 523 +++++++++++++++++++++++++++++++++++ 2 files changed, 1076 insertions(+) create mode 100644 src/prxref/eval_metrics.py create mode 100644 tests/test_eval_metrics.py diff --git a/src/prxref/eval_metrics.py b/src/prxref/eval_metrics.py new file mode 100644 index 00000000..782bc4e5 --- /dev/null +++ b/src/prxref/eval_metrics.py @@ -0,0 +1,553 @@ +"""Deterministic grading and metrics for ``prxref eval score`` (issue #14). + +Two pure layers. Neither does I/O and neither calls an LLM: + +- :func:`match_case` grades every human finding that carries a + ``must_match`` predicate with the deterministic rule of + ``docs/spec-grounded-review.md`` section 7.2: an AI finding in the same + ``file``, within ``quality.DEFAULT_LINE_TOLERANCE`` lines, whose title plus + body satisfies the predicate (:func:`must_match_predicate`). A human finding + with no ``must_match`` is not graded here; it goes to the judge. +- :func:`score_cases` turns graded cases into the JSON-safe metrics and the + per-case rows of ``score.json``. + +A grade is ``full`` (credit 1), ``partial`` (credit :data:`PARTIAL_CREDIT`) or +``none`` (credit 0). ``judge_error`` marks a human finding the judge failed +to grade: it is counted on its own and left out of every denominator, never +scored as 0. A case whose review itself failed is different: the author +received nothing, so its human findings are honestly ``none``. + +One AI finding credits at most :data:`MAX_CREDITS_PER_FINDING` human +findings. A grouped finding (issue #13) that lists extra locations under +:data:`LOCATIONS_FIELD` is credited per location: each location is its own +credit unit with its own cap, exactly as the member findings it replaced +would have been, so a group is never capped as a whole. + +Human findings are duck-typed (:class:`ExpectedLike`, as mappings or +attribute objects). AI findings are the run record's ``findings`` rows, the +CLI's JSON finding shape (``file``, ``line``, ``severity``, ``title``, +``body``, ``drop_reason``), as mappings or attribute objects. Only rows whose +``drop_reason`` is null are ever credited or counted: the eval scores what a +PR author receives, after every quality gate. +""" +from __future__ import annotations + +import math +import re +from collections.abc import Callable, Iterable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, NamedTuple, Protocol + +from .costs import valid_usd +from .quality import DEFAULT_LINE_TOLERANCE, _tokens, normalize_title + +FULL = "full" +PARTIAL = "partial" +NONE = "none" +JUDGE_ERROR = "judge_error" +GRADES: tuple[str, ...] = (FULL, PARTIAL, NONE) +PARTIAL_CREDIT = 0.5 +GRADE_CREDIT: dict[str, float] = {FULL: 1.0, PARTIAL: PARTIAL_CREDIT, NONE: 0.0} +MAX_CREDITS_PER_FINDING = 2 +LOCATIONS_FIELD = "locations" +REGEX_PREFIX = "re:" +MATCH_METHOD = "must_match" +MISSING_LABEL = "(none)" + +_CREDITED = frozenset({FULL, PARTIAL}) +_ALL_GRADES = (*GRADES, JUDGE_ERROR) + + +class ExpectedLike(Protocol): + """The fields of one human finding (``eval_cases.ExpectedFinding``) this module reads.""" + + id: str + file: str + line: int + severity: str + category: str | None + accepted: bool | None + text: str | None + must_match: str | None + + +@dataclass(frozen=True) +class GradedCase: + """One case ready for :func:`score_cases`. + + ``expected`` holds the case's human findings. ``ai_findings`` is the run + record's ``findings`` list, the sequence every grade's ``ai_ref`` indexes. + ``grades`` holds exactly one grade mapping per human finding, from + :func:`match_case` or the judge: ``human_id``, ``grade`` (one of + ``full``, ``partial``, ``none``, ``judge_error``), and for a credited + grade an ``ai_ref`` naming a row whose ``drop_reason`` is null; ``ai_line`` + and ``method`` are optional. ``record`` is the case's run record + (``verdict``, ``chunks_failed``, ``elapsed_ms``, ``cost_usd``, + ``cost_estimated``), or ``None`` when the run left none. + ``judge_cost_usd`` is ``0.0`` when no judge call was made, and ``None`` + when a call was made that nothing could price. + """ + + case_id: str + expected: Sequence[Any] + ai_findings: Sequence[Any] + grades: Sequence[Mapping[str, Any]] + record: Mapping[str, Any] | None = None + judge_cost_usd: float | None = 0.0 + judge_cost_estimated: bool = False + + +class _Unit(NamedTuple): + ref: int + file: str + line: int | None + order: int + + +def _get(obj: Any, name: str) -> Any: + if isinstance(obj, Mapping): + return obj.get(name) + return getattr(obj, name, None) + + +def _int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def _count(value: Any) -> int | None: + number = _int(value) + return number if number is not None and number >= 0 else None + + +def _str(value: Any) -> str: + return value if isinstance(value, str) else "" + + +def _optional_str(value: Any) -> str | None: + return value if isinstance(value, str) else None + + +def _label(value: Any) -> str: + return value if isinstance(value, str) and value.strip() else MISSING_LABEL + + +def must_match_predicate(pattern: str) -> Callable[[str], bool]: + """Compile one ``must_match`` value into a test over a finding's text. + + A value prefixed ``re:`` is a regular expression searched anywhere in the + text, case-insensitively. Any other value is a plain substring, compared + after both sides pass through ``quality.normalize_title``, so case, + backticks, quotes, emphasis marks and whitespace runs never decide a + match. A plain value that normalizes to nothing is compared lowercased as + written. An invalid regular expression raises ``ValueError``. + """ + if pattern.startswith(REGEX_PREFIX): + try: + compiled = re.compile(pattern[len(REGEX_PREFIX):], re.IGNORECASE) + except re.error as exc: + raise ValueError(f"must_match {pattern!r} is not a valid regular expression: {exc}") from exc + return lambda text: compiled.search(text) is not None + needle = normalize_title(pattern) + if not needle: + lowered = pattern.lower() + return lambda text: lowered in text.lower() + return lambda text: needle in normalize_title(text) + + +def _location(entry: Any, default_file: str) -> tuple[str, int] | None: + if isinstance(entry, Mapping): + file, line = entry.get("file"), entry.get("line") + elif isinstance(entry, tuple | list) and len(entry) == 2: + file, line = entry + elif hasattr(entry, "line"): + file, line = getattr(entry, "file", None), entry.line + else: + return None + number = _int(line) + if number is None: + return None + return (file if isinstance(file, str) and file else default_file, number) + + +def credit_locations(finding: Any) -> list[tuple[str, int | None]]: + """Every ``(file, line)`` one AI finding is credited at, deduplicated in order. + + The finding's own ``file`` and ``line`` come first. A grouped finding + adds the entries of its optional :data:`LOCATIONS_FIELD` attribute or + key, each a ``{"file", "line"}`` mapping, a ``(file, line)`` pair, or an + object with those attributes; an entry without a file inherits the + finding's, and an entry without an integer line is skipped. + """ + file = _str(_get(finding, "file")) + locations: list[tuple[str, int | None]] = [(file, _int(_get(finding, "line")))] + extra = _get(finding, LOCATIONS_FIELD) + if isinstance(extra, Iterable) and not isinstance(extra, str | bytes | Mapping): + for entry in extra: + parsed = _location(entry, file) + if parsed is not None and parsed not in locations: + locations.append(parsed) + return locations + + +def _units(ai_findings: Sequence[Any]) -> list[_Unit]: + units: list[_Unit] = [] + for ref, finding in enumerate(ai_findings): + if _get(finding, "drop_reason") is not None: + continue + for order, (file, line) in enumerate(credit_locations(finding)): + units.append(_Unit(ref, file, line, order)) + return units + + +def _distance(human_line: int | None, ai_line: int | None, tolerance: int) -> int | None: + if not human_line: + return 0 + if ai_line is None: + return None + gap = abs(ai_line - human_line) + return gap if gap <= tolerance else None + + +def _finding_text(finding: Any) -> str: + return f"{_str(_get(finding, 'title'))}\n{_str(_get(finding, 'body'))}" + + +def _human_text(human: Any) -> str: + pattern = _str(_get(human, "must_match")).removeprefix(REGEX_PREFIX) + return f"{_str(_get(human, 'text'))}\n{pattern}" + + +def _human_key(human: Any) -> tuple[str, int, str]: + return (_str(_get(human, "file")), _int(_get(human, "line")) or 0, str(_get(human, "id"))) + + +def _assign(order: Sequence[int], prefs: Mapping[int, Sequence[int]], unit_count: int) -> dict[int, int]: + slots: list[list[int]] = [[] for _ in range(unit_count)] + + def augment(human: int, seen: set[int]) -> bool: + for unit in prefs[human]: + if unit in seen: + continue + seen.add(unit) + if len(slots[unit]) < MAX_CREDITS_PER_FINDING: + slots[unit].append(human) + return True + for holder in list(slots[unit]): + if augment(holder, seen): + slots[unit].remove(holder) + slots[unit].append(human) + return True + return False + + for human in order: + augment(human, set()) + return {human: unit for unit, holders in enumerate(slots) for human in holders} + + +def match_case( + expected: Iterable[Any], ai_findings: Sequence[Any], *, tolerance: int = DEFAULT_LINE_TOLERANCE +) -> list[dict[str, Any]]: + """Grade every human finding that has a ``must_match``, deterministically. + + A human finding is credited ``full`` by an AI finding whose + ``drop_reason`` is null, in the same ``file`` (exact), within + ``tolerance`` lines (``quality.DEFAULT_LINE_TOLERANCE``, 5), and whose + title plus body passes :func:`must_match_predicate`. A human ``line`` of + ``0`` is file-level and matches any line of the file. Anything else is + ``none``: this tier never gives partial credit. + + Credit is assigned as a maximum matching under the cap: each credit unit + (an AI finding, or one location of a grouped finding, see + :func:`credit_locations`) credits at most :data:`MAX_CREDITS_PER_FINDING` + human findings, and a human finding that another could only share with + it is re-routed rather than left uncredited. Among a human finding's + candidates the nearest line wins, then the most content tokens shared + with its ``text`` and ``must_match`` (``quality._tokens``), then content + order, so the grades never depend on input order. + + Returns one grade per human finding with a ``must_match``, in input + order: ``{"human_id", "grade", "ai_ref", "ai_line", "method"}``, where + ``ai_ref`` indexes ``ai_findings`` and ``ai_line`` is the credited + location's line (both ``None`` for ``none``) and ``method`` is + :data:`MATCH_METHOD`. An invalid ``re:`` pattern raises ``ValueError``. + """ + humans = [human for human in expected if _str(_get(human, "must_match"))] + order = sorted(range(len(humans)), key=lambda index: _human_key(humans[index])) + units = _units(ai_findings) + texts = {unit.ref: _finding_text(ai_findings[unit.ref]) for unit in units} + tokens = {ref: _tokens(text, split_compounds=True) for ref, text in texts.items()} + prefs: dict[int, list[int]] = {} + for index in order: + human = humans[index] + test = must_match_predicate(_str(_get(human, "must_match"))) + wanted = _tokens(_human_text(human), split_compounds=True) + file = _str(_get(human, "file")) + line = _int(_get(human, "line")) + hits: dict[int, bool] = {} + ranked = [] + for position, unit in enumerate(units): + if unit.file != file: + continue + gap = _distance(line, unit.line, tolerance) + if gap is None: + continue + if unit.ref not in hits: + hits[unit.ref] = test(texts[unit.ref]) + if not hits[unit.ref]: + continue + finding = ai_findings[unit.ref] + content = ( + unit.line if unit.line is not None else -1, + _str(_get(finding, "title")), + _str(_get(finding, "body")), + unit.ref, + unit.order, + ) + ranked.append((gap, -len(wanted & tokens[unit.ref]), content, position)) + prefs[index] = [entry[-1] for entry in sorted(ranked)] + assigned = _assign(order, prefs, len(units)) + grades: list[dict[str, Any]] = [] + for index, human in enumerate(humans): + unit = units[assigned[index]] if index in assigned else None + grades.append({ + "human_id": _get(human, "id"), + "grade": FULL if unit is not None else NONE, + "ai_ref": unit.ref if unit is not None else None, + "ai_line": unit.line if unit is not None else None, + "method": MATCH_METHOD, + }) + return grades + + +def agreement_severity(severity: object) -> str: + """The severity compared for agreement: lowercased, with ``minor`` read as ``warning``. + + Human labels use ``error|warning|minor`` and keep that value everywhere + else (``recall_by_severity`` is keyed by it); only the agreement check + reads a human ``minor`` as prxref's ``warning``. + """ + value = severity.strip().lower() if isinstance(severity, str) else "" + return "warning" if value == "minor" else value + + +def _check_ref(case: GradedCase, human_id: Any, ref: Any) -> None: + number = _int(ref) + if number is None or not 0 <= number < len(case.ai_findings): + raise ValueError( + f"case {case.case_id!r}: grade for {human_id!r} credits ai_ref {ref!r}, " + f"not an index into its {len(case.ai_findings)} AI findings" + ) + if _get(case.ai_findings[number], "drop_reason") is not None: + raise ValueError(f"case {case.case_id!r}: grade for {human_id!r} credits dropped AI finding {number}") + + +def _grades_by_human(case: GradedCase) -> dict[Any, Mapping[str, Any]]: + ids = [_get(human, "id") for human in case.expected] + if len(set(ids)) != len(ids): + raise ValueError(f"case {case.case_id!r}: expected finding ids are not unique") + by_id: dict[Any, Mapping[str, Any]] = {} + for grade in case.grades: + human_id = grade.get("human_id") + if human_id not in ids: + raise ValueError(f"case {case.case_id!r}: grade for unknown expected finding {human_id!r}") + if human_id in by_id: + raise ValueError(f"case {case.case_id!r}: more than one grade for {human_id!r}") + value = grade.get("grade") + if value not in _ALL_GRADES: + raise ValueError( + f"case {case.case_id!r}: grade {value!r} for {human_id!r} is not one of {', '.join(_ALL_GRADES)}" + ) + if value in _CREDITED: + _check_ref(case, human_id, grade.get("ai_ref")) + by_id[human_id] = grade + missing = sorted(str(human_id) for human_id in ids if human_id not in by_id) + if missing: + raise ValueError(f"case {case.case_id!r}: no grade for expected finding(s) {', '.join(missing)}") + return by_id + + +def _recall_block(entries: Iterable[Mapping[str, Any]]) -> dict[str, Any]: + counts = dict.fromkeys(_ALL_GRADES, 0) + credit = 0.0 + for entry in entries: + counts[entry["grade"]] += 1 + if entry["credit"] is not None: + credit += entry["credit"] + scored = counts[FULL] + counts[PARTIAL] + counts[NONE] + return {"recall": credit / scored if scored else None, "credit": credit, "scored": scored, **counts} + + +def case_row(case: GradedCase) -> dict[str, Any]: + """One JSON-safe per-case row of ``score.json``. + + Keys: ``case_id``, ``verdict``, the recall fields (``recall``, ``credit``, + ``scored``, ``full``, ``partial``, ``none``, ``judge_error``), + ``ai_findings`` (rows whose ``drop_reason`` is null), ``unmatched_ai`` + (those no credited grade names; a grouped finding counts once), + ``severity_compared``, ``severity_agreed``, ``chunks_failed``, + ``elapsed_ms``, ``review_cost_usd``, ``review_cost_estimated``, + ``judge_cost_usd``, ``judge_cost_estimated``, and ``findings``: one entry + per human finding, sorted by id, ``{"human_id", "file", "line", + "severity", "category", "accepted", "grade", "credit", "ai_ref", + "ai_line", "ai_severity", "severity_agrees", "method"}``. ``credit`` is + ``None`` for ``judge_error``; the ``ai_*`` fields and + ``severity_agrees`` are ``None`` unless the grade credits. + + Raises ``ValueError`` when the grades do not cover every human finding + exactly once, name an unknown grade, or credit an ``ai_ref`` that is out + of range or dropped. + """ + grades = _grades_by_human(case) + record = case.record if isinstance(case.record, Mapping) else {} + findings: list[dict[str, Any]] = [] + credited_refs: set[int] = set() + for human in sorted(case.expected, key=lambda item: str(_get(item, "id"))): + human_id = _get(human, "id") + grade = grades[human_id] + value = grade["grade"] + ref = _int(grade.get("ai_ref")) if value in _CREDITED else None + ai_severity = None + agrees = None + if ref is not None: + credited_refs.add(ref) + ai_severity = _optional_str(_get(case.ai_findings[ref], "severity")) + agrees = agreement_severity(ai_severity) == agreement_severity(_get(human, "severity")) + accepted = _get(human, "accepted") + findings.append({ + "human_id": human_id, + "file": _optional_str(_get(human, "file")), + "line": _int(_get(human, "line")), + "severity": _optional_str(_get(human, "severity")), + "category": _optional_str(_get(human, "category")), + "accepted": accepted if isinstance(accepted, bool) else None, + "grade": value, + "credit": GRADE_CREDIT.get(value), + "ai_ref": ref, + "ai_line": _int(grade.get("ai_line")) if ref is not None else None, + "ai_severity": ai_severity, + "severity_agrees": agrees, + "method": _optional_str(grade.get("method")), + }) + active = [ref for ref, finding in enumerate(case.ai_findings) if _get(finding, "drop_reason") is None] + compared = [entry["severity_agrees"] for entry in findings if entry["severity_agrees"] is not None] + return { + "case_id": str(case.case_id), + "verdict": _optional_str(record.get("verdict")), + **_recall_block(findings), + "ai_findings": len(active), + "unmatched_ai": sum(1 for ref in active if ref not in credited_refs), + "severity_compared": len(compared), + "severity_agreed": sum(1 for agrees in compared if agrees), + "chunks_failed": _count(record.get("chunks_failed")), + "elapsed_ms": _count(record.get("elapsed_ms")), + "review_cost_usd": valid_usd(record.get("cost_usd")) if case.record is not None else None, + "review_cost_estimated": record.get("cost_estimated") is True, + "judge_cost_usd": valid_usd(case.judge_cost_usd), + "judge_cost_estimated": case.judge_cost_estimated is True, + "findings": findings, + } + + +def _grouped(entries: Sequence[Mapping[str, Any]], key: str) -> dict[str, dict[str, Any]]: + labels = sorted({_label(entry[key]) for entry in entries}) + return {label: _recall_block(e for e in entries if _label(e[key]) == label) for label in labels} + + +def _agreement_block(entries: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + confusion: dict[str, dict[str, int]] = {} + compared = agreed = 0 + for entry in entries: + if entry["severity_agrees"] is None: + continue + compared += 1 + if entry["severity_agrees"]: + agreed += 1 + human = agreement_severity(entry["severity"]) or MISSING_LABEL + ai = agreement_severity(entry["ai_severity"]) or MISSING_LABEL + row = confusion.setdefault(human, {}) + row[ai] = row.get(ai, 0) + 1 + return { + "compared": compared, + "agreed": agreed, + "rate": agreed / compared if compared else None, + "confusion": {human: dict(sorted(confusion[human].items())) for human in sorted(confusion)}, + } + + +def _total_block(rows: Sequence[Mapping[str, Any]], key: str) -> dict[str, int]: + values = [row[key] for row in rows] + return { + "total": sum(value for value in values if value is not None), + "missing": sum(1 for value in values if value is None), + } + + +def _cost_block(rows: Sequence[Mapping[str, Any]], key: str, estimated_key: str) -> dict[str, Any]: + priced = [row[key] for row in rows if row[key] is not None] + total = math.fsum(priced) if rows and len(priced) == len(rows) else None + return { + "total_usd": total, + "per_pr_usd": total / len(rows) if total is not None else None, + "priced": len(priced), + "unpriced": len(rows) - len(priced), + "estimated": sum(1 for row in rows if row[key] is not None and row[estimated_key]), + } + + +def score_cases(cases: Iterable[GradedCase]) -> dict[str, Any]: + """Score graded cases into ``{"metrics": {...}, "cases": [row, ...]}``, JSON-safe. + + ``cases`` holds one :func:`case_row` per case, sorted by ``case_id``. + ``metrics`` holds: + + - ``case_count``. + - ``recall``: the headline, micro over every human finding: + ``{"recall", "credit", "scored", "full", "partial", "none", + "judge_error"}``. ``recall`` is ``credit / scored`` and ``None`` when + nothing was scored; ``judge_error`` findings are outside ``scored``. + - ``recall_by_severity`` and ``recall_by_category``: the same block per + human label as given, keys sorted, a missing label under + :data:`MISSING_LABEL`; the blocks of each partition sum to the + headline. + - ``recall_accepted``: the same block over human findings whose + ``accepted`` is ``True``. + - ``unmatched_ai``: ``{"total", "ai_findings", "per_pr"}``, where + ``per_pr`` is ``total / case_count``. + - ``severity_agreement``: ``{"compared", "agreed", "rate", + "confusion"}`` over credited grades, compared through + :func:`agreement_severity`; ``confusion`` maps human severity to AI + severity to a count. + - ``chunks_failed`` and ``elapsed_ms``: ``{"total", "missing"}``; a case + with no usable value is counted as missing, not as 0. + - ``review_cost`` and ``judge_cost``: ``{"total_usd", "per_pr_usd", + "priced", "unpriced", "estimated"}``. A ``None`` cost is never summed: + one unpriced case makes ``total_usd`` and ``per_pr_usd`` ``None``, as a + run's own cost is (``costs`` module), and ``unpriced`` says how many. + + Raises ``ValueError`` on a duplicate ``case_id`` or on grades + :func:`case_row` rejects. + """ + rows = sorted((case_row(case) for case in cases), key=lambda row: row["case_id"]) + ids = [row["case_id"] for row in rows] + if len(set(ids)) != len(ids): + raise ValueError("case ids are not unique") + entries = [entry for row in rows for entry in row["findings"]] + unmatched = sum(row["unmatched_ai"] for row in rows) + metrics = { + "case_count": len(rows), + "recall": _recall_block(entries), + "recall_by_severity": _grouped(entries, "severity"), + "recall_by_category": _grouped(entries, "category"), + "recall_accepted": _recall_block(entry for entry in entries if entry["accepted"] is True), + "unmatched_ai": { + "total": unmatched, + "ai_findings": sum(row["ai_findings"] for row in rows), + "per_pr": unmatched / len(rows) if rows else None, + }, + "severity_agreement": _agreement_block(entries), + "chunks_failed": _total_block(rows, "chunks_failed"), + "elapsed_ms": _total_block(rows, "elapsed_ms"), + "review_cost": _cost_block(rows, "review_cost_usd", "review_cost_estimated"), + "judge_cost": _cost_block(rows, "judge_cost_usd", "judge_cost_estimated"), + } + return {"metrics": metrics, "cases": rows} diff --git a/tests/test_eval_metrics.py b/tests/test_eval_metrics.py new file mode 100644 index 00000000..5beaecef --- /dev/null +++ b/tests/test_eval_metrics.py @@ -0,0 +1,523 @@ +"""Tests for prxref.eval_metrics: the section 7.2 deterministic matcher and the +``eval score`` metrics (issue #14, task T4). + +Human findings are small frozen-dataclass fakes and AI findings are dict rows +shaped like a run record's ``findings``. Nothing here imports the case loader: +the seam with ``eval_cases`` is exercised by duck typing only, and a grouped +finding (issue #13) is a fake carrying the optional ``locations`` attribute. +""" +from __future__ import annotations + +import json +import math +import random +import re +from dataclasses import dataclass + +import pytest + +from prxref import quality +from prxref.eval_metrics import ( + FULL, + JUDGE_ERROR, + LOCATIONS_FIELD, + MATCH_METHOD, + MAX_CREDITS_PER_FINDING, + MISSING_LABEL, + NONE, + PARTIAL, + PARTIAL_CREDIT, + GradedCase, + agreement_severity, + case_row, + credit_locations, + match_case, + must_match_predicate, + score_cases, +) + +FILE = "src/app.py" + + +@dataclass(frozen=True) +class Human: + """A stand-in for ``eval_cases.ExpectedFinding``.""" + + id: str + file: str = FILE + line: int = 10 + severity: str = "error" + category: str | None = None + accepted: bool | None = None + text: str | None = None + must_match: str | None = None + + +@dataclass(frozen=True) +class GroupedFinding: + """A stand-in for an issue #13 grouped finding with extra locations.""" + + file: str + line: int + title: str + body: str + severity: str = "warning" + drop_reason: str | None = None + locations: tuple = () + + +@dataclass(frozen=True) +class Loc: + file: str + line: int + + +def _ai(line: int = 10, body: str = "", *, file: str = FILE, title: str = "finding", severity: str = "error", + drop_reason: str | None = None) -> dict: + """One run-record finding row, shaped like the CLI's JSON finding row.""" + return { + "file": file, + "line": line, + "severity": severity, + "confidence": 0.9, + "scope": "unknown", + "title": title, + "body": body, + "drop_reason": drop_reason, + } + + +def _grade(human_id: str, grade: str, ai_ref: int | None = None) -> dict: + return {"human_id": human_id, "grade": grade, "ai_ref": ai_ref} + + +def _case(case_id: str = "case-1", expected=(), ai=(), grades=(), record=None, **extra) -> GradedCase: + return GradedCase( + case_id=case_id, expected=list(expected), ai_findings=list(ai), grades=list(grades), record=record, **extra + ) + + +def _by_id(grades: list[dict]) -> dict[str, str]: + return {grade["human_id"]: grade["grade"] for grade in grades} + + +class TestMustMatchPredicate: + def test_plain_substring_ignores_case_and_markup(self): + assert must_match_predicate("clientInfo")("Missing `ClientInfo` in the handshake") + + def test_plain_substring_collapses_whitespace_and_emphasis(self): + assert must_match_predicate("missing header")("a **missing**\n header value") + + def test_plain_substring_rejects_other_text(self): + assert not must_match_predicate("clientInfo")("the client sends no info") + + def test_regex_prefix_is_case_insensitive(self): + raw = "(log|logger|debug|info).{0,80}token|token.{0,80}(log|logger|debug|info)" + text = "Logging the session TOKEN leaks it" + assert re.search(raw, text) is None + assert must_match_predicate("re:" + raw)(text) + + def test_regex_runs_on_the_raw_text(self): + assert must_match_predicate(r"re:headers\.get\(")("use `headers.get(name)` instead") + + def test_invalid_regex_raises_value_error(self): + with pytest.raises(ValueError, match="must_match"): + must_match_predicate("re:(unclosed") + + def test_pattern_that_normalizes_to_nothing_is_compared_as_written(self): + test = must_match_predicate("...") + assert test("then it stops...") + assert not test("then it stops") + + +class TestMatchCase: + def test_tolerance_is_the_quality_default_of_five_lines(self): + assert quality.DEFAULT_LINE_TOLERANCE == 5 + human = Human("H1", line=10, must_match="race") + for line, grade in ((5, FULL), (15, FULL), (4, NONE), (16, NONE)): + assert match_case([human], [_ai(line, "a race here")])[0]["grade"] == grade, line + + def test_full_grade_shape(self): + grades = match_case([Human("H1", must_match="race")], [_ai(13, "a race here")]) + assert grades == [ + {"human_id": "H1", "grade": FULL, "ai_ref": 0, "ai_line": 13, "method": MATCH_METHOD}, + ] + + def test_none_grade_shape(self): + grades = match_case([Human("H1", must_match="race")], []) + assert grades == [ + {"human_id": "H1", "grade": NONE, "ai_ref": None, "ai_line": None, "method": MATCH_METHOD}, + ] + + def test_another_file_never_credits(self): + grades = match_case([Human("H1", must_match="race")], [_ai(10, "a race here", file="src/other.py")]) + assert grades[0]["grade"] == NONE + + def test_the_predicate_must_hold(self): + grades = match_case([Human("H1", must_match="race")], [_ai(10, "an off-by-one here")]) + assert grades[0]["grade"] == NONE + + def test_the_predicate_reads_the_title_too(self): + grades = match_case([Human("H1", must_match="clientInfo")], [_ai(10, "", title="No clientInfo sent")]) + assert grades[0]["grade"] == FULL + + def test_a_dropped_finding_never_credits_and_refs_index_the_whole_list(self): + ai = [_ai(10, "a race here", drop_reason="below confidence floor"), _ai(11, "a race here")] + assert match_case([Human("H1", must_match="race")], ai)[0]["ai_ref"] == 1 + assert match_case([Human("H1", must_match="race")], ai[:1])[0]["grade"] == NONE + + def test_a_human_finding_without_must_match_is_left_to_the_judge(self): + expected = [Human("H1", must_match="race"), Human("H2"), Human("H3", must_match="")] + assert [grade["human_id"] for grade in match_case(expected, [_ai(10, "a race here")])] == ["H1"] + + def test_a_file_level_human_finding_matches_any_line_of_its_file(self): + grades = match_case([Human("H1", line=0, must_match="race")], [_ai(300, "a race here")]) + assert grades[0]["grade"] == FULL + + def test_one_ai_finding_credits_at_most_two_human_findings(self): + assert MAX_CREDITS_PER_FINDING == 2 + expected = [Human(f"H{n}", line=9 + n, must_match="race") for n in (1, 2, 3)] + grades = match_case(expected, [_ai(11, "a race here")]) + assert _by_id(grades) == {"H1": FULL, "H2": FULL, "H3": NONE} + assert sum(1 for grade in grades if grade["grade"] == FULL) == MAX_CREDITS_PER_FINDING + + def test_the_cap_reroutes_a_credit_rather_than_stranding_it(self): + expected = [Human(f"H{n}", line=9 + n, must_match="race") for n in (1, 2, 3)] + ai = [_ai(10, "a race here", title="A"), _ai(5, "a race here", title="B")] + grades = match_case(expected, ai) + assert _by_id(grades) == {"H1": FULL, "H2": FULL, "H3": FULL} + assert [grade["ai_ref"] for grade in grades] == [1, 0, 0] + + def test_a_grouped_finding_is_credited_per_location_not_under_the_cap(self): + expected = [Human(f"H{n}", line=line, must_match="unchecked") for n, line in enumerate((10, 11, 40, 70), 1)] + grouped = GroupedFinding( + FILE, 10, "Unchecked return value", "The return value is unchecked.", + locations=({"file": FILE, "line": 40}, (FILE, 70)), + ) + grades = match_case(expected, [grouped]) + assert _by_id(grades) == {"H1": FULL, "H2": FULL, "H3": FULL, "H4": FULL} + assert [grade["ai_ref"] for grade in grades] == [0, 0, 0, 0] + assert [grade["ai_line"] for grade in grades] == [10, 10, 40, 70] + plain = _ai(10, "The return value is unchecked.", title="Unchecked return value") + assert _by_id(match_case(expected, [plain])) == {"H1": FULL, "H2": FULL, "H3": NONE, "H4": NONE} + + def test_each_location_of_a_group_keeps_its_own_cap(self): + expected = [Human(f"H{n}", line=39 + n, must_match="unchecked") for n in (1, 2, 3)] + grouped = GroupedFinding(FILE, 10, "Unchecked", "unchecked", locations=({"line": 40},)) + assert _by_id(match_case(expected, [grouped])) == {"H1": FULL, "H2": FULL, "H3": NONE} + + def test_grouped_locations_accept_several_shapes_and_skip_bad_entries(self): + finding = { + "file": "a.py", + "line": 3, + LOCATIONS_FIELD: [ + {"line": 9}, + ("b.py", 4), + Loc("a.py", 12), + {"file": "a.py", "line": "7"}, + ("a.py", 3), + {"file": "a.py", "line": True}, + "a.py:5", + ], + } + assert credit_locations(finding) == [("a.py", 3), ("a.py", 9), ("b.py", 4), ("a.py", 12)] + assert credit_locations({"file": "a.py", "line": 3}) == [("a.py", 3)] + assert credit_locations({"file": "a.py", "line": 3, LOCATIONS_FIELD: "a.py:9"}) == [("a.py", 3)] + + def test_shared_tokens_break_a_tie_on_line_distance(self): + human = Human("H1", line=10, text="session token logged in plain text", must_match="re:token") + near_first = _ai(8, "The token value is never validated.", title="Unvalidated token") + on_topic = _ai(12, "The session token is logged in plain text.", title="Secret leak") + wanted = quality._tokens("session token logged in plain text\ntoken", split_compounds=True) + shared = [ + len(wanted & quality._tokens(f"{row['title']}\n{row['body']}", split_compounds=True)) + for row in (near_first, on_topic) + ] + assert shared[1] > shared[0] + assert match_case([human], [near_first, on_topic])[0]["ai_ref"] == 1 + + def test_grades_do_not_depend_on_input_order(self): + expected = [ + Human("H1", line=10, must_match="re:race|lock"), + Human("H2", line=11, must_match="race"), + Human("H3", line=12, must_match="race"), + Human("H4", line=20, must_match="lock"), + Human("H5", line=21, must_match="re:lock|deadlock"), + ] + ai = [ + _ai(11, "a race on the counter", title="A"), + _ai(13, "another race", title="B"), + _ai(20, "lock is never released", title="C"), + _ai(5, "race in setup", title="D"), + _ai(22, "deadlock on shutdown", title="E"), + ] + + def outcome(humans, rows): + return { + grade["human_id"]: ( + grade["grade"], + rows[grade["ai_ref"]]["title"] if grade["ai_ref"] is not None else None, + grade["ai_line"], + ) + for grade in match_case(humans, rows) + } + + baseline = outcome(expected, ai) + assert sum(1 for grade, _, _ in baseline.values() if grade == FULL) == 5 + rng = random.Random(14) + for _ in range(20): + humans, rows = expected[:], ai[:] + rng.shuffle(humans) + rng.shuffle(rows) + assert outcome(humans, rows) == baseline + + +class TestAgreementSeverity: + def test_human_minor_reads_as_warning(self): + assert agreement_severity("minor") == "warning" + assert agreement_severity(" Minor ") == "warning" + + def test_other_severities_are_lowercased_only(self): + assert agreement_severity("Error") == "error" + assert agreement_severity("outofscope") == "outofscope" + assert agreement_severity(None) == "" + + +class TestRecall: + def test_partial_credit_is_half(self): + assert PARTIAL_CREDIT == 0.5 + case = _case( + expected=[Human("H1"), Human("H2"), Human("H3")], + ai=[_ai(10), _ai(11)], + grades=[_grade("H1", FULL, 0), _grade("H2", PARTIAL, 1), _grade("H3", NONE)], + ) + recall = score_cases([case])["metrics"]["recall"] + assert recall == { + "recall": 0.5, "credit": 1.5, "scored": 3, "full": 1, "partial": 1, "none": 1, "judge_error": 0, + } + + def test_a_judge_error_is_excluded_not_scored_as_zero(self): + case = _case( + expected=[Human("H1", accepted=True), Human("H2", accepted=True)], + ai=[_ai(10)], + grades=[_grade("H1", FULL, 0), _grade("H2", JUDGE_ERROR)], + ) + result = score_cases([case]) + metrics = result["metrics"] + assert metrics["recall"]["recall"] == 1.0 + assert metrics["recall"]["scored"] == 1 + assert metrics["recall"]["judge_error"] == 1 + assert metrics["recall_by_severity"]["error"]["recall"] == 1.0 + assert metrics["recall_accepted"]["recall"] == 1.0 + assert result["cases"][0]["findings"][1]["credit"] is None + zeroed = _case( + expected=case.expected, ai=case.ai_findings, grades=[_grade("H1", FULL, 0), _grade("H2", NONE)] + ) + assert score_cases([zeroed])["metrics"]["recall"]["recall"] == 0.5 + + def test_only_judge_errors_leave_recall_unknown_not_zero(self): + case = _case(expected=[Human("H1")], grades=[_grade("H1", JUDGE_ERROR)]) + recall = score_cases([case])["metrics"]["recall"] + assert recall["recall"] is None + assert recall["scored"] == 0 + + def test_the_headline_is_micro_over_human_findings(self): + one = _case("case-1", [Human("H1")], [_ai(10)], [_grade("H1", FULL, 0)]) + three = _case("case-2", [Human(f"H{n}") for n in (1, 2, 3)], [], [_grade(f"H{n}", NONE) for n in (1, 2, 3)]) + result = score_cases([one, three]) + assert result["metrics"]["recall"]["recall"] == 0.25 + assert [row["recall"] for row in result["cases"]] == [1.0, 0.0] + + def test_breakdowns_by_severity_category_and_accepted(self): + expected = [ + Human("H1", severity="error", category="security", accepted=True), + Human("H2", severity="minor", category="style", accepted=False), + Human("H3", severity="warning", accepted=None), + Human("H4", severity="error", category="security", accepted=True), + ] + grades = [_grade("H1", FULL, 0), _grade("H2", PARTIAL, 0), _grade("H3", NONE), _grade("H4", NONE)] + metrics = score_cases([_case(expected=expected, ai=[_ai(10)], grades=grades)])["metrics"] + assert list(metrics["recall_by_severity"]) == ["error", "minor", "warning"] + assert metrics["recall_by_severity"]["error"]["recall"] == 0.5 + assert metrics["recall_by_severity"]["minor"]["recall"] == 0.5 + assert list(metrics["recall_by_category"]) == [MISSING_LABEL, "security", "style"] + assert metrics["recall_by_category"]["security"]["credit"] == 1.0 + assert metrics["recall_accepted"]["scored"] == 2 + assert metrics["recall_accepted"]["recall"] == 0.5 + for partition in ("recall_by_severity", "recall_by_category"): + blocks = metrics[partition].values() + assert sum(block["scored"] for block in blocks) == metrics["recall"]["scored"], partition + assert sum(block["credit"] for block in blocks) == metrics["recall"]["credit"], partition + + +class TestSeverityAgreement: + def test_human_minor_agrees_with_warning_only_for_agreement(self): + expected = [Human("H1", severity="minor"), Human("H2", severity="minor"), Human("H3", severity="error")] + ai = [_ai(10, severity="warning"), _ai(11, severity="outofscope"), _ai(12, severity="error")] + grades = [_grade("H1", FULL, 0), _grade("H2", FULL, 1), _grade("H3", PARTIAL, 2)] + result = score_cases([_case(expected=expected, ai=ai, grades=grades)]) + agreement = result["metrics"]["severity_agreement"] + assert agreement == { + "compared": 3, + "agreed": 2, + "rate": 2 / 3, + "confusion": {"error": {"error": 1}, "warning": {"outofscope": 1, "warning": 1}}, + } + assert [entry["severity_agrees"] for entry in result["cases"][0]["findings"]] == [True, False, True] + assert list(result["metrics"]["recall_by_severity"]) == ["error", "minor"] + assert result["cases"][0]["findings"][0]["severity"] == "minor" + + def test_uncredited_findings_are_not_compared(self): + expected = [Human("H1", severity="error"), Human("H2", severity="error")] + grades = [_grade("H1", NONE), _grade("H2", JUDGE_ERROR)] + agreement = score_cases([_case(expected=expected, ai=[_ai(10)], grades=grades)])["metrics"][ + "severity_agreement" + ] + assert agreement == {"compared": 0, "agreed": 0, "rate": None, "confusion": {}} + + +class TestUnmatchedAi: + def test_unmatched_ai_findings_per_pr(self): + first = _case( + "case-1", + [Human("H1")], + [_ai(10, title="credited"), _ai(20, title="noise"), _ai(30, title="gated", drop_reason="hedge")], + [_grade("H1", FULL, 0)], + ) + second = _case("case-2", [Human("H1")], [_ai(10, title="noise")], [_grade("H1", NONE)]) + result = score_cases([first, second]) + assert result["metrics"]["unmatched_ai"] == {"total": 2, "ai_findings": 3, "per_pr": 1.0} + assert [(row["ai_findings"], row["unmatched_ai"]) for row in result["cases"]] == [(2, 1), (1, 1)] + + def test_a_credited_grouped_finding_counts_once(self): + grouped = GroupedFinding(FILE, 10, "Unchecked", "unchecked", locations=({"line": 40},)) + expected = [Human("H1", line=10, must_match="unchecked"), Human("H2", line=40, must_match="unchecked")] + grades = match_case(expected, [grouped, _ai(90, "noise")]) + row = case_row(_case(expected=expected, ai=[grouped, _ai(90, "noise")], grades=grades)) + assert (row["full"], row["ai_findings"], row["unmatched_ai"]) == (2, 2, 1) + + +class TestRunTotals: + def test_chunks_failed_and_elapsed_totals_skip_missing_values(self): + cases = [ + _case("case-1", record={"chunks_failed": 1, "elapsed_ms": 1000}), + _case("case-2", record={"chunks_failed": 0, "elapsed_ms": 2500}), + _case("case-3", record=None), + _case("case-4", record={"chunks_failed": True, "elapsed_ms": -5}), + ] + metrics = score_cases(cases)["metrics"] + assert metrics["chunks_failed"] == {"total": 1, "missing": 2} + assert metrics["elapsed_ms"] == {"total": 3500, "missing": 2} + + def test_a_none_review_cost_is_never_summed(self): + priced = _case("case-1", record={"cost_usd": 0.25}) + unknown = _case("case-2", record={"cost_usd": None}) + cost = score_cases([priced, unknown])["metrics"]["review_cost"] + assert cost == {"total_usd": None, "per_pr_usd": None, "priced": 1, "unpriced": 1, "estimated": 0} + both = score_cases([priced, _case("case-2", record={"cost_usd": "0.5", "cost_estimated": True})]) + assert both["metrics"]["review_cost"] == { + "total_usd": 0.75, "per_pr_usd": 0.375, "priced": 2, "unpriced": 0, "estimated": 1, + } + + def test_unusable_review_costs_count_as_unpriced(self): + bad = [math.nan, math.inf, -1.0, True, "abc", ""] + cases = [_case(f"case-{n}", record={"cost_usd": value}) for n, value in enumerate(bad)] + cases.append(_case("case-none", record=None)) + cost = score_cases(cases)["metrics"]["review_cost"] + assert (cost["total_usd"], cost["priced"], cost["unpriced"]) == (None, 0, len(bad) + 1) + + def test_judge_cost_defaults_to_a_known_zero(self): + cases = [_case("case-1"), _case("case-2", judge_cost_usd=0.02, judge_cost_estimated=True)] + cost = score_cases(cases)["metrics"]["judge_cost"] + assert cost == {"total_usd": 0.02, "per_pr_usd": 0.01, "priced": 2, "unpriced": 0, "estimated": 1} + cases.append(_case("case-3", judge_cost_usd=None)) + cost = score_cases(cases)["metrics"]["judge_cost"] + assert (cost["total_usd"], cost["per_pr_usd"], cost["unpriced"]) == (None, None, 1) + + +class TestOutputShape: + ROW_KEYS = [ + "case_id", "verdict", "recall", "credit", "scored", "full", "partial", "none", "judge_error", + "ai_findings", "unmatched_ai", "severity_compared", "severity_agreed", "chunks_failed", "elapsed_ms", + "review_cost_usd", "review_cost_estimated", "judge_cost_usd", "judge_cost_estimated", "findings", + ] + ENTRY_KEYS = [ + "human_id", "file", "line", "severity", "category", "accepted", "grade", "credit", "ai_ref", "ai_line", + "ai_severity", "severity_agrees", "method", + ] + METRIC_KEYS = [ + "case_count", "recall", "recall_by_severity", "recall_by_category", "recall_accepted", "unmatched_ai", + "severity_agreement", "chunks_failed", "elapsed_ms", "review_cost", "judge_cost", + ] + + def _result(self): + late = _case( + "case-b", + [Human("H2", severity="minor", must_match="race"), Human("H1", must_match="lock")], + [_ai(10, "a race here", severity="warning"), _ai(40, "noise")], + record={"verdict": "Comment", "chunks_failed": 0, "elapsed_ms": 900, "cost_usd": 0.1}, + ) + late = _case( + late.case_id, late.expected, late.ai_findings, match_case(late.expected, late.ai_findings), late.record + ) + early = _case("case-a", [Human("H1")], [], [_grade("H1", JUDGE_ERROR)], judge_cost_usd=None) + return score_cases([late, early]) + + def test_keys_and_ordering(self): + result = self._result() + assert list(result) == ["metrics", "cases"] + assert list(result["metrics"]) == self.METRIC_KEYS + assert [row["case_id"] for row in result["cases"]] == ["case-a", "case-b"] + for row in result["cases"]: + assert list(row) == self.ROW_KEYS + assert [list(entry) for entry in row["findings"]] == [self.ENTRY_KEYS] * len(row["findings"]) + assert [entry["human_id"] for entry in result["cases"][1]["findings"]] == ["H1", "H2"] + + def test_the_matcher_feeds_the_metrics(self): + row = self._result()["cases"][1] + assert (row["verdict"], row["full"], row["none"], row["unmatched_ai"]) == ("Comment", 1, 1, 1) + credited = row["findings"][1] + assert (credited["grade"], credited["ai_ref"], credited["ai_line"], credited["method"]) == ( + FULL, 0, 10, MATCH_METHOD, + ) + assert (credited["ai_severity"], credited["severity_agrees"]) == ("warning", True) + + def test_the_result_survives_strict_json(self): + result = self._result() + assert json.loads(json.dumps(result, allow_nan=False)) == result + + def test_no_cases(self): + metrics = score_cases([])["metrics"] + assert metrics["case_count"] == 0 + assert metrics["recall"]["recall"] is None + assert metrics["unmatched_ai"]["per_pr"] is None + assert metrics["review_cost"]["total_usd"] is None + + +class TestGradeValidation: + @pytest.mark.parametrize( + ("grades", "message"), + [ + ([_grade("H1", "match", 0)], "'match' for 'H1' is not one of full, partial, none, judge_error"), + ([], "no grade for expected finding"), + ([_grade("H1", FULL, 0), _grade("H1", NONE)], "more than one grade"), + ([_grade("H1", NONE), _grade("H9", NONE)], "unknown expected finding 'H9'"), + ([_grade("H1", FULL)], "credits ai_ref None"), + ([_grade("H1", PARTIAL, 5)], "credits ai_ref 5"), + ([_grade("H1", FULL, True)], "credits ai_ref True"), + ([_grade("H1", FULL, 1)], "credits dropped AI finding 1"), + ], + ) + def test_bad_grades_raise(self, grades, message): + case = _case(expected=[Human("H1")], ai=[_ai(10), _ai(11, drop_reason="hedge")], grades=grades) + with pytest.raises(ValueError, match=re.escape(message)): + score_cases([case]) + + def test_duplicate_case_ids_raise(self): + with pytest.raises(ValueError, match="case ids are not unique"): + score_cases([_case("case-1"), _case("case-1")]) + + def test_duplicate_expected_ids_raise(self): + case = _case(expected=[Human("H1"), Human("H1")], grades=[_grade("H1", NONE)]) + with pytest.raises(ValueError, match="not unique"): + case_row(case) From 0a9c28bfa80d4c6886969bb8a970b07844e9f057 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:44:28 -0700 Subject: [PATCH 12/64] feat(quality): reworded-duplicate tier in apply_sweep_dedup (#10 T2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit apply_sweep_dedup gains a keyword-only `similarity: float | None = None`. None skips the new tier entirely, so the pass is the 0.14.0 exact tier byte for byte (checked against the 8d36b2e function on 20,000 random inputs). With a threshold, a second tier runs after the exact tier over active findings in the same file on the same line (line 0 is never compared) and drops a pair whose titles pass titles_similar: - across the chunk/sweep boundary the chunk copy always survives; the sweep copy is dropped only when it is no more severe, so a more severe sweep copy is kept beside it and the tier never lowers a line's worst severity; - on one side (chunk vs chunk, sweep vs sweep) the more severe copy is kept, then the one ranked first by finding_rank_key. Each finding is compared only with copies already kept, in a fixed content order, so the result does not depend on input order and a dropped copy never drops a third. The reason is `duplicate of finding (reworded, similarity 0.57)`. The module docstring's pass 12 describes the tier, and pass 11 now names the warning and outofscope cap reasons. Not wired into the orchestrator, config or CLI yet. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/quality.py | 133 ++++++++- tests/test_sweep_dedup_similarity.py | 421 +++++++++++++++++++++++++++ 2 files changed, 549 insertions(+), 5 deletions(-) create mode 100644 tests/test_sweep_dedup_similarity.py diff --git a/src/prxref/quality.py b/src/prxref/quality.py index ac60aa3c..8f736404 100644 --- a/src/prxref/quality.py +++ b/src/prxref/quality.py @@ -79,7 +79,9 @@ the spec digest the workers were shown. 11. ``apply_quality_gate``: drop findings below the confidence floor (``confidence 0.40 below floor 0.60``), cap errors per review - (``error cap exceeded (max N)``), and enforce the + (``error cap exceeded (max N)``), optionally cap warnings and + outofscope findings the same way (``warning cap exceeded (max N)``, + ``outofscope cap exceeded (max N)``), and enforce the {error, warning, spec, outofscope} severity vocabulary (``invalid severity: ''``). It RETURNS its findings sorted by ``finding_sort_key``, so the caller re-derives the chunk/sweep @@ -88,7 +90,16 @@ finding which SURVIVED the gate, on file + normalized title (``duplicate of chunk finding``). It runs after the gate so a sub-floor chunk finding cannot suppress its higher-confidence sweep - duplicate and then die at the gate itself. + duplicate and then die at the gate itself. With a similarity + threshold set, a second tier then drops reworded restatements: two + active findings in the same file on the same line (line 0 is never + compared) whose titles pass ``titles_similar`` + (``duplicate of chunk finding (reworded, similarity 0.57)``, or + ``duplicate of sweep finding ...`` between two sweep findings). + Across the boundary the chunk copy always survives and a sweep copy + is dropped only when it is no more severe; on one side the more + severe, then higher-confidence, copy is kept. Without a threshold + the tier does not run. 13. ``apply_containment_note``: a finding that asserts a throw, panic, crash, or unhandled rejection and never names where it is caught or where it propagates to has its body suffixed with @@ -1121,7 +1132,10 @@ def finding_rank_key(finding: Finding) -> tuple[float, str, int, str]: def apply_sweep_dedup( - findings: Sequence[Finding], sweep_start: int + findings: Sequence[Finding], + sweep_start: int, + *, + similarity: float | None = None, ) -> list[Finding]: """Drop systemic-sweep findings that restate a chunk finding. @@ -1131,8 +1145,35 @@ def apply_sweep_dedup( finding adds no recall — it is the same pattern the chunk seat already reported — and is dropped with ``drop_reason="duplicate of chunk finding"``, the same retained-not-silenced convention every other pass - uses. Chunk findings are never dropped by this pass, even when the sweep - phrased the pattern first: the chunk seat cited the exact line. + uses. A sweep finding never causes a chunk finding to be dropped, even + when the sweep phrased the pattern first: the chunk seat cited the exact + line. + + ``similarity`` switches on a second, reworded tier that runs after the + exact tier; ``None`` (the default) skips it entirely, so the pass is + the exact tier alone. The tier compares findings that are still active, + in the same file and on the same line; a line-0 (file-level) finding is + never compared. Two such findings are duplicates when + :func:`titles_similar` holds at ``similarity``. + + - Across the chunk/sweep boundary the chunk copy always survives. The + sweep copy is dropped only when its severity is no higher than the + chunk copy's; a MORE severe sweep copy is kept alongside it, so the + tier can never lower a review's worst severity. Confidence plays no + part here. + - Between two findings on the same side (chunk vs chunk, or sweep vs + sweep), the more severe one is kept, then the one ranked first by + :func:`finding_rank_key`: higher confidence, then content. + + Chunk findings are settled among themselves first; each sweep finding + is then compared with the chunk copies kept on its line before the + sweep copies kept there. A finding is only ever compared with copies + already kept, in a fixed content order, so the result does not depend + on input order and a dropped copy never drops a third. The dropped + copy's ``drop_reason`` names the side of the copy it restates and the + :func:`title_similarity` Jaccard score to two decimals: + ``duplicate of chunk finding (reworded, similarity 0.57)`` or + ``duplicate of sweep finding (reworded, similarity 0.57)``. Runs after :func:`apply_quality_gate`, so the duplicate set is built from chunk findings that SURVIVED it — a sub-floor chunk finding never @@ -1158,6 +1199,88 @@ def apply_sweep_dedup( result.append(replace(f, drop_reason="duplicate of chunk finding")) else: result.append(f) + if similarity is None: + return result + return _dedup_reworded(result, start, similarity) + + +def _reworded_severity_rank(finding: Finding) -> int: + severity = (finding.severity or "").strip().lower() + return _SEVERITY_RANK.get(severity, len(_SEVERITY_RANK)) + + +def _reworded_keep_key( + finding: Finding, +) -> tuple[int, tuple[float, str, int, str], str]: + return ( + _reworded_severity_rank(finding), + finding_rank_key(finding), + repr(finding), + ) + + +def _first_reworded_match( + findings: Sequence[Finding], + candidate: int, + kept: Sequence[int], + threshold: float, +) -> float | None: + title = findings[candidate].title or "" + for k in kept: + kept_title = findings[k].title or "" + if titles_similar(title, kept_title, threshold): + return title_similarity(title, kept_title)[0] + return None + + +def _reworded_drop(finding: Finding, side: str, jaccard: float) -> Finding: + return replace( + finding, + drop_reason=f"duplicate of {side} finding (reworded, similarity {jaccard:.2f})", + ) + + +def _dedup_reworded( + findings: list[Finding], start: int, threshold: float +) -> list[Finding]: + lines: dict[tuple[str, int], list[int]] = {} + for i, f in enumerate(findings): + if f.drop_reason is None and (f.line or 0) > 0: + lines.setdefault((f.file, f.line), []).append(i) + result = list(findings) + for indices in lines.values(): + if len(indices) < 2: + continue + chunk_side = sorted( + (i for i in indices if i < start), + key=lambda i: _reworded_keep_key(findings[i]), + ) + sweep_side = sorted( + (i for i in indices if i >= start), + key=lambda i: _reworded_keep_key(findings[i]), + ) + kept_chunk: list[int] = [] + for i in chunk_side: + jaccard = _first_reworded_match(findings, i, kept_chunk, threshold) + if jaccard is None: + kept_chunk.append(i) + else: + result[i] = _reworded_drop(findings[i], "chunk", jaccard) + kept_sweep: list[int] = [] + for i in sweep_side: + rank = _reworded_severity_rank(findings[i]) + at_least_as_severe = [ + k for k in kept_chunk if _reworded_severity_rank(findings[k]) <= rank + ] + jaccard = _first_reworded_match(findings, i, at_least_as_severe, threshold) + if jaccard is not None: + result[i] = _reworded_drop(findings[i], "chunk", jaccard) + continue + jaccard = _first_reworded_match(findings, i, kept_sweep, threshold) + if jaccard is None: + kept_sweep.append(i) + else: + result[i] = _reworded_drop(findings[i], "sweep", jaccard) return result diff --git a/tests/test_sweep_dedup_similarity.py b/tests/test_sweep_dedup_similarity.py new file mode 100644 index 00000000..052e93c5 --- /dev/null +++ b/tests/test_sweep_dedup_similarity.py @@ -0,0 +1,421 @@ +"""The reworded-duplicate tier of ``apply_sweep_dedup`` (issue #10, T2). + +P1-P3 are the reworded same-line pairs quoted in issue #10; N1 and N2 are +the closest different-problem negatives from the issue map, and S1/S2 the +short-title cases that share fewer than three tokens. They are copied from +``tests/test_title_similarity.py`` so this file reads on its own. +""" +from __future__ import annotations + +import itertools +import random +from collections.abc import Sequence +from dataclasses import replace + +import pytest + +from prxref import quality +from prxref.quality import apply_sweep_dedup, normalize_title +from prxref.triage import Finding + +THRESHOLD = 0.5 + +P1_A = "Replace manual null check with Optional in resolve()" +P1_B = "Use Optional instead of manual null check in adapter lookup" +P2_A = "Restore admin-only scope on destructive migration admin endpoints" +P2_B = "Restore admin-only scope on migration admin controller" +P3_A = "Use a logging-key constant instead of a string literal key" +P3_B = "Use a logging-key constant instead of a string-literal key" +P3_C = "Use the logging-key constant instead of the string literal key" + +REWORDED = { + "P1": (P1_A, P1_B, "0.57"), + "P2": (P2_A, P2_B, "0.57"), + "P3": (P3_A, P3_B, "1.00"), +} + +DIFFERENT_PROBLEM = { + "N1": (P1_A, "Missing null check on resolve() return value"), + "N2": (P2_B, "Add rate limiting to migration admin controller"), + "N3": ( + "Unbounded retry loop in payment webhook handler", + "Payment webhook handler logs the raw card payload", + ), + "N4": ( + "Race condition on cache entry expiry in session store", + "Session store cache key includes the user email", + ), + "S1": ("Missing null check", P1_B), + "S2": ("SQL injection", "SQL injection in search query"), +} + +TRIPLE_A = "Cache entry expiry race" +TRIPLE_B = "Cache entry expiry race in session store" +TRIPLE_C = "Expiry race in session store" + +SEVERITIES = ("error", "warning", "spec", "outofscope") + + +def _base_apply_sweep_dedup( + findings: Sequence[Finding], sweep_start: int +) -> list[Finding]: + """The body of ``apply_sweep_dedup`` at 8d36b2e, copied verbatim.""" + start = max(0, sweep_start) + chunk_keys = { + (f.file, normalize_title(f.title)) + for f in findings[:start] + if f.drop_reason is None + } + result: list[Finding] = [] + for i, f in enumerate(findings): + if ( + i >= start + and f.drop_reason is None + and (f.file, normalize_title(f.title)) in chunk_keys + ): + result.append(replace(f, drop_reason="duplicate of chunk finding")) + else: + result.append(f) + return result + + +def _f(title: str, **kwargs) -> Finding: + defaults = { + "file": "src/app.py", + "line": 10, + "severity": "warning", + "confidence": 0.9, + "title": title, + "body": "Details about the finding.", + } + defaults.update(kwargs) + return Finding(**defaults) + + +def _dedup( + chunk: Sequence[Finding], sweep: Sequence[Finding], similarity: float | None = THRESHOLD +) -> tuple[list[Finding], list[Finding]]: + out = apply_sweep_dedup(list(chunk) + list(sweep), len(chunk), similarity=similarity) + assert len(out) == len(chunk) + len(sweep) + return out[: len(chunk)], out[len(chunk):] + + +def _reason(side: str, score: str) -> str: + return f"duplicate of {side} finding (reworded, similarity {score})" + + +_POOL = ( + P1_A, P1_B, P2_A, P2_B, P3_A, P3_B, + "use a logging-key constant instead of a `string literal` key.", + DIFFERENT_PROBLEM["N1"][1], DIFFERENT_PROBLEM["N2"][1], + TRIPLE_A, TRIPLE_B, TRIPLE_C, "SQL injection", "", +) + + +def _random_findings(rng: random.Random) -> list[Finding]: + return [ + Finding( + file=rng.choice(["a.py", "b.py"]), + line=rng.choice([0, 1, 1, 2]), + severity=rng.choice(SEVERITIES), + confidence=rng.choice([0.6, 0.7, 0.9, 1.0]), + title=rng.choice(_POOL), + body=rng.choice(["b1", "b2"]), + drop_reason=rng.choice([None, None, None, None, 'hedged: "if"']), + ) + for _ in range(rng.randint(0, 12)) + ] + + +class TestNoneIsTheExactTier: + def test_none_equals_the_pre_change_function_on_randomized_input(self): + rng = random.Random(20260924) + cases = 3000 + same = 0 + control_differs = 0 + for _ in range(cases): + findings = _random_findings(rng) + start = rng.randint(-2, len(findings) + 2) + expected = repr(_base_apply_sweep_dedup(list(findings), start)) + default = repr(apply_sweep_dedup(list(findings), start)) + explicit = repr(apply_sweep_dedup(list(findings), start, similarity=None)) + if expected == default == explicit: + same += 1 + if repr(apply_sweep_dedup(list(findings), start, similarity=THRESHOLD)) != expected: + control_differs += 1 + assert same == cases + assert control_differs > 0 + + def test_none_never_scores_a_title(self, monkeypatch): + def refuse(*_args, **_kwargs): + raise AssertionError("the similarity tier ran") + + monkeypatch.setattr(quality, "titles_similar", refuse) + monkeypatch.setattr(quality, "title_similarity", refuse) + chunk, sweep = [_f(P1_A)], [_f(P1_B)] + _dedup(chunk, sweep, similarity=None) + with pytest.raises(AssertionError, match="the similarity tier ran"): + _dedup(chunk, sweep, similarity=THRESHOLD) + + def test_similarity_is_keyword_only(self): + with pytest.raises(TypeError): + apply_sweep_dedup([_f(P1_A), _f(P1_B)], 1, THRESHOLD) + + +class TestRewordedPairs: + @pytest.mark.parametrize("name", sorted(REWORDED)) + def test_reworded_sweep_copy_is_dropped(self, name): + chunk_title, sweep_title, score = REWORDED[name] + chunk, sweep = _dedup([_f(chunk_title)], [_f(sweep_title)]) + assert chunk[0].drop_reason is None + assert sweep[0].drop_reason == _reason("chunk", score) + + @pytest.mark.parametrize("name", sorted(REWORDED)) + def test_either_wording_on_the_chunk_side(self, name): + sweep_title, chunk_title, score = REWORDED[name] + chunk, sweep = _dedup([_f(chunk_title)], [_f(sweep_title)]) + assert chunk[0].drop_reason is None + assert sweep[0].drop_reason == _reason("chunk", score) + + @pytest.mark.parametrize("name", sorted(DIFFERENT_PROBLEM)) + def test_different_problem_pair_is_kept(self, name): + a, b = DIFFERENT_PROBLEM[name] + chunk, sweep = _dedup([_f(a)], [_f(b)]) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + chunk, sweep = _dedup([_f(a), _f(b)], []) + assert [f.drop_reason for f in chunk] == [None, None] + + def test_without_a_threshold_the_reworded_pair_is_kept(self): + chunk, sweep = _dedup([_f(P1_A)], [_f(P1_B)], similarity=None) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + + def test_threshold_is_inclusive_and_read(self): + _, sweep = _dedup([_f(P1_A)], [_f(P1_B)], similarity=4 / 7) + assert sweep[0].drop_reason == _reason("chunk", "0.57") + _, sweep = _dedup([_f(P1_A)], [_f(P1_B)], similarity=0.6) + assert sweep[0].drop_reason is None + + def test_dropped_finding_is_marked_not_removed(self): + original = _f(P1_B, severity="warning", confidence=0.7, body="sweep body") + _, sweep = _dedup([_f(P1_A)], [original]) + assert sweep[0] == replace(original, drop_reason=_reason("chunk", "0.57")) + + def test_exact_title_keeps_the_exact_tier_reason(self): + chunk, sweep = _dedup( + [_f(P3_A, line=10)], + [_f(P3_A.upper(), line=10), _f(P3_A, line=40)], + ) + assert chunk[0].drop_reason is None + assert [f.drop_reason for f in sweep] == ["duplicate of chunk finding"] * 2 + + def test_a_finding_dropped_earlier_keeps_its_reason_and_suppresses_nothing(self): + gated = _f(P1_A, drop_reason="confidence 0.40 below floor 0.60") + chunk, sweep = _dedup([gated], [_f(P1_B)]) + assert chunk[0].drop_reason == "confidence 0.40 below floor 0.60" + assert sweep[0].drop_reason is None + hedged = _f(P1_B, drop_reason='hedged: "if"') + _, sweep = _dedup([_f(P1_A)], [hedged]) + assert sweep[0].drop_reason == 'hedged: "if"' + + +class TestLineRule: + def test_line_zero_pair_is_never_compared(self): + chunk, sweep = _dedup([_f(P3_A, line=0)], [_f(P3_B, line=0)]) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + chunk, _ = _dedup([_f(P1_A, line=0), _f(P1_B, line=0)], []) + assert [f.drop_reason for f in chunk] == [None, None] + + def test_line_zero_against_an_anchored_copy_is_not_compared(self): + chunk, sweep = _dedup([_f(P1_A, line=10)], [_f(P1_B, line=0)]) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + + def test_same_line_control_is_dropped(self): + _, sweep = _dedup([_f(P3_A, line=10)], [_f(P3_B, line=10)]) + assert sweep[0].drop_reason == _reason("chunk", "1.00") + + @pytest.mark.parametrize("sweep_line", [9, 11, 15]) + def test_a_nearby_line_is_not_the_same_line(self, sweep_line): + chunk, sweep = _dedup([_f(P3_A, line=10)], [_f(P3_B, line=sweep_line)]) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + + def test_another_file_is_never_compared(self): + chunk, sweep = _dedup([_f(P3_A, file="src/a.py")], [_f(P3_B, file="src/b.py")]) + assert [f.drop_reason for f in chunk + sweep] == [None, None] + + +class TestKeepRuleAcrossTheBoundary: + @pytest.mark.parametrize( + ("chunk_severity", "sweep_severity"), + list(itertools.product(SEVERITIES, repeat=2)), + ) + def test_the_chunk_copy_always_survives(self, chunk_severity, sweep_severity): + chunk, sweep = _dedup( + [_f(P1_A, severity=chunk_severity, confidence=0.6)], + [_f(P1_B, severity=sweep_severity, confidence=1.0)], + ) + assert chunk[0].drop_reason is None + sweep_more_severe = SEVERITIES.index(sweep_severity) < SEVERITIES.index(chunk_severity) + expected = None if sweep_more_severe else _reason("chunk", "0.57") + assert sweep[0].drop_reason == expected + + def test_a_more_severe_sweep_copy_survives_beside_the_chunk_copy(self): + chunk, sweep = _dedup( + [_f(P2_A, severity="warning")], [_f(P2_B, severity="error")] + ) + assert [(f.severity, f.drop_reason) for f in chunk + sweep] == [ + ("warning", None), + ("error", None), + ] + + def test_confidence_does_not_decide_across_the_boundary(self): + chunk, sweep = _dedup( + [_f(P1_A, confidence=0.6)], [_f(P1_B, confidence=1.0)] + ) + assert chunk[0].drop_reason is None + assert sweep[0].drop_reason == _reason("chunk", "0.57") + + def test_randomized_invariants_of_the_keep_rule(self): + rng = random.Random(3) + tier_drops = 0 + for _ in range(2000): + findings = _random_findings(rng) + start = rng.randint(0, len(findings)) + exact = apply_sweep_dedup(findings, start) + tiered = apply_sweep_dedup(findings, start, similarity=THRESHOLD) + worst_before: dict[tuple[str, int], int] = {} + worst_after: dict[tuple[str, int], int] = {} + for i, (before, after) in enumerate(zip(exact, tiered, strict=True)): + if before.drop_reason is not None: + assert after == before + continue + rank = SEVERITIES.index(before.severity) + key = (before.file, before.line) + worst_before[key] = min(worst_before.get(key, rank), rank) + if after.drop_reason is None: + worst_after[key] = min(worst_after.get(key, rank), rank) + continue + tier_drops += 1 + assert before.line > 0 + assert after.drop_reason.startswith("duplicate of ") + if i < start: + assert after.drop_reason.startswith("duplicate of chunk finding (reworded") + assert worst_after == worst_before + assert tier_drops > 0 + + def test_the_sweep_copy_is_compared_with_every_kept_chunk_copy(self): + chunk, sweep = _dedup( + [_f(P2_A, severity="warning"), _f(P1_A, severity="error")], + [_f(P2_B, severity="warning")], + ) + assert [f.drop_reason for f in chunk] == [None, None] + assert sweep[0].drop_reason == _reason("chunk", "0.57") + + def test_a_chunk_match_outranks_a_sweep_match_in_the_reason(self): + _, sweep = _dedup( + [_f(P3_A, severity="warning")], + [_f(P3_B, severity="error"), _f(P3_C, severity="warning")], + ) + assert sweep[0].drop_reason is None + assert sweep[1].drop_reason == _reason("chunk", "1.00") + + +class TestKeepRuleOnOneSide: + @pytest.mark.parametrize("side", ["chunk", "sweep"]) + def test_higher_confidence_wins_at_equal_severity(self, side): + pair = [_f(P1_A, confidence=0.7), _f(P1_B, confidence=0.9)] + chunk, sweep = _dedup(pair, []) if side == "chunk" else _dedup([], pair) + out = chunk + sweep + assert out[0].drop_reason == _reason(side, "0.57") + assert out[1].drop_reason is None + + @pytest.mark.parametrize("side", ["chunk", "sweep"]) + def test_higher_severity_beats_higher_confidence(self, side): + pair = [_f(P1_A, severity="error", confidence=0.6), _f(P1_B, severity="warning", confidence=1.0)] + chunk, sweep = _dedup(pair, []) if side == "chunk" else _dedup([], pair) + out = chunk + sweep + assert out[0].drop_reason is None + assert out[1].drop_reason == _reason(side, "0.57") + + def test_full_ties_break_on_content_not_arrival(self): + pair = [_f(P1_A), _f(P1_B)] + forward, _ = _dedup(pair, []) + backward, _ = _dedup(list(reversed(pair)), []) + kept_forward = [f.title for f in forward if f.drop_reason is None] + kept_backward = [f.title for f in backward if f.drop_reason is None] + assert kept_forward == kept_backward == [P1_A] + + def test_a_dropped_copy_never_drops_a_third(self): + chunk, _ = _dedup( + [ + _f(TRIPLE_C, confidence=0.8), + _f(TRIPLE_B, confidence=0.9), + _f(TRIPLE_A, confidence=0.95), + ], + [], + ) + assert [f.drop_reason for f in chunk] == [None, _reason("chunk", "0.67"), None] + + def test_a_sweep_copy_kept_for_severity_still_absorbs_a_weaker_sweep_copy(self): + chunk, sweep = _dedup( + [_f(P3_A, severity="spec")], + [_f(P3_B, severity="error"), _f(P3_C, severity="warning")], + ) + assert chunk[0].drop_reason is None + assert sweep[0].drop_reason is None + assert sweep[1].drop_reason == _reason("sweep", "1.00") + + +class TestOrderIndependence: + def _cluster(self) -> tuple[list[Finding], list[Finding]]: + chunk = [ + _f(P1_A, severity="warning", confidence=0.7), + _f(P1_B, severity="warning", confidence=0.9), + _f(TRIPLE_A, severity="warning", confidence=0.95), + _f(TRIPLE_B, severity="warning", confidence=0.9), + _f(TRIPLE_C, severity="warning", confidence=0.8), + _f(P3_A, line=0), + ] + sweep = [ + _f("Replace the manual null check with Optional in resolve() call", confidence=1.0), + _f(P1_B + " path", severity="error", confidence=0.6), + _f("Session store cache entry expiry race", severity="error", confidence=0.9), + _f("Expiry race in the session store", severity="error", confidence=0.7), + _f(P3_B, line=0), + ] + return chunk, sweep + + def test_the_cluster_resolves_to_the_expected_survivors(self): + chunk, sweep = self._cluster() + out_c, out_s = _dedup(chunk, sweep) + assert [f.drop_reason for f in out_c] == [ + _reason("chunk", "0.57"), None, None, _reason("chunk", "0.67"), None, None, + ] + assert [f.drop_reason for f in out_s] == [ + _reason("chunk", "0.50"), None, None, _reason("sweep", "0.67"), None, + ] + + def test_every_shuffle_of_each_side_gives_one_result(self): + chunk, sweep = self._cluster() + out_c, out_s = _dedup(chunk, sweep) + reference = sorted(map(repr, out_c + out_s)) + rng = random.Random(7) + for _ in range(200): + c, s = list(chunk), list(sweep) + rng.shuffle(c) + rng.shuffle(s) + out_c, out_s = _dedup(c, s) + assert sorted(map(repr, out_c + out_s)) == reference + for before, after in zip(c + s, out_c + out_s, strict=True): + assert replace(after, drop_reason=None) == before + + def test_randomized_inputs_are_order_independent(self): + rng = random.Random(10) + for _ in range(1000): + findings = _random_findings(rng) + start = rng.randint(0, len(findings)) + chunk, sweep = findings[:start], findings[start:] + reference = sorted(map(repr, apply_sweep_dedup(findings, start, similarity=THRESHOLD))) + rng.shuffle(chunk) + rng.shuffle(sweep) + shuffled = apply_sweep_dedup(chunk + sweep, start, similarity=THRESHOLD) + assert sorted(map(repr, shuffled)) == reference From 8ac7cd84fe0f5ffab5915b7300f1d6c95227c975 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:46:44 -0700 Subject: [PATCH 13/64] feat(cli): prxref prompts export DIR [--force] (#11 T5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add prompt_templates.export_prompt_templates(dest, *, force=False), which writes the packaged worker.md, systemic.md and summary.md into DIR byte for byte and returns the written paths. It iterates TEMPLATE_NAMES by name and never lists the package directory, so the judge prompt (not overridable, decisions #11) is never exported. DIR is created when missing. Without force, any existing target (a dangling symlink included) is refused with a ConfigError naming each existing file and --force, before anything is written. With force a target is overwritten, and a symlink is replaced by a regular file instead of being written through. An empty DIR, or one that cannot be created or written, is a ConfigError too. Wire it as the `prompts export` subcommand, shaped like `trace render`: the handler prints one written path per line and exits 2 on ConfigError with the same "configuration error: ..." line review prints; a bare `prxref prompts` prints usage and exits 2, as a bare `prxref trace` does. README `## CLI Flags` documents the subcommand and --force. tests/test_prompts_export.py: byte equality with reviewer.load_prompt, exactly three files and no judge.md, full and partial refusal with no file modified, --force, round-trip through load_prompt_templates with zero warnings and overrides equal to the packaged text, and the CLI and module entry points. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- README.md | 2 + src/prxref/cli.py | 39 ++++++ src/prxref/prompt_templates.py | 50 +++++++ tests/test_prompts_export.py | 247 +++++++++++++++++++++++++++++++++ 4 files changed, 338 insertions(+) create mode 100644 tests/test_prompts_export.py diff --git a/README.md b/README.md index a660fc04..bb30279c 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,8 @@ Replay flags, for evaluation (see [Replay Mode (Evaluation)](#replay-mode-evalua The other subcommands: `prxref serve [--port N] [--host H]` runs the [webhook server](#webhook-server) (default port `8080`, default host `0.0.0.0`); `prxref trace render FILE [-o OUT]` renders a JSONL run trace (`PRXREF_TRACE_FILE`) to a standalone HTML pipeline view, written next to the trace unless `-o`/`--out` names the output; and `prxref --version` prints the version. +`prxref prompts export DIR [--force]` writes the packaged `worker.md`, `systemic.md` and `summary.md` prompt templates into `DIR`, byte for byte, as the starting point for a `PRXREF_PROMPTS_DIR` override directory, and prints each path it wrote. It creates `DIR` when it is missing. When any of the three files already exists it overwrites nothing, writes nothing, and exits `2` naming the file; `--force` overwrites them. The judge prompt of `prxref eval` is never exported, because it cannot be overridden. + ## Replay Mode (Evaluation) A replay reviews a pinned, reproducible input instead of a PR as it stands, so one change can be reviewed again later, by another model or another prxref build, and compared. Three invocations cover it: diff --git a/src/prxref/cli.py b/src/prxref/cli.py index 9e67a145..b3f102ac 100644 --- a/src/prxref/cli.py +++ b/src/prxref/cli.py @@ -76,6 +76,7 @@ from prxref.forges.base import detect_forge from prxref.forges.replay import LocalDiffForge, ReplayForge from prxref.llm import ConfigError +from prxref.prompt_templates import export_prompt_templates from prxref.rules import load_review_rules from prxref.ticket import load_ticket_context from prxref.triage import SCOPE_IN, SCOPE_OUT, normalize_scope @@ -242,6 +243,21 @@ def _build_parser() -> argparse.ArgumentParser: help="output HTML path (default: the trace path with an .html suffix)", ) + pr = sub.add_parser("prompts", help="work with the overridable prompt templates") + pr_sub = pr.add_subparsers(dest="prompts_command") + pr_export = pr_sub.add_parser( + "export", + help="write the packaged worker, systemic and summary templates to DIR as an override starting point", + ) + pr_export.add_argument( + "prompts_export_dir", metavar="DIR", help="directory to write into (created when missing)" + ) + pr_export.add_argument( + "--force", + action="store_true", + help="overwrite templates already in DIR (without it, an existing one is refused and nothing is written)", + ) + return parser @@ -956,6 +972,24 @@ class of mistake as a malformed env var, not a review failure. return 0 +def _cmd_prompts_export(args: argparse.Namespace) -> int: + """Write the packaged prompt templates to ``DIR``, printing one written path per line. + + Exit 2 when an existing template would be overwritten without + ``--force``, or when ``DIR`` cannot be created or written: both are + ``ConfigError``, reported as ``review`` reports one, and the refusal + writes nothing. + """ + try: + written = export_prompt_templates(args.prompts_export_dir, force=args.force) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + for path in written: + print(path) + return 0 + + def main(argv: list[str] | None = None) -> int: """CLI entry point dispatching ``review``, ``serve``, or ``--version``.""" parser = _build_parser() @@ -980,6 +1014,11 @@ def main(argv: list[str] | None = None) -> int: return _cmd_trace_render(args) parser.print_help(sys.stderr) return 2 + if args.command == "prompts": + if args.prompts_command == "export": + return _cmd_prompts_export(args) + parser.print_help(sys.stderr) + return 2 parser.print_help(sys.stderr) return 2 diff --git a/src/prxref/prompt_templates.py b/src/prxref/prompt_templates.py index 76c84d6b..707c4e24 100644 --- a/src/prxref/prompt_templates.py +++ b/src/prxref/prompt_templates.py @@ -336,3 +336,53 @@ def _is_within(path: str, root: str) -> bool: return os.path.commonpath([path, root]) == root except ValueError: return False + + +def export_prompt_templates(dest: str | os.PathLike[str], *, force: bool = False) -> list[str]: + """Write the packaged ``worker.md``, ``systemic.md`` and ``summary.md`` into directory ``dest``. + + The starting point for an override directory: each template is written + byte for byte as packaged, so the exported directory loads through + :func:`load_prompt_templates` without a warning. Only the + :data:`TEMPLATE_NAMES` are written, each by name; the package directory + is never listed, so the judge prompt, which is not overridable, is never + exported. ``dest`` is created, with any missing parents, when it does + not exist. Returns the written paths, ``dest`` joined with each file + name, in :data:`TEMPLATE_NAMES` order. + + Without ``force``, any target that already exists (a dangling symlink + included) is refused before anything is written or created, with a + :class:`~prxref.llm.ConfigError` naming every existing file and + ``--force``. With ``force`` each target is overwritten, and a symlink is + replaced by a regular file rather than written through. An empty + ``dest``, or one that cannot be created or written, is a ``ConfigError`` + too. + """ + raw = os.fspath(dest) + if not raw.strip(): + raise ConfigError("prompts export: DIR must name a directory, got an empty path") + targets = [os.path.join(raw, f"{name}.md") for name in TEMPLATE_NAMES] + if not force: + existing = [target for target in targets if os.path.lexists(target)] + if existing: + raise ConfigError( + f"prompts export: refusing to overwrite {', '.join(repr(t) for t in existing)}; " + "pass --force to overwrite" + ) + written: list[str] = [] + try: + os.makedirs(raw, exist_ok=True) + for name, target in zip(TEMPLATE_NAMES, targets, strict=True): + if os.path.islink(target): + os.unlink(target) + with open(target, "wb") as fh: + fh.write(_packaged_bytes(name)) + written.append(target) + except OSError as exc: + where = exc.filename if exc.filename is not None else raw + raise ConfigError(f"prompts export: cannot write {os.fspath(where)!r}: {_reason(exc)}") from exc + return written + + +def _packaged_bytes(name: str) -> bytes: + return resources.files("prxref").joinpath("prompts").joinpath(f"{name}.md").read_bytes() diff --git a/tests/test_prompts_export.py b/tests/test_prompts_export.py new file mode 100644 index 00000000..023ea2c5 --- /dev/null +++ b/tests/test_prompts_export.py @@ -0,0 +1,247 @@ +"""``prxref prompts export DIR [--force]`` (#11 T5): the packaged templates as an override starting point.""" +from __future__ import annotations + +import hashlib +import logging +import os +import subprocess +import sys +from importlib import resources +from pathlib import Path + +import pytest + +from prxref import reviewer +from prxref.cli import _build_parser, main +from prxref.llm import ConfigError +from prxref.prompt_templates import TEMPLATE_NAMES, export_prompt_templates, load_prompt_templates + +FILES = tuple(f"{name}.md" for name in TEMPLATE_NAMES) +MINE = b"our team's own template\n" + + +def _packaged(name: str) -> bytes: + return reviewer.load_prompt(name).encode("utf-8") + + +def _resource(fname: str) -> bytes: + return resources.files("prxref").joinpath("prompts").joinpath(fname).read_bytes() + + +def _state(d: Path) -> dict[str, tuple[bool, bytes, int]]: + return { + p.name: (p.is_symlink(), os.readlink(p).encode() if p.is_symlink() else p.read_bytes(), p.lstat().st_mtime_ns) + for p in sorted(d.iterdir()) + } + + +def _seed(d: Path, names: tuple[str, ...]) -> None: + d.mkdir(parents=True, exist_ok=True) + for fname in names: + (d / fname).write_bytes(MINE) + + +@pytest.fixture +def work(tmp_path, monkeypatch) -> Path: + root = tmp_path / "work" + root.mkdir() + monkeypatch.chdir(root) + return root + + +class TestExportedFiles: + @pytest.mark.parametrize("name", TEMPLATE_NAMES) + def test_reviewer_load_prompt_text_encodes_back_to_the_raw_packaged_bytes(self, name): + assert _packaged(name) == _resource(f"{name}.md") + + @pytest.mark.parametrize("name", TEMPLATE_NAMES) + def test_each_exported_file_is_byte_equal_to_the_packaged_template(self, work, name): + export_prompt_templates("prompts") + assert (work / "prompts" / f"{name}.md").read_bytes() == _packaged(name) + + def test_exactly_the_three_overridable_templates_are_written_and_never_judge(self, work): + packaged = {p.name for p in resources.files("prxref").joinpath("prompts").iterdir() if p.name.endswith(".md")} + assert "judge.md" in packaged + assert packaged > set(FILES) + export_prompt_templates("prompts") + assert sorted(os.listdir(work / "prompts")) == sorted(FILES) + assert not (work / "prompts" / "judge.md").exists() + + def test_it_returns_the_written_paths_as_given_in_template_order(self, work): + assert export_prompt_templates("prompts") == [os.path.join("prompts", f) for f in FILES] + + def test_a_path_object_is_accepted_and_returned_as_strings(self, work): + written = export_prompt_templates(work / "prompts") + assert written == [str(work / "prompts" / f) for f in FILES] + + def test_a_missing_directory_is_created_with_its_parents(self, work): + dest = work / "a" / "b" / "prompts" + export_prompt_templates(dest) + assert sorted(os.listdir(dest)) == sorted(FILES) + + def test_an_existing_directory_with_other_files_keeps_them(self, work): + _seed(work / "prompts", ("README.md",)) + export_prompt_templates("prompts") + assert sorted(os.listdir(work / "prompts")) == sorted((*FILES, "README.md")) + assert (work / "prompts" / "README.md").read_bytes() == MINE + + +class TestRefusal: + def test_every_existing_target_is_refused_naming_each_and_force_and_nothing_is_modified(self, work): + d = work / "prompts" + _seed(d, FILES) + before = _state(d) + with pytest.raises(ConfigError, match="--force") as exc: + export_prompt_templates("prompts") + for fname in FILES: + assert repr(os.path.join("prompts", fname)) in str(exc.value) + assert _state(d) == before + + @pytest.mark.parametrize("fname", FILES) + def test_one_existing_target_refuses_the_whole_export(self, work, fname): + d = work / "prompts" + _seed(d, (fname,)) + before = _state(d) + with pytest.raises(ConfigError, match="--force") as exc: + export_prompt_templates("prompts") + assert repr(os.path.join("prompts", fname)) in str(exc.value) + for other in set(FILES) - {fname}: + assert other not in str(exc.value) + assert _state(d) == before + assert os.listdir(d) == [fname] + + def test_a_dangling_symlink_counts_as_an_existing_target(self, work): + d = work / "prompts" + d.mkdir() + (d / "worker.md").symlink_to(work / "nowhere.md") + before = _state(d) + with pytest.raises(ConfigError, match="worker.md"): + export_prompt_templates("prompts") + assert _state(d) == before + assert not (work / "nowhere.md").exists() + + def test_a_second_export_into_the_same_directory_is_refused(self, work): + export_prompt_templates("prompts") + before = _state(work / "prompts") + with pytest.raises(ConfigError, match="--force"): + export_prompt_templates("prompts") + assert _state(work / "prompts") == before + + def test_force_overwrites_every_existing_target(self, work): + d = work / "prompts" + _seed(d, FILES) + export_prompt_templates("prompts", force=True) + for name in TEMPLATE_NAMES: + assert (d / f"{name}.md").read_bytes() == _packaged(name) + + def test_force_fills_a_partly_seeded_directory(self, work): + d = work / "prompts" + _seed(d, ("summary.md",)) + assert export_prompt_templates("prompts", force=True) == [os.path.join("prompts", f) for f in FILES] + for name in TEMPLATE_NAMES: + assert (d / f"{name}.md").read_bytes() == _packaged(name) + + def test_force_replaces_a_symlink_rather_than_writing_through_it(self, work): + shared = work / "shared.md" + shared.write_bytes(MINE) + d = work / "prompts" + d.mkdir() + (d / "worker.md").symlink_to(shared) + export_prompt_templates("prompts", force=True) + assert not (d / "worker.md").is_symlink() + assert (d / "worker.md").read_bytes() == _packaged("worker") + assert shared.read_bytes() == MINE + + @pytest.mark.parametrize("dest", ["blocker", os.path.join("blocker", "sub")]) + def test_a_destination_that_cannot_be_a_directory_is_a_config_error(self, work, dest): + (work / "blocker").write_bytes(MINE) + with pytest.raises(ConfigError, match="prompts export: cannot write"): + export_prompt_templates(dest, force=True) + assert (work / "blocker").read_bytes() == MINE + + def test_a_directory_standing_in_a_targets_place_is_a_config_error_under_force(self, work): + (work / "prompts" / "worker.md").mkdir(parents=True) + with pytest.raises(ConfigError, match="prompts export: cannot write"): + export_prompt_templates("prompts", force=True) + + @pytest.mark.parametrize("dest", ["", " "]) + def test_an_empty_destination_is_a_config_error_and_writes_nothing(self, work, dest): + with pytest.raises(ConfigError, match="DIR"): + export_prompt_templates(dest) + assert os.listdir(work) == [] + + +class TestRoundTrip: + @pytest.mark.parametrize("source", ["--prompts-dir", "PRXREF_PROMPTS_DIR"]) + def test_the_exported_directory_loads_silently_and_overrides_nothing(self, work, caplog, source): + export_prompt_templates("prompts") + with caplog.at_level(logging.DEBUG, logger="prxref"): + loaded = load_prompt_templates("prompts", source=source) + assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == [] + assert loaded is not None + assert loaded.overridden == TEMPLATE_NAMES + for name in TEMPLATE_NAMES: + assert loaded.override(name) == reviewer.load_prompt(name) + assert loaded.text(name) == reviewer.load_prompt(name) + recorded = loaded.record()["templates"][name] + assert recorded["sha256"] == hashlib.sha256(_resource(f"{name}.md")).hexdigest() + + +class TestCli: + def test_export_exits_0_and_prints_one_line_per_written_file(self, work, capsys): + assert main(["prompts", "export", "prompts"]) == 0 + out, err = capsys.readouterr() + assert out.splitlines() == [os.path.join("prompts", f) for f in FILES] + assert err == "" + for name in TEMPLATE_NAMES: + assert (work / "prompts" / f"{name}.md").read_bytes() == _packaged(name) + + @pytest.mark.parametrize("seeded", [FILES, ("systemic.md",)], ids=["all", "one"]) + def test_an_existing_template_exits_2_naming_force_and_modifies_nothing(self, work, capsys, seeded): + d = work / "prompts" + _seed(d, seeded) + before = _state(d) + assert main(["prompts", "export", "prompts"]) == 2 + out, err = capsys.readouterr() + assert out == "" + assert err.startswith("configuration error: prompts export: refusing to overwrite") + assert "--force" in err + for fname in seeded: + assert fname in err + assert _state(d) == before + + def test_force_overwrites_and_exits_0(self, work, capsys): + d = work / "prompts" + _seed(d, FILES) + assert main(["prompts", "export", "prompts", "--force"]) == 0 + assert len(capsys.readouterr().out.splitlines()) == 3 + for name in TEMPLATE_NAMES: + assert (d / f"{name}.md").read_bytes() == _packaged(name) + + def test_an_unwritable_destination_exits_2(self, work, capsys): + (work / "blocker").write_bytes(MINE) + assert main(["prompts", "export", os.path.join("blocker", "sub")]) == 2 + assert capsys.readouterr().err.startswith("configuration error: prompts export: cannot write") + + def test_prompts_without_an_action_exits_2_with_usage(self, capsys): + assert main(["prompts"]) == 2 + assert "usage:" in capsys.readouterr().err.lower() + + def test_export_without_dir_is_an_argparse_usage_error(self, capsys): + with pytest.raises(SystemExit) as exc: + main(["prompts", "export"]) + assert exc.value.code == 2 + assert "DIR" in capsys.readouterr().err + + def test_force_defaults_off(self): + assert _build_parser().parse_args(["prompts", "export", "d"]).force is False + assert _build_parser().parse_args(["prompts", "export", "d", "--force"]).force is True + + def test_the_module_entry_point_exports_then_refuses(self, work): + cmd = [sys.executable, "-m", "prxref.cli", "prompts", "export", "prompts"] + first = subprocess.run(cmd, capture_output=True, text=True, env=dict(os.environ)) + assert first.returncode == 0, first.stderr + assert first.stdout.splitlines() == [os.path.join("prompts", f) for f in FILES] + second = subprocess.run(cmd, capture_output=True, text=True, env=dict(os.environ)) + assert second.returncode == 2 + assert "--force" in second.stderr From 18e365947319192be3f92950c3f7d1015b3e1dd6 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:47:23 -0700 Subject: [PATCH 14/64] feat: render worker/systemic prompt overrides from PromptContext (#11 T2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PromptContext gains worker_template and systemic_template, one contiguous block after the existing fields. Each holds an operator override's text (PromptTemplates.override(name)), or "" when the template is packaged. _render_prompt and _render_systemic_prompt use the override when the field is non-empty and load_prompt("worker.md"/"systemic.md") exactly as before when it is empty, so an unset run is byte-identical and never reads a template it does not use. An override is split at the "## Review Context" marker exactly like the packaged text: the stripped head is the system half with the rules and ticket-scope blocks appended, and the tail is filled by the one-pass keyed fill_template, never str.format, so stray braces, {0}, {diff.__class__} and format-spec shapes render literally. tests/test_prompt_template_render.py covers both renderers, field independence, the load_prompt bypass, injection probes, byte identity of an empty field against the packaged render, and an end-to-end override loaded by prompt_templates.load_prompt_templates. The pinned PromptContext field-set list in tests/test_prompt_context.py gains the two names. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/reviewer.py | 21 +- tests/test_prompt_context.py | 1 + tests/test_prompt_template_render.py | 374 +++++++++++++++++++++++++++ 3 files changed, 393 insertions(+), 3 deletions(-) create mode 100644 tests/test_prompt_template_render.py diff --git a/src/prxref/reviewer.py b/src/prxref/reviewer.py index 181ccc26..4b37be88 100644 --- a/src/prxref/reviewer.py +++ b/src/prxref/reviewer.py @@ -162,8 +162,21 @@ class PromptContext: lines: ``ticket_context`` (the fenced ticket text), then ``spec_digest`` (the Spec constraints block), then the diff or digest. + TEMPLATES: ``worker_template`` replaces ``prompts/worker.md`` for chunk + units and ``systemic_template`` replaces ``prompts/systemic.md`` for the + sweep, each holding an operator override's text as + :meth:`prxref.prompt_templates.PromptTemplates.override` returns it. An + override is split at the ``## Review Context`` marker exactly as the + packaged template is: its head becomes the SYSTEM half, with the rules and + ``ticket_scope`` blocks appended in the order above and nothing filled in, + and its tail is the USER half, filled in one :func:`fill_template` pass + and never through :meth:`str.format`, so stray or foreign braces in the + override render literally. + Every field defaults to ``""``, which injects nothing; ``spec_digest`` - empty renders ``(no specs provided for this review)`` as before. + empty renders ``(no specs provided for this review)`` as before, and an + empty template field renders the packaged template read by + :func:`load_prompt`, byte for byte as before. :attr:`scope_active` is true only when the scope instructions are in the prompt, and it alone decides whether a model-supplied ``scope`` is read and whether the ``## Output Format`` example finding shows a ``"scope"`` @@ -175,6 +188,8 @@ class PromptContext: ticket_scope: str = "" ticket_context: str = "" spec_digest: str = "" + worker_template: str = "" + systemic_template: str = "" @property def scope_active(self) -> bool: @@ -247,7 +262,7 @@ def _render_prompt( *, prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> tuple[str, str]: - template = load_prompt("worker.md") + template = prompt_context.worker_template or load_prompt("worker.md") head, marker, tail = template.partition(_CONTEXT_MARKER) if not marker: raise ValueError(f"worker.md is missing the {_CONTEXT_MARKER!r} split marker") @@ -310,7 +325,7 @@ def _render_systemic_prompt( *, prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> tuple[str, str]: - template = load_prompt("systemic.md") + template = prompt_context.systemic_template or load_prompt("systemic.md") head, marker, tail = template.partition(_CONTEXT_MARKER) if not marker: raise ValueError(f"systemic.md is missing the {_CONTEXT_MARKER!r} split marker") diff --git a/tests/test_prompt_context.py b/tests/test_prompt_context.py index 47b7a804..0f088379 100644 --- a/tests/test_prompt_context.py +++ b/tests/test_prompt_context.py @@ -146,6 +146,7 @@ def test_the_field_set_is_the_contract(self): assert [f.name for f in dataclasses.fields(PromptContext)] == [ "rules_worker", "rules_sweep", "ticket_scope", "ticket_context", "spec_digest", + "worker_template", "systemic_template", ] def test_it_is_frozen(self): diff --git a/tests/test_prompt_template_render.py b/tests/test_prompt_template_render.py new file mode 100644 index 00000000..cd62d1b8 --- /dev/null +++ b/tests/test_prompt_template_render.py @@ -0,0 +1,374 @@ +"""Prompt-template overrides (#11 T2): the two renderers honour ``PromptContext`` templates. + +``PromptContext.worker_template`` and ``systemic_template`` carry an +operator's override of ``worker.md`` / ``systemic.md`` into +``_render_prompt`` and ``_render_systemic_prompt``. These tests pin that an +override is split at the ``## Review Context`` marker exactly like the +packaged text (head to the system half with the rules and ticket-scope blocks +appended, tail filled by the one-pass keyed ``fill_template``), that each +field reaches only its own renderer, that override text is never run through +``str.format``, and that an empty field is the packaged render byte for byte. + +None of these tests uses the ``contract_stubs`` fixture: it replaces +``reviewer.load_prompt`` with a summary-only stub, and every test here renders +the real review templates. +""" +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest + +from prxref import reviewer +from prxref.chunk_context import sibling_summary_block +from prxref.forges.base import Thread +from prxref.prompt_templates import load_prompt_templates +from prxref.reviewer import ( + _CONTEXT_MARKER, + _NO_SPECS_TEXT, + _SCOPE_EXAMPLE, + NO_PROMPT_CONTEXT, + PromptContext, + _render_prompt, + _render_systemic_prompt, + load_prompt, + render_chunk, +) +from prxref.triage import parse_unified_diff + +MINI_DIFF = """\ +diff --git a/src/app.py b/src/app.py +--- a/src/app.py ++++ b/src/app.py +@@ -1,2 +1,3 @@ + import os ++import sys + print(os.name) +""" + +DIGEST = "## src/app.py\n@@ -1,2 +1,3 @@\n+import sys" + +RULES_WORKER = "## Team review rules\n\nWorker framing: never log tokens." +RULES_SWEEP = "## Team review rules\n\nSweep framing: never log tokens." +TICKET_SCOPE = "## Ticket scope\n\nAdd a \"scope\" key to every finding." +TICKET_BLOCK = "### Ticket context\n\n```text\nShip the header flag.\n```" +SPEC = "[spec:spec.md#L1] (MUST) tools MUST be named with the mcp prefix" + +FULL = PromptContext( + rules_worker=RULES_WORKER, + rules_sweep=RULES_SWEEP, + ticket_scope=TICKET_SCOPE, + ticket_context=TICKET_BLOCK, + spec_digest=SPEC, +) + +WORKER_HEAD = "You are the ACME chunk reviewer.\n\nStyle-guide findings are welcome." +SYSTEMIC_HEAD = "You are the ACME sweep reviewer.\n\nFlag cross-file drift only." + +WORKER_OVERRIDE = ( + f"{WORKER_HEAD}\n\n" + f"{_CONTEXT_MARKER}\n\n" + "TITLE={pr_title}\nDESC={pr_description}\nREPO={repo_hint}\n" + "{ticket_context}SPEC={spec_digest}\n" + "DIFF:\n{diff}\nCTX:\n{context_blocks}\n" + 'EXAMPLE={"title": "x"{scope_example}}\n' +) +SYSTEMIC_OVERRIDE = ( + f"{SYSTEMIC_HEAD}\n\n" + f"{_CONTEXT_MARKER}\n\n" + "TITLE={pr_title}\nDESC={pr_description}\nREPO={repo_hint}\n" + "{ticket_context}SPEC={spec_digest}\n" + "DIGEST:\n{digest}\n" + 'EXAMPLE={"title": "x"{scope_example}}\n' +) + +STYLE_LINE = "no style-guide nits that change neither behavior nor risk" +STYLE_RELAXED = "style-guide findings that enforce the team standard are welcome" + +PROBES = ("{ stray", "{0}", "{diff.__class__}", "{pr_title!r}", "{pr_title:>9}", r"\g<0> \1") + + +def _chunk(): + return parse_unified_diff(MINI_DIFF) + + +def _worker(ctx=NO_PROMPT_CONTEXT, *, title="t", description="d", context_blocks="CTX"): + return _render_prompt( + _chunk(), title, description, "r", context_blocks=context_blocks, prompt_context=ctx, + ) + + +def _sweep(ctx=NO_PROMPT_CONTEXT, *, title="t", description="d", threads=()): + return _render_systemic_prompt(DIGEST, title, description, "r", threads, prompt_context=ctx) + + +def _blocks(context_blocks="CTX") -> str: + return "\n\n".join(b for b in (sibling_summary_block(_chunk(), ()), context_blocks) if b) + + +def _packaged(name: str) -> str: + return load_prompt(f"{name}.md") + + +def _with(ctx: PromptContext, **fields: str) -> PromptContext: + return dataclasses.replace(ctx, **fields) + + +class TestWorkerOverride: + def test_the_override_head_is_the_system_half(self): + system, _ = _worker(PromptContext(worker_template=WORKER_OVERRIDE)) + assert system == WORKER_HEAD + + def test_the_override_tail_is_filled_by_every_keyed_slot(self): + _, user = _worker(PromptContext(worker_template=WORKER_OVERRIDE)) + assert user == ( + f"{_CONTEXT_MARKER}\n\n" + f"TITLE=t\nDESC=d\nREPO=r\nSPEC={_NO_SPECS_TEXT}\n" + f"DIFF:\n{render_chunk(_chunk())}\nCTX:\n{_blocks()}\n" + 'EXAMPLE={"title": "x"}' + ) + + def test_an_active_ticket_fills_the_override_tail_slots(self): + _, user = _worker(_with(FULL, worker_template=WORKER_OVERRIDE)) + assert f"REPO=r\n{TICKET_BLOCK}\n\nSPEC={SPEC}\n" in user + assert user.endswith('EXAMPLE={"title": "x"' + _SCOPE_EXAMPLE + "}") + + def test_rules_then_ticket_scope_are_appended_to_the_override_head(self): + system, user = _worker(_with(FULL, worker_template=WORKER_OVERRIDE)) + assert system == f"{WORKER_HEAD}\n\n{RULES_WORKER}\n\n{TICKET_SCOPE}" + assert RULES_SWEEP not in system + assert RULES_WORKER not in user and TICKET_SCOPE not in user + + def test_nothing_of_the_packaged_template_survives(self): + system, user = _worker(PromptContext(worker_template=WORKER_OVERRIDE)) + packaged_head = _packaged("worker").partition(_CONTEXT_MARKER)[0].strip() + assert packaged_head not in system and STYLE_LINE not in system + user + assert "## Output Format" not in user + + def test_an_override_without_the_marker_raises_naming_the_marker(self): + with pytest.raises(ValueError, match=_CONTEXT_MARKER): + _worker(PromptContext(worker_template="no marker here {diff}")) + + +class TestSystemicOverride: + def test_the_override_head_is_the_system_half(self): + system, _ = _sweep(PromptContext(systemic_template=SYSTEMIC_OVERRIDE)) + assert system == SYSTEMIC_HEAD + + def test_the_override_tail_is_filled_by_every_keyed_slot(self): + _, user = _sweep(PromptContext(systemic_template=SYSTEMIC_OVERRIDE)) + assert user == ( + f"{_CONTEXT_MARKER}\n\n" + f"TITLE=t\nDESC=d\nREPO=r\nSPEC={_NO_SPECS_TEXT}\n" + f"DIGEST:\n{DIGEST}\n" + 'EXAMPLE={"title": "x"}' + ) + + def test_an_active_ticket_fills_the_override_tail_slots(self): + _, user = _sweep(_with(FULL, systemic_template=SYSTEMIC_OVERRIDE)) + assert f"REPO=r\n{TICKET_BLOCK}\n\nSPEC={SPEC}\n" in user + assert user.endswith('EXAMPLE={"title": "x"' + _SCOPE_EXAMPLE + "}") + + def test_rules_then_ticket_scope_are_appended_to_the_override_head(self): + system, user = _sweep(_with(FULL, systemic_template=SYSTEMIC_OVERRIDE)) + assert system == f"{SYSTEMIC_HEAD}\n\n{RULES_SWEEP}\n\n{TICKET_SCOPE}" + assert RULES_WORKER not in system + assert RULES_SWEEP not in user and TICKET_SCOPE not in user + + def test_the_discussion_block_still_follows_the_override_tail(self): + thread = Thread(path="src/app.py", line=None, resolved=False, author="ana", body_snippet="why sys?") + ctx = PromptContext(systemic_template=SYSTEMIC_OVERRIDE) + _, user = _sweep(ctx, threads=[thread]) + _, bare = _sweep(ctx) + assert user == f"{bare}\n\n### Existing discussion\n\n- src/app.py: ana: why sys?" + + def test_an_override_without_the_marker_raises_naming_the_marker(self): + with pytest.raises(ValueError, match=_CONTEXT_MARKER): + _sweep(PromptContext(systemic_template="no marker here {digest}")) + + +class TestEachFieldReachesOnlyItsRenderer: + @pytest.mark.parametrize("base", [NO_PROMPT_CONTEXT, FULL], ids=["unset", "full"]) + def test_a_worker_override_leaves_the_systemic_render_unchanged(self, base): + assert _sweep(_with(base, worker_template=WORKER_OVERRIDE)) == _sweep(base) + + @pytest.mark.parametrize("base", [NO_PROMPT_CONTEXT, FULL], ids=["unset", "full"]) + def test_a_systemic_override_leaves_the_worker_render_unchanged(self, base): + assert _worker(_with(base, systemic_template=SYSTEMIC_OVERRIDE)) == _worker(base) + + def test_both_overrides_together_each_reach_their_own_renderer(self): + ctx = PromptContext(worker_template=WORKER_OVERRIDE, systemic_template=SYSTEMIC_OVERRIDE) + assert _worker(ctx) == _worker(PromptContext(worker_template=WORKER_OVERRIDE)) + assert _sweep(ctx) == _sweep(PromptContext(systemic_template=SYSTEMIC_OVERRIDE)) + + +class TestLoadPromptIsReadOnlyForAnEmptyField: + @pytest.fixture + def calls(self, monkeypatch) -> list[str]: + seen: list[str] = [] + real = reviewer.load_prompt + + def spy(name: str) -> str: + seen.append(name) + return real(name) + + monkeypatch.setattr(reviewer, "load_prompt", spy) + return seen + + def test_an_empty_worker_field_reads_the_packaged_worker_template(self, calls): + _worker(PromptContext(systemic_template=SYSTEMIC_OVERRIDE)) + assert calls == ["worker.md"] + + def test_a_worker_override_never_reads_the_packaged_template(self, calls): + _worker(PromptContext(worker_template=WORKER_OVERRIDE)) + assert calls == [] + + def test_an_empty_systemic_field_reads_the_packaged_systemic_template(self, calls): + _sweep(PromptContext(worker_template=WORKER_OVERRIDE)) + assert calls == ["systemic.md"] + + def test_a_systemic_override_never_reads_the_packaged_template(self, calls): + _sweep(PromptContext(systemic_template=SYSTEMIC_OVERRIDE)) + assert calls == [] + + +def _probe_override(slots: str) -> str: + probes = " | ".join(PROBES) + return ( + f"HEAD {probes} {{pr_title}}\n\n{_CONTEXT_MARKER}\n\n" + f"TAIL {probes}\nTITLE={{pr_title}}\nDESC={{pr_description}}\nREPO={{repo_hint}}\n{slots}\n" + ) + + +WORKER_PROBE = _probe_override("{ticket_context}{spec_digest}\n{diff}\n{context_blocks}{scope_example}") +SYSTEMIC_PROBE = _probe_override("{ticket_context}{spec_digest}\n{digest}{scope_example}") + +RENDERS = [ + pytest.param(lambda ctx, **kw: _worker(ctx, **kw), "worker_template", WORKER_PROBE, id="worker"), + pytest.param(lambda ctx, **kw: _sweep(ctx, **kw), "systemic_template", SYSTEMIC_PROBE, id="systemic"), +] + + +class TestOverrideTextIsNeverFormatted: + @pytest.mark.parametrize(("render", "field", "override"), RENDERS) + def test_stray_and_foreign_braces_render_literally_in_both_halves(self, render, field, override): + system, user = render(PromptContext(**{field: override})) + for probe in PROBES: + assert probe in system + assert probe in user + assert "TITLE=t\n" in user and "TITLE={pr_title}" not in user + + @pytest.mark.parametrize(("render", "field", "override"), RENDERS) + def test_the_head_is_never_filled_so_pr_data_stays_out_of_the_system_half(self, render, field, override): + system, _ = render(PromptContext(**{field: override}), title="SECRET-TITLE") + assert system.endswith("{pr_title}") + assert "SECRET-TITLE" not in system + + def test_a_pr_title_naming_the_diff_slot_renders_literally(self): + diff = render_chunk(_chunk()) + _, user = _worker(PromptContext(worker_template=WORKER_OVERRIDE), title="{diff} {0}") + assert "TITLE={diff} {0}\n" in user + assert user.count(diff) == 1 + + def test_a_pr_title_naming_the_digest_slot_renders_literally(self): + _, user = _sweep(PromptContext(systemic_template=SYSTEMIC_OVERRIDE), title="{digest} {0}") + assert "TITLE={digest} {0}\n" in user + assert user.count(DIGEST) == 1 + + @pytest.mark.parametrize(("render", "field", "override"), RENDERS) + def test_a_description_full_of_probes_renders_literally(self, render, field, override): + description = " ".join(PROBES) + " {pr_title} {spec_digest}" + _, user = render(PromptContext(**{field: override}), description=description) + assert f"TITLE=t\nDESC={description}\nREPO=r\n" in user + + +class TestAnEmptyFieldIsThePackagedRender: + def test_the_empty_fields_are_the_shared_default(self): + assert PromptContext(worker_template="", systemic_template="") == NO_PROMPT_CONTEXT + + @pytest.mark.parametrize("base", [NO_PROMPT_CONTEXT, FULL], ids=["unset", "full"]) + def test_the_packaged_text_as_a_worker_override_renders_byte_identically(self, base): + assert _worker(base) == _worker(_with(base, worker_template=_packaged("worker"))) + + @pytest.mark.parametrize("base", [NO_PROMPT_CONTEXT, FULL], ids=["unset", "full"]) + def test_the_packaged_text_as_a_systemic_override_renders_byte_identically(self, base): + assert _sweep(base) == _sweep(_with(base, systemic_template=_packaged("systemic"))) + + @pytest.mark.parametrize("name", ["worker", "systemic"]) + def test_an_empty_field_renders_the_packaged_head_and_tail(self, name): + render = _worker if name == "worker" else _sweep + system, user = render(PromptContext()) + head, marker, _ = _packaged(name).partition(_CONTEXT_MARKER) + assert system == head.strip() + assert user.startswith(marker) and "{" + "pr_title" + "}" not in user + + +@pytest.fixture +def work(tmp_path, monkeypatch) -> Path: + root = tmp_path / "work" + root.mkdir() + monkeypatch.chdir(root) + return root + + +def _relaxed(name: str) -> str: + text = _packaged(name) + head, marker, tail = text.partition(_CONTEXT_MARKER) + assert STYLE_LINE in head + return head.replace(STYLE_LINE, STYLE_RELAXED) + marker + tail + + +def _write(work: Path, files: dict[str, bytes]) -> str: + d = work / "prompts" + d.mkdir() + for fname, data in files.items(): + (d / fname).write_bytes(data) + return str(d) + + +class TestEndToEndWithTheRealLoader: + def test_a_worker_only_override_relaxes_the_worker_head_and_leaves_the_sweep_packaged(self, work): + prompts = load_prompt_templates( + _write(work, {"worker.md": _relaxed("worker").encode()}), source="--prompts-dir", + ) + assert prompts is not None and prompts.overridden == ("worker",) + ctx = _with( + FULL, + worker_template=prompts.override("worker"), + systemic_template=prompts.override("systemic"), + ) + assert ctx.systemic_template == "" + system, user = _worker(ctx) + packaged_system, packaged_user = _worker(FULL) + assert system == packaged_system.replace(STYLE_LINE, STYLE_RELAXED) + assert STYLE_LINE not in system and system.endswith(f"\n\n{RULES_WORKER}\n\n{TICKET_SCOPE}") + assert user == packaged_user + assert _sweep(ctx) == _sweep(FULL) + + def test_both_overrides_reach_their_renderers(self, work): + prompts = load_prompt_templates( + _write(work, { + "worker.md": _relaxed("worker").encode(), + "systemic.md": _relaxed("systemic").encode(), + }), + source="PRXREF_PROMPTS_DIR", + ) + assert prompts is not None and prompts.overridden == ("worker", "systemic") + ctx = PromptContext( + worker_template=prompts.override("worker"), + systemic_template=prompts.override("systemic"), + ) + for render in (_worker, _sweep): + system, user = render(ctx) + packaged_system, packaged_user = render() + assert system == packaged_system.replace(STYLE_LINE, STYLE_RELAXED) != packaged_system + assert user == packaged_user + + def test_a_crlf_override_renders_as_its_lf_twin(self, work): + crlf = _relaxed("worker").replace("\n", "\r\n").encode() + prompts = load_prompt_templates(_write(work, {"worker.md": crlf}), source="--prompts-dir") + assert prompts is not None + system, user = _worker(PromptContext(worker_template=prompts.override("worker"))) + assert "\r" not in system + user + assert (system, user) == _worker(PromptContext(worker_template=_relaxed("worker"))) From 8548225e6c56a48efa7f95c827eae038f36f8087 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:48:03 -0700 Subject: [PATCH 15/64] feat(github): rebuild a PR diff from the /pulls/{n}/files listing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ForgeImpl._get_diff_from_files, the fallback get_diff will call when GitHub refuses a PR diff as too large for the diff media type (#15, T3). It pages /pulls/{n}/files through _iter_pages ("changed-file listing"), maps each entry into the shared renderer's entry (filename -> new_path, previous_filename or filename -> old_path; added/removed/renamed pick the new, deleted and rename headers; every other status renders plain; patch -> diff) and renders it with render_diff_entries, so the headers match GitLab's byte for byte. A file listed without a patch renders header-only with one WARNING naming it, worded like GitLab's "has no inline diff". Completeness is asserted, never assumed: one get_pr read supplies changed_files, and a listing shorter than it (GitHub caps the listing at 3,000 files) raises ValueError naming both counts, as does PR metadata with no integer changed_files. A PR past 3,000 files therefore fails the run instead of being reviewed partially. The check runs before any per-file WARNING is logged. A failed listing read raises FeedReadError; a failed PR read raises as get_pr does (requests.HTTPError, or the transport error). Both reach orchestrate_review's get_diff error run. get_diff and get_compare_diff are untouched; wiring the 406 too_large detection is F15-C's task. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/forges/github.py | 67 ++++++ tests/test_github_files_fallback.py | 349 ++++++++++++++++++++++++++++ 2 files changed, 416 insertions(+) create mode 100644 tests/test_github_files_fallback.py diff --git a/src/prxref/forges/github.py b/src/prxref/forges/github.py index bc9a4570..7e900950 100644 --- a/src/prxref/forges/github.py +++ b/src/prxref/forges/github.py @@ -13,6 +13,7 @@ from prxref.retry_logging import LoggingRetry +from ._diff_render import render_diff_entries from .base import ( ATTRIBUTION_MARKER, SUMMARY_MARKER, @@ -199,6 +200,72 @@ def get_compare_diff(self, ref: PRRef, *, base_sha: str, head_sha: str) -> str: resp.raise_for_status() return resp.text + def _get_diff_from_files(self, ref: PRRef) -> str: + """Rebuild the PR's unified diff from its ``/pulls/{n}/files`` listing. + + The fallback for a diff too large for the diff media type. Each listed + file is mapped into the renderer's per-file entry: ``filename`` is the + new path, ``previous_filename`` (else ``filename``) the old path, and + the ``added``, ``removed`` and ``renamed`` statuses pick the new, deleted + and rename headers; every other status renders plain ``---``/``+++``. + ``patch`` supplies the hunks. A file listed without a ``patch`` (a + binary, a patch GitHub withheld as too large, a pure rename) renders + header-only, with one WARNING naming it. + + Completeness is checked, not assumed: the listing is compared with the + PR's ``changed_files``, and a shorter listing (GitHub caps it at 3,000 + files) raises ``ValueError`` instead of returning a partial diff, as + does PR metadata carrying no integer ``changed_files``. A failed + listing read raises ``FeedReadError``; a failed PR metadata read raises + as ``get_pr`` does. + """ + url = ( + f"{self._api_base(ref)}/repos/{ref.owner}/{ref.repo}" + f"/pulls/{ref.number}/files" + ) + listed: list[dict] = [] + for page in self._iter_pages( + ref, url, self._headers(ref.host), what="changed-file listing" + ): + listed.extend(page) + + where = f"{ref.owner}/{ref.repo}#{ref.number}" + changed_files = self.get_pr(ref).raw.get("changed_files") + if not isinstance(changed_files, int) or isinstance(changed_files, bool): + raise ValueError( + f"GitHub PR diff for {where} cannot be checked for completeness: " + "the PR metadata carries no integer changed_files; refusing to " + "review a diff that may be partial" + ) + if len(listed) < changed_files: + raise ValueError( + f"GitHub PR diff for {where} cannot be rebuilt whole: " + f"changed_files={changed_files} but the changed-file listing " + f"returned {len(listed)} (GitHub caps the listing at 3,000 " + "files); refusing to review a partial diff" + ) + + entries: list[dict] = [] + for f in listed: + filename = f.get("filename") or "" + status = f.get("status") + patch = f.get("patch") + if not patch: + logger.warning( + "GitHub PR diff: %s has no inline diff (no patch in the " + "files listing); it is reviewed as header-only", + filename, + ) + entries.append({ + "old_path": f.get("previous_filename") or filename, + "new_path": filename, + "new_file": status == "added", + "deleted_file": status == "removed", + "renamed_file": status == "renamed", + "diff": patch, + }) + return render_diff_entries(entries) + def _iter_pages( self, ref: PRRef, diff --git a/tests/test_github_files_fallback.py b/tests/test_github_files_fallback.py new file mode 100644 index 00000000..fdd477d0 --- /dev/null +++ b/tests/test_github_files_fallback.py @@ -0,0 +1,349 @@ +"""Tests for GitHub's PR diff rebuilt from the ``/pulls/{n}/files`` listing. + +``ForgeImpl._get_diff_from_files`` is the fallback for a PR whose diff GitHub +refuses to serve through the diff media type. It pages the changed-file +listing, maps each entry into the shared renderer's GitLab-shaped entry, and +refuses to return a diff the listing may have cut short. +""" +from __future__ import annotations + +import json +import logging +from unittest.mock import MagicMock + +import pytest +import requests + +from prxref.forges import github, gitlab +from prxref.forges.base import FeedReadError +from prxref.forges.github import ForgeImpl +from prxref.triage import parse_unified_diff + +PAGE_SIZE = 100 +API = "https://api.github.com/repos/acme/api" +FILES_URL = f"{API}/pulls/42/files" +PR_URL = f"{API}/pulls/42" +REQUEST_TIMEOUT = (10.0, 30.0) + + +def _mock_response(status_code=200, json_data=None, text=""): + resp = MagicMock(spec=requests.Response) + resp.status_code = status_code + resp.ok = 200 <= status_code < 300 + resp.headers = {} + if json_data is not None: + resp.json.return_value = json_data + resp.text = json.dumps(json_data) + else: + resp.text = text + resp.json.side_effect = ValueError("No JSON") + resp.raise_for_status.side_effect = ( + None if resp.ok else requests.HTTPError(response=resp) + ) + return resp + + +def _ref(url="https://github.com/acme/api/pull/42"): + ref = ForgeImpl.parse_pr_url(url) + assert ref is not None + return ref + + +# ``changed_files`` sentinels: the listing's own length, or no key at all. +_LISTED = object() +_ABSENT = object() + + +def _pr(changed_files): + pr = { + "title": "t", "body": "", "user": {"login": "dev"}, + "head": {"ref": "feat", "sha": "b" * 40}, + "base": {"ref": "main", "sha": "a" * 40}, + } + if changed_files is not _ABSENT: + pr["changed_files"] = changed_files + return pr + + +def _session(listing, changed_files=_LISTED, *, listing_status=200, pr_status=200): + """A Session double serving ``listing`` a page at a time, plus the PR itself. + + Routes on URL and the ``page`` parameter, never on call order, so the order + of the two reads is not load-bearing. A page past the end of ``listing`` + comes back as ``[]``, which is how GitHub answers it. + """ + if changed_files is _LISTED: + changed_files = len(listing) + + def get(url, headers=None, params=None, **kwargs): + if url.endswith("/pulls/42/files"): + if listing_status != 200: + return _mock_response(listing_status, json_data={"message": "boom"}) + start = (params["page"] - 1) * params["per_page"] + return _mock_response(json_data=listing[start:start + params["per_page"]]) + if url.endswith("/pulls/42"): + if pr_status != 200: + return _mock_response(pr_status, json_data={"message": "Not Found"}) + return _mock_response(json_data=_pr(changed_files)) + raise AssertionError(f"unrouted GET {url}") + + session = MagicMock(spec=requests.Session) + session.get.side_effect = get + return session + + +def _file(name, status="modified", patch="@@ -1 +1 @@\n-old\n+new", **extra): + entry = {"filename": name, "status": status, **extra} + if patch is not None: + entry["patch"] = patch + return entry + + +def _rebuild(session, ref=None): + return ForgeImpl(session=session)._get_diff_from_files(ref or _ref()) + + +def _gets_to(session, url): + return [c for c in session.get.call_args_list if c[0][0] == url] + + +def _warnings(caplog): + return [r for r in caplog.records if r.levelno == logging.WARNING] + + +# --- paging ------------------------------------------------------------------- + + +def test_reads_the_listing_to_its_last_page(monkeypatch): + monkeypatch.setenv("PRXREF_GITHUB_TOKEN", "t0ken") + listing = [_file(f"src/mod_{i:03d}.py") for i in range(130)] + session = _session(listing) + + diff = _rebuild(session) + + listing_gets = _gets_to(session, FILES_URL) + assert [c[1]["params"]["page"] for c in listing_gets] == [1, 2] + assert all(c[1]["params"]["per_page"] == PAGE_SIZE for c in listing_gets) + assert len(_gets_to(session, PR_URL)) == 1 + for call in session.get.call_args_list: + assert call[1]["headers"]["Accept"] == "application/vnd.github+json" + assert call[1]["headers"]["Authorization"] == "Bearer t0ken" + assert call[1]["timeout"] == REQUEST_TIMEOUT == github._REQUEST_TIMEOUT + + files = parse_unified_diff(diff) + assert [f.path for f in files] == [f"src/mod_{i:03d}.py" for i in range(130)] + assert all(f.status == "modified" and f.added_lines == {1} for f in files) + + +def test_uses_the_enterprise_api_base(monkeypatch): + monkeypatch.setenv("PRXREF_GITHUB_ENTERPRISE_TOKEN", "ghes-t0ken") + session = _session([_file("src/app.py")]) + + _rebuild(session, _ref("https://git.corp.example/acme/api/pull/42")) + + urls = {c[0][0] for c in session.get.call_args_list} + assert urls == { + "https://git.corp.example/api/v3/repos/acme/api/pulls/42/files", + "https://git.corp.example/api/v3/repos/acme/api/pulls/42", + } + assert all( + c[1]["headers"]["Authorization"] == "Bearer ghes-t0ken" + for c in session.get.call_args_list + ) + + +# --- header shapes, byte-for-byte against the GitLab renderer ------------------- + + +def _gitlab_entry(old, new, *, new_file=False, deleted_file=False, renamed_file=False, + diff): + return { + "old_path": old, "new_path": new, "new_file": new_file, + "deleted_file": deleted_file, "renamed_file": renamed_file, "diff": diff, + } + + +HEADER_CASES = { + "added": ( + _file("src/new.py", "added", "@@ -0,0 +1 @@\n+x"), + _gitlab_entry("src/new.py", "src/new.py", new_file=True, + diff="@@ -0,0 +1 @@\n+x\n"), + ["diff --git a/src/new.py b/src/new.py", "new file mode 100644", + "--- /dev/null", "+++ b/src/new.py"], + "added", + ), + "removed": ( + _file("src/gone.py", "removed", "@@ -1 +0,0 @@\n-x"), + _gitlab_entry("src/gone.py", "src/gone.py", deleted_file=True, + diff="@@ -1 +0,0 @@\n-x\n"), + ["diff --git a/src/gone.py b/src/gone.py", "deleted file mode 100644", + "--- a/src/gone.py", "+++ /dev/null"], + "removed", + ), + "renamed": ( + _file("src/new_name.py", "renamed", "@@ -1 +1 @@\n-a\n+b", + previous_filename="src/old_name.py"), + _gitlab_entry("src/old_name.py", "src/new_name.py", renamed_file=True, + diff="@@ -1 +1 @@\n-a\n+b\n"), + ["diff --git a/src/old_name.py b/src/new_name.py", + "rename from src/old_name.py", "rename to src/new_name.py", + "--- a/src/old_name.py", "+++ b/src/new_name.py"], + "renamed", + ), + "modified": ( + _file("src/app.py", "modified", "@@ -1 +1 @@\n-a\n+b"), + _gitlab_entry("src/app.py", "src/app.py", diff="@@ -1 +1 @@\n-a\n+b\n"), + ["diff --git a/src/app.py b/src/app.py", "--- a/src/app.py", + "+++ b/src/app.py"], + "modified", + ), + "changed": ( + _file("bin/run.sh", "changed", "@@ -1 +1 @@\n-a\n+b"), + _gitlab_entry("bin/run.sh", "bin/run.sh", diff="@@ -1 +1 @@\n-a\n+b\n"), + ["diff --git a/bin/run.sh b/bin/run.sh", "--- a/bin/run.sh", + "+++ b/bin/run.sh"], + "modified", + ), +} + + +@pytest.mark.parametrize("status", sorted(HEADER_CASES)) +def test_each_status_renders_the_gitlab_header_shape(status): + github_entry, gitlab_entry, header, parsed_status = HEADER_CASES[status] + + diff = _rebuild(_session([github_entry])) + + assert diff == gitlab._render_diff_entries([gitlab_entry]) + assert diff.splitlines()[:len(header)] == header + (parsed,) = parse_unified_diff(diff) + assert parsed.status == parsed_status + assert parsed.path == github_entry["filename"] + + +def test_a_copied_file_renders_plain_from_its_source(): + entry = _file("src/copy.py", "copied", "@@ -1 +1 @@\n-a\n+b", + previous_filename="src/orig.py") + + diff = _rebuild(_session([entry])) + + assert diff == gitlab._render_diff_entries([ + _gitlab_entry("src/orig.py", "src/copy.py", diff="@@ -1 +1 @@\n-a\n+b\n") + ]) + assert diff.splitlines()[:3] == [ + "diff --git a/src/orig.py b/src/copy.py", "--- a/src/orig.py", + "+++ b/src/copy.py", + ] + + +def test_mixed_statuses_render_in_listing_order(): + listing = [HEADER_CASES[s][0] for s in ("added", "removed", "renamed", "modified")] + expected = [HEADER_CASES[s][1] for s in ("added", "removed", "renamed", "modified")] + + diff = _rebuild(_session(listing)) + + assert diff == gitlab._render_diff_entries(expected) + + +# --- a file listed without a patch ------------------------------------------------ + + +@pytest.mark.parametrize("patch_form", ["absent", "none", "empty"]) +def test_a_patchless_file_is_header_only_with_one_warning(caplog, patch_form): + binary = {"filename": "assets/logo.png", "status": "modified"} + if patch_form == "none": + binary["patch"] = None + elif patch_form == "empty": + binary["patch"] = "" + listing = [binary, _file("src/app.py")] + + with caplog.at_level(logging.DEBUG, logger="prxref.forges.github"): + diff = _rebuild(_session(listing)) + + header_only = ( + "diff --git a/assets/logo.png b/assets/logo.png\n" + "--- a/assets/logo.png\n+++ b/assets/logo.png\n" + ) + assert diff.startswith(header_only) + assert diff == gitlab._render_diff_entries([ + _gitlab_entry("assets/logo.png", "assets/logo.png", diff=""), + _gitlab_entry("src/app.py", "src/app.py", diff="@@ -1 +1 @@\n-old\n+new\n"), + ]) + (warning,) = _warnings(caplog) + assert warning.getMessage() == ( + "GitHub PR diff: assets/logo.png has no inline diff (no patch in the " + "files listing); it is reviewed as header-only" + ) + assert [f.path for f in parse_unified_diff(diff)] == ["assets/logo.png", "src/app.py"] + + +def test_a_complete_rebuild_logs_nothing_at_info_or_above(caplog): + listing = [_file(f"src/mod_{i}.py") for i in range(3)] + + with caplog.at_level(logging.DEBUG, logger="prxref.forges.github"): + _rebuild(_session(listing)) + + assert [r for r in caplog.records if r.levelno >= logging.INFO] == [] + + +# --- completeness is asserted, never assumed ----------------------------------------- + + +def test_a_listing_shorter_than_changed_files_raises_with_the_counts(caplog): + listing = [_file(f"src/mod_{i}.py") for i in range(4)] + listing.append({"filename": "assets/logo.png", "status": "modified"}) + + with caplog.at_level(logging.WARNING, logger="prxref.forges.github"): + with pytest.raises(ValueError) as exc: + _rebuild(_session(listing, changed_files=7)) + + message = str(exc.value) + assert "acme/api#42" in message + assert "changed_files=7" in message + assert "returned 5" in message + assert "3,000" in message + assert _warnings(caplog) == [] + + +def test_a_pr_past_the_three_thousand_file_cap_fails_rather_than_going_partial(): + listing = [_file(f"src/mod_{i:04d}.py") for i in range(3000)] + session = _session(listing, changed_files=3001) + + with pytest.raises( + ValueError, + match=r"changed_files=3001 but the changed-file listing returned 3000 " + r"\(GitHub caps the listing at 3,000 files\)", + ): + _rebuild(session) + + pages = [c[1]["params"]["page"] for c in _gets_to(session, FILES_URL)] + assert pages == list(range(1, 32)) + assert len(_gets_to(session, PR_URL)) == 1 + + +def test_a_listing_matching_changed_files_at_the_cap_is_whole(): + listing = [_file(f"src/mod_{i:04d}.py") for i in range(3000)] + + diff = _rebuild(_session(listing, changed_files=3000)) + + assert len(parse_unified_diff(diff)) == 3000 + + +@pytest.mark.parametrize( + "changed_files", [_ABSENT, None, "1", True], ids=["absent", "null", "str", "bool"] +) +def test_pr_metadata_without_an_integer_changed_files_raises(changed_files): + with pytest.raises(ValueError, match="no integer changed_files"): + _rebuild(_session([_file("src/app.py")], changed_files=changed_files)) + + +# --- failures ------------------------------------------------------------------------ + + +def test_a_non_ok_listing_raises_feed_read_error(): + with pytest.raises(FeedReadError, match=r"changed-file listing for acme/api#42 .*500"): + _rebuild(_session([_file("src/app.py")], listing_status=500)) + + +def test_a_failed_pr_metadata_read_raises_http_error(): + with pytest.raises(requests.HTTPError): + _rebuild(_session([_file("src/app.py")], pr_status=404)) From fc457bee7fa070c549aa679eaa00f7421ca996af Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:50:20 -0700 Subject: [PATCH 16/64] fix: GitHub diff past 20,000 lines falls back to the files listing (#15 T4-T6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit get_diff now reads a 406 whose JSON errors[].code includes too_large as GitHub's diff line limit, logs one DEBUG line and returns _get_diff_from_files(ref) (seat F15-B's listing rebuild). Any other 406, including a non-JSON body, and every other status still go through raise_for_status(); under the limit the request is the same single GET with the same Accept header and no params. The edit stays inside get_diff's body; get_compare_diff is untouched because the orchestrator's probe showed the compare endpoint returns 200 past a million lines. tests/test_github_get_diff.py holds the first direct get_diff tests: the one-GET acceptance test, the too_large hand-off, nine 406 shapes that must still raise, real requests.Response objects for the JSON decode path, the DEBUG-only log, propagation of the rebuild's ValueError/FeedReadError, and a CLI run over the real orchestrator where a rebuild past the file cap ends as an Error run with exit 0 (with a control whose rebuild returns a diff that is then reviewed). docs/forges.md replaces the "no fallback yet" limitation with the fallback behaviour and records the compare probe under the replay bullet. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- docs/forges.md | 4 +- src/prxref/forges/github.py | 28 ++- tests/test_github_get_diff.py | 328 ++++++++++++++++++++++++++++++++++ 3 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 tests/test_github_get_diff.py diff --git a/docs/forges.md b/docs/forges.md index efed4665..994093a5 100644 --- a/docs/forges.md +++ b/docs/forges.md @@ -53,7 +53,7 @@ Every host is covered, but not by the same means. GitHub and GitLab are host-agn - **API Endpoints & Behavior:** - **Base URL:** `https://api.github.com` for `github.com`, or `https://{host}/api/v3` for GHES. - **Metadata:** `GET /repos/{owner}/{repo}/pulls/{number}` - - **Diffs:** `GET /repos/{owner}/{repo}/pulls/{number}` with `Accept: application/vnd.github.v3.diff, application/vnd.diff`. GitHub refuses this diff for a pull request whose diff runs past 20,000 lines (HTTP `406`, error code `too_large`), so such a PR ends as an `Error` run and gets the error notice. There is no fallback to the paged `/pulls/{number}/files` listing yet. + - **Diffs:** `GET /repos/{owner}/{repo}/pulls/{number}` with `Accept: application/vnd.github.v3.diff, application/vnd.diff`. GitHub refuses this diff for a pull request whose diff runs past 20,000 lines, answering HTTP `406` with error code `too_large`. On exactly that answer the diff is rebuilt from the paged `GET /repos/{owner}/{repo}/pulls/{number}/files` listing (`per_page=100`, read to the last page), with the same header reconstruction as the GitLab adapter's **Diffs**. The switch is logged at debug only and adds nothing to the summary. A file GitHub lists without a `patch` (binary, or too large for GitHub to render) is logged as a warning and reviewed as a header-only file. GitHub's listing stops at 3,000 files, so it is checked against the pull request's `changed_files` count: a pull request past 3,000 files, or a listing page that cannot be read, fails the review with verdict `Error` rather than reviewing part of it. A `406` without `too_large` (a wrong media type, say) still fails the review, and a pull request under the limit makes the same single request as before. The pinned-range replay below reads the compare endpoint instead, which has no such limit. - **Summary Comments:** Managed on the issue comments endpoint (`/repos/{owner}/{repo}/issues/{number}/comments`). Summary deduplication is handled via the embedded hidden HTML marker ``. If an existing review comment contains this marker, it is updated via `PATCH /repos/{owner}/{repo}/issues/comments/{comment_id}` instead of creating a duplicate comment. - **Inline Comments:** `POST /repos/{owner}/{repo}/pulls/{number}/comments` with `body`, `path`, `line`, and `side` (`RIGHT`). HTTP 422 errors (e.g. comment line not part of diff hunk) are gracefully skipped. - **Thread List:** `GET /repos/{owner}/{repo}/pulls/{number}/comments`. @@ -62,7 +62,7 @@ Every host is covered, but not by the same means. GitHub and GitLab are host-agn - **Event Header:** `X-GitHub-Event` (must equal `pull_request`) - **Accepted Actions:** `opened`, `synchronize` - **Signature Header:** `X-Hub-Signature-256` (HMAC-SHA256) validated against `PRXREF_GITHUB_WEBHOOK_SECRET`. -- **Pinned Commit Range (Replay):** `GET /repos/{owner}/{repo}/compare/{base_sha}...{head_sha}` on the same base URL (GHES included), with `Accept: application/vnd.github.diff`. The three dots are the merge-base form and are required, because the two-dot spelling returns 404. Without the diff media type the endpoint returns its JSON comparison object rather than a diff. The text is returned unmodified, an empty range (a head already merged into the base) returns empty text, and an HTTP or transport error raises. +- **Pinned Commit Range (Replay):** `GET /repos/{owner}/{repo}/compare/{base_sha}...{head_sha}` on the same base URL (GHES included), with `Accept: application/vnd.github.diff`. The three dots are the merge-base form and are required, because the two-dot spelling returns 404. Without the diff media type the endpoint returns its JSON comparison object rather than a diff. The text is returned unmodified, an empty range (a head already merged into the base) returns empty text, and an HTTP or transport error raises. The compare endpoint does not apply the pull-request diff's 20,000-line limit: two read-only probes of public tag ranges on 2026-09-24, one of about 1.0 million diff lines across 3,316 files and one of about 1.05 million lines across 10,373 files, each returned HTTP `200` with the whole diff. So a `--base-sha`/`--head-sha` replay is unaffected by the limit and has no files-listing fallback. --- diff --git a/src/prxref/forges/github.py b/src/prxref/forges/github.py index bc9a4570..8054683a 100644 --- a/src/prxref/forges/github.py +++ b/src/prxref/forges/github.py @@ -170,13 +170,39 @@ def get_pr(self, ref: PRRef) -> PRData: ) def get_diff(self, ref: PRRef) -> str: - """Fetch the raw unified diff of the PR (all files).""" + """Fetch the raw unified diff of the PR (all files). + + Under GitHub's diff limit this is one GET with the diff media type, + and its body is returned as-is. Past 20,000 lines GitHub refuses that + request with HTTP 406 and a JSON ``errors`` entry whose ``code`` is + ``too_large``. Exactly that answer is logged at DEBUG and the diff is + rebuilt from the changed-file listing by ``_get_diff_from_files``, + whose ``ValueError`` (a listing cut short by GitHub's 3,000-file cap) + or ``FeedReadError`` propagates. Any other 406, including one whose + body is not JSON, and every other non-OK status raise ``HTTPError`` + as before. + """ url = f"{self._api_base(ref)}/repos/{ref.owner}/{ref.repo}/pulls/{ref.number}" headers = self._headers( ref.host, {"Accept": "application/vnd.github.v3.diff, application/vnd.diff"}, ) resp = self.session.get(url, headers=headers, timeout=_REQUEST_TIMEOUT) + if resp.status_code == 406: + try: + body = resp.json() + except ValueError: + body = None + errors = body.get("errors") if isinstance(body, dict) else None + if isinstance(errors, list) and any( + isinstance(e, dict) and e.get("code") == "too_large" for e in errors + ): + logger.debug( + "diff for %s/%s#%s exceeded GitHub's line limit (406 too_large); " + "rebuilding it from the files listing", + ref.owner, ref.repo, ref.number, + ) + return self._get_diff_from_files(ref) resp.raise_for_status() return resp.text diff --git a/tests/test_github_get_diff.py b/tests/test_github_get_diff.py new file mode 100644 index 00000000..f10ace9f --- /dev/null +++ b/tests/test_github_get_diff.py @@ -0,0 +1,328 @@ +"""Direct tests of ``github.ForgeImpl.get_diff``: the diff request and its 406 fallback. + +GitHub refuses the diff media type for a pull request whose diff runs past +20,000 lines, with HTTP 406 and an ``errors`` entry coded ``too_large``. Exactly +that answer hands the read to ``_get_diff_from_files``, which rebuilds the diff +from the changed-file listing; every other answer is the single GET it always +was. ``_get_diff_from_files`` is a double set on the instance in every test +here, so these pin the detection and the hand-off, not the listing itself. +""" +from __future__ import annotations + +import json +import logging +import sys +import types +from unittest.mock import MagicMock + +import pytest +import requests + +from prxref import orchestrator as real_orchestrator +from prxref.cli import main +from prxref.forges.base import FeedReadError +from prxref.forges.github import ForgeImpl +from tests.test_forge_github import REQUEST_TIMEOUT, _mock_response, _ref, _routed_session +from tests.test_orchestrator import FakeLLM + +DIFF_ACCEPT = "application/vnd.github.v3.diff, application/vnd.diff" +PR_API_URL = "https://api.github.com/repos/acme/api/pulls/42" +DIFF = ( + "diff --git a/src/app.py b/src/app.py\n" + "--- a/src/app.py\n+++ b/src/app.py\n@@ -1 +1 @@\n-x\n+y\n" +) +# The body GitHub sent for the 0.14.0 release PR, verbatim. +TOO_LARGE = { + "message": "Sorry, the diff exceeded the maximum number of lines (20000)", + "errors": [{"resource": "PullRequest", "field": "diff", "code": "too_large"}], +} +DEBUG_LINE = ( + "diff for acme/api#42 exceeded GitHub's line limit (406 too_large); " + "rebuilding it from the files listing" +) +CAP_ERROR = ( + "acme/api#42: changed_files=3200 but the files listing returned 3000 " + "(GitHub caps it at 3,000)" +) + + +@pytest.fixture(autouse=True) +def _no_github_token(monkeypatch): + monkeypatch.delenv("PRXREF_GITHUB_TOKEN", raising=False) + monkeypatch.delenv("PRXREF_GITHUB_ENTERPRISE_TOKEN", raising=False) + + +def _forge(resp): + session = MagicMock(spec=requests.Session) + session.get.return_value = resp + return ForgeImpl(session=session), session + + +def _fallback(monkeypatch, forge, **kwargs): + """Stand in for ``_get_diff_from_files``, which is another seat's method.""" + fallback = MagicMock(**kwargs) + monkeypatch.setattr(forge, "_get_diff_from_files", fallback, raising=False) + return fallback + + +# --- under the limit: the request as it always was ----------------------------- + + +@pytest.mark.parametrize( + ("url", "token_env", "expected_url", "expected_headers"), + [ + ( + "https://github.com/acme/api/pull/42", + None, + PR_API_URL, + {"Accept": DIFF_ACCEPT}, + ), + ( + "https://git.corp.example/acme/api/pull/42", + ("PRXREF_GITHUB_ENTERPRISE_TOKEN", "placeholder-token"), + "https://git.corp.example/api/v3/repos/acme/api/pulls/42", + {"Accept": DIFF_ACCEPT, "Authorization": "Bearer placeholder-token"}, + ), + ], + ids=["github.com-anonymous", "ghes-token"], +) +def test_under_the_limit_is_exactly_one_get_with_the_diff_accept_header( + monkeypatch, caplog, url, token_env, expected_url, expected_headers +): + """The issue's acceptance test: a pull request under the limit makes exactly + the request it made before the fallback existed, and returns its body.""" + if token_env is not None: + monkeypatch.setenv(*token_env) + resp = _mock_response(text=DIFF) + forge, session = _forge(resp) + fallback = _fallback(monkeypatch, forge) + + with caplog.at_level(logging.DEBUG, logger="prxref.forges.github"): + assert forge.get_diff(_ref(url)) == DIFF + + session.get.assert_called_once_with( + expected_url, headers=expected_headers, timeout=REQUEST_TIMEOUT + ) + resp.json.assert_not_called() + fallback.assert_not_called() + assert [r for r in caplog.records if r.name == "prxref.forges.github"] == [] + + +# --- 406 too_large: rebuild from the listing ----------------------------------- + + +@pytest.mark.parametrize( + "body", + [ + TOO_LARGE, + {"errors": [{"code": "custom"}, {"field": "diff", "code": "too_large"}]}, + ], + ids=["github-body", "too_large-among-other-errors"], +) +def test_406_too_large_rebuilds_the_diff_from_the_files_listing(monkeypatch, body): + resp = _mock_response(406, json_data=body) + forge, session = _forge(resp) + fallback = _fallback(monkeypatch, forge, return_value=DIFF) + ref = _ref() + + assert forge.get_diff(ref) == DIFF + + fallback.assert_called_once_with(ref) + assert session.get.call_count == 1 + resp.raise_for_status.assert_not_called() + + +def test_the_fallback_logs_one_debug_line_and_nothing_louder(monkeypatch, caplog): + """A successful fallback is DEBUG only (decisions #15): the rebuilt diff is + the same content, so it earns no warning and no summary line.""" + forge, _ = _forge(_mock_response(406, json_data=TOO_LARGE)) + _fallback(monkeypatch, forge, return_value=DIFF) + + with caplog.at_level(logging.DEBUG, logger="prxref.forges.github"): + forge.get_diff(_ref()) + + ours = [ + (r.levelno, r.getMessage()) + for r in caplog.records + if r.name == "prxref.forges.github" + ] + assert ours == [(logging.DEBUG, DEBUG_LINE)] + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + +@pytest.mark.parametrize( + "exc", + [ + ValueError(CAP_ERROR), + FeedReadError("pull request files for acme/api#42 returned HTTP 502 at page 7"), + ], + ids=["past-the-file-cap", "listing-read-failed"], +) +def test_a_failed_rebuild_propagates(monkeypatch, exc): + """A listing that cannot be read whole fails the read; it never returns short.""" + forge, _ = _forge(_mock_response(406, json_data=TOO_LARGE)) + _fallback(monkeypatch, forge, side_effect=exc) + + with pytest.raises(type(exc)) as info: + forge.get_diff(_ref()) + + assert info.value is exc + + +# --- every other refusal raises as before -------------------------------------- + + +@pytest.mark.parametrize( + "resp_kwargs", + [ + {"json_data": {"message": "Unsupported 'Accept' header"}}, + {"json_data": {"message": "Not Acceptable", "errors": []}}, + {"json_data": {"errors": [{"field": "diff", "code": "invalid"}]}}, + {"json_data": {"message": TOO_LARGE["message"]}}, + {"json_data": {"errors": {"code": "too_large"}}}, + {"json_data": {"errors": ["too_large"]}}, + {"json_data": [{"code": "too_large"}]}, + {"text": "Not Acceptable"}, + {"text": ""}, + ], + ids=[ + "no-errors-key", + "empty-errors", + "another-code", + "too-large-only-in-the-message", + "errors-not-a-list", + "error-entries-not-objects", + "body-is-a-list", + "body-not-json", + "empty-body", + ], +) +def test_a_406_without_too_large_still_raises(monkeypatch, resp_kwargs): + """A bare 406 (a wrong media type, say) is a real misconfiguration, not a + large pull request, so the fallback must not mask it.""" + resp = _mock_response(406, **resp_kwargs) + forge, session = _forge(resp) + fallback = _fallback(monkeypatch, forge) + + with pytest.raises(requests.HTTPError): + forge.get_diff(_ref()) + + fallback.assert_not_called() + assert session.get.call_count == 1 + + +def _real_406(content: bytes) -> requests.Response: + resp = requests.Response() + resp.status_code = 406 + resp.reason = "Not Acceptable" + resp.url = PR_API_URL + resp.encoding = "utf-8" + resp._content = content + return resp + + +@pytest.mark.parametrize( + ("content", "falls_back"), + [ + (json.dumps(TOO_LARGE).encode(), True), + (b"Not Acceptable", False), + (b"", False), + ], + ids=["too_large", "not-json", "empty"], +) +def test_a_real_406_response_is_read_the_same_way(monkeypatch, content, falls_back): + """A real ``requests.Response``, not a double: its JSON decode error is the + one the detection must survive.""" + forge, _ = _forge(_real_406(content)) + fallback = _fallback(monkeypatch, forge, return_value=DIFF) + + if falls_back: + assert forge.get_diff(_ref()) == DIFF + else: + with pytest.raises(requests.HTTPError, match="406"): + forge.get_diff(_ref()) + + assert fallback.call_count == (1 if falls_back else 0) + + +@pytest.mark.parametrize("status", [401, 404, 422, 500]) +def test_every_other_status_raises_as_before_even_with_a_too_large_body( + monkeypatch, status +): + resp = _mock_response(status, json_data=TOO_LARGE) + forge, session = _forge(resp) + fallback = _fallback(monkeypatch, forge) + + with pytest.raises(requests.HTTPError): + forge.get_diff(_ref()) + + resp.json.assert_not_called() + fallback.assert_not_called() + assert session.get.call_count == 1 + + +# --- through the CLI and the real orchestrator --------------------------------- + + +def _session_refusing_the_diff(): + """The adapter's routed session double, with the diff answering 406 too_large.""" + session = _routed_session(summary_feed=[]) + routed = session.get.side_effect + + def get(url, headers=None, params=None, **kwargs): + if url == PR_API_URL and "diff" in (headers or {}).get("Accept", ""): + return _mock_response(406, json_data=TOO_LARGE) + return routed(url, headers=headers, params=params, **kwargs) + + session.get.side_effect = get + return session + + +class TestThroughTheCli: + """``prxref review`` over the real orchestrator and the real GitHub adapter. + + Only the HTTP session, the model and ``_get_diff_from_files`` are doubles. + A rebuild that raises is a failed ``get_diff``: verdict ``Error`` and exit + 0, never a partial review. The control lets the same rebuild return a + diff, and the same rig then reviews it. + """ + + @pytest.fixture + def rig(self, monkeypatch): + assert sys.modules["prxref.orchestrator"] is real_orchestrator + forge = ForgeImpl(session=_session_refusing_the_diff()) + fallback = _fallback(monkeypatch, forge) + llm = FakeLLM('{"findings": []}') + ref = _ref() + monkeypatch.setattr("prxref.cli.detect_forge", lambda url: ref) + monkeypatch.setattr("prxref.cli.make_forge", lambda ref: forge) + monkeypatch.setattr("prxref.llm_backends.create_llm_client", lambda cfg: llm) + return types.SimpleNamespace(fallback=fallback, llm=llm, ref=ref) + + @staticmethod + def _review(ref): + return main(["review", "--pr-url", ref.url, "--no-post", "--format", "json"]) + + def test_a_listing_past_the_file_cap_ends_as_an_error_run_that_exits_0( + self, rig, capsys, caplog + ): + rig.fallback.side_effect = ValueError(CAP_ERROR) + + with caplog.at_level(logging.ERROR, logger="prxref"): + assert self._review(rig.ref) == 0 + + out, _ = capsys.readouterr() + assert json.loads(out)["verdict"] == "Error" + rig.fallback.assert_called_once_with(rig.ref) + assert rig.llm.calls == 0 + assert f"get_diff failed: {CAP_ERROR}" in caplog.text + + def test_control_a_rebuilt_diff_is_reviewed(self, rig, capsys): + rig.fallback.return_value = DIFF + + assert self._review(rig.ref) == 0 + + out, _ = capsys.readouterr() + assert json.loads(out)["verdict"] != "Error" + rig.fallback.assert_called_once_with(rig.ref) + assert rig.llm.calls >= 1 From 6ae4b6124a8f1ca3bf4dc0f3b5b5517d0d45706e Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:54:18 -0700 Subject: [PATCH 17/64] feat(cli): --as-of, --description-file and --no-description replay flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #16 T3, without the stamp change (decisions.md folds that into R16-E). The three flags choose which PR description a replay shows. They are CLI-only, mutually exclusive, and each one makes the run a replay (posting off). _resolve_replay stays pure and validates them after the SHA checks: two or more exit 2 naming every one given; --as-of must be ISO-8601 with a UTC offset (a date alone or a naive time exits 2 instead of being read in the host's time zone, per map-16), is normalised to a tz-aware UTC datetime, and needs --pr-url. _run_review reads --description-file before the URL is parsed, through text_inputs.check_readable_path plus decode_text, the same path as the rules file: missing, unreadable, non-regular, non-UTF-8, NUL bytes and a symlink out of the cwd each exit 2. _ReplayRequest gains as_of, description_file, description_text and no_description as trailing fields, so a run without the new flags resolves to the same request, or None, as before. _replay_forge, the stamp and _print_summary are untouched. README '## CLI Flags' documents the flags. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- README.md | 5 + src/prxref/cli.py | 175 ++++++++- tests/test_replay_description_flags.py | 522 +++++++++++++++++++++++++ 3 files changed, 694 insertions(+), 8 deletions(-) create mode 100644 tests/test_replay_description_flags.py diff --git a/README.md b/README.md index a660fc04..3ca79757 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,11 @@ Replay flags, for evaluation (see [Replay Mode (Evaluation)](#replay-mode-evalua - `--base-sha SHA` / `--head-sha SHA` — review the pinned range `BASE...HEAD` of the `--pr-url` repository (the merge-base diff, as the PR's own diff is), with file context read at `HEAD`. The two come as a pair, must be full 40- or 64-character hex commit SHAs, must differ, and need `--pr-url`. - `--no-threads` — hide the PR's existing threads from the prompt and from the thread-dedup passes. - `--diff-file PATH` — review this unified diff (`git diff` or `git format-patch` output) instead of fetching one; `--pr-url` becomes optional. +- `--as-of TIME` — show the reviewer the PR's title and description as they were at `TIME`: an ISO-8601 time with a UTC offset, such as `2026-05-01T09:30:00Z` or `2026-05-01T11:30:00+02:00`. A date alone or a time without an offset exits `2` rather than being read in the local time zone, and so does a value that is not ISO-8601. Needs `--pr-url`. +- `--description-file PATH` — use this file's text as the PR description, with or without `--pr-url`. It is read like `--rules-file`: a missing, unreadable or non-regular file, a file that is not UTF-8 or contains NUL bytes, and a path under the working directory that symlinks out of it each exit `2`. A blank file is an empty description. +- `--no-description` — review with an empty PR description. + +`--as-of`, `--description-file` and `--no-description` are mutually exclusive: giving two or more exits `2` naming each one given. They are CLI-only, with no environment variable. The other subcommands: `prxref serve [--port N] [--host H]` runs the [webhook server](#webhook-server) (default port `8080`, default host `0.0.0.0`); `prxref trace render FILE [-o OUT]` renders a JSONL run trace (`PRXREF_TRACE_FILE`) to a standalone HTML pipeline view, written next to the trace unless `-o`/`--out` names the output; and `prxref --version` prints the version. diff --git a/src/prxref/cli.py b/src/prxref/cli.py index 9e67a145..d1cb3a74 100644 --- a/src/prxref/cli.py +++ b/src/prxref/cli.py @@ -21,7 +21,10 @@ ``--base-sha`` / ``--head-sha`` a commit range in the ``--pr-url`` repository, ``--diff-file PATH`` a diff on disk (``--pr-url`` is then optional, and no forge is contacted without it), and ``--no-threads`` hides -the PR's existing threads. Any of them makes the run a replay: it never +the PR's existing threads. At most one of ``--as-of TIME`` (the description +in force at that time), ``--description-file PATH`` and ``--no-description`` +chooses the PR description a replay shows; each is CLI-only, with no +environment variable. Any replay flag makes the run a replay: it never posts, and its run record gains a ``replay`` stamp. They are validated before the URL is parsed, and a bad set exits 2 naming the flag. The webhook daemon never replays. @@ -59,14 +62,18 @@ from __future__ import annotations import argparse +import codecs +import errno import importlib import json import logging import os import re +import stat import sys import time from dataclasses import dataclass, replace +from datetime import UTC, date, datetime from pathlib import Path from typing import Any @@ -77,6 +84,7 @@ from prxref.forges.replay import LocalDiffForge, ReplayForge from prxref.llm import ConfigError from prxref.rules import load_review_rules +from prxref.text_inputs import check_readable_path, decode_text from prxref.ticket import load_ticket_context from prxref.triage import SCOPE_IN, SCOPE_OUT, normalize_scope from prxref.viz import render_file @@ -193,6 +201,34 @@ def _build_parser() -> argparse.ArgumentParser: "implies no posting" ), ) + rev.add_argument( + "--as-of", + default=None, + metavar="TIME", + help=( + "replay: show the PR's title and description as they were at TIME, " + "an ISO-8601 time with a UTC offset (2026-05-01T09:30:00Z); needs " + "--pr-url; excludes --description-file and --no-description; " + "implies no posting" + ), + ) + rev.add_argument( + "--description-file", + default=None, + metavar="PATH", + help=( + "replay: use this UTF-8 text file as the PR description; excludes " + "--as-of and --no-description; implies no posting" + ), + ) + rev.add_argument( + "--no-description", + action="store_true", + help=( + "replay: review with an empty PR description; excludes --as-of " + "and --description-file; implies no posting" + ), + ) rev.add_argument( "-v", "--verbose", @@ -506,6 +542,14 @@ class _ReplayRequest: ``None``. ``diff_file`` is the path exactly as the operator typed it, and ``diff_text`` is that file's text once ``_run_review`` has read it (``None`` until then, and without ``--diff-file``). + + The description fields (issue #16) hold at most one choice, because + ``--as-of``, ``--description-file`` and ``--no-description`` are mutually + exclusive. ``as_of`` is the ``--as-of`` time as a timezone-aware UTC + ``datetime``. ``description_file`` is the path as typed, and + ``description_text`` is its decoded text once ``_run_review`` has read it + (``None`` until then, and without ``--description-file``). + ``no_description`` is ``--no-description``. """ base_sha: str | None = None @@ -513,6 +557,10 @@ class _ReplayRequest: no_threads: bool = False diff_file: str | None = None diff_text: str | None = None + as_of: datetime | None = None + description_file: str | None = None + description_text: str | None = None + no_description: bool = False def stamp(self, *, has_forge: bool) -> dict[str, Any]: """The run record's ``replay`` stamp: four keys, in a fixed order, all present. @@ -536,20 +584,28 @@ def _resolve_replay( head_sha: str | None = None, no_threads: bool = False, diff_file: str | None = None, + as_of: str | None = None, + description_file: str | None = None, + no_description: bool = False, ) -> _ReplayRequest | None: """Validate the replay flags; ``None`` means a normal, non-replay run. Pure: it reads nothing and calls nothing. ``_run_review`` calls it first, before ``detect_forge``, so a bad set of replay flags exits 2 even next to an unrecognised URL. A flag counts as given whenever it is not - ``None``, so an empty value is validated rather than ignored. + ``None`` (``--no-description`` whenever it is true), so an empty value + is validated rather than ignored. The checks run in this order, each a ``ConfigError`` naming its flag: no ``--pr-url`` and no ``--diff-file``; only one of ``--base-sha`` / ``--head-sha``; either one not a full 40- or 64-character hex SHA; the - two naming the same commit (compared lowercased); and a range without - ``--pr-url`` to resolve it in. Whether the forge can fetch the range is - only known once it exists, so ``_run_review`` checks that. + two naming the same commit (compared lowercased); a range without + ``--pr-url`` to resolve it in; two or more of ``--as-of``, + ``--description-file`` and ``--no-description``, naming every one + given; an ``--as-of`` that :func:`_parse_as_of` refuses; and an + ``--as-of`` without ``--pr-url``, whose history it reads. Whether the + forge can fetch the range is only known once it exists, so + ``_run_review`` checks that. """ if url is None and diff_file is None: raise ConfigError("--pr-url: required unless --diff-file is given") @@ -571,14 +627,70 @@ def _resolve_replay( "--base-sha/--head-sha: need --pr-url (the range is resolved in " "that PR's repository)" ) - if head_sha is None and not no_threads and diff_file is None: + chosen = [ + flag for flag, given in ( + ("--as-of", as_of is not None), + ("--description-file", description_file is not None), + ("--no-description", bool(no_description)), + ) if given + ] + if len(chosen) > 1: + raise ConfigError( + f"{'/'.join(chosen)}: cannot be combined (give at most one of " + "--as-of, --description-file and --no-description)" + ) + cutoff = _parse_as_of(as_of) if as_of is not None else None + if cutoff is not None and url is None: + raise ConfigError( + "--as-of: needs --pr-url (the description history is read from " + "that PR)" + ) + if head_sha is None and not no_threads and diff_file is None and not chosen: return None return _ReplayRequest( base_sha=base_sha, head_sha=head_sha, no_threads=bool(no_threads), - diff_file=diff_file, + diff_file=diff_file, as_of=cutoff, description_file=description_file, + no_description=bool(no_description), ) +def _parse_as_of(value: str) -> datetime: + """Parse ``--as-of`` into a timezone-aware UTC ``datetime``; refusals are a ``ConfigError``. + + The value is ISO-8601 as :meth:`datetime.fromisoformat` reads it, and it + must carry a UTC offset (``Z`` or ``+02:00``). A date alone + (``2026-05-01``) and a time without an offset are refused rather than + read in this machine's time zone or at an assumed hour, because either + guess would move the cutoff with the host running the replay. A time + whose UTC equivalent falls outside ``datetime``'s range is refused too. + """ + try: + day = date.fromisoformat(value) + except ValueError: + pass + else: + raise ConfigError( + f"--as-of: {value!r} is a date without a time; give a time with a " + f"UTC offset, such as '{day.isoformat()}T00:00:00Z'" + ) + try: + parsed = datetime.fromisoformat(value) + except ValueError: + raise ConfigError( + "--as-of: must be an ISO-8601 time with a UTC offset, such as " + f"'2026-05-01T09:30:00Z', got {value!r}" + ) from None + if parsed.utcoffset() is None: + raise ConfigError( + f"--as-of: {value!r} has no UTC offset; add one, such as 'Z' for " + "UTC or '+02:00'" + ) + try: + return parsed.astimezone(UTC) + except (OverflowError, ValueError): + raise ConfigError(f"--as-of: {value!r} is out of range in UTC") from None + + def _read_diff_file(path: str) -> str: """Read the ``--diff-file`` text; a file that cannot be read is a ``ConfigError``. @@ -597,6 +709,42 @@ def _read_diff_file(path: str) -> str: ) from exc +def _read_description_file(path: str) -> str: + """Read the ``--description-file`` text; every failure is a ``ConfigError`` naming the flag. + + The file is configuration, not review input, so it is read as the rules + file is: through :func:`prxref.text_inputs.check_readable_path` (a path + under the working directory that symlinks out of it is refused, and so + is anything but a regular file), then decoded strictly by + :func:`prxref.text_inputs.decode_text` (a leading BOM dropped, CRLF and + CR folded to LF). A missing, unreadable or non-regular file, invalid + UTF-8 and NUL bytes each raise. A blank file is not an error: it is an + empty description. + """ + try: + resolved = check_readable_path(path, confine=True) + with open(resolved, "rb") as fh: + if not stat.S_ISREG(os.fstat(fh.fileno()).st_mode): + raise OSError(errno.EINVAL, "not a regular file", path) + raw = fh.read() + except OSError as exc: + raise ConfigError( + f"--description-file: cannot read {path!r}: {exc.strerror or exc}" + ) from exc + try: + text = decode_text(raw) + except UnicodeDecodeError as exc: + offset = exc.start + (len(codecs.BOM_UTF8) if raw.startswith(codecs.BOM_UTF8) else 0) + raise ConfigError( + f"--description-file: {path!r} is not UTF-8 text ({exc.reason} at byte {offset})" + ) from exc + if "\x00" in text: + raise ConfigError( + f"--description-file: {path!r} contains NUL bytes; expected Markdown or plain text" + ) + return text + + def _replay_forge(forge: Any, ref: Any, replay: _ReplayRequest) -> ReplayForge: """Wrap the ``--pr-url`` forge in a :class:`ReplayForge` for this replay. @@ -665,16 +813,24 @@ def _run_review( head_sha: str | None = None, no_threads: bool = False, diff_file: str | None = None, + as_of: str | None = None, + description_file: str | None = None, + no_description: bool = False, ) -> Any: replay = _resolve_replay( url, base_sha=base_sha, head_sha=head_sha, no_threads=no_threads, - diff_file=diff_file, + diff_file=diff_file, as_of=as_of, description_file=description_file, + no_description=no_description, ) # The diff file is read with the flags, before the URL is parsed, so an # unreadable one exits 2 whatever the URL. Without --pr-url it is the # whole input: a synthetic "local" ref, and no forge is ever built. if replay is not None and replay.diff_file is not None: replay = replace(replay, diff_text=_read_diff_file(replay.diff_file)) + if replay is not None and replay.description_file is not None: + replay = replace( + replay, description_text=_read_description_file(replay.description_file), + ) if url is None: ref = LocalDiffForge.ref_for(replay.diff_file) else: @@ -879,6 +1035,9 @@ def _cmd_review(args: argparse.Namespace) -> int: head_sha=args.head_sha, no_threads=args.no_threads, diff_file=args.diff_file, + as_of=args.as_of, + description_file=args.description_file, + no_description=args.no_description, ) except ConfigError as exc: print(f"configuration error: {exc}", file=sys.stderr) diff --git a/tests/test_replay_description_flags.py b/tests/test_replay_description_flags.py new file mode 100644 index 00000000..542f962a --- /dev/null +++ b/tests/test_replay_description_flags.py @@ -0,0 +1,522 @@ +"""The replay description flags through the CLI (issue #16 T3): parser, request, validation. + +``--as-of TIME``, ``--description-file PATH`` and ``--no-description`` choose +which PR description a replay shows. These tests pin what ``prxref.cli`` does +with them before any history is read: each flag alone makes the run a replay +whose ``_ReplayRequest`` carries it; they are mutually exclusive; every bad +value exits 2 naming its flag before the URL is parsed and before any forge +or LLM client exists; and a run without them is exactly what it was. How the +request is resolved into a pinned title and description (``_replay_forge``) +and how it is stamped belong to later tasks, so ``_replay_forge`` is replaced +by a recorder here and the stamp is not asserted. +""" +from __future__ import annotations + +import dataclasses +import logging +import os +import sys +import types +from datetime import UTC, datetime, timedelta +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from prxref import cli +from prxref.cli import main +from prxref.forges.base import detect_forge +from prxref.forges.replay import LocalDiffForge +from prxref.llm import ConfigError +from tests.test_orchestrator import FakeForge, _added_file_diff + +URL = "https://github.com/acme/widget/pull/7" +BASE = "d" * 40 +HEAD = "c" * 40 +APP_DIFF = _added_file_diff("src/app.py", 12) +FILE_DIFF = _added_file_diff("src/from_file.py", 5) +POSTING_OFF = "replay run: posting to the forge is disabled" +AS_OF = "2026-05-01T09:30:00Z" +AS_OF_UTC = datetime(2026, 5, 1, 9, 30, tzinfo=UTC) +EXCLUSIVE = "cannot be combined (give at most one of --as-of, --description-file and --no-description)" +NOT_ISO = "--as-of: must be an ISO-8601 time with a UTC offset, such as '2026-05-01T09:30:00Z', got {!r}" + + +def _install_fake_module(monkeypatch, fullname: str, **attrs) -> types.ModuleType: + mod = types.ModuleType(fullname) + for k, v in attrs.items(): + setattr(mod, k, v) + monkeypatch.setitem(sys.modules, fullname, mod) + return mod + + +@pytest.fixture +def runtime(monkeypatch): + """Doubles for everything past config. ``detect_forge`` is the real parser + behind a call counter, ``make_forge`` hands back ``rec.made``, and + ``_replay_forge`` records the ``_ReplayRequest`` it is handed and returns + ``rec.wrapped``, so these tests observe the request exactly as the + resolution step will receive it.""" + rec = types.SimpleNamespace( + orchestrate=[], llm=[], forge=[], detect=[], replayed=[], + made=FakeForge(diff=APP_DIFF), wrapped=MagicMock(name="ReplayForge"), + ) + + def fake_orchestrate_review(**kwargs): + rec.orchestrate.append(kwargs) + return {"verdict": "Approved", "findings_active": [], "findings_dropped": []} + + def fake_create_llm_client(cfg): + rec.llm.append(cfg) + return MagicMock(name="LLMClient") + + def spy_make_forge(ref): + rec.forge.append(ref) + return rec.made + + def spy_detect_forge(url): + rec.detect.append(url) + return detect_forge(url) + + def spy_replay_forge(forge, ref, replay): + rec.replayed.append(replay) + return rec.wrapped + + monkeypatch.setattr("prxref.cli.make_forge", spy_make_forge) + monkeypatch.setattr("prxref.cli.detect_forge", spy_detect_forge) + monkeypatch.setattr("prxref.cli._replay_forge", spy_replay_forge) + _install_fake_module( + monkeypatch, "prxref.llm_backends", create_llm_client=fake_create_llm_client, + ) + _install_fake_module( + monkeypatch, "prxref.orchestrator", orchestrate_review=fake_orchestrate_review, + ) + return rec + + +@pytest.fixture +def desc_file(tmp_path) -> str: + path = tmp_path / "description.md" + path.write_text("The original description.\n", encoding="utf-8") + return str(path) + + +@pytest.fixture +def diff_file(tmp_path) -> str: + path = tmp_path / "case.patch" + path.write_text(FILE_DIFF, encoding="utf-8") + return str(path) + + +def _assert_nothing_ran(rec) -> None: + """A configuration error exits before the URL is parsed and before the + forge or the LLM client exists.""" + assert rec.detect == [] + assert rec.forge == [] + assert rec.llm == [] + assert rec.orchestrate == [] + + +def _exit_2(runtime, capsys, args: list[str]) -> str: + assert main(["review", *args]) == 2 + err = capsys.readouterr().err + _assert_nothing_ran(runtime) + return err + + +class TestParser: + def test_the_flags_default_to_off(self): + args = cli._build_parser().parse_args(["review", "--pr-url", URL]) + assert args.as_of is None + assert args.description_file is None + assert args.no_description is False + + def test_the_flags_parse_as_typed(self): + args = cli._build_parser().parse_args([ + "review", "--as-of", AS_OF, "--description-file", "d.md", "--no-description", + ]) + assert (args.as_of, args.description_file, args.no_description) == (AS_OF, "d.md", True) + + +class TestEachFlagMakesAReplay: + """One flag alone is a replay, and the request carries exactly that flag.""" + + @pytest.mark.parametrize(("kwargs", "expected"), [ + pytest.param({"as_of": AS_OF}, cli._ReplayRequest(as_of=AS_OF_UTC), id="as-of"), + pytest.param( + {"description_file": "d.md"}, cli._ReplayRequest(description_file="d.md"), + id="description-file", + ), + pytest.param({"no_description": True}, cli._ReplayRequest(no_description=True), id="no-description"), + ]) + def test_the_resolver_returns_a_request_carrying_the_flag(self, kwargs, expected): + req = cli._resolve_replay(URL, **kwargs) + assert req == expected + assert req.description_text is None + + @pytest.mark.parametrize("kwargs", [ + {"description_file": "d.md"}, {"no_description": True}, + ], ids=["description-file", "no-description"]) + def test_description_flags_need_no_pr_url_with_a_diff_file(self, kwargs): + req = cli._resolve_replay(None, diff_file="case.patch", **kwargs) + assert req == cli._ReplayRequest(diff_file="case.patch", **kwargs) + + def test_the_resolver_stays_pure(self, monkeypatch, tmp_path): + monkeypatch.chdir(tmp_path) + req = cli._resolve_replay(URL, description_file="nowhere.md") + assert req == cli._ReplayRequest(description_file="nowhere.md") + assert list(tmp_path.iterdir()) == [] + + def test_as_of_rides_along_with_the_other_replay_flags(self): + req = cli._resolve_replay( + URL, base_sha=BASE, head_sha=HEAD, no_threads=True, as_of=AS_OF, + ) + assert req == cli._ReplayRequest( + base_sha=BASE, head_sha=HEAD, no_threads=True, as_of=AS_OF_UTC, + ) + + @pytest.mark.parametrize("args", [ + ["--as-of", AS_OF], ["--description-file", "{desc}"], ["--no-description"], + ], ids=["as-of", "description-file", "no-description"]) + def test_each_flag_alone_turns_posting_off_and_wraps_the_forge( + self, runtime, caplog, desc_file, args, + ): + args = [desc_file if a == "{desc}" else a for a in args] + with caplog.at_level(logging.INFO, logger="prxref"): + assert main(["review", "--pr-url", URL, *args]) == 0 + call = runtime.orchestrate[0] + assert call["post"] is False + assert POSTING_OFF in caplog.text + assert call["forge"] is runtime.wrapped + assert call["replay"] is not None + assert len(runtime.replayed) == 1 + + def test_the_as_of_request_reaches_the_resolution_step(self, runtime): + assert main(["review", "--pr-url", URL, "--as-of", "2026-05-01T11:30:00+02:00"]) == 0 + (req,) = runtime.replayed + assert req == cli._ReplayRequest(as_of=AS_OF_UTC) + + def test_the_no_description_request_reaches_the_resolution_step(self, runtime): + assert main(["review", "--pr-url", URL, "--no-description"]) == 0 + (req,) = runtime.replayed + assert req == cli._ReplayRequest(no_description=True) + + def test_the_description_file_is_read_into_the_request(self, runtime, tmp_path): + path = tmp_path / "pr.md" + path.write_bytes(b"\xef\xbb\xbfFixes the parser.\r\n\r\nSee #12.\r") + assert main(["review", "--pr-url", URL, "--description-file", str(path)]) == 0 + (req,) = runtime.replayed + assert req == cli._ReplayRequest( + description_file=str(path), description_text="Fixes the parser.\n\nSee #12.\n", + ) + + def test_the_description_file_is_kept_as_typed(self, runtime, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + Path("cases").mkdir() + Path("cases/pr.md").write_text("text\n", encoding="utf-8") + assert main(["review", "--pr-url", URL, "--description-file", "cases/pr.md"]) == 0 + assert runtime.replayed[0].description_file == "cases/pr.md" + + def test_a_blank_description_file_is_an_empty_description(self, runtime, tmp_path): + path = tmp_path / "blank.md" + path.write_bytes(b"") + assert main(["review", "--pr-url", URL, "--description-file", str(path)]) == 0 + assert runtime.replayed[0].description_text == "" + + @pytest.mark.parametrize("args", [ + ["--description-file", "{desc}"], ["--no-description"], + ], ids=["description-file", "no-description"]) + def test_a_lone_diff_file_accepts_the_description_flags( + self, runtime, diff_file, desc_file, args, + ): + args = [desc_file if a == "{desc}" else a for a in args] + assert main(["review", "--diff-file", diff_file, *args]) == 0 + call = runtime.orchestrate[0] + assert isinstance(call["forge"], LocalDiffForge) + assert call["post"] is False + assert call["replay"] is not None + assert runtime.detect == [] and runtime.forge == [] and runtime.replayed == [] + + +class TestMutualExclusion: + """Two or more of the three exit 2 naming every one given, whatever the argv order.""" + + @pytest.mark.parametrize(("args", "named"), [ + pytest.param(["--as-of", AS_OF, "--description-file", "{desc}"], "--as-of/--description-file", + id="as-of+description-file"), + pytest.param(["--as-of", AS_OF, "--no-description"], "--as-of/--no-description", + id="as-of+no-description"), + pytest.param(["--description-file", "{desc}", "--no-description"], "--description-file/--no-description", + id="description-file+no-description"), + pytest.param(["--no-description", "--description-file", "{desc}"], "--description-file/--no-description", + id="reversed-argv"), + pytest.param(["--no-description", "--description-file", "{desc}", "--as-of", AS_OF], + "--as-of/--description-file/--no-description", id="all-three"), + ]) + def test_a_pair_exits_2_naming_both(self, runtime, capsys, desc_file, args, named): + args = [desc_file if a == "{desc}" else a for a in args] + err = _exit_2(runtime, capsys, ["--pr-url", URL, *args]) + assert err == f"configuration error: {named}: {EXCLUSIVE}\n" + + def test_the_conflict_is_reported_before_a_bad_as_of(self, runtime, capsys): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--as-of", "yesterday", "--no-description"]) + assert err == f"configuration error: --as-of/--no-description: {EXCLUSIVE}\n" + + def test_the_conflict_is_reported_before_the_file_is_read(self, runtime, capsys, tmp_path): + missing = str(tmp_path / "absent.md") + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", missing, "--no-description"]) + assert err == f"configuration error: --description-file/--no-description: {EXCLUSIVE}\n" + + def test_the_resolver_refuses_a_pair_for_direct_callers(self): + with pytest.raises(ConfigError, match="^--as-of/--no-description: cannot be combined"): + cli._resolve_replay(URL, as_of=AS_OF, no_description=True) + + def test_an_empty_description_file_value_still_counts_as_given(self, runtime, capsys): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", "", "--no-description"]) + assert err == f"configuration error: --description-file/--no-description: {EXCLUSIVE}\n" + + +class TestAsOf: + """``--as-of`` must be an ISO-8601 time WITH a UTC offset; it is stored in UTC. + + A date alone and a time with no offset are refused (map-16: "a naive time + or bad ISO exits 2"), never read in the host's time zone or at an assumed + hour, so one replay command gives one cutoff on every machine. + """ + + @pytest.mark.parametrize("value", [ + "yesterday", "", "2026-05", "2026-05-01T24:00:00Z", "2026-05-01T09:30:00z", + " 2026-05-01T09:30:00Z", "2026-13-01T00:00:00Z", + ]) + def test_a_value_that_is_not_iso_8601_exits_2(self, runtime, capsys, value): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--as-of", value]) + assert err == f"configuration error: {NOT_ISO.format(value)}\n" + + @pytest.mark.parametrize(("value", "day"), [ + ("2026-05-01", "2026-05-01"), ("20260501", "2026-05-01"), ("2026-W18-5", "2026-05-01"), + ]) + def test_a_date_alone_exits_2(self, runtime, capsys, value, day): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--as-of", value]) + assert err == ( + f"configuration error: --as-of: {value!r} is a date without a time; give a time " + f"with a UTC offset, such as '{day}T00:00:00Z'\n" + ) + + @pytest.mark.parametrize("value", [ + "2026-05-01T09:30:00", "2026-05-01 09:30", "2026-05-01T09:30:00.250", + ]) + def test_a_time_without_an_offset_exits_2(self, runtime, capsys, value): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--as-of", value]) + assert err == ( + f"configuration error: --as-of: {value!r} has no UTC offset; add one, such as " + "'Z' for UTC or '+02:00'\n" + ) + + @pytest.mark.parametrize("value", ["0001-01-01T00:00:00+01:00", "9999-12-31T23:59:59-01:00"]) + def test_a_time_outside_the_utc_range_exits_2(self, runtime, capsys, value): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--as-of", value]) + assert err == f"configuration error: --as-of: {value!r} is out of range in UTC\n" + + @pytest.mark.parametrize(("value", "expected"), [ + (AS_OF, AS_OF_UTC), + ("2026-05-01T09:30Z", AS_OF_UTC), + ("2026-05-01T11:30:00+02:00", AS_OF_UTC), + ("2026-05-01T04:30:00-05:00", AS_OF_UTC), + ("2026-05-01 09:30:00+00:00", AS_OF_UTC), + ("2026-05-01T09:30:00.123456Z", AS_OF_UTC.replace(microsecond=123456)), + ]) + def test_an_offset_time_is_accepted_and_normalised_to_utc(self, value, expected): + req = cli._resolve_replay(URL, as_of=value) + assert req.as_of == expected + assert req.as_of.tzinfo is UTC + assert req.as_of.utcoffset() == timedelta(0) + + def test_the_parsed_time_is_the_same_instant_as_typed(self): + typed = datetime.fromisoformat("2026-05-01T23:30:00-03:00") + assert cli._resolve_replay(URL, as_of="2026-05-01T23:30:00-03:00").as_of == typed + assert cli._resolve_replay(URL, as_of="2026-05-01T23:30:00-03:00").as_of.day == 2 + + def test_as_of_needs_a_pr_url(self, runtime, capsys, diff_file): + err = _exit_2(runtime, capsys, ["--diff-file", diff_file, "--as-of", AS_OF]) + assert err == ( + "configuration error: --as-of: needs --pr-url (the description history is " + "read from that PR)\n" + ) + + def test_a_bad_value_is_reported_before_a_missing_pr_url(self, runtime, capsys, diff_file): + err = _exit_2(runtime, capsys, ["--diff-file", diff_file, "--as-of", "2026-05-01"]) + assert "is a date without a time" in err + + def test_a_bad_as_of_beats_an_unrecognized_url(self, runtime, capsys): + url = "https://example.com/not/a/pr" + assert detect_forge(url) is None + err = _exit_2(runtime, capsys, ["--pr-url", url, "--as-of", "2026-05-01T09:30:00"]) + assert "--as-of:" in err and "has no UTC offset" in err + + def test_the_sha_checks_still_come_first(self, runtime, capsys): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--base-sha", BASE, "--as-of", "nope"]) + assert err == ( + "configuration error: --base-sha/--head-sha: must be given together " + "(got only --base-sha)\n" + ) + + +class TestDescriptionFile: + """The file is read like the rules file; every failure exits 2 naming the flag.""" + + def test_a_missing_file_exits_2(self, runtime, capsys, tmp_path): + missing = str(tmp_path / "absent.md") + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", missing]) + assert err == ( + f"configuration error: --description-file: cannot read {missing!r}: " + "No such file or directory\n" + ) + + def test_an_empty_path_exits_2(self, runtime, capsys): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", ""]) + assert err == "configuration error: --description-file: cannot read '': No such file or directory\n" + + def test_a_directory_exits_2(self, runtime, capsys, tmp_path): + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", str(tmp_path)]) + assert err == f"configuration error: --description-file: cannot read {str(tmp_path)!r}: Is a directory\n" + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="needs os.mkfifo") + def test_a_fifo_is_refused_without_blocking(self, runtime, capsys, tmp_path): + fifo = tmp_path / "pipe.md" + os.mkfifo(fifo) + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", str(fifo)]) + assert err == f"configuration error: --description-file: cannot read {str(fifo)!r}: not a regular file\n" + + @pytest.mark.skipif(os.name == "nt" or os.geteuid() == 0, reason="needs POSIX permissions as non-root") + def test_an_unreadable_file_exits_2(self, runtime, capsys, tmp_path): + path = tmp_path / "locked.md" + path.write_text("secret\n", encoding="utf-8") + path.chmod(0) + try: + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", str(path)]) + finally: + path.chmod(0o600) + assert err == f"configuration error: --description-file: cannot read {str(path)!r}: Permission denied\n" + + @pytest.mark.parametrize(("raw", "byte"), [ + (b"caf\xe9 au lait\n", 3), + (b"\xef\xbb\xbfcaf\xe9 au lait\n", 6), + ], ids=["latin-1", "latin-1-after-bom"]) + def test_a_file_that_is_not_utf_8_exits_2(self, runtime, capsys, tmp_path, raw, byte): + path = tmp_path / "latin1.md" + path.write_bytes(raw) + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", str(path)]) + assert err == ( + f"configuration error: --description-file: {str(path)!r} is not UTF-8 text " + f"(invalid continuation byte at byte {byte})\n" + ) + + def test_a_file_with_nul_bytes_exits_2(self, runtime, capsys, tmp_path): + path = tmp_path / "binary.md" + path.write_bytes(b"PNG\x00\x00\x01 header\n") + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", str(path)]) + assert err == ( + f"configuration error: --description-file: {str(path)!r} contains NUL bytes; " + "expected Markdown or plain text\n" + ) + + def test_a_symlink_out_of_the_working_directory_exits_2(self, runtime, capsys, tmp_path, monkeypatch): + work = tmp_path / "checkout" + work.mkdir() + outside = tmp_path / "private.md" + outside.write_text("not for the prompt\n", encoding="utf-8") + (work / "PR.md").symlink_to(outside) + monkeypatch.chdir(work) + err = _exit_2(runtime, capsys, ["--pr-url", URL, "--description-file", "PR.md"]) + assert err == ( + "configuration error: --description-file: cannot read 'PR.md': " + "resolves outside the working directory\n" + ) + + def test_a_symlink_inside_the_working_directory_is_read(self, runtime, tmp_path, monkeypatch): + work = tmp_path / "checkout" + (work / "docs").mkdir(parents=True) + (work / "docs" / "real.md").write_text("inside\n", encoding="utf-8") + (work / "PR.md").symlink_to(work / "docs" / "real.md") + monkeypatch.chdir(work) + assert main(["review", "--pr-url", URL, "--description-file", "PR.md"]) == 0 + assert runtime.replayed[0].description_text == "inside\n" + + def test_an_absolute_path_outside_the_working_directory_is_read( + self, runtime, tmp_path, monkeypatch, + ): + work = tmp_path / "checkout" + work.mkdir() + outside = tmp_path / "ci-temp" / "description.md" + outside.parent.mkdir() + outside.write_text("from the CI step\n", encoding="utf-8") + monkeypatch.chdir(work) + assert main(["review", "--pr-url", URL, "--description-file", str(outside)]) == 0 + assert runtime.replayed[0].description_text == "from the CI step\n" + + def test_the_file_is_read_before_the_url_is_parsed(self, runtime, capsys, tmp_path): + url = "https://example.com/not/a/pr" + assert detect_forge(url) is None + err = _exit_2(runtime, capsys, ["--pr-url", url, "--description-file", str(tmp_path / "absent.md")]) + assert err.startswith("configuration error: --description-file: cannot read ") + + def test_the_diff_file_is_reported_first(self, runtime, capsys, tmp_path): + err = _exit_2(runtime, capsys, [ + "--diff-file", str(tmp_path / "absent.patch"), + "--description-file", str(tmp_path / "absent.md"), + ]) + assert err.startswith("configuration error: --diff-file: cannot read ") + + def test_a_lone_diff_file_run_reads_it_too(self, runtime, capsys, diff_file, tmp_path): + err = _exit_2(runtime, capsys, [ + "--diff-file", diff_file, "--description-file", str(tmp_path / "absent.md"), + ]) + assert err.startswith("configuration error: --description-file: cannot read ") + + def test_the_reader_returns_the_decoded_text(self, tmp_path): + path = tmp_path / "pr.md" + path.write_bytes(b"line one\r\nline two\rline three\n") + assert cli._read_description_file(str(path)) == "line one\nline two\nline three\n" + + +class TestNonReplayRunsAreUnchanged: + """Without the new flags, the resolver returns what it returned in 0.14.""" + + OLD_FIELDS = ["base_sha", "head_sha", "no_threads", "diff_file", "diff_text"] + + def test_no_replay_flag_is_still_none(self): + assert cli._resolve_replay(URL) is None + assert cli._resolve_replay( + URL, as_of=None, description_file=None, no_description=False, + ) is None + + @pytest.mark.parametrize(("kwargs", "expected"), [ + ({"no_threads": True}, cli._ReplayRequest(None, None, True, None)), + ({"base_sha": BASE, "head_sha": HEAD}, cli._ReplayRequest(BASE, HEAD, False, None)), + ({"diff_file": "case.patch"}, cli._ReplayRequest(None, None, False, "case.patch")), + ], ids=["no-threads", "pinned", "diff-file"]) + def test_an_old_replay_request_is_the_same_request(self, kwargs, expected): + req = cli._resolve_replay(URL, **kwargs) + assert req == expected + assert (req.as_of, req.description_file, req.description_text, req.no_description) == ( + None, None, None, False, + ) + + def test_the_old_fields_keep_their_positions(self): + names = [f.name for f in dataclasses.fields(cli._ReplayRequest)] + assert names[: len(self.OLD_FIELDS)] == self.OLD_FIELDS + + def test_a_normal_run_passes_replay_none_and_posts(self, runtime, caplog): + with caplog.at_level(logging.INFO, logger="prxref"): + assert main(["review", "--pr-url", URL]) == 0 + call = runtime.orchestrate[0] + assert call["replay"] is None + assert call["post"] is True + assert call["forge"] is runtime.made + assert runtime.replayed == [] + assert POSTING_OFF not in caplog.text + + def test_a_caller_that_passes_no_new_kwarg_is_not_a_replay(self, runtime): + cli._run_review(URL, post=False) + assert runtime.orchestrate[0]["replay"] is None + assert runtime.replayed == [] From e733301d701a8eb40f6ff5ab11969a515bfdd743 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Thu, 24 Sep 2026 10:54:59 -0700 Subject: [PATCH 18/64] feat(eval): judge client, per-case judge call, grade cache, cost and stamp (#14 T6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/prxref/eval_judge.py, the live half of `prxref eval score`'s judge: - build_judge_client(cfg, judge_model) builds the judge from the review's own config with only llm_models replaced, through the same prxref.llm_backends.create_llm_client lookup the CLI uses. An empty judge model (which the factory would silently replace with the reviewer's PRXREF_LLM_MODELS chain) or a comma chain is a ConfigError naming --judge-model. - judge_case(...) makes at most one single-shot json_mode call per case, following prxref.judge's call sequence. A client exception or a response parse_judge_response rejects is a judge error with no grades, never a score of 0, and is never cached. - Grades are cached as one JSON file per judge_cache_key, written atomically (temp file, then os.replace). A hit makes no call and re-parses the stored response. A corrupt entry is a miss with a WARNING. - The judge cost goes through costs.run_cost with the price table, so an unknown cost stays None. judge_cost totals a scoring run. - check_self_judging warns when the judge is one of the reviewer's sampling.models. judge_stamp records the judge sampling (the same shape as the run record), JUDGE_PROMPT_VERSION, the template sha256 and self_judged. The eval module does not import prxref.cli. Tests use a stub factory keyed on cfg["llm_models"] and make no network calls. 🤖 Authored with Claude Code — Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/eval_judge.py | 409 ++++++++++++++++++++++++++++++++++++ tests/test_eval_judge.py | 442 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 851 insertions(+) create mode 100644 src/prxref/eval_judge.py create mode 100644 tests/test_eval_judge.py diff --git a/src/prxref/eval_judge.py b/src/prxref/eval_judge.py new file mode 100644 index 00000000..84077352 --- /dev/null +++ b/src/prxref/eval_judge.py @@ -0,0 +1,409 @@ +"""The judge side of ``prxref eval score``: client, one call per case, cache, cost, stamp. + +:mod:`prxref.judge` holds the pure half of a judge grading (prompt, parser, +per-ref cap, cache key). This module runs it against a live client: + +1. :func:`build_judge_client` builds the judge from the review's own config + with only ``llm_models`` replaced, through the same + ``prxref.llm_backends.create_llm_client`` factory the CLI looks up at call + time. The judge therefore shares the review's backend, base URL and key + path, and one test stub keyed on ``cfg["llm_models"]`` serves both. +2. :func:`judge_case` grades one case with at most one single-shot + ``json_mode`` call. A cached grading makes no call. A response that + :func:`prxref.judge.parse_judge_response` rejects, and any exception the + client raises, make the case a judge error (:attr:`JudgeOutcome.error`), + never a score of 0. +3. The cache is one JSON file per :func:`prxref.judge.judge_cache_key` under + a directory the caller chooses; :func:`judge_cache_dir` names the + conventional one, ``/