From 3ccc52f1b961425b61c9372900531d8c84b7fba6 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:04:50 -0700 Subject: [PATCH 01/61] feat: spec-grounded review (rebased onto 0.13.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squash of the two rebased spec-branch patches (REB 0001 + 0002) onto the 0.13.0 release, so the foundation lands as one bisectable commit. Both original commits were authored in session 68f0a704-ce12-4ecc-878a-f432b0aac171; their bodies follow. feat: spec-grounded review (WIP, pre-rebase snapshot) Adds a second review axis: the operator supplies scope/intent via repeatable --spec / PRXREF_SPEC_SOURCES (web URLs, local files/dirs, Jira ticket URLs), which a new src/prxref/specs.py fetches, prunes to a deterministic diff-relevant constraint digest, and injects into the existing worker and systemic prompts. Violations surface as a new πŸ” `spec` severity ranked below warning, non-blocking on fetch failure, with a golden eval dataset under tests/evals/. fix: post-rebase repairs for spec-grounded review on 0.12.0 Teach the release-shape-foldin test doubles the new spec_digest keyword, and add `spec` to the severity vocabulary row in docs/quality.md, which landed on main after the feature branch was cut. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- .env.example | 25 + docs/env-vars.md | 18 +- docs/quality.md | 2 +- docs/spec-grounded-review.md | 699 ++++++++++++++++++ src/prxref/cli.py | 33 +- src/prxref/config.py | 48 +- src/prxref/formatter.py | 46 +- src/prxref/orchestrator.py | 149 +++- src/prxref/prompts/summary.md | 4 +- src/prxref/prompts/systemic.md | 7 +- src/prxref/prompts/worker.md | 9 + src/prxref/quality.py | 21 +- src/prxref/reviewer.py | 25 + src/prxref/specs.py | 614 +++++++++++++++ tests/evals/README.md | 96 +++ tests/evals/__init__.py | 0 .../case-001-mcp-protocol-upgrade/diff.patch | 32 + .../docs/mcp-client-spec.md | 35 + .../expected.json | 18 + .../case-001-mcp-protocol-upgrade/meta.json | 13 + .../case-001-mcp-protocol-upgrade/ticket.md | 26 + .../case-002-session-token-logging/diff.patch | 49 ++ .../docs/session-security.md | 20 + .../expected.json | 26 + .../case-002-session-token-logging/meta.json | 18 + .../case-002-session-token-logging/ticket.md | 24 + .../case-003-config-schema-pin/diff.patch | 27 + .../docs/config-schema.md | 22 + .../case-003-config-schema-pin/expected.json | 26 + .../case-003-config-schema-pin/meta.json | 18 + .../case-003-config-schema-pin/ticket.md | 26 + tests/evals/test_evals.py | 164 ++++ tests/test_cli.py | 75 ++ tests/test_config.py | 139 ++++ tests/test_formatter.py | 23 +- tests/test_orchestrator.py | 215 +++++- tests/test_quality.py | 64 ++ tests/test_reviewer.py | 65 ++ tests/test_specs.py | 426 +++++++++++ 39 files changed, 3277 insertions(+), 70 deletions(-) create mode 100644 docs/spec-grounded-review.md create mode 100644 src/prxref/specs.py create mode 100644 tests/evals/README.md create mode 100644 tests/evals/__init__.py create mode 100644 tests/evals/case-001-mcp-protocol-upgrade/diff.patch create mode 100644 tests/evals/case-001-mcp-protocol-upgrade/docs/mcp-client-spec.md create mode 100644 tests/evals/case-001-mcp-protocol-upgrade/expected.json create mode 100644 tests/evals/case-001-mcp-protocol-upgrade/meta.json create mode 100644 tests/evals/case-001-mcp-protocol-upgrade/ticket.md create mode 100644 tests/evals/case-002-session-token-logging/diff.patch create mode 100644 tests/evals/case-002-session-token-logging/docs/session-security.md create mode 100644 tests/evals/case-002-session-token-logging/expected.json create mode 100644 tests/evals/case-002-session-token-logging/meta.json create mode 100644 tests/evals/case-002-session-token-logging/ticket.md create mode 100644 tests/evals/case-003-config-schema-pin/diff.patch create mode 100644 tests/evals/case-003-config-schema-pin/docs/config-schema.md create mode 100644 tests/evals/case-003-config-schema-pin/expected.json create mode 100644 tests/evals/case-003-config-schema-pin/meta.json create mode 100644 tests/evals/case-003-config-schema-pin/ticket.md create mode 100644 tests/evals/test_evals.py create mode 100644 tests/test_specs.py diff --git a/.env.example b/.env.example index 8eb4fce3..c834797a 100644 --- a/.env.example +++ b/.env.example @@ -128,6 +128,31 @@ PRXREF_MAX_CHUNKS=8 # total-failure notice always names its status regardless. # PRXREF_POST_VERDICT=1 +# Spec/ticket sources to review against: web URLs, local file or directory +# paths, or Jira ticket URLs. Comma- or whitespace-separated when set here; +# repeatable `--spec` flags replace this list entirely (no merge). +# PRXREF_SPEC_SOURCES= + +# Raw fetched characters kept per spec source before pruning. Must be > 0. +# Truncation at the cap is announced in the fetched text, never silent. +# PRXREF_SPEC_MAX_CHARS=120000 + +# Token budget for the spec digest injected into worker prompts. Must be > 0. +# PRXREF_SPEC_DIGEST_TOKENS=3000 + +# Jira REST base for ticket fetches, overriding a ticket URL's own host (a +# self-hosted board often sits behind a different REST host than its browse +# URL). Empty uses the ticket URL's own host. +# PRXREF_JIRA_BASE_URL= + +# Jira account email for HTTP basic auth on ticket fetches. Leave empty for +# anonymous access to public boards. Missing credentials are a fetch failure +# (the review proceeds un-grounded with a note), never a configuration error. +# PRXREF_JIRA_EMAIL= + +# Jira API token paired with PRXREF_JIRA_EMAIL for HTTP basic auth. +# PRXREF_JIRA_API_TOKEN= + # ------------------------------------------------------------------------------ # Per-Forge Authentication Tokens # ------------------------------------------------------------------------------ diff --git a/docs/env-vars.md b/docs/env-vars.md index 3eda76da..8bf35c9e 100644 --- a/docs/env-vars.md +++ b/docs/env-vars.md @@ -33,6 +33,9 @@ Configuration is loaded from built-in defaults, overridden by environment variab | `PRXREF_FAIL_ON` | `never` | Exit-code policy for `prxref review`. `never` (the default) keeps the advisory contract β€” the exit code never reflects findings. `error` exits `1` when the completed review carries an active error-severity finding; `any` exits `1` on any active finding. Under either value a review that fails to complete also exits `1`, so a gating lane cannot read a broken run as green. The webhook daemon has no exit code and is unaffected. See [Bad Configuration Is the Only Thing That Fails a Build](#bad-configuration-is-the-only-thing-that-fails-a-build). | | `PRXREF_POST_MODE` | `summary+inline` | What gets posted to the forge: `summary+inline` (the summary comment, then inline comments only if the summary landed), `summary` (the summary comment only β€” inline comments are never posted), or `inline` (inline comments only β€” no summary is posted on any path, including the error notice). Any other value raises `ConfigError` and `prxref review` exits `2`. A dry run posts nothing in any mode. | | `PRXREF_POST_VERDICT` | `True` | Set to the literal `1` to keep the verdict stamp in the posted summary; any other value renders the summary without it (no `Approved` / `Request-Changes` heading), keeping the findings, counts, and attribution. The computed verdict printed to stdout and the total-failure notice are unaffected. | +| `PRXREF_SPEC_SOURCES` | *(empty)* | Spec/ticket sources to review against: public web URLs, local file or directory paths, or Jira ticket URLs. Comma- **and** whitespace-separated when set through the environment. The repeatable `--spec` flag replaces this list entirely when given β€” there is no merge. | +| `PRXREF_SPEC_MAX_CHARS` | `120000` | Raw fetched characters kept per spec source (after decoding), before pruning. Must be **greater than 0**. Truncation at the cap is announced in the fetched text, never silent. | +| `PRXREF_SPEC_DIGEST_TOKENS` | `3000` | Token budget for the final spec-constraint digest injected into worker prompts (estimated at 4 characters per token, like the systemic digest). Must be **greater than 0**. | ### Per-Forge Authentication @@ -48,6 +51,14 @@ Configuration is loaded from built-in defaults, overridden by environment variab | `PRXREF_GITHUB_ENTERPRISE_TOKEN` | *(empty)* | GitHub Enterprise token for custom/self-hosted GitHub Enterprise Server domains. Falls back to `PRXREF_GITHUB_TOKEN` if unset. | | `PRXREF_GITLAB_TOKEN` | *(empty)* | GitLab Personal, Project, or Group Access Token (sent via `PRIVATE-TOKEN` header) for `gitlab.com` or self-hosted GitLab. | +### Spec Sources / Jira + +| Variable | Default | Purpose | +|---|---|---| +| `PRXREF_JIRA_BASE_URL` | *(empty)* | REST base for Jira ticket fetches, overriding a ticket URL's own host β€” a self-hosted board often sits behind a different REST host than its browse URL. Empty resolves the ticket URL's own `scheme://host`. | +| `PRXREF_JIRA_EMAIL` | *(empty)* | Jira account email for HTTP basic authentication when fetching a ticket, used together with `PRXREF_JIRA_API_TOKEN`. When either is empty the fetch is anonymous, which public boards accept. | +| `PRXREF_JIRA_API_TOKEN` | *(empty)* | Jira API token for HTTP basic authentication, paired with `PRXREF_JIRA_EMAIL`. Missing credentials are a fetch failure (the review proceeds un-grounded with a note naming these variables), not a configuration error. | + ### Webhook Receiver | Variable | Default | Purpose | @@ -126,10 +137,11 @@ The two knobs above are the only configuration that touches the filtering. The e ## Environment Cross-Check & Defaults -The tables above define all **36** configuration keys in `src/prxref/config.py` (`_DEFAULTS`), and every one of them appears in `.env.example`: +The tables above define all **42** configuration keys in `src/prxref/config.py` (`_DEFAULTS`), and every one of them appears in `.env.example`: -- **LLM / Pipeline (23):** `PRXREF_LLM_BACKEND`, `PRXREF_LLM_BASE_URL`, `PRXREF_LLM_API_KEY`, `PRXREF_LLM_MODELS`, `PRXREF_LLM_REASONING_EFFORT`, `PRXREF_LLM_MAX_TOKENS`, `PRXREF_LLM_TIMEOUT`, `PRXREF_LLM_TEMPERATURE`, `PRXREF_LLM_SEED`, `PRXREF_CONFIDENCE_FLOOR`, `PRXREF_MAX_ERROR_FINDINGS`, `PRXREF_MAX_CHUNKS`, `PRXREF_CHUNK_TOKEN_BUDGET`, `PRXREF_CHUNK_MAX_FILES`, `PRXREF_CHUNK_CONTEXT_LINES`, `PRXREF_MAX_WORKERS`, `PRXREF_MAX_INLINE_COMMENTS`, `PRXREF_TRACE_FILE`, `PRXREF_TRACE_DIR`, `PRXREF_DRY_RUN`, `PRXREF_FAIL_ON`, `PRXREF_POST_MODE`, `PRXREF_POST_VERDICT` +- **LLM / Pipeline (26):** `PRXREF_LLM_BACKEND`, `PRXREF_LLM_BASE_URL`, `PRXREF_LLM_API_KEY`, `PRXREF_LLM_MODELS`, `PRXREF_LLM_REASONING_EFFORT`, `PRXREF_LLM_MAX_TOKENS`, `PRXREF_LLM_TIMEOUT`, `PRXREF_LLM_TEMPERATURE`, `PRXREF_LLM_SEED`, `PRXREF_CONFIDENCE_FLOOR`, `PRXREF_MAX_ERROR_FINDINGS`, `PRXREF_MAX_CHUNKS`, `PRXREF_CHUNK_TOKEN_BUDGET`, `PRXREF_CHUNK_MAX_FILES`, `PRXREF_CHUNK_CONTEXT_LINES`, `PRXREF_MAX_WORKERS`, `PRXREF_MAX_INLINE_COMMENTS`, `PRXREF_TRACE_FILE`, `PRXREF_TRACE_DIR`, `PRXREF_DRY_RUN`, `PRXREF_FAIL_ON`, `PRXREF_POST_MODE`, `PRXREF_POST_VERDICT`, `PRXREF_SPEC_SOURCES`, `PRXREF_SPEC_MAX_CHARS`, `PRXREF_SPEC_DIGEST_TOKENS` - **Per-Forge Auth (9):** `PRXREF_BITBUCKET_TOKEN`, `PRXREF_BITBUCKET_USER`, `PRXREF_BITBUCKET_APP_PASSWORD`, `PRXREF_BITBUCKET_SERVER_TOKEN`, `PRXREF_BITBUCKET_SERVER_USER`, `PRXREF_BITBUCKET_SERVER_PASSWORD`, `PRXREF_GITHUB_TOKEN`, `PRXREF_GITHUB_ENTERPRISE_TOKEN`, `PRXREF_GITLAB_TOKEN` +- **Spec Sources / Jira (3):** `PRXREF_JIRA_BASE_URL`, `PRXREF_JIRA_EMAIL`, `PRXREF_JIRA_API_TOKEN` - **Webhooks (4):** `PRXREF_BITBUCKET_WEBHOOK_SECRET`, `PRXREF_GITHUB_WEBHOOK_SECRET`, `PRXREF_GITLAB_WEBHOOK_SECRET`, `PRXREF_ALLOW_UNSIGNED` -*(36 configuration keys, plus one deprecated alias β€” `PRXREF_MAX_ERRORS` for `PRXREF_MAX_ERROR_FINDINGS` β€” for 37 accepted variable names.)* +*(42 configuration keys, plus one deprecated alias β€” `PRXREF_MAX_ERRORS` for `PRXREF_MAX_ERROR_FINDINGS` β€” for 43 accepted variable names.)* diff --git a/docs/quality.md b/docs/quality.md index b469f046..aea8daf6 100644 --- a/docs/quality.md +++ b/docs/quality.md @@ -64,7 +64,7 @@ its own findings against prxref's own stale comments and then delete them. | `settled in thread: ` | `apply_settled_thread_suppression` | A thread on the same path already argued this subject out. A **resolved** thread still settles it β€” resolution is a decision, not an expiry. | | `claims removal of a path present in the post-image: ` | `apply_removal_claim_check` | A removal verb governs this path, and every path the claim names is still present after the PR lands. | | `hedged: ""` | `apply_hedge_gate` | The finding's own text conditions the defect on something the model never established. | -| `invalid severity: ''` | `apply_quality_gate` | Severity outside {`error`, `warning`, `outofscope`}. | +| `invalid severity: ''` | `apply_quality_gate` | Severity outside {`error`, `warning`, `spec`, `outofscope`}. | | `confidence below floor ` | `apply_quality_gate` | Below `PRXREF_CONFIDENCE_FLOOR`. | | `error cap exceeded (max )` | `apply_quality_gate` | Beyond `PRXREF_MAX_ERROR_FINDINGS`. Ties break on finding content, not arrival order, so the cap is reproducible. | | `duplicate of chunk finding` | `apply_sweep_dedup` | A whole-diff sweep finding restates a chunk finding that already survived the gate. | diff --git a/docs/spec-grounded-review.md b/docs/spec-grounded-review.md new file mode 100644 index 00000000..b8f2b841 --- /dev/null +++ b/docs/spec-grounded-review.md @@ -0,0 +1,699 @@ +# Spec-Grounded Review β€” Design + +Status: design for implementation (v1). Owner: prxref. +Grounded against the tree at 0.11.x; `file:line` citations refer to the current source. + +## 0. What this feature is + +Today every prxref seat reviews the diff against generic bug classes only +(`prompts/worker.md:1-23`). Spec-grounded review adds a second axis: the +operator supplies scope/intent β€” a ticket plus a docs/spec corpus β€” and the +reviewer must additionally catch **violations of that spec**, emitted as a new +`spec` severity (πŸ”), distinct from `error` πŸŸ₯ / `warning` 🟧 / `outofscope` 🟦. + +Motivating case: a repo adapting to the MCP spec version `2026-07-28` is +reviewed against the spec's client/server best-practices docs, so "the diff +sends the protocol-version header the spec forbids" surfaces as a spec finding +rather than nothing. + +### Locked decisions (user-approved, not to be re-litigated) + +1. **Input**: repeatable `--spec ` flag + `PRXREF_SPEC_SOURCES` + (comma/space separated). Accepts public web URLs, local file/dir paths, and + Jira ticket URLs. +2. **Docs handling**: fetch + prune. Extract headings/constraints, prune to + diff-relevant slices, inject the result. Never post full docs raw. No + persistent graph index in v1 (Β§10). +3. **Jira auth**: REST + HTTP basic auth via `PRXREF_JIRA_BASE_URL` / + `PRXREF_JIRA_EMAIL` / `PRXREF_JIRA_API_TOKEN`, plus anonymous access for + public boards. (MCP noted as an alternative fetch path, not designed here.) +4. **Severity**: new `spec` severity, symbol πŸ”, own ordering and fallback + rules (Β§5). + +### Non-negotiable posture carried over + +- stdlib + `requests` only in core; the spec fetcher uses the same zero-extra + dependency stack as the forges (`forges/github.py:10`). +- Non-blocking: any spec-fetch failure degrades to "reviewed un-grounded, with + a note in the summary". Exit 2 remains reserved for config errors + (`cli.py:8-15`). +- Every posted comment keeps model attribution (`forges/base.py:72`). + +--- + +## 1. Input plumbing + +### 1.1 CLI flag + +Add to the `review` subparser in `_build_parser` (`cli.py:61-86`), beside +`--max-chunks`: + +```python +rev.add_argument( + "--spec", + action="append", + default=None, + metavar="URL_OR_PATH", + help="spec/ticket source to review against; repeatable (PRXREF_SPEC_SOURCES otherwise)", +) +``` + +`_cmd_review` passes `spec_sources=args.spec` into `_run_review`, which +forwards it as a `load_config` override exactly the way `max_chunks` does +today (`cli.py:175-180`), with +`source_labels={"spec_sources": "--spec"}` so a malformed value is reported as +the flag the operator typed. This keeps `--spec` on the identical +range-checked path as the env var and preserves the existing exit-2 contract +(`cli.py:271-284`). + +Precedence: **`--spec` replaces `PRXREF_SPEC_SOURCES` when given; there is no merge.** +This matches the documented precedence "built-in defaults < environment < +overrides" (`config.py:104`). + +The webhook daemon inherits the feature for free: `_webhook_handler` calls the +same `_run_review` (`cli.py:220-231`), so `PRXREF_SPEC_SOURCES` in the daemon's +environment grounds every webhook-triggered review. + +### 1.2 Config keys + +Six new keys, all in `config._DEFAULTS` (`config.py:136-180`): + +| key | env var | type | default | +|---|---|---|---| +| `spec_sources` | `PRXREF_SPEC_SOURCES` | list (comma **and** whitespace separated) | `[]` | +| `spec_max_chars` | `PRXREF_SPEC_MAX_CHARS` | int | `120000` | +| `spec_digest_tokens` | `PRXREF_SPEC_DIGEST_TOKENS` | int | `3000` | +| `jira_base_url` | `PRXREF_JIRA_BASE_URL` | str | `""` | +| `jira_email` | `PRXREF_JIRA_EMAIL` | str | `""` | +| `jira_api_token` | `PRXREF_JIRA_API_TOKEN` | str | `""` | + +Table placement: + +- `_LIST_KEYS` (`config.py:189`) += `spec_sources`. The existing list + coercion (`_coerce_env`, `config.py:295-296`) splits on commas only; widen it + to `re.split(r"[,\s]+")` so `PRXREF_SPEC_SOURCES` accepts space-separated values as + locked. The only other list key is `llm_models`, whose entries can never + contain spaces, so the widened split is a no-op for it. +- `_INT_KEYS` (`config.py:182-186`) += `spec_max_chars`, `spec_digest_tokens`; + `_RANGES` (`config.py:252-264`) += `_Range(0)` for both (positive, + unbounded above, like every other size knob β€” `config.py:206-244`). +- `jira_*` are plain string keys: no int/float/choice table entry, same as + every token key today. + +Semantics of the two ints: `spec_max_chars` bounds **raw fetched bytes per +source** (post-decode); `spec_digest_tokens` bounds the **final digest text** +injected into prompts, converted at the same 4-chars-per-token estimate the +systemic digest uses (`systemic.py:70`). + +### 1.3 What is deliberately NOT a config error + +Missing/partial Jira credentials are **not** `ConfigError`. `load_config` +validates values, not combinations; a ticket URL with no credentials takes the +fetch-failure path (Β§2.5) and the review proceeds un-grounded with a note. The +one thing the CLI will refuse is a malformed `--spec`/`PRXREF_SPEC_SOURCES` value β€” +but since the value is an opaque string list, there is nothing to range-check; +v1 adds no URL validation at config time. (Judgment call J2, Β§11.) + +--- + +## 2. Fetch layer: new module `src/prxref/specs.py` + +One module, mirroring the shape of `systemic.py`: deterministic helpers, data +in / text out, no LLM in the loop, docstrings on public API, no inline +comments. + +### 2.1 Data shape + +```python +@dataclass +class SpecSource: + origin: str # the string the operator supplied, verbatim + kind: str # "file" | "dir" | "url" | "jira" + text: str # extracted plain text; "" when failed + error: str # "" on success, human-readable reason otherwise +``` + +```python +@dataclass +class TicketRef: + base_url: str # resolved REST base (env override or URL host) + key: str # e.g. "PROJ-123" + url: str # original ticket URL +``` + +### 2.2 Dispatch + +`parse_ticket_url(url: str) -> TicketRef | None` recognizes: + +- `{host}/browse/{KEY}-{n}` (Jira Cloud + Server classic), +- `{host}/rest/api/{2|3}/issue/{KEY}-{n}` (raw REST links), +- `{host}/jira/software/c/projects/{KEY}/issues/{KEY}-{n}` (Cloud new UI). + +Resolution: `PRXREF_JIRA_BASE_URL` wins when set (self-hosted boards often sit +behind a different REST host than the browse URL); otherwise the ticket URL's +own `scheme://host`. + +`fetch_specs(sources, *, max_chars, jira_base_url, jira_email, jira_api_token, session=None) -> list[SpecSource]` +dispatches per source, in the order given: + +- `http://`/`https://` prefix + ticket match β†’ Jira REST (Β§2.4). +- `http(s)` otherwise β†’ plain GET (Β§2.3). +- existing filesystem path β†’ file or directory (Β§2.3). +- anything else β†’ `SpecSource(error="not a URL or path: …")` β€” a per-source + failure, never an abort. + +### 2.3 Web + local fetching + +- **HTTP**: one `requests.Session` built like the forge sessions β€” + `LoggingRetry(total=3, backoff_factor=1, status_forcelist=[429,500,502,503,504], + allowed_methods=frozenset({"GET","HEAD","OPTIONS"}))` β€” reusing the read-only + retry policy verbatim from `forges/github.py:57-86` (same duplicate-POST + reasoning; the spec fetcher only ever GETs). Timeout + `SPEC_FETCH_TIMEOUT_S = 15` (module constant, not a config key in v1 β€” + Β§10). Content must arrive as `text/*`, `application/json`, or a text-like + markdown/html type; other content types fail that source. +- **Size cap**: stream via `iter_content`, decode incrementally, stop at + `max_chars` and append an explicit `[source truncated at N chars]` marker β€” + truncation is announced, never silent (the same doctrine as + `systemic.TRUNCATION_MARKER`, `systemic.py:72-74`). +- **HTML**: strip tags with `html.parser` (stdlib) to plain text before + extraction β€” spec pages are HTML more often than markdown. +- **Local path**: read UTF-8 (errors β†’ per-source failure), same + `max_chars` cap. A directory reads `*.md`, `*.markdown`, `*.txt`, `*.adoc` + in sorted filename order, capped at the first 20 files + (`SPEC_DIR_MAX_FILES = 20`), each through the same cap. + +### 2.4 Jira REST (primary path) + +``` +GET {base}/rest/api/2/issue/{key}?fields=summary,description,issuetype,labels +Authorization: Basic base64(email:api_token) +``` + +- Credentials present (both `jira_email` and `jira_api_token` non-empty) β†’ + basic auth; absent β†’ anonymous request (public boards work with no config). +- Render the ticket as plain text: `Summary: …`, `Type: …`, `Labels: …`, + then the description body verbatim (Atlassian wiki-markup or ADF-plain β€” + v1 passes the text through; it is prose, and the extractor Β§3.2 reads prose + fine). +- HTTP 401/403 without credentials β†’ error text that names the three + `PRXREF_JIRA_*` variables, because that is the fix an operator can act on. + The reason may end up posted (Β§6.4), so it is written to survive + `redact_for_post` (`orchestrator.py:202-235`): variable NAMES only, never + values, host kept (a host is not a credential and `_URL_RE` will strip the + original ticket URL if it reappears). +- MCP as an alternative fetch path is noted here as a future option only + (Β§10); REST basic-auth is the designed primary per the locked decisions. + +### 2.5 Failure doctrine + +`fetch_specs` never raises. Every exception inside a source becomes that +source's `error` string. A run where **all** sources failed is not an error +run: the pipeline behaves exactly like a run with no specs, plus a summary +note listing the failures (Β§6.4). This is the same shape as `list_threads` +best-effort failure (`orchestrator.py:447-451`). + +--- + +## 3. Prune/digest: constraint extraction and diff-relevance pruning + +### 3.1 Goal + +Each fetched source becomes a compact **spec constraint digest**: a bounded, +deterministic text the worker prompts can hold, containing the constraints +that could plausibly be violated by *this* diff. Deterministic and model-free, +like `systemic.build_digest` (`systemic.py:326-414`): same input β†’ same text, +so evals and traces stay stable. + +### 3.2 Extraction (regex pass, no model) + +Per source, in document order, collect: + +- **Headings**: `^#{1,6} ` (markdown), `^\n[A-Z][^\n]{0,80}\n[-=]{3,}$` + (setext/asciidoc) β€” kept as `[heading]` scoping lines so a constraint stays + attached to its section (e.g. "Client BEST PRACTICES" vs "Server"). +- **Normative statements**: lines containing RFC-2119-strength keywords β€” + `MUST`, `MUST NOT`, `SHALL`, `SHALL NOT`, `REQUIRED`, `SHALL NOT`, + `FORBIDDEN`, `MUST NEVER` (strength 3); `SHOULD`, `SHOULD NOT`, + `RECOMMENDED`, `recommended to`, `forbidden to` (strength 2); `MAY`, + `can`, `discouraged` (strength 1, lowest keep-priority). Case-sensitive for + the RFC-2119 all-caps forms, case-insensitive for the prose forms. +- **Version pins**: `\b\d{4}-\d{2}-\d{2}\b` (spec revisions like + `2026-07-28`), `\b(?:v?\d+\.\d+(?:\.\d+)?)\b` adjacent to the words + `version|protocol|revision|draft`. A pinned version inside a kept constraint + is quoted verbatim; a version pin on its own line is kept as its own + constraint. +- **Naming/shape rules**: lines matching + `(?:MUST|SHOULD|SHALL)[^.]{0,120}(?:named|name|prefix|suffix|header|field|snake_case|camelCase|lowercase|uppercase)` + β€” the "tools MUST be named `mcp__`"-class constraints. +- **Ticket text** (`kind == "jira"`): the summary always, and the description + kept in full up to a per-source sub-budget (`min(spec_max_chars // 4, 6000)` + chars) β€” the ticket is the *scope/intent*, small and high-value; it is not + pruned to keywords. + +Each kept unit renders as one line: + +``` +[spec:{origin-short}#{anchor-or-line-N}] (MUST) +``` + +`origin-short` is the source's basename or URL path tail; the Jira ticket's +tag is `[ticket:{KEY}]`. + +### 3.3 Diff-relevance pruning + +Rank kept constraints, then keep until budget: + +1. **Ticket constraints** (from `jira` sources): always kept first β€” scope + beats relevance. +2. **Relevant spec constraints**: score = overlap between the constraint's + content tokens and the diff's token set (file path segments + changed-line + text, both compound-split). Tokenization mirrors the existing evidence + vocabulary β€” 4-char floor, stopword-filtered, snake/camel parts split β€” + which `quality.py:443-475` (`_tokens`, `_evidence_tokens`) already + implements; v1 imports that logic (promote to a small shared helper or + duplicate narrowly; implementer's choice, pinned by a parity test). + Constraints with score β‰₯ 1 are "relevant". +3. **Unmatched MUSTs** (strength 3, score 0): kept after relevant ones, + ordered by source order β€” a MUST the diff doesn't obviously touch is still + the cheapest place a sweep can find a violation of the "absence is + evidence" kind, exactly the migration-DDL argument in + `systemic.py:17-22`. +4. SHOULDs (2) then MAYs (1) with score 0 are dropped first, then trailing + unmatched SHOULDs, as budget runs out. + +### 3.4 Budget + +`build_spec_digest(sources, files, token_budget) -> str` enforces +`token_budget Γ— 4` chars (`PRXREF_SPEC_DIGEST_TOKENS`, default 3000 β†’ ~12k +chars), walking the ranked list and stopping with a final +`[spec digest truncated: budget reached]` line. Every source that contributed +at least one line gets an origin tag so the model can cite *which* spec a +constraint came from; a source that contributed nothing after pruning gets a +one-line `[spec:{origin}: nothing diff-relevant kept]` so silence is +explained, not inferred away. + +The digest is built once per review, after `parse_unified_diff` (files are the +pruning input) and before the worker fan-out. + +--- + +## 4. Prompt integration + +### 4.1 Where the digest enters + +Zero extra LLM calls in v1: the digest rides the **existing** per-chunk calls +and the **existing** systemic sweep. (A dedicated spec-sweep call is Β§10 / +judgment call J6.) + +Plumbing: + +- `reviewer.review_chunk` (`reviewer.py:285-337`) and + `reviewer.review_systemic` (`reviewer.py:340-371`) gain a + `spec_digest: str = ""` keyword. +- `_render_prompt` (`reviewer.py:117-139`) and `_render_systemic_prompt` + (`reviewer.py:142-163`) add `.replace("{spec_digest}", …)` next to the + existing placeholder fills; empty digest renders the literal + `(no specs provided for this review)`. +- The prompt templates gain a `### Spec constraints` block inside the + `## Review Context` section (between `{pr_description}` and the diff/digest + block), containing `{spec_digest}`. +- `orchestrate_review` (`orchestrator.py:262-280`) gains + `spec_sources: Sequence[str] = ()`; it calls `specs.fetch_specs` + builds + the digest inside the same never-raise fence as every other stage + (`orchestrator.py:67-71`), then threads the digest into `_run_workers` β†’ + `_run_worker` β†’ `_invoke_chunk` β†’ `review_chunk` and into `_run_sweep` β†’ + `review_systemic`. Trace: one `tracer.event("specs", …)` recording + sources/ok/fail counts and final digest chars, between `build_chunks` and + the worker span. + +### 4.2 What the prompts say + +`prompts/worker.md` β€” extend `## Severity Vocabulary` (`worker.md:7-11`) with: + +> - `spec` β€” the diff violates a constraint quoted in the Spec constraints +> block below: a MUST/SHALL/required behaviour not implemented, a +> forbidden behaviour implemented, a version pin or naming rule broken. +> Only when specs were provided. Quote the violated constraint verbatim in +> the body, prefixed `Spec: "`. + +and a new `## Spec-grounded rules` section: emit `spec` **only** for a +conflict between the diff and a quoted constraint β€” never for a generic best +practice not present in the block; when the block reads `(no specs provided +for this review)`, `spec` is not a legal severity. Cite the diff line that +violates it (same `file`/`line` contract as every finding, `worker.md:62`). + +`prompts/systemic.md` β€” same vocabulary bullet, plus one mission line: with +the whole-diff digest plus constraints in view, the sweep is the natural seat +for cross-file spec classes (naming rules, version pins, "no component may do +X" rules), while per-chunk seats catch line-local violations. Nothing else in +the sweep's class list changes (`systemic.md:5-15`). + +`prompts/summary.md` β€” see Β§6.1 (counts line only). + +Severity wording matters for eval scoring: the constraint quote convention +(`Spec: "…"`) gives expected.json a machine-checkable field and gives the +severity-consistency pass distinctive titles (Β§5.3). + +--- + +## 5. Quality-gate integration: the `spec` severity + +### 5.1 Vocabulary and ordering + +`quality.SEVERITIES` (`quality.py:52`) becomes +`{"error", "warning", "spec", "outofscope"}`. + +**Ordering** β€” `spec` sits below `warning`, above `outofscope`: + +``` +error(0) > warning(1) > spec(2) > outofscope(3) +``` + +Rationale: a spec violation is an operator-requested contract breach β€” always +worth reporting β€” but it is not claimed to break at runtime, so it does not +outrank a generic warning. Updated everywhere the rank table is restated: + +- `quality._SEVERITY_RANK` (`quality.py:556`) β€” drives severity-consistency + max-raise; `spec: 2`, `outofscope: 3`. +- `orchestrator._SEVERITY_RANK` (`orchestrator.py:129`) β€” inline-comment + priority; `.get(f.severity, 3)` at `orchestrator.py:522` keeps unknown + severities last without further edit. +- `formatter._SEVERITY_ORDER` (`formatter.py:25`) β€” summary table order. + +**Unknown-severity fallback**: unchanged in kind. The gate drops any severity +outside `SEVERITIES` with `drop_reason="invalid severity: …"` +(`quality.py:890-895`), so unknown strings never post; the rendering-layer +fallbacks (`formatter._norm_severity` β†’ `"outofscope"`, `formatter.py:44-49`; +`orchestrator._SEVERITY_MARKERS.get(…, "🟦")`, `orchestrator.py:971,:1070`) +continue to map unknown β†’ outofscope 🟦 for anything that reaches them. A +model that misspells `spec` therefore loses the finding loudly (drop, audit +trail) rather than silently mis-rendering it. + +### 5.2 Gate mechanics per pass + +- **Location validation** (`quality.py:70-93`): severity-agnostic β€” a spec + finding must name a diff path like any other. +- **Line align** (`quality.py:349-411`): severity-agnostic β€” body-citation + precedence, `snap_line` tolerance 5 (`quality.py:62`), blank-anchor guard + all apply unchanged. The `Spec: "…"` quote in the body must not be mistaken + for a location; it is prose, and the citation regexes + (`quality.py:258-265`) only match `path:line` / `line N` shapes. +- **Thread dedup** (`quality.py:527-553`): unchanged; a spec finding + duplicating an existing human thread is suppressed like any other. +- **Severity consistency** (`quality.py:718-838`): `spec` joins the rank map, + so a title-group containing both `spec` and `warning` members raises to + `warning`, `spec`+`error` raises to `error`, etc. β€” current max-raise + semantics preserved (judgment call J5). +- **Quality gate** (`quality.py:866-928`): confidence floor applies to spec + findings unchanged. The **error cap ignores `spec`** β€” it counts + `severity == "error"` only (`quality.py:913`) β€” so a spec-heavy review is + not crowded out by, nor crowding out, `PRXREF_MAX_ERROR_FINDINGS`. If spec + findings ever need their own cap, that is a future `PRXREF_MAX_SPEC_FINDINGS` + (not in v1). +- **Sweep dedup** (`quality.py:577-615`): unchanged; a sweep-emitted spec + finding restating a surviving chunk finding still dedups on + `(file, normalize_title)`. + +### 5.3 Verdict and exit codes + +- **Verdict unchanged**: `Request-Changes` iff an active `error` survives + (`orchestrator.py:480-484`). `spec` findings do **not** move the verdict in + v1 (judgment call J3). `PRXREF_FAIL_ON=any` already gates on spec findings + for lanes that want a hard signal (`cli.py:250-260`); `FAIL_ON=error` + ignores them, exactly as designed. + +--- + +## 6. Output + +### 6.1 Summary counts line + +`prompts/summary.md:5` and the fallback template +(`orchestrator.py:237-243`) become: + +``` +πŸŸ₯ {error_count} error Β· 🟧 {warning_count} warning Β· πŸ” {spec_count} spec Β· 🟦 {outofscope_count} outofscope +``` + +`_render_summary` (`orchestrator.py:965-993`): initialize +`counts = {"error": 0, "warning": 0, "spec": 0, "outofscope": 0}` and add the +`.replace("{spec_count}", …)` to the chain. The findings-bullet marker lookup +(`orchestrator.py:971`) and inline renderer (`orchestrator.py:1069-1076`) +need `"spec": "πŸ”"` in `_SEVERITY_MARKERS` (`orchestrator.py:124`); the +inline header renders `[SPEC]`. + +`formatter.py` mirrors all of this (`:20-25`, counts at `:172-180`) so the +forge-neutral renderer and the orchestrator renderer cannot drift; the +existing orchestrator-template pin tests +(`tests/test_orchestrator.py:30`, `tests/test_formatter.py:103,:115`) are +updated to the new counts line in the same change. + +### 6.2 Inline rendering + +Spec findings post as ordinary inline comments (`orchestrator.py:519-538`) +with the πŸ” marker and `[SPEC]` label, anchored like any finding. They count +against `PRXREF_MAX_INLINE_COMMENTS` by confidence+severity rank; at rank 2 +they yield the anchor to errors/warnings first, which is the intended +posture. + +### 6.3 Attribution + +Unchanged: `_format_finding`'s trailing +`Reviewed by prxref Β· model=…` (`orchestrator.py:1075`) and the summary's +`{attribution}` line carry model attribution as required by convention. + +### 6.4 Grounding note in the summary + +When `spec_sources` was non-empty, the summary gains one blockquote line after +the counts (via a new `{spec_note}` placeholder that renders `""` when no +specs were requested, keeping today's output byte-identical otherwise): + +``` +> πŸ” Spec-grounded: 3 source(s) Β· 41 constraint(s) injected +> ⚠️ Spec fetch failed for 1 source(s): ticket PROJ-9 (HTTP 401 β€” set PRXREF_JIRA_EMAIL/PRXREF_JIRA_API_TOKEN) +``` + +Failure reasons pass through `redact_for_post` before the post +(`orchestrator.py:202-235` doctrine: everything interpolated into a posted +comment is redacted; URLs are stripped by `_URL_RE`). A total fetch failure +renders only the failure line, and the review reads as un-grounded β€” which it +was. + +--- + +## 7. Golden eval dataset: `tests/evals/` + +The sibling worker lands three cases under `tests/evals//` with this +contract (defined here so runner and data agree): + +``` +tests/evals// + ticket.md # ticket text; fed to specs.py as a file source + docs/ # spec corpus (e.g. trimmed MCP best-practice pages) + diff.patch # unified diff with planted violations + clean lines + expected.json # ground truth (shape below) + meta.json # case name, spec version pin, planted-line inventory, floor +``` + +`expected.json`: + +```json +{ + "findings": [ + {"file": "src/mcp/server.py", "line": 42, "severity": "spec", + "title_hint": "forbidden header", "constraint_ref": "docs/lifecycle.md#MUST-close", + "source": "spec"} + ], + "nonfindings": [ + {"file": "src/mcp/util.py", "line": 7, "source": "generic", + "note": "clean refactor; must NOT attract a spec finding"} + ] +} +``` + +`source` labels ground truth as `"spec"` (violation of a corpus/ticket +constraint) or `"generic"` (ordinary bug the *unguided* reviewer should also +catch, kept to measure that grounding does not cost generic recall). + +### 7.1 Runner + +`tests/evals/harness.py` (data-local, not shipped) + a thin +`tests/test_evals.py` wrapper so `uv run pytest tests/test_evals.py` runs it +in CI: + +1. Load the case; build sources as `["/ticket.md", "/docs"]` + (file sources β€” the fetch layer is exercised by `test_specs.py` with + mocked sessions, not here). +2. Drive the real pipeline: `parse_unified_diff(diff.patch)` β†’ + `specs.build_spec_digest` β†’ `reviewer.render prompt` paths β†’ **stub LLM** + β†’ the full quality pass chain (`apply_location_validation` β†’ + `apply_line_align` β†’ … β†’ `apply_sweep_dedup`, `orchestrator.py:453-475`) + β†’ active findings. +3. **Stub LLM modes**: + - *plumbing mode* β€” per-case `stub_response.json` hand-written findings + exercising gate/anchor/dedup edges (drifted lines, sub-floor confidence, + invalid severities, sweep-vs-chunk duplicates); + - *recorded mode* β€” a real model's captured response for the case, + replayed verbatim (regression mode for model drift). + The stub satisfies the `LLMClient.invoke` shape (`reviewer.py:219-224`), + is injected the way `test_reviewer.py` doubles are today. +4. Score **post-gate, post-alignment** output against `expected.json` β€” + never raw model output, so the eval measures what a PR author receives. + +### 7.2 Scoring metric + +Matching rule, in order: same `file` (exact) **and** +`|pred.line βˆ’ exp.line| ≀ quality.DEFAULT_LINE_TOLERANCE` (5, +`quality.py:62`) **and** token overlap between title+body and +`title_hint + constraint_ref` using `quality.normalize_title` / +`_tokens` (`quality.py:443-475, :561`). Expected `line: 0` (file-level) +matches any line in the same file. + +- **Spec recall** = matched expected `source:"spec"` entries Γ· total spec + entries. +- **Spec precision** = predicted `severity=="spec"` findings that match a spec + entry Γ· all predicted spec findings. +- **Class-miss** (counted, not folded in): a prediction that matches an + expected spec entry but carries a non-spec severity. Binary good/bad hides + this third outcome β€” a "recall hit" that arrives as a generic warning is a + grounding failure the F1 must not launder. +- **Specificity**: any predicted spec finding matching a `nonfindings` entry + is a counted false-positive of the worst kind. +- **Anchor check**: predicted lines are snapped through the production + `apply_line_align` before scoring, so eval anchor tolerance can never + diverge from shipped behavior. Self-check: each expected line must be an + added line of the case diff (or 0) β€” validated when the case is loaded, so + a stale `expected.json` fails loudly, not as a mysterious 0-recall. +- Aggregate: per-case P/R/F1 table plus means; the pytest wrapper asserts a + per-case floor (F1 β‰₯ 0.8, class-miss ≀ 1, zero specificity hits) from + `meta.json`. + +--- + +## 8. Config surfaces checklist + docs updates + +The four-surface rule is enforced, not aspirational: +`tests/test_docs_consistency.py:58-118` fails the build when a `_DEFAULTS` key +misses any surface, and when `docs/env-vars.md`'s **stated counts** go stale. +For six new keys: + +- [ ] `src/prxref/config.py`: `_DEFAULTS` + six keys; `_LIST_KEYS` += + `spec_sources`; `_INT_KEYS` += `spec_max_chars`, `spec_digest_tokens`; + `_RANGES` += both; module docstring env table (lines 5-83) += six + entries. +- [ ] `.env.example`: six commented entries with defaults (file pattern + `.env.example:11-60`). +- [ ] `docs/env-vars.md`: six table rows (LLM & Pipeline section for the + three spec keys; a new "Spec Sources / Jira" subsection for the three + `PRXREF_JIRA_*` keys); update the stated totals β€” + `**35** configuration keys` β†’ `**41**`, and + `for 36 accepted variable names` β†’ `for 42` (`docs/env-vars.md:122-128`; + the test asserts these strings, `test_docs_consistency.py:103-118`). +- [ ] `README.md`: short "Review against a spec or ticket" section with one + copy-paste example. +- [ ] `src/prxref/cli.py`: `--spec` flag + plumbing (Β§1.1). +- [ ] Prompt templates: `worker.md`, `systemic.md` (Β§4.2), `summary.md` (Β§6.1). +- [ ] No new `_CHOICE_KEYS` entry is needed (no enum-valued key in this + feature). + +--- + +## 9. Testing plan and rollout + +### 9.1 Unit tests + +- `tests/test_specs.py` (new): dispatch (file / dir / URL / ticket-URL + shapes / garbage); Jira auth present vs anonymous vs 401-message content; + size caps + truncation markers; HTML stripping; dir file cap + sort order; + `fetch_specs` never raises; digest determinism (same inputs twice β†’ + identical bytes); pruning rank order (ticket > relevant > unmatched-MUST > + SHOULD); budget truncation marker; relevance scoring parity with + `quality._tokens`. +- `tests/test_quality.py`: `spec` passes the gate; rank order + error > warning > spec > outofscope in `apply_severity_consistency`; unknown + severity still dropped; error cap ignores spec findings. +- `tests/test_orchestrator.py`: `{spec_count}` + `{spec_note}` in the summary; + πŸ” marker in bullets and inline cards; verdict NOT moved by spec-only + findings; all-sources-failed run completes with the failure note; digest + reaches the worker prompt (captured via stub LLM) and the sweep prompt; + `redact_for_post` applied to fetch-failure notes. +- `tests/test_reviewer.py`: `{spec_digest}` replacement; `(no specs provided…)` + default; `spec` severity passes through unfiltered (reviewer never gates, + `reviewer.py:299-300`). +- `tests/test_cli.py`: `--spec` repeatable; override replaces env; + `--spec` values survive into `orchestrate_review` kwargs. +- `tests/test_docs_consistency.py`: must stay green untouched β€” it is the + checklist enforcer (Β§8). + +### 9.2 Rollout + +- Version: `0.11.1` β†’ **`0.12.0`** (`pyproject.toml:3`). New user-facing + flag + new severity = feature minor; no breaking change (the severity + vocabulary grows, but unknown-severity handling was already + drop-with-audit, so older consumers of run records degrade safely). +- `CHANGELOG.md`: one feature entry in the style of the severity-rename entry + (`CHANGELOG.md:173-184`): what `--spec`/`PRXREF_SPEC_SOURCES` accept, the πŸ” + `spec` severity and its ordering, Jira env vars, fetch-failure + non-blocking behavior, and the eval harness. Explicit note that verdict and + exit codes are unchanged (advisory doctrine preserved), and that + `PRXREF_FAIL_ON=any` is the opt-in gate for spec findings. + +--- + +## 10. Open questions / explicit non-goals (v1) + +- **No persistent graph index** (locked): specs are fetched, pruned, and + discarded per run. A future `PRXREF_SPEC_INDEX` (cache extracted constraint + sets keyed by URL hash with TTL) is the natural v2 β€” the extraction pass is + already deterministic and pure. +- **No auto-discovery of specs from the PR body** (non-goal): if a PR + description links a ticket, prxref does not fetch it. Operators are + explicit. +- **No multi-ticket traversal** (non-goal): multiple `--spec` sources are + allowed, but each is fetched independently; no epicβ†’story expansion, no + linked-issue walks. +- **No dedicated spec-sweep LLM call** (judgment call J6): v1 injects the + digest into existing chunk + sweep prompts. If recorded-mode evals show the + sweep prompt too loaded to catch spec classes, v2 adds a third single-shot + unit (mirroring `_run_sweep`, `orchestrator.py:847-918`) β€” one more + `chunk_count` unit, same failure shape. +- **No MCP ticket fetch** in v1 (locked: REST basic-auth is primary; MCP noted + as an alternative path for a future backend). +- **No per-source timeout/Retry config knobs** in v1: module constants + (`SPEC_FETCH_TIMEOUT_S`, `SPEC_DIR_MAX_FILES`); promote to env vars only if + real usage demands it. +- **Jira comments not fetched** (v1 keeps summary+description): comment + threads are noisy and frequently carry the debate the review is supposed to + settle. + +--- + +## 11. Judgment calls for the user to confirm + +- **J1 β€” `--spec` replaces `PRXREF_SPEC_SOURCES`; no merge.** Matches load_config + override precedence (`config.py:104`). Alternative: flag values append to + env values. +- **J2 β€” missing Jira credentials are a fetch failure, not exit 2.** The + review proceeds un-grounded with a note naming the three env vars. + Alternative: a Jira ticket URL with zero `PRXREF_JIRA_*` config could be a + config error (exit 2) on the "required value missing" theory. +- **J3 β€” `spec` findings do not move the verdict.** Verdict stays + error-only (`orchestrator.py:480-484`); `PRXREF_FAIL_ON=any` is the hard + gate. Alternative: any active `spec` finding also yields + `Request-Changes`. +- **J4 β€” ordering `error > warning > spec > outofscope`; unknown still falls + back to outofscope/🟦 at the render layer and is dropped by the gate.** +- **J5 β€” severity-consistency can rewrite a `spec` finding to `warning`/ + `error` on a title collision** (current max-raise semantics with `spec` at + rank 2). Alternative: make `spec` sticky (exempt from raises), at the cost + of the pass's group-coherence guarantee. +- **J6 β€” no extra LLM call in v1**: the digest rides chunk prompts + the + systemic sweep. Alternative: a dedicated third sweep unit for spec classes. +- **J7 β€” list coercion widened to comma-or-whitespace for ALL list keys** + (touches `llm_models`' coercion too; behavior-identical for it). + Alternative: a `spec_sources`-only split rule. +- **J8 β€” digest budget default 3000 tokens (~12k chars)**, on top of a + 25k-token chunk budget (`triage.py:17`). If prompts grow too large on the + smallest configured budgets, the digest could be charged a fixed share of + `PRXREF_CHUNK_TOKEN_BUDGET` instead of being an independent knob. diff --git a/src/prxref/cli.py b/src/prxref/cli.py index bc974f72..a10fcaea 100644 --- a/src/prxref/cli.py +++ b/src/prxref/cli.py @@ -90,6 +90,16 @@ def _build_parser() -> argparse.ArgumentParser: default=None, help="override the per-model request deadline in seconds", ) + rev.add_argument( + "--spec", + action="append", + default=None, + metavar="URL_OR_PATH", + help=( + "spec/ticket source to review against; repeatable " + "(PRXREF_SPEC_SOURCES otherwise)" + ), + ) rev.add_argument( "-v", "--verbose", @@ -287,23 +297,31 @@ def _run_review( max_chunks: int | None = None, timeout: float | None = None, trace_dir: str | None = None, + spec_sources: list[str] | None = None, ) -> Any: ref = detect_forge(url) if ref is None: return None - # --max-chunks and --timeout arrive as load_config overrides (None is - # ignored), so each flag is range-checked on exactly the same path as its - # environment variable and its precedence is derived once, here. There is - # deliberately no way to inject a pre-built config dict: that would bypass - # _check_ranges and make every range guarantee conditional on nobody using - # the bypass. + # --max-chunks, --timeout, and --spec arrive as load_config overrides (None + # is ignored), so each flag rides exactly the path its environment variable + # does: --max-chunks and --timeout are range-checked on the same pass as + # PRXREF_MAX_CHUNKS and PRXREF_LLM_TIMEOUT, and --spec replaces + # PRXREF_SPEC_SOURCES wholesale rather than merging with it. Precedence is + # derived once, here. There is deliberately no way to inject a pre-built + # config dict: that would bypass _check_ranges and make every range + # guarantee conditional on nobody using the bypass. cfg = load_config( max_chunks=max_chunks, llm_timeout=timeout, trace_dir=trace_dir, + spec_sources=spec_sources, # The operator typed a flag, so a rejection has to name the flag. Only # the CLI knows that spelling; config takes the label and reports it. - source_labels={"max_chunks": "--max-chunks", "llm_timeout": "--timeout"}, + source_labels={ + "max_chunks": "--max-chunks", + "llm_timeout": "--timeout", + "spec_sources": "--spec", + }, ) # PRXREF_DRY_RUN is the standing "never write to the forge" switch and # --no-post is the per-invocation one; either alone suppresses posting, so @@ -408,6 +426,7 @@ def _cmd_review(args: argparse.Namespace) -> int: max_chunks=args.max_chunks, timeout=args.timeout, trace_dir=args.trace_dir, + spec_sources=args.spec, ) except ConfigError as exc: print(f"configuration error: {exc}", file=sys.stderr) diff --git a/src/prxref/config.py b/src/prxref/config.py index 62e798b8..985fc6fe 100644 --- a/src/prxref/config.py +++ b/src/prxref/config.py @@ -91,9 +91,35 @@ PRXREF_DRY_RUN / ``--no-post``, which post nothing in any mode. PRXREF_POST_VERDICT literal "1" keeps the verdict stamp in the - posted summary; any other value renders the - summary without it (default on). The - total-failure notice always names its status. + posted summary; any other value renders the + summary without it (default on). The + total-failure notice always names its status. + PRXREF_SPEC_SOURCES Spec/ticket sources to review against, as + comma- or whitespace-separated web URLs and + local file/dir paths; the repeatable + ``--spec`` flag replaces (never merges) this + list. Jira ticket URLs are routed to the + Jira REST fetcher below. + PRXREF_SPEC_MAX_CHARS Raw fetched characters kept per spec source + before pruning; positive int (default + 120000) + PRXREF_SPEC_DIGEST_TOKENS Token budget for the spec digest injected + into worker prompts; positive int (default + 3000) + +Spec sources / Jira: + PRXREF_JIRA_BASE_URL REST base for Jira ticket fetches, + overriding a ticket URL's own host (a + self-hosted board often sits behind a + different REST host than its browse URL); + empty = the ticket URL's own host + PRXREF_JIRA_EMAIL Jira account email for HTTP basic auth; + empty fetches anonymously where the board + allows it. Missing credentials are a fetch + failure (the review proceeds un-grounded), + never a configuration error. + PRXREF_JIRA_API_TOKEN Jira API token paired with + PRXREF_JIRA_EMAIL for HTTP basic auth Per-forge auth: PRXREF_BITBUCKET_TOKEN Bitbucket Cloud bearer token @@ -132,6 +158,7 @@ import math import os +import re from typing import NamedTuple from prxref.forges.base import Forge, PRRef @@ -179,6 +206,12 @@ "trace_dir": "", "post_mode": "summary+inline", "post_verdict": True, + "spec_sources": [], + "spec_max_chars": 120000, + "spec_digest_tokens": 3000, + "jira_base_url": "", + "jira_email": "", + "jira_api_token": "", "bitbucket_token": "", "bitbucket_user": "", "bitbucket_app_password": "", @@ -198,10 +231,11 @@ "max_error_findings", "max_chunks", "llm_max_tokens", "llm_seed", "chunk_token_budget", "chunk_max_files", "chunk_context_lines", "max_workers", "max_inline_comments", + "spec_max_chars", "spec_digest_tokens", }) _FLOAT_KEYS = frozenset({"confidence_floor", "llm_timeout"}) _BOOL_KEYS = frozenset({"allow_unsigned", "dry_run", "post_verdict"}) -_LIST_KEYS = frozenset({"llm_models"}) +_LIST_KEYS = frozenset({"llm_models", "spec_sources"}) # An enum-valued key has no numeric interval to check, so its legal vocabulary # is declared here instead and enforced on the same pass as the ranges. A @@ -275,6 +309,8 @@ def describe(self) -> str: "chunk_context_lines": _Range(0, low_inclusive=True), "max_error_findings": _Range(0, low_inclusive=True), "llm_seed": _Range(0, low_inclusive=True), + "spec_max_chars": _Range(0), + "spec_digest_tokens": _Range(0), "confidence_floor": _Range(0.0, 1.0, low_inclusive=True), } @@ -308,7 +344,9 @@ def _coerce_env(key: str, raw: str, source: str) -> object: if key in _BOOL_KEYS: return _truthy(raw) if key in _LIST_KEYS: - return [part.strip() for part in raw.split(",") if part.strip()] + return [ + part.strip() for part in re.split(r"[,\s]+", raw) if part.strip() + ] return raw except ValueError as exc: raise ConfigError(f"{source}: {exc}") from exc diff --git a/src/prxref/formatter.py b/src/prxref/formatter.py index 9d043e74..dcb26f57 100644 --- a/src/prxref/formatter.py +++ b/src/prxref/formatter.py @@ -20,25 +20,26 @@ _SEVERITY_MARKERS: dict[str, str] = { "error": "πŸŸ₯", "warning": "🟧", + "spec": "πŸ”", "outofscope": "🟦", } -_SEVERITY_ORDER: dict[str, int] = {"error": 0, "warning": 1, "outofscope": 2} - -_DEFAULT_SUMMARY_TEMPLATE = """\ -## {verdict_banner} - -**Findings:** πŸŸ₯ {error_count} error Β· 🟧 {warning_count} warning Β· 🟦 {outofscope_count} outofscope - -{active_count} active of {total_count} raw - -{findings_table} -{dropped_section} ---- - -*chunks {chunk_count} Β· {input_tokens} in / {output_tokens} out tokens Β· {elapsed_s}s Β· model {model}* +_SEVERITY_ORDER: dict[str, int] = { + "error": 0, "warning": 1, "spec": 2, "outofscope": 3, +} -*{attribution}* -""" +_DEFAULT_SUMMARY_TEMPLATE = ( + "## {verdict_banner}\n\n" + "**Findings:** πŸŸ₯ {error_count} error Β· 🟧 {warning_count} warning Β· " + "πŸ” {spec_count} spec Β· 🟦 {outofscope_count} outofscope\n" + "{spec_note}\n" + "{active_count} active of {total_count} raw\n\n" + "{findings_table}\n" + "{dropped_section}\n" + "---\n\n" + "*chunks {chunk_count} Β· {input_tokens} in / {output_tokens} out tokens" + " Β· {elapsed_s}s Β· model {model}*\n\n" + "*{attribution}*\n" +) def _norm_severity(severity: str) -> str: @@ -123,10 +124,11 @@ def _load_summary_template() -> str: """Load ``prompts/summary.md`` via the shared loader, else inline default. Placeholder contract for the template owner: ``verdict_banner``, - ``error_count``, ``warning_count``, ``outofscope_count``, ``active_count``, - ``total_count``, ``findings_table``, ``dropped_section``, - ``chunk_count``, ``input_tokens``, ``output_tokens``, ``elapsed_s``, - ``model``, ``attribution``. + ``error_count``, ``warning_count``, ``spec_count``, ``spec_note``, + ``outofscope_count``, ``active_count``, ``total_count``, + ``findings_table``, ``dropped_section``, ``chunk_count``, + ``input_tokens``, ``output_tokens``, ``elapsed_s``, ``model``, + ``attribution``. """ if _reviewer_load_prompt is not None: try: @@ -177,9 +179,13 @@ def format_summary( "warning_count": sum( 1 for f in findings_active if _norm_severity(f.severity) == "warning" ), + "spec_count": sum( + 1 for f in findings_active if _norm_severity(f.severity) == "spec" + ), "outofscope_count": sum( 1 for f in findings_active if _norm_severity(f.severity) == "outofscope" ), + "spec_note": "", "active_count": len(findings_active), "total_count": len(findings_active) + len(findings_dropped), "findings_table": _findings_table(findings_active), diff --git a/src/prxref/orchestrator.py b/src/prxref/orchestrator.py index ba3454bf..d7f269f7 100644 --- a/src/prxref/orchestrator.py +++ b/src/prxref/orchestrator.py @@ -22,7 +22,13 @@ ``context_lines=0`` rendering β€” a strictly smaller prompt attacks the prefill-side share of the wall clock, and a truncated completion (the response-side budget) is not a timeout and never reaches this retry. -4. Systemic sweep: after the chunk workers, ONE more worker-style +4. Spec grounding (best-effort, only when ``spec_sources`` is non-empty): + ``specs.fetch_specs`` + ``specs.build_spec_digest`` run inside the same + never-raise fence as every other stage, and the digest rides the + existing chunk calls and the systemic sweep β€” no extra LLM unit. A run + whose every source failed behaves exactly like a run with no specs, + plus a grounding note in the summary. +5. Systemic sweep: after the chunk workers, ONE more worker-style single-shot call over the whole-PR digest built by ``systemic.build_digest`` (every file with hunk headers; short files and migrations render their full added content, the rest only the @@ -31,7 +37,7 @@ chunk results, and counts as one more review unit: ``chunk_count`` is ``len(chunks) + 1`` whenever the sweep ran, and a sweep failure is one failed chunk in the partial-review banner. -5. Deterministic checks and quality passes, in exactly this order β€” the +6. Deterministic checks and quality passes, in exactly this order β€” the raw chunk + sweep findings first gain ``heuristics.release_shape_findings(files)`` (a pure, no-LLM finding about a PR that is β‰₯80% release machinery yet also touches source), @@ -74,7 +80,7 @@ sorted by ``finding_sort_key``. Every result β€” including an error or summary-only exit β€” carries a ``sampling`` record naming the temperature, seed, and model chain actually in force. -6. Verdict: ``"Error"`` when every CHUNK review failed (a sweep success +7. Verdict: ``"Error"`` when every CHUNK review failed (a sweep success on a dead worker pool cannot carry the run); ``"Request-Changes"`` iff any active error-severity finding survives; else ``"Approved"``. A partial failure keeps the verdict but the summary @@ -83,9 +89,10 @@ blockquote) β€” a partial review reads as a successful one, so a failure left only in the logs reaches nobody, and a file list left out of it leaves the operator guessing which files went unreviewed. -7. Post: summary rendered from ``reviewer.load_prompt("summary")`` with +8. Post: summary rendered from ``reviewer.load_prompt("summary")`` with placeholders ``{verdict} {title} {file_count} {error_count} - {warning_count} {outofscope_count} {findings} {attribution}`` filled, plus + {warning_count} {spec_count} {spec_note} {outofscope_count} {findings} + {attribution}`` filled, plus inline comments for up to ``max_inline_comments`` active findings. ``post_mode`` narrows what is written: ``"summary+inline"`` (default) is that full behaviour, ``"summary"`` skips the inline batch, ``"inline"`` @@ -114,7 +121,7 @@ from concurrent.futures import ThreadPoolExecutor from typing import Any -from . import chunk_context, heuristics, reviewer, systemic +from . import chunk_context, heuristics, reviewer, specs, systemic from .forges.base import ( ATTRIBUTION_MARKER, Forge, @@ -169,12 +176,14 @@ # letting a pathological run bury the findings under its own diagnostics. MAX_REPORTED_REASONS = 3 -_SEVERITY_MARKERS = {"error": "πŸŸ₯", "warning": "🟧", "outofscope": "🟦"} +_SEVERITY_MARKERS = {"error": "πŸŸ₯", "warning": "🟧", "spec": "πŸ”", "outofscope": "🟦"} # Inline-comment priority: the most severe findings get the anchor first, so # a cap or a rejected anchor costs the run its least-important comments -# rather than whatever happened to sit at the tail of chunk order. -_SEVERITY_RANK = {"error": 0, "warning": 1, "outofscope": 2} +# rather than whatever happened to sit at the tail of chunk order. spec sits +# below warning (a spec violation is an operator-requested contract breach, +# but not claimed to break at runtime) and above outofscope. +_SEVERITY_RANK = {"error": 0, "warning": 1, "spec": 2, "outofscope": 3} _REDACTED = "[redacted]" @@ -286,7 +295,9 @@ def redact_for_post(reason: str) -> str: "πŸ€– **prxref review β€” {verdict}**\n\n" "PR: {title}\n\n" "Files reviewed: {file_count} Β· πŸŸ₯ {error_count} error Β· " - "🟧 {warning_count} warning Β· 🟦 {outofscope_count} outofscope\n\n" + "🟧 {warning_count} warning Β· πŸ” {spec_count} spec Β· " + "🟦 {outofscope_count} outofscope\n" + "{spec_note}\n" "{findings}\n\n{attribution}" ) @@ -326,6 +337,12 @@ def orchestrate_review( post_verdict: bool = True, trace_file: str | None = None, trace_dir: str | None = None, + spec_sources: Sequence[str] = (), + spec_max_chars: int = 120000, + spec_digest_tokens: int = 3000, + jira_base_url: str = "", + jira_email: str = "", + jira_api_token: str = "", ) -> dict: """Run one full review pass over a PR and optionally post results. @@ -370,6 +387,21 @@ def orchestrate_review( (model, token counts, elapsed, error) under that directory, labelled ``chunk0`` … ``chunkN-1`` and ``sweep``. Empty (the default) traces nothing; a write failure is a logged warning, never a review failure. + + ``spec_sources`` grounds the review against written specs: each entry is + fetched by :func:`prxref.specs.fetch_specs` and the pruned constraint + digest (:func:`prxref.specs.build_spec_digest`) is injected into every + worker prompt and the sweep prompt β€” no extra LLM unit. The fetch never + raises and never fails the run: sources that fail become a grounding + note in the summary (failure reasons pass through + :func:`redact_for_post` before posting), and a run whose every source + failed is exactly a run with no specs plus that note. The remaining + spec keywords mirror the config keys of the same names + (``spec_max_chars``, ``spec_digest_tokens``, ``jira_base_url``, + ``jira_email``, ``jira_api_token``); the defaults restate + ``config._DEFAULTS`` the way ``MAX_WORKERS`` does. ``spec_sources`` is + deliberately absent from the returned dict, like every other request + knob. """ t0 = time.perf_counter() tracer = get_tracer(trace_file) @@ -469,11 +501,39 @@ def orchestrate_review( logger.warning("list_threads failed (best-effort): %s", e) threads = [] + # Best-effort, like the thread listing: a spec-fetch failure is data for + # the grounding note, never a failed review. The digest is built once, + # after parse_unified_diff (the files are the pruning input) and before + # the worker fan-out, then rides the existing chunk + sweep calls. + spec_digest = "" + spec_note = "" + if spec_sources: + try: + fetched = specs.fetch_specs( + list(spec_sources), + max_chars=spec_max_chars, + jira_base_url=jira_base_url, + jira_email=jira_email, + jira_api_token=jira_api_token, + ) + spec_digest = specs.build_spec_digest(fetched, files, spec_digest_tokens) + ok = sum(1 for s in fetched if not s.error) + tracer.event( + "specs", "ok", + sources=len(fetched), ok=ok, fail=len(fetched) - ok, + digest_chars=len(spec_digest), + ) + spec_note = _spec_note(fetched, spec_digest) + except Exception as e: # noqa: BLE001 + logger.error("spec grounding failed (best-effort): %s", e) + tracer.event("specs", "fail", error=e.__class__.__name__) + reader = _make_file_reader(forge, ref, pr) results = _run_workers( llm, chunks, pr, max_tokens=max_tokens, max_workers=max_workers, context_lines=context_lines, tracer=tracer, reader=reader, all_files=files, trace_dir=trace_dir, + spec_digest=spec_digest, ) # One more worker-style unit, not inside the pool: the sweep digests the @@ -486,6 +546,7 @@ def orchestrate_review( llm, files, pr, max_tokens=max_tokens, token_budget=token_budget, tracer=tracer, threads=threads, trace_dir=trace_dir, + spec_digest=spec_digest, ) ) @@ -635,6 +696,7 @@ def orchestrate_review( # "findings may be incomplete" without which-files acts on nothing. failed_chunks=failed_chunks, include_verdict=post_verdict, + spec_note=spec_note, ) try: forge.post_summary(ref, summary) @@ -681,6 +743,7 @@ def orchestrate_review( chunks_reviewed=chunks_reviewed, chunks_failed=chunks_failed, failed_chunks=failed_chunks, include_verdict=post_verdict, + spec_note=spec_note, inline_accounting=_inline_accounting( len(findings_active), inline_attempted, inline_posted, failed=inline_failed, cap=max_inline_comments, @@ -865,7 +928,7 @@ def _run_workers( llm: LLMClient, chunks, pr: PRData, *, max_tokens: int | None = None, max_workers: int = MAX_WORKERS, context_lines: int | None = None, tracer: Tracer | None = None, reader=None, all_files=None, - trace_dir: str | None = None, + trace_dir: str | None = None, spec_digest: str = "", ) -> list[dict]: # Never below 1: ThreadPoolExecutor rejects a zero-width pool, and a # library caller is not gated by config's range check. @@ -901,6 +964,7 @@ def _heartbeat() -> None: _run_worker, i + 1, len(chunks), llm, chunk, pr, max_tokens, context_lines, tracer, reader, all_files, trace_label=f"chunk{i}", trace_dir=trace_dir, + spec_digest=spec_digest, ) for i, chunk in enumerate(chunks) ] @@ -947,6 +1011,7 @@ def _invoke_chunk( max_tokens: int | None, context_lines: int | None, reader=None, *, include_definitions: bool = True, all_files=None, trace_label: str = "", trace_dir: str | None = None, + spec_digest: str = "", ) -> dict: """One normalized :func:`reviewer.review_chunk` call; never raises. @@ -971,6 +1036,7 @@ def _invoke_chunk( max_tokens=max_tokens, context_lines=context_lines, context_blocks=blocks, sibling_files=all_files or (), trace_label=trace_label, trace_dir=trace_dir or "", + spec_digest=spec_digest, ) except Exception as e: # noqa: BLE001 return { @@ -1012,6 +1078,7 @@ def _run_worker( max_tokens: int | None = None, context_lines: int | None = None, tracer: Tracer | None = None, reader=None, all_files=None, trace_label: str = "", trace_dir: str | None = None, + spec_digest: str = "", ) -> dict: tracer = tracer if tracer is not None else get_tracer() t0 = time.perf_counter() @@ -1028,7 +1095,7 @@ def _run_worker( ) res = _invoke_chunk( llm, chunk, pr, max_tokens, context_lines, reader, all_files=all_files, - trace_label=trace_label, trace_dir=trace_dir, + trace_label=trace_label, trace_dir=trace_dir, spec_digest=spec_digest, ) if ( res["error"] @@ -1054,6 +1121,7 @@ def _run_worker( llm, chunk, pr, max_tokens, _TIMEOUT_RETRY_CONTEXT_LINES, reader, include_definitions=False, all_files=all_files, trace_label=trace_label, trace_dir=trace_dir, + spec_digest=spec_digest, ) error = res["error"] @@ -1092,6 +1160,7 @@ def _run_sweep( tracer: Tracer | None = None, threads: Sequence[Thread] = (), trace_dir: str | None = None, + spec_digest: str = "", ) -> dict: """Run the whole-PR systemic sweep as one worker-style review unit. @@ -1099,7 +1168,8 @@ def _run_sweep( ``token_budget``), makes ONE single-shot call through :func:`reviewer.review_systemic` β€” so ``PRXREF_LLM_MAX_TOKENS``, the timeout, and the model fallback chain all apply as to any chunk β€” and - returns the same result shape a chunk worker does. A failure is that + returns the same result shape a chunk worker does. ``spec_digest`` rides + along into the sweep prompt's Spec constraints block. A failure is that shape with ``error`` set prefixed ``systemic sweep:``, so the partial-review banner names the unit that failed; it counts as one failed chunk in the caller's coverage accounting. @@ -1122,6 +1192,7 @@ def _run_sweep( llm, digest, pr_title=pr.title, pr_description=pr.description, max_tokens=max_tokens, threads=discussion, trace_label="sweep", trace_dir=trace_dir or "", + spec_digest=spec_digest, ) except Exception as e: # noqa: BLE001 logger.error("[sweep] raised: %s", e) @@ -1203,6 +1274,7 @@ def _render_summary( failed_chunks: Sequence[tuple[str, Sequence[str]]] = (), include_verdict: bool = True, inline_accounting: str | None = None, + spec_note: str = "", ) -> str: try: template = reviewer.load_prompt("summary") @@ -1212,7 +1284,7 @@ def _render_summary( if not include_verdict: template = _strip_verdict_stamp(template) - counts = {"error": 0, "warning": 0, "outofscope": 0} + counts = {"error": 0, "warning": 0, "spec": 0, "outofscope": 0} for f in findings_active: counts[f.severity] = counts.get(f.severity, 0) + 1 @@ -1237,6 +1309,8 @@ def _render_summary( .replace("{file_count}", str(len(files))) .replace("{error_count}", str(counts["error"])) .replace("{warning_count}", str(counts["warning"])) + .replace("{spec_count}", str(counts["spec"])) + .replace("{spec_note}", spec_note) .replace("{outofscope_count}", str(counts["outofscope"])) .replace("{findings}", bullets) .replace("{attribution}", attribution) @@ -1260,6 +1334,53 @@ def _render_summary( return rendered +# A digest line that IS a constraint: a spec unit +# (``[spec:origin#anchor] (MUST|SHOULD|MAY) statement``) or a ticket line +# (``[ticket:KEY] statement``). Heading lines, the truncation marker, and +# the ``nothing diff-relevant kept`` lines are scoping or bookkeeping, not +# injected constraints, and never match. +_SPEC_CONSTRAINT_RE = re.compile( + r"^\[(?:spec:[^\]]*#\S+|ticket:[^\]]+)\] \((?:MUST|SHOULD|MAY)\) " +) + + +def _spec_constraint_count(digest: str) -> int: + """The number of constraint lines the digest actually injects.""" + return sum( + 1 for line in digest.splitlines() if _SPEC_CONSTRAINT_RE.match(line) + ) + + +def _spec_note(sources: Sequence[Any], digest: str) -> str: + """Render the summary's grounding note, ``""`` when nothing was requested. + + One blockquote line counts what was injected; one names every failed + source, each reason through :func:`redact_for_post` first, because this + text is posted. A run whose every source failed renders ONLY the failure + line β€” the review was un-grounded, and the note must not dress it up as + grounded. The note rides the ``{spec_note}`` placeholder on its own line + between the counts and the findings, and a non-empty note carries its own + trailing newline, so an empty return leaves the summary byte-identical to + an ungrounded run's. + """ + if not sources: + return "" + total = len(sources) + failed = [s for s in sources if s.error] + lines: list[str] = [] + if len(failed) < total: + lines.append( + f"> πŸ” Spec-grounded: {total} source(s) Β· " + f"{_spec_constraint_count(digest)} constraint(s) injected" + ) + if failed: + reasons = "; ".join(redact_for_post(s.error) for s in failed) + lines.append( + f"> ⚠️ Spec fetch failed for {len(failed)} source(s): {reasons}" + ) + return "\n".join(lines) + "\n" + + def _chunk_files_label(files: Sequence[str]) -> str: """``chunk of 3 files (a.py, b.py, c.py)``, first three then a count.""" listing = ", ".join(files[:3]) diff --git a/src/prxref/prompts/summary.md b/src/prxref/prompts/summary.md index 6032eaa8..9c0fb891 100644 --- a/src/prxref/prompts/summary.md +++ b/src/prxref/prompts/summary.md @@ -2,8 +2,8 @@ PR: {title} Β· files reviewed: {file_count} -πŸŸ₯ {error_count} error Β· 🟧 {warning_count} warning Β· 🟦 {outofscope_count} outofscope - +πŸŸ₯ {error_count} error Β· 🟧 {warning_count} warning Β· πŸ” {spec_count} spec Β· 🟦 {outofscope_count} outofscope +{spec_note} {findings} --- diff --git a/src/prxref/prompts/systemic.md b/src/prxref/prompts/systemic.md index 829f4ce3..cc675d2d 100644 --- a/src/prxref/prompts/systemic.md +++ b/src/prxref/prompts/systemic.md @@ -13,7 +13,7 @@ Per-chunk reviewers each see one slice of the diff and reliably miss classes tha - A removed guard: a deleted numeric limit constant (`MAX_*_LENGTH`, `*_SIZE`, `*_BYTES`, `*_TIMEOUT`) or a deleted validator/sanitiser definition (`isValid*`, `validate*`, `sanitize*`, `check*`, `assert*`, `escape*`) on a path that consumes remote or third-party input. The digest shows these as `-` lines; the code that remains says nothing about the bound that is gone, so the deletion itself is the finding. - Repo-config drift: two lockfiles for one package manager root β€” a lockfile newly added while another lockfile or a `packageManager` pin also appears in the PR. The digest states this collision on a `! repo-config:` line. -Nothing else. Per-file bugs inside one chunk are the chunk workers' job; repeating them here only duplicates their findings, which are deduplicated away. +Nothing else. Per-file bugs inside one chunk are the chunk workers' job; repeating them here only duplicates their findings, which are deduplicated away. One cross-file addition: with the whole-diff digest plus the spec constraints in view, this sweep is the natural seat for cross-file spec classes β€” naming rules, version pins, and `no component may` rules β€” while per-chunk seats catch line-local violations. Do not raise a subject the reviewers already argued out under `### Existing discussion` β€” that decision was made with more context than the digest carries. If you raise it anyway, say in the body why the discussion's conclusion is wrong. @@ -21,6 +21,7 @@ Do not raise a subject the reviewers already argued out under `### Existing disc - `error` β€” the change will break at runtime or is a real bug: crash, wrong result, data loss, security hole, broken contract. - `warning` β€” risk or smell the diff introduces or worsens: race-prone pattern, resource leak, missing error handling, load-bearing duplication. +- `spec` β€” the diff violates a constraint quoted in the Spec constraints block below: a MUST/SHALL/required behaviour not implemented, a forbidden behaviour implemented, a version pin or naming rule broken. Only when specs were provided. Quote the violated constraint verbatim in the body, prefixed `Spec: "`. - `outofscope` β€” minor: misleading naming, a TODO without context, dead code the diff adds. ## Confidence @@ -44,6 +45,10 @@ PR description: Repo: {repo_hint} +### Spec constraints + +{spec_digest} + The digest below lists every changed file (`## path`) with its hunk headers (`@@`), then its lines: a short file (or any file the migration DDL pattern touches) shows its FULL added content β€” every `+` line β€” so a statement you would expect and do not see inside such a file is evidence of absence; larger files show only the added (`+|`) and removed (`-|`) lines that matched a high-signal pattern, with secret, auth, and entry-point lines listed ahead of noisier matches in a capped file. A `! repo-config:` line is a synthetic note, not a diff line β€” cite it as a file-level finding (`line: 0`). `[full content omitted: ...]` means that file degraded to pattern lines only. `[digest truncated: token budget reached]` means the cap cut the text short; there is no more. ### Digest diff --git a/src/prxref/prompts/worker.md b/src/prxref/prompts/worker.md index b6ef7158..2bb52177 100644 --- a/src/prxref/prompts/worker.md +++ b/src/prxref/prompts/worker.md @@ -8,8 +8,13 @@ Verify every claim against the diff itself. Every finding must cite a file and l - `error` β€” the change will break at runtime or is a real bug: crash, wrong result, data loss, security hole, broken contract. - `warning` β€” risk or smell the diff introduces or worsens: race-prone pattern, resource leak, missing error handling, load-bearing duplication. +- `spec` β€” the diff violates a constraint quoted in the Spec constraints block below: a MUST/SHALL/required behaviour not implemented, a forbidden behaviour implemented, a version pin or naming rule broken. Only when specs were provided. Quote the violated constraint verbatim in the body, prefixed `Spec: "`. - `outofscope` β€” minor: misleading naming, a TODO without context, dead code the diff adds. +## Spec-grounded rules + +Emit `spec` only for a conflict between the diff and a constraint quoted in the Spec constraints block β€” never for a generic best practice not present in the block. When the block reads `(no specs provided for this review)`, `spec` is not a legal severity. Cite the diff line that violates it β€” the same `file`/`line` contract as every finding β€” and quote the violated constraint verbatim in the body, prefixed `Spec: "`. + ## Confidence Each finding carries a confidence from 0.0 to 1.0 β€” how certain you are from this diff alone. Findings below the quality floor (default 0.6) are dropped downstream. 0.5 means "plausible but unverified". Reserve 0.9+ for defects provable from the diff text alone. @@ -37,6 +42,10 @@ PR description: Repo: {repo_hint} +### Spec constraints + +{spec_digest} + The input stays under roughly 30k tokens; the diff below is the complete chunk. ### Diff diff --git a/src/prxref/quality.py b/src/prxref/quality.py index 5c468cb5..47a353de 100644 --- a/src/prxref/quality.py +++ b/src/prxref/quality.py @@ -62,7 +62,7 @@ 9. ``apply_quality_gate``: drop findings below the confidence floor (``confidence 0.40 below floor 0.60``), cap errors per review (``error cap exceeded (max N)``), and enforce the - {error, warning, outofscope} severity vocabulary + {error, warning, spec, outofscope} severity vocabulary (``invalid severity: ''``). It RETURNS its findings sorted by ``finding_sort_key``, so the caller re-derives the chunk/sweep boundary from finding identity rather than carrying an index across it. @@ -95,7 +95,7 @@ from .forges.base import Thread from .triage import DiffLine, FileDiff, Finding, Hunk -SEVERITIES: frozenset[str] = frozenset({"error", "warning", "outofscope"}) +SEVERITIES: frozenset[str] = frozenset({"error", "warning", "spec", "outofscope"}) DEFAULT_CONFIDENCE_FLOOR: float = 0.6 DEFAULT_MAX_ERRORS: int = 10 @@ -967,7 +967,9 @@ def apply_settled_thread_suppression( return result -_SEVERITY_RANK: dict[str, int] = {"error": 0, "warning": 1, "outofscope": 2} +_SEVERITY_RANK: dict[str, int] = { + "error": 0, "warning": 1, "spec": 2, "outofscope": 3, +} _TITLE_PUNCT_RE = re.compile(r"[`*\"'\u2018\u2019\u201c\u201d]") @@ -1225,8 +1227,8 @@ def apply_severity_consistency(findings: Sequence[Finding]) -> list[Finding]: describing different problems stay apart. Components bind transitively (A shares a token with B, B with C, so - all three group). Each component is rewritten to its highest - severity (error > warning > outofscope). A rewritten finding keeps + all three group). Each component is rewritten to its highest severity + (error > warning > spec > outofscope). A rewritten finding keeps its own file, line, body, and confidence; only severity changes. Findings carrying a ``drop_reason`` or a severity outside the vocabulary pass through untouched. One summary line is logged when @@ -1562,12 +1564,15 @@ def apply_quality_gate( """Filter findings through vocabulary, confidence, and per-review error caps. Order: - 1. Severity vocabulary: non-empty lowercase must be in {error, warning, note}; - case-mismatches are normalized; invalid severities are dropped. + 1. Severity vocabulary: non-empty lowercase must be in + {error, warning, spec, outofscope}; case-mismatches are normalized; + invalid severities are dropped. 2. Confidence floor: drop findings below the threshold (default 0.6). 3. Error cap: among surviving errors, keep the top N ranked by :func:`finding_rank_key` and drop the rest, so ties are broken by - content rather than by arrival order. + content rather than by arrival order. ``spec`` findings never count + toward the cap: a spec-heavy review is neither crowded out by it nor + crowding it out. The returned list is sorted by :func:`finding_sort_key`. """ diff --git a/src/prxref/reviewer.py b/src/prxref/reviewer.py index ed224ea9..db780d8d 100644 --- a/src/prxref/reviewer.py +++ b/src/prxref/reviewer.py @@ -56,6 +56,8 @@ _CONTEXT_MARKER = "## Review Context" +_NO_SPECS_TEXT = "(no specs provided for this review)" + _MAX_TOKENS_ENV = "PRXREF_LLM_MAX_TOKENS" # Caps on the ``### Existing discussion`` block appended to the sweep prompt. @@ -155,6 +157,7 @@ def _render_prompt( context_lines: int | None = None, context_blocks: str = "", sibling_files: Sequence[FileDiff] = (), + spec_digest: str = "", ) -> tuple[str, str]: template = load_prompt("worker.md") head, marker, tail = template.partition(_CONTEXT_MARKER) @@ -172,6 +175,8 @@ def _render_prompt( "{repo_hint}", repo_hint.strip() or "(unspecified)" ).replace( "{context_blocks}", blocks + ).replace( + "{spec_digest}", spec_digest.strip() or _NO_SPECS_TEXT ).replace( "{diff}", render_chunk(chunk, context_lines) or "(empty chunk)" ) @@ -217,6 +222,7 @@ def _render_systemic_prompt( pr_description: str, repo_hint: str, threads: Sequence[Thread] = (), + spec_digest: str = "", ) -> tuple[str, str]: template = load_prompt("systemic.md") head, marker, tail = template.partition(_CONTEXT_MARKER) @@ -230,6 +236,8 @@ def _render_systemic_prompt( "{pr_description}", pr_description.strip() or "(none)" ).replace( "{repo_hint}", repo_hint.strip() or "(unspecified)" + ).replace( + "{spec_digest}", spec_digest.strip() or _NO_SPECS_TEXT ).replace( "{digest}", digest.strip() or "(empty digest)" ) @@ -434,6 +442,7 @@ def review_chunk( sibling_files: Sequence[FileDiff] = (), trace_dir: str = "", trace_label: str = "", + spec_digest: str = "", ) -> tuple[list[Finding], dict]: """Review one chunk with a single LLM call. @@ -485,6 +494,13 @@ def review_chunk( that directory (:func:`_write_trace_files`); the empty default traces nothing. The orchestrator passes both, so ``PRXREF_TRACE_DIR`` covers every chunk without any per-caller wiring. + + ``spec_digest`` is the spec-grounding text injected into the prompt's + Spec constraints block (built by + :func:`prxref.specs.build_spec_digest`); empty renders the literal + ``(no specs provided for this review)`` instead, and the prompt tells + the model ``spec`` is then not a legal severity. The orchestrator always + passes this keyword too. """ system, user = _render_prompt( chunk=chunk, @@ -494,6 +510,7 @@ def review_chunk( context_lines=context_lines, context_blocks=context_blocks, sibling_files=sibling_files, + spec_digest=spec_digest, ) budget = MAX_TOKENS if max_tokens is None else max_tokens return _invoke_and_parse( @@ -513,6 +530,7 @@ def review_systemic( threads: Sequence[Thread] = (), trace_dir: str = "", trace_label: str = "", + spec_digest: str = "", ) -> tuple[list[Finding], dict]: """Review the whole-PR systemic digest with a single LLM call. @@ -524,6 +542,12 @@ def review_systemic( :func:`prxref.systemic.build_digest`; the prompt (``prompts/systemic.md``) restricts findings to those systemic classes. + ``spec_digest`` rides the same prompt under the Spec constraints block, + under the :func:`review_chunk` contract for it: empty renders + ``(no specs provided for this review)``, and with the whole-diff digest + plus constraints in view the sweep is the natural seat for cross-file + spec classes. + Returns ``(findings, meta)`` under exactly the :func:`review_chunk` contract β€” never raises, ``meta["error"]`` empty on success, truncation named when the budget is why β€” so the orchestrator can treat the sweep @@ -539,6 +563,7 @@ def review_systemic( pr_description=pr_description, repo_hint=repo_hint, threads=threads, + spec_digest=spec_digest, ) budget = MAX_TOKENS if max_tokens is None else max_tokens return _invoke_and_parse( diff --git a/src/prxref/specs.py b/src/prxref/specs.py new file mode 100644 index 00000000..1e630b15 --- /dev/null +++ b/src/prxref/specs.py @@ -0,0 +1,614 @@ +"""Spec-source fetching and constraint digest for spec-grounded reviews. + +A review can be grounded in written specs: files, directories, web pages, and +Jira tickets supplied by the operator. Workers each see one slice of the diff, +so a constraint that lives only in a spec document has no seat that knows it β€” +this module fetches those sources and reduces them to a compact, deterministic +digest of the constraints this diff could plausibly violate. + +Two halves, both deterministic and model-free, mirroring +:mod:`prxref.systemic`: + +- The fetch layer (:func:`fetch_specs`) turns each configured source into one + :class:`SpecSource` of plain text, or a human-readable error. It never + raises: a failed source is data, not an exception, and a run whose sources + all failed behaves exactly like a run with no specs. HTTP goes through a + read-only retried session (GET/HEAD/OPTIONS only, the same policy the forge + adapters use), streams at most ``max_chars`` per source with an explicit + truncation marker, accepts only text-like content types, strips HTML to + text, and reads local files as UTF-8. Jira tickets are fetched over REST + with basic auth when credentials are configured, anonymously otherwise; + a 401/403 without credentials names the ``PRXREF_JIRA_*`` variables β€” the + fix an operator can act on β€” never their values. + +- The digest (:func:`build_spec_digest`) extracts constraints per source in + document order β€” headings, RFC-2119 normative statements, version pins, and + naming/shape rules β€” then ranks them: ticket scope first, then constraints + whose tokens overlap the diff's token set, then unmatched MUST-level rules, + with weaker unmatched strengths dropped first as the budget runs out. + Same input, same text, so evals and traces stay stable. +""" +from __future__ import annotations + +import codecs +import json +import os +import re +from dataclasses import dataclass +from html.parser import HTMLParser +from urllib.parse import urlparse + +import requests +from requests.adapters import HTTPAdapter + +from .quality import _evidence_tokens, _tokens +from .retry_logging import LoggingRetry +from .triage import FileDiff + +SPEC_FETCH_TIMEOUT_S = 15 + +SPEC_DIR_MAX_FILES = 20 + +CHARS_PER_TOKEN = 4 + +TRUNCATION_MARKER = "[spec digest truncated: budget reached]" + +SOURCE_TRUNCATION_MARKER = "[source truncated at {n} chars]" + +TICKET_DESC_BUDGET_CHARS = 6000 + +_STATEMENT_MAX_CHARS = 400 + +_KEY = r"[A-Z][A-Z0-9_]*-\d+" + +_TICKET_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(rf"^(?Phttps?://[^/]+)/browse/(?P{_KEY})(?:[/?#]|$)"), + re.compile(rf"^(?Phttps?://[^/]+)/rest/api/(?:2|3)/issue/(?P{_KEY})(?:[/?#]|$)"), + re.compile( + rf"^(?Phttps?://[^/]+)/jira/software/c/projects/[A-Z][A-Z0-9_]*/issues/(?P{_KEY})(?:[/?#]|$)" + ), +) + +_MD_HEADING_RE = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$") + +_SETEXT_UNDER_RE = re.compile(r"^[-=]{3,}\s*$") + +_SETEXT_TEXT_RE = re.compile(r"^[A-Z][^\n]{0,80}$") + +_STRENGTH3_RE = re.compile(r"\bMUST\b|\bSHALL\b|\bREQUIRED\b|\bFORBIDDEN\b") + +_STRENGTH2_RE = re.compile(r"\bSHOULD\b|\bRECOMMENDED\b") + +_STRENGTH2_PROSE_RE = re.compile(r"recommended to|forbidden to", re.IGNORECASE) + +_STRENGTH1_RE = re.compile(r"\bMAY\b") + +_STRENGTH1_PROSE_RE = re.compile(r"\bcan\b|discouraged", re.IGNORECASE) + +_NAMING_RE = re.compile( + r"(?:MUST|SHOULD|SHALL)[^.]{0,120}" + r"(?:named|name|prefix|suffix|header|field|snake_case|camelCase|lowercase|uppercase)" +) + +_VERSION_DATE_RE = re.compile(r"\b\d{4}-\d{2}-\d{2}\b") + +_VERSION_NUM_RE = re.compile(r"\bv?\d+\.\d+(?:\.\d+)?\b") + +_VERSION_CONTEXT_RE = re.compile(r"version|protocol|revision|draft", re.IGNORECASE) + +_STANDALONE_PIN_RE = re.compile( + rf"^(?:{_VERSION_DATE_RE.pattern}|{_VERSION_NUM_RE.pattern})[.:]?\s*$" +) + +_DIR_SUFFIXES = (".md", ".markdown", ".txt", ".adoc") + +_TEXT_TYPE_MARKS = ("json", "xml", "html", "markdown", "javascript", "yaml") + + +@dataclass +class SpecSource: + """One fetched spec source: what was asked for, what came back.""" + + origin: str + kind: str + text: str + error: str + + +@dataclass +class TicketRef: + """A Jira ticket recognized inside a URL.""" + + base_url: str + key: str + url: str + + +def parse_ticket_url(url: str) -> TicketRef | None: + """Recognize a Jira ticket URL, returning its REST base and key. + + Three shapes are recognized: ``{host}/browse/{KEY}-{n}`` (Jira Cloud and + Server classic), ``{host}/rest/api/{2|3}/issue/{KEY}-{n}`` (raw REST + links), and ``{host}/jira/software/c/projects/{KEY}/issues/{KEY}-{n}`` + (Cloud new UI). Project keys are uppercase letters, digits, and + underscores; the numeric suffix is required. Anything else returns + ``None``. + """ + text = (url or "").strip() + for pattern in _TICKET_PATTERNS: + m = pattern.match(text) + if m: + return TicketRef(base_url=m.group("base"), key=m.group("key"), url=text) + return None + + +def _create_default_session() -> requests.Session: + """Build the read-only HTTP session spec fetching uses. + + The retry policy is the forge adapters' verbatim: read verbs only, since + urllib3 retries beneath the requests adapter and a re-sent write is sent + whole. Spec fetching only ever GETs. + """ + session = requests.Session() + retry = LoggingRetry( + total=3, + backoff_factor=1, + status_forcelist=[429, 500, 502, 503, 504], + respect_retry_after_header=True, + allowed_methods=frozenset(["GET", "HEAD", "OPTIONS"]), + ) + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def _is_text_like(content_type: str) -> bool: + return bool(content_type) and ( + content_type.startswith("text/") or any(mark in content_type for mark in _TEXT_TYPE_MARKS) + ) + + +def _read_stream(resp: requests.Response, max_chars: int) -> str: + """Decode the streamed body up to ``max_chars``, announcing truncation.""" + decoder = codecs.getincrementaldecoder(resp.encoding or "utf-8")(errors="replace") + parts: list[str] = [] + total = 0 + for chunk in resp.iter_content(chunk_size=8192): + if not chunk: + continue + decoded = decoder.decode(chunk, final=False) + if decoded: + parts.append(decoded) + total += len(decoded) + if total >= max_chars: + break + parts.append(decoder.decode(b"", final=True)) + text = "".join(parts) + if len(text) > max_chars: + text = text[:max_chars] + SOURCE_TRUNCATION_MARKER.format(n=max_chars) + return text + + +class _HTMLTextExtractor(HTMLParser): + """Strip tags to text: block tags become line breaks, script/style drop.""" + + _BLOCK = frozenset( + { + "address", "article", "aside", "blockquote", "br", "caption", "dd", "div", "dl", "dt", + "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", + "h6", "head", "header", "hr", "li", "main", "nav", "ol", "p", "pre", "section", + "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul", + } + ) + _DROP = frozenset({"script", "style", "template"}) + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._parts: list[str] = [] + self._skip = 0 + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag in self._DROP: + self._skip += 1 + elif tag in self._BLOCK: + self._parts.append("\n") + + def handle_endtag(self, tag: str) -> None: + if tag in self._DROP: + self._skip = max(0, self._skip - 1) + elif tag in self._BLOCK: + self._parts.append("\n") + + def handle_data(self, data: str) -> None: + if not self._skip: + self._parts.append(data) + + def text(self) -> str: + collapsed: list[str] = [] + blank = True + for line in "".join(self._parts).splitlines(): + stripped = line.strip() + if not stripped: + if not blank: + collapsed.append("") + blank = True + continue + blank = False + collapsed.append(stripped) + while collapsed and not collapsed[-1]: + collapsed.pop() + return "\n".join(collapsed) + + +def _strip_html(text: str) -> str: + extractor = _HTMLTextExtractor() + extractor.feed(text) + extractor.close() + return extractor.text() + + +def _read_capped(path: str, max_chars: int) -> str: + with open(path, encoding="utf-8") as fh: + data = fh.read() + if len(data) > max_chars: + return data[:max_chars] + SOURCE_TRUNCATION_MARKER.format(n=max_chars) + return data + + +def _fetch_url(src: SpecSource, url: str, max_chars: int, session: requests.Session) -> None: + resp = session.get(url, timeout=SPEC_FETCH_TIMEOUT_S, stream=True) + content_type = (resp.headers.get("Content-Type") or "").split(";")[0].strip().lower() + if resp.status_code != 200: + src.error = f"HTTP {resp.status_code} fetching {url}" + return + if not _is_text_like(content_type): + src.error = f"not a text content type: {content_type or 'unknown'}" + return + text = _read_stream(resp, max_chars) + if "html" in content_type: + text = _strip_html(text) + src.text = text + + +def _fetch_file(src: SpecSource, path: str, max_chars: int) -> None: + src.text = _read_capped(path, max_chars) + + +def _fetch_dir(src: SpecSource, path: str, max_chars: int) -> None: + names = sorted( + name + for name in os.listdir(path) + if name.endswith(_DIR_SUFFIXES) and os.path.isfile(os.path.join(path, name)) + )[:SPEC_DIR_MAX_FILES] + if not names: + src.error = f"no {'/'.join(_DIR_SUFFIXES)} files in {path}" + return + parts = [f"## {name}\n\n{_read_capped(os.path.join(path, name), max_chars)}" for name in names] + src.text = "\n\n".join(parts) + + +def _fetch_jira( + src: SpecSource, + ref: TicketRef, + jira_base_url: str, + jira_email: str, + jira_api_token: str, + session: requests.Session, +) -> None: + base = (jira_base_url or ref.base_url).rstrip("/") + url = f"{base}/rest/api/2/issue/{ref.key}?fields=summary,description,issuetype,labels" + auth = (jira_email, jira_api_token) if jira_email and jira_api_token else None + resp = session.get(url, timeout=SPEC_FETCH_TIMEOUT_S, auth=auth) + if resp.status_code in (401, 403) and auth is None: + src.error = ( + f"Jira returned {resp.status_code} for {ref.key} without credentials; set " + "PRXREF_JIRA_BASE_URL, PRXREF_JIRA_EMAIL and PRXREF_JIRA_API_TOKEN to authenticate." + ) + return + if resp.status_code != 200: + src.error = f"Jira returned {resp.status_code} for {ref.key}" + return + payload = resp.json() + fields = payload.get("fields") or {} if isinstance(payload, dict) else {} + summary = fields.get("summary") or "" + issuetype = fields.get("issuetype") + issue_type = issuetype.get("name", "") if isinstance(issuetype, dict) else "" + labels = ", ".join(str(label) for label in fields.get("labels") or []) + description = fields.get("description") + if description is None: + description = "" + elif not isinstance(description, str): + description = json.dumps(description) + src.text = "\n".join([f"Summary: {summary}", f"Type: {issue_type}", f"Labels: {labels}", "", description]).strip() + + +def _dispatch( + src: SpecSource, + raw: str, + max_chars: int, + jira_base_url: str, + jira_email: str, + jira_api_token: str, + session: requests.Session, +) -> None: + text = raw.strip() + if text.startswith(("http://", "https://")): + ref = parse_ticket_url(text) + if ref is not None: + src.kind = "jira" + _fetch_jira(src, ref, jira_base_url, jira_email, jira_api_token, session) + else: + src.kind = "url" + _fetch_url(src, text, max_chars, session) + return + if os.path.isdir(text): + src.kind = "dir" + _fetch_dir(src, text, max_chars) + return + if os.path.isfile(text): + src.kind = "file" + _fetch_file(src, text, max_chars) + return + src.error = f"not a URL or path: {text}" + + +def fetch_specs( + sources: list[str], + *, + max_chars: int, + jira_base_url: str = "", + jira_email: str = "", + jira_api_token: str = "", + session: requests.Session | None = None, +) -> list[SpecSource]: + """Fetch every spec source, in the order given, never raising. + + Each source string is dispatched as: a URL matching a Jira ticket shape + (:func:`parse_ticket_url`) goes to Jira REST; any other ``http(s)`` URL is + fetched as a web page; an existing filesystem path is read as a file or a + directory of spec files; anything else fails that source with a + ``not a URL or path`` error. Every exception β€” network, decode, bad + credentials β€” becomes that source's ``error`` string with empty ``text``; + a source never aborts the run. ``jira_base_url`` overrides the ticket + URL's own host when non-empty (self-hosted boards behind a different REST + host); Jira authenticates with basic auth only when both ``jira_email`` + and ``jira_api_token`` are non-empty, anonymously otherwise. + """ + active = session if session is not None else _create_default_session() + out: list[SpecSource] = [] + for raw in sources: + src = SpecSource(origin=raw, kind="", text="", error="") + try: + _dispatch(src, raw, max_chars, jira_base_url, jira_email, jira_api_token, active) + except Exception as exc: # noqa: BLE001 - a failed source is data, not an abort + src.text = "" + src.error = f"{type(exc).__name__}: {exc}" + out.append(src) + return out + + +@dataclass +class _Unit: + """One kept digest line with its rank inputs.""" + + origin: str + source_idx: int + doc_idx: int + strength: int + is_ticket: bool + render: str + heading_render: str | None = None + heading_key: str | None = None + tokens: frozenset[str] = frozenset() + score: int = 0 + + +def _origin_short(origin: str) -> str: + parsed = urlparse(origin) + if parsed.scheme: + tail = parsed.path.rstrip("/").rsplit("/", 1)[-1] + return tail or parsed.netloc + return origin.rsplit("/", 1)[-1] or origin + + +def _heading_slug(text: str) -> str: + slug = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-") + return slug[:40].strip("-") + + +def _one_line(text: str, limit: int) -> str: + collapsed = " ".join(text.split()) + return collapsed if len(collapsed) <= limit else collapsed[:limit] + + +def _strength(line: str) -> int | None: + if _STRENGTH3_RE.search(line): + return 3 + if _STRENGTH2_RE.search(line) or _STRENGTH2_PROSE_RE.search(line): + return 2 + if _STRENGTH1_RE.search(line) or _STRENGTH1_PROSE_RE.search(line): + return 1 + return None + + +def _is_version_pin_line(line: str) -> bool: + if _VERSION_DATE_RE.search(line): + return True + return bool(_VERSION_NUM_RE.search(line) and _VERSION_CONTEXT_RE.search(line)) + + +def _spec_units(src: SpecSource, source_idx: int) -> list[_Unit]: + short = _origin_short(src.origin) + lines = src.text.splitlines() + units: list[_Unit] = [] + heading: tuple[str, str] | None = None + i = 0 + while i < len(lines): + line = lines[i] + md = _MD_HEADING_RE.match(line) + setext = ( + md is None + and _SETEXT_TEXT_RE.match(line) is not None + and i + 1 < len(lines) + and _SETEXT_UNDER_RE.match(lines[i + 1]) is not None + and _strength(line) is None + ) + if md or setext: + text = (md.group(1) if md else line).strip() + slug = _heading_slug(text) + heading = ( + f"[spec:{short}#{slug}] (heading) {_one_line(text, _STATEMENT_MAX_CHARS)}", + slug, + ) + i += 2 if setext else 1 + continue + i += 1 + stripped = line.strip() + if not stripped: + continue + strength = 3 if _NAMING_RE.search(stripped) else _strength(stripped) + if strength is None and _is_version_pin_line(stripped): + strength = 3 + if strength is None: + continue + anchor = f"L{i}" + statement = _one_line(stripped, _STATEMENT_MAX_CHARS) + label = {3: "MUST", 2: "SHOULD", 1: "MAY"}[strength] + units.append( + _Unit( + origin=src.origin, + source_idx=source_idx, + doc_idx=i, + strength=strength, + is_ticket=False, + render=f"[spec:{short}#{anchor}] ({label}) {statement}", + heading_render=heading[0] if heading else None, + heading_key=heading[1] if heading else None, + tokens=frozenset(_evidence_tokens(statement)), + ) + ) + return units + + +def _ticket_units(src: SpecSource, source_idx: int) -> list[_Unit]: + ref = parse_ticket_url(src.origin) + tag = f"ticket:{ref.key}" if ref else f"ticket:{_origin_short(src.origin)}" + units: list[_Unit] = [] + used = 0 + for doc_idx, line in enumerate(line for line in src.text.splitlines() if line.strip()): + if used >= TICKET_DESC_BUDGET_CHARS: + break + statement = " ".join(line.split()) + used += len(statement) + 1 + units.append( + _Unit( + origin=src.origin, + source_idx=source_idx, + doc_idx=doc_idx, + strength=3, + is_ticket=True, + render=f"[{tag}] {statement}", + tokens=frozenset(_evidence_tokens(statement)), + ) + ) + return units + + +def _diff_tokens(files: list[FileDiff]) -> frozenset[str]: + toks: set[str] = set() + for f in files: + toks |= _tokens(f.path, split_compounds=True) + for h in f.hunks: + for ln in h.lines: + if ln.kind != " ": + toks |= _tokens(ln.text, split_compounds=True) + return frozenset(toks) + + +def _rank_units(sources: list[SpecSource], diff_toks: frozenset[str]) -> list[_Unit]: + units: list[_Unit] = [] + for idx, src in enumerate(sources): + if src.error or not src.text: + continue + if src.kind == "jira": + units.extend(_ticket_units(src, idx)) + else: + units.extend(_spec_units(src, idx)) + ticket = [u for u in units if u.is_ticket] + rest: list[_Unit] = [] + for u in units: + if u.is_ticket: + continue + u.score = len(u.tokens & diff_toks) + rest.append(u) + relevant = sorted( + (u for u in rest if u.score >= 1), + key=lambda u: (-u.score, u.source_idx, u.doc_idx), + ) + unmatched = [u for u in rest if u.score == 0] + return ( + ticket + + relevant + + [u for u in unmatched if u.strength >= 3] + + [u for u in unmatched if u.strength == 2] + + [u for u in unmatched if u.strength == 1] + ) + + +def build_spec_digest(sources: list[SpecSource], files: list[FileDiff], token_budget: int) -> str: + """Reduce the fetched sources to a bounded digest of this diff's constraints. + + Per source, in document order, headings, RFC-2119 normative statements, + version pins, and naming/shape rules are extracted; each kept unit + renders as one ``[spec:{origin}#anchor] (STRENGTH) statement`` line, its + nearest preceding heading riding along as a scoping line. Units are + ranked: ticket constraints first (scope beats relevance), then units + sharing at least one content token with the diff's token set (file paths + plus changed lines, compound-split, the same evidence vocabulary + :mod:`prxref.quality` uses β€” higher overlap first), then unmatched + MUST-level rules in source order, then unmatched SHOULD- and MAY-level + rules, which the budget exhausts first. The walk stops at + ``token_budget * 4`` chars with :data:`TRUNCATION_MARKER`. Every source + that contributed nothing after pruning gets an explicit + ``[spec:{origin}: nothing diff-relevant kept]`` line, so silence is + explained. A ``token_budget`` below 1 degrades to a truncated one-line + digest rather than raising. + """ + budget = max(1, token_budget) * CHARS_PER_TOKEN + intro = ( + "Spec constraints ranked for this diff: ticket scope first, then " + "constraints sharing tokens with the diff, then unmatched MUST-level rules." + ) + out = [intro] + used = len(intro) + contributed: set[str] = set() + emitted_headings: set[tuple[int, str]] = set() + truncated = False + for unit in _rank_units(sources, _diff_tokens(files)): + parts: list[str] = [] + if unit.heading_render is not None: + key = (unit.source_idx, unit.heading_key or "") + if key not in emitted_headings: + emitted_headings.add(key) + parts.append(unit.heading_render) + parts.append(unit.render) + block = "\n".join(parts) + if used + len(block) + 1 > budget: + out.append(TRUNCATION_MARKER) + truncated = True + break + out.append(block) + used += len(block) + 1 + contributed.add(unit.origin) + if not truncated: + for src in sources: + if src.origin in contributed: + continue + line = f"[spec:{src.origin}: nothing diff-relevant kept]" + if used + len(line) + 1 > budget: + out.append(TRUNCATION_MARKER) + break + out.append(line) + used += len(line) + 1 + return "\n".join(out) diff --git a/tests/evals/README.md b/tests/evals/README.md new file mode 100644 index 00000000..8701be48 --- /dev/null +++ b/tests/evals/README.md @@ -0,0 +1,96 @@ +# Golden offline eval dataset + +Self-contained eval cases for **spec-grounded review**: the future reviewer is +given a Jira-style ticket (scope/intent) plus a docs/spec corpus and must +review a unified diff against that context β€” catching spec violations, not +just generic bugs. Every planted spec violation is detectable ONLY by reading +the case's `docs/` corpus; from the diff alone it looks like correct code. + +`test_evals.py` here is a STRUCTURAL scorer only: it proves the dataset is +well-formed and self-consistent. It runs NO LLM. A later pass will run the +real pipeline per case and score its findings against `expected.json`. + +## Case layout + +``` +tests/evals/case-NNN-/ +β”œβ”€β”€ ticket.md Jira-style ticket: summary, description, acceptance criteria +β”œβ”€β”€ docs/ 1-3 short normative markdown docs the feature must conform to +β”œβ”€β”€ diff.patch git-style unified diff implementing the ticket +β”œβ”€β”€ expected.json machine-checkable expected findings (the scoring key) +└── meta.json case metadata incl. planted-violation manifest +``` + +## `ticket.md` + +Free-form markdown with a `## Summary`, a `## Description`, and a +`## Acceptance Criteria` section. The ticket may REFERENCE the spec corpus +("per docs/session-security.md") but must not restate the violated rule β€” +otherwise the violation stops being doc-detectable. + +## `docs/` + +1-3 short markdown files. Concrete and checkable: exact version strings, +required fields, naming rules, forbidden patterns. Normative keywords MUST / +MUST NOT / FORBIDDEN mark every rule a finding can be grounded on. + +## `diff.patch` + +One raw git-style unified diff, parseable by the production parser +(`prxref.triage.parse_unified_diff`). Each diff contains: + +- at least one PLANTED spec violation (something `docs/` forbids or requires + that the diff gets wrong), and +- at most one generic bug (a plain code defect visible without the docs). + +## `expected.json` + +A JSON array. Each entry is a MUST-FIND item β€” the floor a correct review +must meet, NOT an exhaustive list of every possible finding. Extra true +positives are allowed. + +| field | type | rules | +|--------------|--------|-------| +| `id` | string | unique within the case; referenced by `meta.json` | +| `file` | string | path as it appears in the diff (post-`b/` form); must be a file the diff touches | +| `line_hint` | int | 1-based line in the NEW (post-image) file; must land on a line the diff ADDS (`FileDiff.added_lines`) | +| `severity` | string | `error` \| `warning` \| `outofscope` \| `spec` | +| `must_match` | string | acceptance predicate for the finding body: plain substring, or regex when prefixed `re:` | +| `source` | string | `spec` (grounded in the docs corpus; planted) or `generic` (plain bug) | + +Severity vocabulary: `error` (clear defect), `warning` (probable issue), +`outofscope` (change outside the ticket's declared scope β€” reserved, unused +in the current cases), `spec` (spec-grounded finding; used by every planted +violation). + +## `meta.json` + +```json +{ + "id": "case-NNN-", + "title": "one-line description of the failure mode", + "source_prs": [], + "planted_violations": [ + { + "id": "V1", + "description": "what the diff gets wrong and which doc rule it breaks", + "expected_ref": "S1" + } + ], + "notes": "context for future scorers" +} +``` + +`source_prs` is optional. Every `planted_violations[].expected_ref` must be +the `id` of exactly one `expected.json` entry with `source: "spec"`, and +every `spec`-sourced entry must be referenced by exactly one planted +violation β€” the structural scorer enforces this bidirectionally. + +## Adding a case + +1. Create `case-NNN-/` with all five artifacts above. +2. Pick ONE failure mode not yet covered (required-omission, forbidden + pattern, naming/version pin, scope creep, ...). +3. Write the planted violation so the diff alone looks correct. +4. Compute `line_hint` against the diff's new-image line numbers. +5. Run `uv run pytest tests/evals/ -q` β€” the structural scorer must pass. diff --git a/tests/evals/__init__.py b/tests/evals/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/evals/case-001-mcp-protocol-upgrade/diff.patch b/tests/evals/case-001-mcp-protocol-upgrade/diff.patch new file mode 100644 index 00000000..654a6abe --- /dev/null +++ b/tests/evals/case-001-mcp-protocol-upgrade/diff.patch @@ -0,0 +1,32 @@ +diff --git a/src/mcp_client/session.py b/src/mcp_client/session.py +--- a/src/mcp_client/session.py ++++ b/src/mcp_client/session.py +@@ -1,6 +1,6 @@ + """Session handling for the MCP client.""" + +-PROTOCOL_VERSION = "2025-06-18" ++PROTOCOL_VERSION = "2026-07-28" + + +class McpSession: +@@ -12,9 +12,19 @@ + self.session_id = None + + def initialize_payload(self): + return { + "protocolVersion": PROTOCOL_VERSION, ++ "capabilities": {"roots": {"listChanged": True}}, + } + + def request_headers(self): +- return {"Authorization": f"Bearer {self.token}"} ++ headers = {"Authorization": f"Bearer {self.token}"} ++ if self.session_id is not None: ++ headers["Mcp-Session-Id"] = self.session_id ++ return headers ++ ++ def mark_initialized(self, response): ++ if response.ok: ++ self.session_id = response.headers["Mcp-Session-Id"].strip() ++ return True ++ return False diff --git a/tests/evals/case-001-mcp-protocol-upgrade/docs/mcp-client-spec.md b/tests/evals/case-001-mcp-protocol-upgrade/docs/mcp-client-spec.md new file mode 100644 index 00000000..fed9e68b --- /dev/null +++ b/tests/evals/case-001-mcp-protocol-upgrade/docs/mcp-client-spec.md @@ -0,0 +1,35 @@ +# MCP Client Wire Spec β€” release 2026-07-28 (excerpt) + +Scope: client-side requirements for protocol version `2026-07-28`. This +excerpt is normative for the client upgrade epic. + +## 1. Protocol version + +- The initialize request MUST carry `protocolVersion` exactly + `"2026-07-28"`. +- On a server rejection, the client MAY retry once with `"2025-06-18"` + (legacy fallback). + +## 2. Initialize params (required fields) + +The initialize params object MUST include ALL of: + +- `protocolVersion`: string (see section 1) +- `capabilities`: object (MAY be empty) +- `clientInfo`: object with REQUIRED string fields: + - `name`: the client product name + - `version`: the client semantic version + +An initialize request missing any required field MUST be treated as a spec +violation; conforming servers SHOULD reject it. + +## 3. Session continuity + +- The server MAY return an `Mcp-Session-Id` response header on initialize. +- If present, the client MUST store it and send it as the `Mcp-Session-Id` + request header on every subsequent request. Omitting it is a protocol + violation. + +## 4. Forbidden + +- Clients MUST NOT send `initialize` more than once per session. diff --git a/tests/evals/case-001-mcp-protocol-upgrade/expected.json b/tests/evals/case-001-mcp-protocol-upgrade/expected.json new file mode 100644 index 00000000..a36dae20 --- /dev/null +++ b/tests/evals/case-001-mcp-protocol-upgrade/expected.json @@ -0,0 +1,18 @@ +[ + { + "id": "S1", + "file": "src/mcp_client/session.py", + "line_hint": 17, + "severity": "spec", + "must_match": "clientInfo", + "source": "spec" + }, + { + "id": "G1", + "file": "src/mcp_client/session.py", + "line_hint": 28, + "severity": "error", + "must_match": "re:KeyError|\\.get\\(|headers\\.get|missing header", + "source": "generic" + } +] diff --git a/tests/evals/case-001-mcp-protocol-upgrade/meta.json b/tests/evals/case-001-mcp-protocol-upgrade/meta.json new file mode 100644 index 00000000..fab6a55b --- /dev/null +++ b/tests/evals/case-001-mcp-protocol-upgrade/meta.json @@ -0,0 +1,13 @@ +{ + "id": "case-001-mcp-protocol-upgrade", + "title": "MCP client upgrade to wire spec 2026-07-28 omits a required initialize field", + "source_prs": [], + "planted_violations": [ + { + "id": "V1", + "description": "initialize params omit the required clientInfo object (docs/mcp-client-spec.md section 2 requires name and version); the diff adds capabilities but never clientInfo", + "expected_ref": "S1" + } + ], + "notes": "Failure mode: required-field omission. The version bump itself is CORRECT (2026-07-28) and the session-id echo is implemented, so the violation is invisible without docs/mcp-client-spec.md section 2. Generic bug G1: mark_initialized indexes response.headers directly for a header the spec says is MAY-present, crashing on legacy servers (hence S1 anchors the initialize payload at the capabilities line, new-image line 17)." +} diff --git a/tests/evals/case-001-mcp-protocol-upgrade/ticket.md b/tests/evals/case-001-mcp-protocol-upgrade/ticket.md new file mode 100644 index 00000000..68098a59 --- /dev/null +++ b/tests/evals/case-001-mcp-protocol-upgrade/ticket.md @@ -0,0 +1,26 @@ +# PROJ-4821: Upgrade MCP client to wire spec 2026-07-28 + +**Type:** Task **Priority:** P1 **Epic:** MCP protocol adoption + +## Summary + +Bump our MCP client to the 2026-07-28 wire spec release so we can talk to +servers that require the new version, keeping the legacy fallback for older +servers. + +## Description + +The platform team cut the 2026-07-28 wire spec. Servers on the new version +reject our current initialize handshake. We need the client's session layer +updated to the new spec's client requirements: version pinning, the required +initialize params, and session-id continuity after the handshake. + +Out of scope: server-side changes, transport (HTTP/SSE) replacement. + +## Acceptance Criteria + +- [ ] Client sends `protocolVersion` "2026-07-28" on initialize +- [ ] Initialize params carry every field the spec requires +- [ ] Session id returned by the server is echoed on later requests +- [ ] Legacy fallback to "2025-06-18" still available for old servers +- [ ] Conformance per docs/mcp-client-spec.md (excerpt, normative) diff --git a/tests/evals/case-002-session-token-logging/diff.patch b/tests/evals/case-002-session-token-logging/diff.patch new file mode 100644 index 00000000..2a66e3ab --- /dev/null +++ b/tests/evals/case-002-session-token-logging/diff.patch @@ -0,0 +1,49 @@ +diff --git a/src/admin/app.py b/src/admin/app.py +--- a/src/admin/app.py ++++ b/src/admin/app.py +@@ -1,7 +1,10 @@ + """Admin console app.""" + + from flask import Flask + ++from admin.session_store import SessionStore ++ + app = Flask(__name__) ++store = SessionStore("sessions.db") + + sessions = {} +diff --git a/src/admin/session_store.py b/src/admin/session_store.py +--- /dev/null ++++ b/src/admin/session_store.py +@@ -0,0 +1,31 @@ ++"""Durable session store for the admin console.""" ++ ++import logging ++import os ++import random ++import time ++ ++logger = logging.getLogger(__name__) ++ ++SESSION_TTL_SECONDS = 3600 ++ ++ ++class SessionStore: ++ """SQLite-backed session store surviving process restarts.""" ++ ++ def __init__(self, db_path): ++ self.db_path = db_path ++ self.secret_key = os.environ["VITE_SESSION_SECRET"] ++ ++ def issue(self, user_id): ++ token = self._mint_token() ++ session_id = f"{user_id}-{random.random()}" ++ self._persist(session_id, token, user_id, time.time() + SESSION_TTL_SECONDS) ++ logger.debug("issued session token %s for user %s", token, user_id) ++ return session_id ++ ++ def _mint_token(self): ++ return os.urandom(32).hex() ++ ++ def _persist(self, session_id, token, user_id, expires_at): ++ raise NotImplementedError diff --git a/tests/evals/case-002-session-token-logging/docs/session-security.md b/tests/evals/case-002-session-token-logging/docs/session-security.md new file mode 100644 index 00000000..77c6af8d --- /dev/null +++ b/tests/evals/case-002-session-token-logging/docs/session-security.md @@ -0,0 +1,20 @@ +# Platform Security Spec β€” sessions and secrets (excerpt) + +Normative for all services under `admin/` and `ingest/`. A +session-handling change is not mergeable until it passes S-1 and S-2. + +## S-1 Secret handling in logs + +Tokens, session tokens, bearer credentials, and `Authorization` header +values MUST NEVER be written to logs, metrics, traces, or error reports, at +any level including DEBUG. Log the opaque session ID instead; the session ID +is not a secret. + +## S-2 Environment variable prefixes + +- Server-side secrets MUST be read from environment variables carrying the + `APP_` prefix only (e.g. `APP_SESSION_SECRET`). +- `VITE_`-prefixed variables are inlined into the browser bundle by the + build pipeline and are FORBIDDEN as a source of server-side secrets. + Reading any `VITE_*` variable from a server process is a violation + regardless of the variable's contents. diff --git a/tests/evals/case-002-session-token-logging/expected.json b/tests/evals/case-002-session-token-logging/expected.json new file mode 100644 index 00000000..dafdaef8 --- /dev/null +++ b/tests/evals/case-002-session-token-logging/expected.json @@ -0,0 +1,26 @@ +[ + { + "id": "S1", + "file": "src/admin/session_store.py", + "line_hint": 24, + "severity": "spec", + "must_match": "re:(log|logger|debug|info).{0,80}token|token.{0,80}(log|logger|debug|info)", + "source": "spec" + }, + { + "id": "S2", + "file": "src/admin/session_store.py", + "line_hint": 18, + "severity": "spec", + "must_match": "re:VITE_", + "source": "spec" + }, + { + "id": "G1", + "file": "src/admin/session_store.py", + "line_hint": 31, + "severity": "error", + "must_match": "re:NotImplementedError|not implemented|stub|does not persist|persistence", + "source": "generic" + } +] diff --git a/tests/evals/case-002-session-token-logging/meta.json b/tests/evals/case-002-session-token-logging/meta.json new file mode 100644 index 00000000..de5dda68 --- /dev/null +++ b/tests/evals/case-002-session-token-logging/meta.json @@ -0,0 +1,18 @@ +{ + "id": "case-002-session-token-logging", + "title": "Durable session store logs the token and reads its secret from a forbidden env prefix", + "source_prs": [], + "planted_violations": [ + { + "id": "V1", + "description": "issue() logs the session token at DEBUG level, forbidden by docs/session-security.md S-1 (log the opaque session ID instead)", + "expected_ref": "S1" + }, + { + "id": "V2", + "description": "the signing key is read from VITE_SESSION_SECRET; docs/session-security.md S-2 forbids VITE_* as a server-side secret source (APP_ prefix only)", + "expected_ref": "S2" + } + ], + "notes": "Failure mode: forbidden pattern. Both violations look like reasonable code from the diff alone (a log line for support, a secret from the environment); only docs/session-security.md makes them violations. Generic bug G1: _persist raises NotImplementedError, so sessions do NOT actually survive a restart despite the ticket's first acceptance criterion. The weak session id (f-string with user_id and random.random()) is intentionally unlisted; expected.json is a floor, not a ceiling." +} diff --git a/tests/evals/case-002-session-token-logging/ticket.md b/tests/evals/case-002-session-token-logging/ticket.md new file mode 100644 index 00000000..2e80a71e --- /dev/null +++ b/tests/evals/case-002-session-token-logging/ticket.md @@ -0,0 +1,24 @@ +# AUTH-991: Persist admin console sessions across restarts + +**Type:** Story **Priority:** P0 **Epic:** Admin console GA + +## Summary + +Move admin session state out of the in-process dict into a durable store so +deployments no longer log every user out. + +## Description + +Support tickets spike after every deploy: the admin console keeps sessions in +a process-local dict, so any restart invalidates them. Introduce a durable +session store and wire it into the app. Add an issuance log line so support +can correlate a session with a user when triaging incidents. + +Non-goals: SSO, MFA changes. + +## Acceptance Criteria + +- [ ] Sessions survive an app restart +- [ ] Session issuance is logged for support correlation +- [ ] Secrets are wired from the environment +- [ ] Passes the platform security checklist (docs/session-security.md) diff --git a/tests/evals/case-003-config-schema-pin/diff.patch b/tests/evals/case-003-config-schema-pin/diff.patch new file mode 100644 index 00000000..951bd5e1 --- /dev/null +++ b/tests/evals/case-003-config-schema-pin/diff.patch @@ -0,0 +1,27 @@ +diff --git a/config/ingest.yaml b/config/ingest.yaml +--- a/config/ingest.yaml ++++ b/config/ingest.yaml +@@ -1,4 +1,4 @@ + # Ingest service configuration. +-schema_version: "2.9" ++schema_version: "3" + retention: + days: 30 +diff --git a/migrations/V24-add-retention-index.sql b/migrations/V24-add-retention-index.sql +--- /dev/null ++++ b/migrations/V24-add-retention-index.sql +@@ -0,0 +1,5 @@ ++-- Backfill index for retention lookups. ++CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_events_retention ++ ON events (tenant_id, created_at) ++ WHERE created_at > now() - interval '30 days'; ++ +diff --git a/src/ingest/runner.py b/src/ingest/runner.py +--- a/src/ingest/runner.py ++++ b/src/ingest/runner.py +@@ -8,4 +8,4 @@ + return [ + name +- for name in os.listdir(directory) ++ for name in sorted(os.listdir(directory)) + if name.endswith(".sql") diff --git a/tests/evals/case-003-config-schema-pin/docs/config-schema.md b/tests/evals/case-003-config-schema-pin/docs/config-schema.md new file mode 100644 index 00000000..a66aa51c --- /dev/null +++ b/tests/evals/case-003-config-schema-pin/docs/config-schema.md @@ -0,0 +1,22 @@ +# Ingest Config Schema Spec β€” v3.0 (excerpt) + +Normative for `config/ingest.yaml` and `migrations/`. + +## C-1 Pinned schema version + +`schema_version` MUST be the exact string `"3.0"`. + +- `"2.9"` (or anything older) is rejected as unsupported. +- `"3"` and `"3.0.0"` are rejected: the field is an exact-match string, not a + semver range. The registry compares string equality and nothing else. + +## C-2 Migration file naming + +Migration files MUST be named `V__.sql` where: + +- `` is the migration's integer version, written without zero padding, +- the separator between `` and `` is EXACTLY two underscores (`__`), +- `` uses lowercase words separated by single underscores. + +`V24-add-retention-index.sql` (hyphen separator) and `V24_add_retention.sql` +(single underscore separator) are both invalid and rejected by the registry. diff --git a/tests/evals/case-003-config-schema-pin/expected.json b/tests/evals/case-003-config-schema-pin/expected.json new file mode 100644 index 00000000..159c3d23 --- /dev/null +++ b/tests/evals/case-003-config-schema-pin/expected.json @@ -0,0 +1,26 @@ +[ + { + "id": "S1", + "file": "config/ingest.yaml", + "line_hint": 2, + "severity": "spec", + "must_match": "re:\"3\\.0\"|3\\.0|exact|string equality", + "source": "spec" + }, + { + "id": "S2", + "file": "migrations/V24-add-retention-index.sql", + "line_hint": 1, + "severity": "spec", + "must_match": "re:V24-add|hyphen|double underscore|__|naming", + "source": "spec" + }, + { + "id": "G1", + "file": "src/ingest/runner.py", + "line_hint": 10, + "severity": "error", + "must_match": "re:lexicographic|sort.{0,60}order|V10|natural", + "source": "generic" + } +] diff --git a/tests/evals/case-003-config-schema-pin/meta.json b/tests/evals/case-003-config-schema-pin/meta.json new file mode 100644 index 00000000..3171f03c --- /dev/null +++ b/tests/evals/case-003-config-schema-pin/meta.json @@ -0,0 +1,18 @@ +{ + "id": "case-003-config-schema-pin", + "title": "Config pin uses a near-miss schema version and a migration file the naming rule rejects", + "source_prs": [], + "planted_violations": [ + { + "id": "V1", + "description": "schema_version is set to \"3\"; docs/config-schema.md C-1 pins the exact string \"3.0\" and explicitly rejects \"3\"", + "expected_ref": "S1" + }, + { + "id": "V2", + "description": "migration file V24-add-retention-index.sql uses a hyphen separator; docs/config-schema.md C-2 requires V__.sql with exactly two underscores", + "expected_ref": "S2" + } + ], + "notes": "Failure mode: naming/version constraint. Both violations are near-misses that read as a diligent upgrade from the diff alone (\"3\" IS newer than \"2.9\"; the filename follows V-.sql intuition); only docs/config-schema.md makes them wrong. Generic bug G1: sorted() orders migration filenames lexicographically, so V10 sorts before V9 on any fleet past nine migrations." +} diff --git a/tests/evals/case-003-config-schema-pin/ticket.md b/tests/evals/case-003-config-schema-pin/ticket.md new file mode 100644 index 00000000..7f89c63a --- /dev/null +++ b/tests/evals/case-003-config-schema-pin/ticket.md @@ -0,0 +1,26 @@ +# DATA-177: Pin ingest config to schema 3.0 and ship retention index + +**Type:** Task **Priority:** P1 **Epic:** Retention queries + +## Summary + +Adopt the ingest config schema pinned by the platform registry and add the +backfill migration for retention queries. + +## Description + +The config registry started validating `schema_version` strictly as part of +the 3.0 rollout. Our service config must be updated to the pinned value, and +the retention dashboard needs its backing index, delivered as a migration +file the registry will accept. + +Naming and version rules are owned by the config schema spec; follow it +exactly β€” the registry rejects anything else at startup. + +## Acceptance Criteria + +- [ ] `config/ingest.yaml` passes registry validation on boot +- [ ] New migration file is accepted by the migration registry +- [ ] Retention queries use the new index +- [ ] Migrations apply in order on a fresh database +- [ ] Conformance per docs/config-schema.md (excerpt, normative) diff --git a/tests/evals/test_evals.py b/tests/evals/test_evals.py new file mode 100644 index 00000000..f16b49e8 --- /dev/null +++ b/tests/evals/test_evals.py @@ -0,0 +1,164 @@ +"""Golden offline eval dataset: loader, schema validation, structural scorer. + +Each ``case-*`` directory under ``tests/evals/`` is a self-contained eval case +for the future spec-grounded review pipeline: a Jira-style ticket +(``ticket.md``), a small normative spec corpus (``docs/``), a unified diff +implementing the ticket with at least one planted spec violation +(``diff.patch``), machine-checkable expected findings (``expected.json``), and +case metadata (``meta.json``). See ``README.md`` for the full case format. + +This module deliberately runs NO LLM. It proves the dataset is well-formed and +self-consistent: every case parses with the production diff parser +(:func:`prxref.triage.parse_unified_diff`), both JSON schemas hold, every +planted violation in ``meta.json`` maps to exactly one ``spec``-sourced entry +in ``expected.json`` and vice versa, and every expected finding anchors on a +line the diff actually adds. A later pass runs the real pipeline per case and +scores its findings against ``expected.json``; ``must_match`` is the +acceptance predicate for that pass (plain substring, or a regex when prefixed +``re:``). +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from prxref.triage import FileDiff, parse_unified_diff + +EVALS_DIR = Path(__file__).resolve().parent +SEVERITY_VOCABULARY = frozenset({"error", "warning", "outofscope", "spec"}) +SOURCE_VOCABULARY = frozenset({"spec", "generic"}) +EXPECTED_KEYS = frozenset( + {"id", "file", "line_hint", "severity", "must_match", "source"} +) +META_KEYS = frozenset({"id", "title", "source_prs", "planted_violations", "notes"}) +META_REQUIRED_KEYS = frozenset({"id", "title", "planted_violations", "notes"}) +VIOLATION_KEYS = frozenset({"id", "description", "expected_ref"}) +CASE_DIRS = sorted( + p for p in EVALS_DIR.iterdir() if p.is_dir() and p.name.startswith("case-") +) + + +@dataclass +class EvalCase: + """One loaded eval case: metadata, expected findings, parsed diff.""" + + directory: Path + meta: dict + expected: list[dict] + files: list[FileDiff] + + @property + def file_map(self) -> dict[str, FileDiff]: + return {f.path: f for f in self.files} + + +def _read_json(path: Path): + return json.loads(path.read_text(encoding="utf-8")) + + +@pytest.fixture(params=CASE_DIRS, ids=[p.name for p in CASE_DIRS]) +def case(request) -> EvalCase: + directory = request.param + files = parse_unified_diff((directory / "diff.patch").read_text(encoding="utf-8")) + return EvalCase( + directory=directory, + meta=_read_json(directory / "meta.json"), + expected=_read_json(directory / "expected.json"), + files=files, + ) + + +def test_dataset_has_cases(): + assert len(CASE_DIRS) >= 3, "eval dataset needs at least three case-* directories" + + +def test_case_layout(case: EvalCase): + assert (case.directory / "ticket.md").is_file() + assert (case.directory / "diff.patch").is_file() + docs = sorted((case.directory / "docs").glob("*.md")) + assert docs, "docs corpus is empty" + for doc in docs: + text = doc.read_text(encoding="utf-8") + assert "MUST" in text or "FORBIDDEN" in text, doc.name + ticket = (case.directory / "ticket.md").read_text(encoding="utf-8") + assert "Acceptance Criteria" in ticket + assert len(ticket) > 200 + + +def test_diff_parses_with_production_parser(case: EvalCase): + assert case.files, "diff.patch produced no FileDiff records" + assert any(f.hunks for f in case.files), "diff.patch contains no hunks" + + +def test_expected_json_schema(case: EvalCase): + assert isinstance(case.expected, list) and case.expected + seen_ids: set[str] = set() + for entry in case.expected: + assert set(entry) == EXPECTED_KEYS, entry + assert isinstance(entry["id"], str) and entry["id"] + assert isinstance(entry["file"], str) and entry["file"] + assert isinstance(entry["line_hint"], int) + assert not isinstance(entry["line_hint"], bool) + assert entry["line_hint"] >= 1 + assert entry["severity"] in SEVERITY_VOCABULARY + assert isinstance(entry["must_match"], str) and entry["must_match"] + assert entry["source"] in SOURCE_VOCABULARY + assert entry["id"] not in seen_ids, f"duplicate expected id {entry['id']}" + seen_ids.add(entry["id"]) + + +def test_meta_json_schema(case: EvalCase): + meta = case.meta + assert set(meta) <= META_KEYS + assert META_REQUIRED_KEYS <= set(meta) + assert meta["id"] == case.directory.name + assert isinstance(meta["title"], str) and meta["title"] + assert isinstance(meta["notes"], str) and meta["notes"] + assert isinstance(meta.get("source_prs", []), list) + violations = meta["planted_violations"] + assert isinstance(violations, list) and violations + violation_ids: set[str] = set() + for violation in violations: + assert set(violation) == VIOLATION_KEYS, violation + assert isinstance(violation["id"], str) and violation["id"] + assert isinstance(violation["description"], str) and violation["description"] + assert isinstance(violation["expected_ref"], str) and violation["expected_ref"] + assert violation["id"] not in violation_ids + violation_ids.add(violation["id"]) + + +def test_planted_violations_map_to_spec_entries(case: EvalCase): + by_id = {entry["id"]: entry for entry in case.expected} + for violation in case.meta["planted_violations"]: + entry = by_id.get(violation["expected_ref"]) + assert entry is not None, f"unknown expected_ref {violation['expected_ref']}" + assert entry["source"] == "spec", entry + spec_ids = {entry["id"] for entry in case.expected if entry["source"] == "spec"} + referenced = {v["expected_ref"] for v in case.meta["planted_violations"]} + assert spec_ids, "case has no spec-sourced expected findings" + assert referenced == spec_ids, "spec entries and planted violations diverge" + + +def test_expected_files_present_in_diff(case: EvalCase): + file_map = case.file_map + for entry in case.expected: + assert entry["file"] in file_map, entry["file"] + + +def test_line_hints_anchor_added_lines(case: EvalCase): + file_map = case.file_map + for entry in case.expected: + added = file_map[entry["file"]].added_lines + assert entry["line_hint"] in added, entry + + +def test_must_match_patterns_compile(case: EvalCase): + for entry in case.expected: + pattern = entry["must_match"] + if pattern.startswith("re:"): + re.compile(pattern[3:]) diff --git a/tests/test_cli.py b/tests/test_cli.py index 26a69787..02fb14df 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -774,6 +774,81 @@ def test_legal_edge_values_still_run( assert len(fake_runtime["orchestrate_calls"]) == 1 +class TestSpecFlag: + """``--spec`` collects repeatable sources and rides the ``load_config`` + override path, exactly like ``--max-chunks``: the flag replaces the + PRXREF_SPEC_SOURCES environment list wholesale (no merge).""" + + REF = PRRef( + forge="github", + host="github.com", + owner="org", + repo="repo", + number=7, + url="https://github.com/org/repo/pull/7", + ) + URL = "https://github.com/org/repo/pull/7" + + @pytest.fixture(autouse=True) + def _detect(self, monkeypatch): + monkeypatch.setattr("prxref.cli.detect_forge", lambda url: self.REF) + + def test_the_flag_is_repeatable_and_defaults_to_none(self): + parser = cli._build_parser() + args = parser.parse_args(["review", "--pr-url", self.URL]) + assert args.spec is None + args = parser.parse_args([ + "review", "--pr-url", self.URL, + "--spec", "https://a/spec.md", "--spec", "docs/specs", + ]) + assert args.spec == ["https://a/spec.md", "docs/specs"] + + def test_flag_values_reach_the_resolved_config(self, fake_runtime): + """The override is observed on the config the pipeline is actually + handed, not on the parser: fake_create_llm_client records it.""" + assert main([ + "review", "--pr-url", self.URL, "--no-post", + "--spec", "https://a/spec.md", "--spec", "docs/specs", + ]) == 0 + assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + "https://a/spec.md", "docs/specs", + ] + + def test_no_flag_and_no_env_leaves_the_default_empty(self, fake_runtime): + assert main(["review", "--pr-url", self.URL, "--no-post"]) == 0 + assert fake_runtime["llm_calls"][0]["spec_sources"] == [] + + def test_env_sources_load_through_the_normal_path( + self, fake_runtime, monkeypatch + ): + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://a/spec.md docs/specs") + assert main(["review", "--pr-url", self.URL, "--no-post"]) == 0 + assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + "https://a/spec.md", "docs/specs", + ] + + def test_the_flag_replaces_the_environment_without_merging( + self, fake_runtime, monkeypatch + ): + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://env/only.md") + assert main([ + "review", "--pr-url", self.URL, "--no-post", + "--spec", "https://flag/only.md", + ]) == 0 + assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + "https://flag/only.md" + ] + + def test_the_webhook_daemon_gets_the_environment_sources( + self, fake_runtime, monkeypatch + ): + """The daemon passes no flags, so PRXREF_SPEC_SOURCES in its environment is + the only way it can ground a review β€” and it must reach the pipeline.""" + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://a/spec.md") + cli._webhook_handler(self.URL) + assert fake_runtime["llm_calls"][0]["spec_sources"] == ["https://a/spec.md"] + + class TestDryRun: """PRXREF_DRY_RUN must reach BOTH review paths, daemon included. diff --git a/tests/test_config.py b/tests/test_config.py index 84beb9bd..33f5ebd4 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -518,6 +518,145 @@ def test_no_upper_bound_is_invented(self, monkeypatch): assert cfg["chunk_context_lines"] == 100_000 +class TestSpecSourceConfig: + """PRXREF_SPEC_SOURCES / PRXREF_SPEC_MAX_CHARS / PRXREF_SPEC_DIGEST_TOKENS and the + three PRXREF_JIRA_* keys: the spec-grounded review's input surface.""" + + def test_defaults_are_inert(self): + cfg = load_config() + assert cfg["spec_sources"] == [] + assert cfg["spec_max_chars"] == 120000 + assert isinstance(cfg["spec_max_chars"], int) + assert cfg["spec_digest_tokens"] == 3000 + assert isinstance(cfg["spec_digest_tokens"], int) + assert cfg["jira_base_url"] == "" + assert cfg["jira_email"] == "" + assert cfg["jira_api_token"] == "" + + def test_the_new_keys_are_declared_in_their_tables(self): + """The four-surface rule starts here: list key in _LIST_KEYS, both ints + in _INT_KEYS and _RANGES, the three strings in none of them.""" + assert "spec_sources" in config._LIST_KEYS + assert {"spec_max_chars", "spec_digest_tokens"} <= config._INT_KEYS + assert {"spec_max_chars", "spec_digest_tokens"} <= set(config._RANGES) + for key in ("jira_base_url", "jira_email", "jira_api_token"): + assert key not in ( + config._INT_KEYS | config._FLOAT_KEYS | config._BOOL_KEYS + | config._LIST_KEYS + ) + + def test_env_values_coerce(self, monkeypatch): + monkeypatch.setenv("PRXREF_SPEC_MAX_CHARS", "5000") + monkeypatch.setenv("PRXREF_SPEC_DIGEST_TOKENS", "500") + monkeypatch.setenv("PRXREF_JIRA_BASE_URL", "https://jira.example.com") + monkeypatch.setenv("PRXREF_JIRA_EMAIL", "ops@example.com") + monkeypatch.setenv("PRXREF_JIRA_API_TOKEN", "token") + cfg = load_config() + assert cfg["spec_max_chars"] == 5000 + assert isinstance(cfg["spec_max_chars"], int) + assert cfg["spec_digest_tokens"] == 500 + assert isinstance(cfg["spec_digest_tokens"], int) + assert cfg["jira_base_url"] == "https://jira.example.com" + assert cfg["jira_email"] == "ops@example.com" + assert cfg["jira_api_token"] == "token" + + @pytest.mark.parametrize("raw,expected", [ + ("a", ["a"]), + ("a,b", ["a", "b"]), + ("a, b", ["a", "b"]), + ("a b", ["a", "b"]), + ("a b,c d", ["a", "b", "c", "d"]), + (" a , b ", ["a", "b"]), + ("a,,b", ["a", "b"]), + ("a,b,", ["a", "b"]), + ]) + def test_spec_sources_split_on_comma_and_whitespace( + self, monkeypatch, raw, expected + ): + """A PRXREF_SPEC_SOURCES value may separate sources with commas, whitespace, + or both β€” URLs and paths are opaque strings, so the only split that + cannot corrupt one is a run of separators.""" + monkeypatch.setenv("PRXREF_SPEC_SOURCES", raw) + assert load_config()["spec_sources"] == expected + + @pytest.mark.parametrize("name,key,expected", [ + ("PRXREF_SPEC_SOURCES", "spec_sources", []), + ("PRXREF_SPEC_MAX_CHARS", "spec_max_chars", 120000), + ("PRXREF_SPEC_DIGEST_TOKENS", "spec_digest_tokens", 3000), + ]) + def test_whitespace_only_env_reads_as_unset(self, monkeypatch, name, key, expected): + monkeypatch.setenv(name, " ") + assert load_config()[key] == expected + + @pytest.mark.parametrize("name", [ + "PRXREF_SPEC_MAX_CHARS", + "PRXREF_SPEC_DIGEST_TOKENS", + ]) + def test_malformed_int_names_the_variable(self, monkeypatch, name): + monkeypatch.setenv(name, "lots") + with pytest.raises(ConfigError, match=name): + load_config() + + @pytest.mark.parametrize("name", [ + "PRXREF_SPEC_MAX_CHARS", + "PRXREF_SPEC_DIGEST_TOKENS", + ]) + @pytest.mark.parametrize("raw", ["0", "-1"]) + def test_non_positive_spec_ints_rejected(self, monkeypatch, name, raw): + """Same bound as every other size knob: positive, unbounded above.""" + monkeypatch.setenv(name, raw) + with pytest.raises(ConfigError, match=name): + load_config() + + @pytest.mark.parametrize("key,env", [ + ("spec_max_chars", "PRXREF_SPEC_MAX_CHARS"), + ("spec_digest_tokens", "PRXREF_SPEC_DIGEST_TOKENS"), + ]) + def test_overrides_cannot_smuggle_a_degenerate_value(self, key, env): + """An override is still range-checked, and reported as the override.""" + with pytest.raises(ConfigError, match=key) as exc: + load_config(**{key: 0}) + assert env not in str(exc.value) + + def test_no_upper_bound_is_invented(self, monkeypatch): + """A ceiling would be corpus-size-specific; a huge cap is a cap.""" + monkeypatch.setenv("PRXREF_SPEC_MAX_CHARS", "10000000") + monkeypatch.setenv("PRXREF_SPEC_DIGEST_TOKENS", "1000000") + cfg = load_config() + assert cfg["spec_max_chars"] == 10_000_000 + assert cfg["spec_digest_tokens"] == 1_000_000 + + def test_overrides_accept_the_new_keys(self): + cfg = load_config( + spec_sources=["https://a/spec.md"], + spec_max_chars=1, + spec_digest_tokens=1, + jira_base_url="https://jira.example.com", + jira_email="ops@example.com", + jira_api_token="token", + ) + assert cfg["spec_sources"] == ["https://a/spec.md"] + assert cfg["spec_max_chars"] == 1 + assert cfg["spec_digest_tokens"] == 1 + assert cfg["jira_base_url"] == "https://jira.example.com" + assert cfg["jira_email"] == "ops@example.com" + assert cfg["jira_api_token"] == "token" + + def test_an_override_replaces_the_environment(self, monkeypatch): + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "env-a env-b") + assert load_config(spec_sources=["flag-a"])["spec_sources"] == ["flag-a"] + + def test_a_none_override_is_ignored(self, monkeypatch): + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "env-a") + assert load_config(spec_sources=None)["spec_sources"] == ["env-a"] + + def test_the_widened_list_split_is_a_noop_for_llm_models(self, monkeypatch): + """The comma-or-whitespace split applies to every list key; model + names can never contain spaces, so ``llm_models`` only gains slack.""" + monkeypatch.setenv("PRXREF_LLM_MODELS", "m1 m2,m3") + assert load_config()["llm_models"] == ["m1", "m2", "m3"] + + class TestPreExistingNumericRanges: """The three numeric keys that predate the range check are now covered too. diff --git a/tests/test_formatter.py b/tests/test_formatter.py index a0fe592d..dfa2424c 100644 --- a/tests/test_formatter.py +++ b/tests/test_formatter.py @@ -68,6 +68,7 @@ def test_error_marker_bold_title_body_attribution(self): def test_warning_and_note_markers(self): assert format_inline_comment(_f(severity="warning"), "a").startswith("🟧 **") + assert format_inline_comment(_f(severity="spec"), "a").startswith("πŸ” **") assert format_inline_comment(_f(severity="outofscope"), "a").startswith("🟦 **") def test_unknown_severity_defaults_to_note(self): @@ -112,10 +113,19 @@ def test_empty_findings_approved_zero_counts(self): findings_dropped=[], ) assert "βœ… Approved" in text - assert "πŸŸ₯ 0 error Β· 🟧 0 warning Β· 🟦 0 outofscope" in text + assert "πŸŸ₯ 0 error Β· 🟧 0 warning Β· πŸ” 0 spec Β· 🟦 0 outofscope" in text assert "0 active of 0 raw" in text assert "No findings survived the quality passes." in text + def test_spec_count_present(self): + text = _summary(findings_active=[_f(severity="spec", title="Spec breach")]) + assert "πŸ” 1 spec" in text + + def test_no_specs_requested_leaves_no_grounding_note(self): + """The renderer owns the grounding note; the forge-neutral formatter + always renders it empty, so its output matches an ungrounded run.""" + assert "Spec-grounded" not in _summary() + class TestFormatSummaryTables: def test_table_rows_sorted_error_first(self): @@ -130,6 +140,17 @@ def test_table_rows_sorted_error_first(self): assert text.index("z.py:2") < text.index("z.py:1") assert "| πŸŸ₯ | z.py:2 | Boom |" in text + def test_spec_rows_order_between_warning_and_outofscope(self): + text = _summary( + findings_active=[ + _f(severity="outofscope", file="z.py", line=1), + _f(severity="spec", file="z.py", line=2, title="Spec breach"), + _f(severity="warning", file="z.py", line=3), + ] + ) + assert text.index("z.py:3") < text.index("z.py:2") < text.index("z.py:1") + assert "| πŸ” | z.py:2 | Spec breach |" in text + def test_file_level_finding_omits_line_zero(self): text = _summary(findings_active=[_f(line=0)]) assert "src/app.py:" not in text diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 75d054b4..91c069df 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -21,13 +21,16 @@ import prxref from prxref.forges.base import InlineComment, PRData, PRRef, Thread from prxref.llm import InvokeResult +from prxref.specs import SpecSource from prxref.triage import DEFAULT_TOKEN_BUDGET, Finding, build_chunks, parse_unified_diff SUMMARY_TEMPLATE = ( "πŸ€– **prxref review β€” {verdict}**\n\n" "PR: {title}\n\n" "Files reviewed: {file_count} Β· πŸŸ₯ {error_count} error Β· " - "🟧 {warning_count} warning Β· 🟦 {outofscope_count} outofscope\n\n" + "🟧 {warning_count} warning Β· πŸ” {spec_count} spec Β· " + "🟦 {outofscope_count} outofscope\n" + "{spec_note}\n\n" "{findings}\n\n{attribution}" ) @@ -35,7 +38,7 @@ def _contract_review_chunk( llm, files, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, context_lines=None, context_blocks="", sibling_files=(), - trace_label="", trace_dir="", + trace_label="", trace_dir="", spec_digest="", ): result = llm.invoke( system="review the chunk", @@ -67,7 +70,7 @@ def _contract_review_chunk( def _contract_review_systemic( llm, digest, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, - threads=(), trace_label="", trace_dir="", + threads=(), trace_label="", trace_dir="", spec_digest="", ): return [], { "escalations": [], "input_tokens": 0, "output_tokens": 0, @@ -411,7 +414,7 @@ def test_three_chunks_run_concurrently_and_all_findings_collected(self, monkeypa def barrier_review_chunk( llm, files, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, context_lines=None, context_blocks="", - sibling_files=(), trace_label="", trace_dir="", + sibling_files=(), trace_label="", trace_dir="", spec_digest="", ): barrier.wait() return [Finding( @@ -2067,6 +2070,197 @@ def test_the_shipped_template_style_strips_to_a_clean_header(self): assert out == "## prxref automated review\n" +class TestSpecGrounding: + """``spec_sources`` grounds the run without adding a review unit. + + The fetch + digest ride the never-raise fence; the digest threads into + every chunk prompt and the sweep prompt; the summary gains the grounding + note (failure reasons redacted) and the ``spec`` severity renders πŸ” + without ever moving the verdict. + """ + + SPEC_TEXT = "## Rules\n\nTools MUST be named with an mcp prefix.\n" + + SPEC_FINDINGS = { + "src/app.py": [ + {"file": "src/app.py", "line": 3, "severity": "spec", + "confidence": 0.9, "title": "Forbidden header sent", + "body": "Spec: \"tools MUST be named with an mcp prefix\"; " + "the diff adds it to the data path."}, + ], + } + + def _fetched(self, *, failed: bool = False, origin: str = "docs/spec.md"): + if failed: + return [SpecSource( + origin=origin, kind="url", text="", + error="HTTP 404 fetching https://secret.example.invalid/spec.md", + )] + return [SpecSource( + origin=origin, kind="file", text=self.SPEC_TEXT, error="", + )] + + def _run( + self, monkeypatch, *, fetched=None, llm=None, post=True, **kw, + ): + sources = fetched if fetched is not None else self._fetched() + monkeypatch.setattr( + orchestrator.specs, "fetch_specs", lambda *a, **k: sources, + ) + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + res = orchestrate_review( + forge, REF, llm or FakeLLM(self.SPEC_FINDINGS), post=post, + spec_sources=["docs/spec.md"], **kw, + ) + return forge, res + + def test_the_counts_line_and_note_reach_the_summary(self, monkeypatch): + forge, _res = self._run(monkeypatch) + summary = forge.summaries[0] + assert "πŸ” 1 spec" in summary + assert ( + "> πŸ” Spec-grounded: 1 source(s) Β· 1 constraint(s) injected" + in summary + ) + + def test_no_specs_keeps_the_note_empty_and_the_placeholder_filled( + self, monkeypatch, + ): + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + orchestrate_review(forge, REF, FakeLLM(self.SPEC_FINDINGS)) + summary = forge.summaries[0] + assert "πŸ” 1 spec" in summary + assert "Spec-grounded" not in summary + assert "{spec_note}" not in summary + assert "{spec_count}" not in summary + + def test_the_marker_reaches_bullets_and_inline_cards(self, monkeypatch): + forge, _res = self._run(monkeypatch) + assert "- πŸ” `src/app.py:3` β€” Forbidden header sent" in forge.summaries[0] + body = forge.inline_batches[0][0].body + assert body.startswith("πŸ€– πŸ” **[SPEC] Forbidden header sent**") + + def test_a_spec_only_review_does_not_move_the_verdict(self, monkeypatch): + forge, res = self._run(monkeypatch) + assert len(res["findings_active"]) == 1 + assert res["findings_active"][0].severity == "spec" + assert res["verdict"] == "Approved" + + def test_all_sources_failing_completes_with_only_the_failure_note( + self, monkeypatch, + ): + forge, res = self._run( + monkeypatch, fetched=self._fetched(failed=True), + llm=FakeLLM("{}"), + ) + assert res["verdict"] == "Approved" + summary = forge.summaries[0] + assert "> ⚠️ Spec fetch failed for 1 source(s)" in summary + assert "Spec-grounded:" not in summary + + def test_a_partial_failure_shows_the_note_and_the_failures( + self, monkeypatch, + ): + fetched = self._fetched() + self._fetched(failed=True, origin="x/other.md") + forge, _res = self._run(monkeypatch, fetched=fetched) + summary = forge.summaries[0] + assert "> πŸ” Spec-grounded: 2 source(s) Β· 1 constraint(s) injected" in summary + assert "> ⚠️ Spec fetch failed for 1 source(s)" in summary + + def test_fetch_failure_reasons_are_redacted(self, monkeypatch): + leaked = SpecSource( + origin="https://secret.example.invalid/browse/PROJ-9", + kind="jira", text="", + error=( + "Jira returned 401 for PROJ-9 contacting " + "https://secret.example.invalid/browse/PROJ-9 with " + "opaquetoken0123456789abcdef012345" + ), + ) + forge, _res = self._run( + monkeypatch, fetched=[leaked], llm=FakeLLM("{}"), + ) + body = forge.summaries[0] + assert "> ⚠️ Spec fetch failed for 1 source(s)" in body + assert "secret.example.invalid" not in body + assert "opaquetoken0123456789abcdef012345" not in body + assert "PROJ-9" in body + + def test_the_digest_reaches_the_worker_prompt(self, monkeypatch): + prompts: list[str] = [] + + class RecordingLLM: + def invoke(self, system, user, *, max_tokens=4096, json_mode=False, + timeout_s=60.0): + prompts.append(user) + return InvokeResult( + text="{}", input_tokens=1, output_tokens=1, + model="m", backend="b", elapsed_ms=1, + ) + + monkeypatch.setattr( + orchestrator.reviewer, "review_chunk", REAL_REVIEW_CHUNK, + ) + monkeypatch.setattr( + orchestrator.reviewer, "load_prompt", REAL_LOAD_PROMPT, + ) + self._run(monkeypatch, llm=RecordingLLM(), post=False) + assert len(prompts) == 1 + assert "### Spec constraints" in prompts[0] + assert "(MUST) Tools MUST be named with an mcp prefix" in prompts[0] + assert "(no specs provided for this review)" not in prompts[0] + + def test_the_digest_reaches_the_sweep(self, monkeypatch): + double, calls = _sweep_double([("findings", [])]) + monkeypatch.setattr(orchestrator.reviewer, "review_systemic", double) + self._run(monkeypatch, llm=FakeLLM("{}"), post=False) + assert len(calls) == 1 + assert "Tools MUST be named with an mcp prefix" in calls[0]["spec_digest"] + + def test_the_specs_trace_event_records_source_counts(self, monkeypatch, tmp_path): + target = tmp_path / "run.jsonl" + self._run(monkeypatch, llm=FakeLLM("{}"), post=False, trace_file=str(target)) + events = [json.loads(x) for x in target.read_text().splitlines() if x.strip()] + specs_events = [e for e in events if e["node"] == "specs"] + assert len(specs_events) == 1 + assert specs_events[0]["phase"] == "ok" + meta = specs_events[0]["meta"] + assert meta["sources"] == 1 + assert meta["ok"] == 1 + assert meta["fail"] == 0 + assert meta["digest_chars"] > 0 + + def test_a_specs_crash_never_fails_the_run(self, monkeypatch): + def boom(*a, **kw): + raise RuntimeError("boom specs") + + monkeypatch.setattr(orchestrator.specs, "fetch_specs", boom) + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + res = orchestrate_review( + forge, REF, FakeLLM(self.SPEC_FINDINGS), post=True, + spec_sources=["docs/spec.md"], + ) + assert res["verdict"] == "Approved" + summary = forge.summaries[0] + assert "Spec-grounded" not in summary + assert "Spec fetch failed" not in summary + assert "Forbidden header sent" in summary + + def test_no_specs_asked_runs_no_spec_stage(self, monkeypatch, tmp_path): + monkeypatch.setattr( + orchestrator.specs, "fetch_specs", + lambda *a, **k: (_ for _ in ()).throw(AssertionError("called")), + ) + target = tmp_path / "run.jsonl" + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + orchestrate_review( + forge, REF, FakeLLM(self.SPEC_FINDINGS), post=False, + trace_file=str(target), + ) + events = [json.loads(x) for x in target.read_text().splitlines() if x.strip()] + assert [e for e in events if e["node"] == "specs"] == [] + + class TestRunTrace: """Every exit closes the ``run`` node, and says which kind of exit it was. @@ -2307,8 +2501,12 @@ def _sweep_double(results: list, **meta_overrides): def _review_systemic( llm, digest, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, threads=(), trace_label="", trace_dir="", + spec_digest="", ): - calls.append({"digest": digest, "max_tokens": max_tokens, "threads": list(threads)}) + calls.append({ + "digest": digest, "max_tokens": max_tokens, + "threads": list(threads), "spec_digest": spec_digest, + }) kind, payload = results.pop(0) meta = { "escalations": [], "input_tokens": 7, "output_tokens": 3, @@ -2566,7 +2764,7 @@ def test_a_chunk_and_sweep_echo_never_drops_the_deterministic_finding( def _review_chunk(llm, files, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, context_lines=None, context_blocks="", sibling_files=(), - trace_label="", trace_dir=""): + trace_label="", trace_dir="", spec_digest=""): return [self._matching_finding("chunk worker restatement")], { "input_tokens": 10, "output_tokens": 5, "model": "m", "elapsed_ms": 1, "error": "", @@ -2574,7 +2772,7 @@ def _review_chunk(llm, files, *, pr_title="", pr_description="", def _review_systemic(llm, digest, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, threads=(), - trace_label="", trace_dir=""): + trace_label="", trace_dir="", spec_digest=""): return [self._matching_finding("sweep restatement")], { "input_tokens": 7, "output_tokens": 3, "model": "sweep-model", "elapsed_ms": 1, "error": "", @@ -2707,7 +2905,8 @@ def _scripted_double(outcomes: list[dict]): def _rc(llm, files, *, pr_title="", pr_description="", repo_hint="", max_tokens=None, context_lines=None, context_blocks="", - sibling_files=(), trace_label="", trace_dir=""): + sibling_files=(), trace_label="", trace_dir="", + spec_digest=""): calls.append({ "context_lines": context_lines, "context_blocks": context_blocks, diff --git a/tests/test_quality.py b/tests/test_quality.py index a9719653..ee2fe9a6 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -571,6 +571,70 @@ def test_no_token_rewrites_logs_nothing(self, caplog): assert not [r for r in caplog.records if r.name == quality_logger.name] +class TestSpecSeverity: + """The ``spec`` severity: gate vocabulary, rank order, and cap exemption.""" + + def test_spec_passes_the_quality_gate(self): + f = _f(severity="Spec", confidence=0.9) + result = apply_quality_gate([f]) + assert result[0].drop_reason is None + assert result[0].severity == "spec" + + def test_unknown_severity_is_still_dropped(self): + f = _f(severity="sepc", confidence=0.95) + result = apply_quality_gate([f]) + assert result[0].drop_reason == "invalid severity: 'sepc'" + + def test_rank_order_in_severity_consistency(self): + spec = _f(file="a.ts", line=1, severity="spec", title="Shared pattern") + warning = _f(file="b.ts", line=1, severity="warning", title="shared pattern") + error = _f(file="c.ts", line=1, severity="error", title="shared pattern") + outofscope = _f( + file="d.ts", line=1, severity="outofscope", title="shared pattern", + ) + result = apply_severity_consistency([spec, warning, error, outofscope]) + assert [f.severity for f in result] == [ + "error", "error", "error", "error", + ] + + def test_spec_outranks_outofscope_in_a_title_group(self): + spec = _f(file="a.ts", line=1, severity="spec", title="Shared pattern") + outofscope = _f( + file="b.ts", line=1, severity="outofscope", title="shared pattern", + ) + result = apply_severity_consistency([spec, outofscope]) + assert [f.severity for f in result] == ["spec", "spec"] + + def test_error_cap_ignores_spec_findings(self): + specs = [ + _f(severity="spec", confidence=0.9, title=f"Spec breach {i}") + for i in range(5) + ] + result = apply_quality_gate(specs, max_errors=2) + assert len(active(result)) == 5 + assert all(f.drop_reason is None for f in result) + + def test_error_cap_still_caps_errors_alongside_spec(self): + errors = [ + _f(severity="error", confidence=0.9, title=f"Err {i}") + for i in range(3) + ] + specs = [ + _f(severity="spec", confidence=0.9, title=f"Spec breach {i}") + for i in range(4) + ] + result = apply_quality_gate(errors + specs, max_errors=2) + survivors = active(result) + assert sum(1 for f in survivors if f.severity == "error") == 2 + assert sum(1 for f in survivors if f.severity == "spec") == 4 + + def test_a_title_collision_raises_spec_to_the_group_max(self): + spec = _f(file="a.ts", line=1, severity="spec", title="Shared pattern") + error = _f(file="b.ts", line=1, severity="error", title="shared pattern") + result = apply_severity_consistency([spec, error]) + assert [f.severity for f in result] == ["error", "error"] + + class TestQualityGate: def test_drops_below_confidence_floor(self): # Default floor is 0.60 diff --git a/tests/test_reviewer.py b/tests/test_reviewer.py index 66689ae8..e7846f4e 100644 --- a/tests/test_reviewer.py +++ b/tests/test_reviewer.py @@ -660,6 +660,71 @@ def test_max_tokens_none_uses_the_module_default(self): assert llm.calls[0]["max_tokens"] == 1234 +class TestSpecDigestPrompt: + """The spec digest rides the existing prompts; empty renders the marker.""" + + def test_the_shipped_worker_template_has_the_constraints_block(self): + template = load_prompt("worker.md") + assert "### Spec constraints" in template + assert "{spec_digest}" in template + assert "`spec`" in template + + def test_the_shipped_systemic_template_has_the_constraints_block(self): + template = load_prompt("systemic.md") + assert "### Spec constraints" in template + assert "{spec_digest}" in template + + def test_digest_replaces_the_placeholder_in_the_worker_prompt(self): + llm = FakeLLM(CLEAN_RESPONSE) + review_chunk( + llm, parse_unified_diff(MINI_DIFF), + spec_digest="[spec:spec.md#L1] (MUST) tools MUST be named with the mcp prefix", + ) + user = llm.calls[0]["user"] + assert "tools MUST be named with the mcp prefix" in user + assert "{spec_digest}" not in user + assert "(no specs provided for this review)" not in user + + def test_empty_digest_renders_the_no_specs_marker_in_the_worker_prompt(self): + llm = FakeLLM(CLEAN_RESPONSE) + review_chunk(llm, parse_unified_diff(MINI_DIFF)) + assert "(no specs provided for this review)" in llm.calls[0]["user"] + + def test_digest_replaces_the_placeholder_in_the_sweep_prompt(self): + llm = FakeLLM(CLEAN_RESPONSE) + review_systemic( + llm, TestReviewSystemic.DIGEST, + spec_digest="[ticket:PROJ-9] ship the header flag", + ) + user = llm.calls[0]["user"] + assert "ship the header flag" in user + assert "{spec_digest}" not in user + assert "(no specs provided for this review)" not in user + + def test_empty_digest_renders_the_no_specs_marker_in_the_sweep_prompt(self): + llm = FakeLLM(CLEAN_RESPONSE) + review_systemic(llm, TestReviewSystemic.DIGEST) + assert "(no specs provided for this review)" in llm.calls[0]["user"] + + def test_spec_severity_passes_through_unfiltered(self): + payload = json.dumps({ + "findings": [ + {"file": "src/app.py", "line": 3, "severity": "spec", + "confidence": 0.9, "title": "Forbidden header sent", + "body": "Spec: \"clients MUST NOT send the protocol header\"."}, + ], + "escalations": [], + }) + llm = FakeLLM(payload) + findings, meta = review_chunk( + llm, parse_unified_diff(MINI_DIFF), + spec_digest="clients MUST NOT send the protocol header", + ) + assert meta["error"] == "" + assert len(findings) == 1 + assert findings[0].severity == "spec" + + class _RaisingLLM: def invoke(self, system, user, *, max_tokens=4096, json_mode=False, timeout_s=60.0): diff --git a/tests/test_specs.py b/tests/test_specs.py new file mode 100644 index 00000000..95539f20 --- /dev/null +++ b/tests/test_specs.py @@ -0,0 +1,426 @@ +"""Spec fetch + digest tests: prxref.specs dispatch, failure doctrine, ranking.""" +from __future__ import annotations + +import pytest + +from prxref.specs import ( + SOURCE_TRUNCATION_MARKER, + SPEC_DIR_MAX_FILES, + TRUNCATION_MARKER, + SpecSource, + build_spec_digest, + fetch_specs, + parse_ticket_url, +) +from prxref.triage import parse_unified_diff + + +class _FakeResponse: + def __init__( + self, + status_code: int = 200, + content_type: str = "text/plain", + body: bytes = b"", + chunks: list[bytes] | None = None, + encoding: str = "utf-8", + payload: object | None = None, + ): + self.status_code = status_code + self.headers = {"Content-Type": content_type} + self._chunks = chunks if chunks is not None else [body] + self.encoding = encoding + self._payload = payload if payload is not None else {} + + def iter_content(self, chunk_size: int = 8192, **kwargs): + return iter(self._chunks) + + def json(self): + return self._payload + + +class _FakeSession: + def __init__(self, response: _FakeResponse | None = None, exc: Exception | None = None): + self.response = response + self.exc = exc + self.calls: list[tuple[str, dict]] = [] + + def get(self, url: str, **kwargs): + self.calls.append((url, kwargs)) + if self.exc is not None: + raise self.exc + return self.response + + +def _added_file(path: str, lines: list[str]) -> str: + body = "".join(f"+{text}\n" for text in lines) + return ( + f"diff --git a/{path} b/{path}\n" + "new file mode 100644\n" + "--- /dev/null\n" + f"+++ b/{path}\n" + f"@@ -0,0 +1,{len(lines)} @@\n" + f"{body}" + ) + + +class TestParseTicketUrl: + def test_browse_shape(self): + ref = parse_ticket_url("https://jira.acme.com/browse/PROJ-123") + assert ref is not None + assert ref.base_url == "https://jira.acme.com" + assert ref.key == "PROJ-123" + assert ref.url == "https://jira.acme.com/browse/PROJ-123" + + def test_rest_api_v2_and_v3(self): + for version in ("2", "3"): + ref = parse_ticket_url(f"https://jira.internal/rest/api/{version}/issue/OPS-4") + assert ref is not None + assert ref.base_url == "https://jira.internal" + assert ref.key == "OPS-4" + + def test_cloud_new_ui(self): + ref = parse_ticket_url( + "https://acme.atlassian.net/jira/software/c/projects/ENG/issues/ENG-9?jql=all" + ) + assert ref is not None + assert ref.base_url == "https://acme.atlassian.net" + assert ref.key == "ENG-9" + + def test_non_matches(self): + assert parse_ticket_url("https://github.com/o/r/pull/1") is None + assert parse_ticket_url("https://x.com/browse/proj-1") is None + assert parse_ticket_url("https://x.com/browse/PROJ") is None + assert parse_ticket_url("/browse/PROJ-1") is None + assert parse_ticket_url("") is None + + +class TestFetchSpecsNeverRaises: + def test_garbage_sources_become_errors(self): + sources = ["", " ", "not a path or url", "\x00bad"] + session = _FakeSession(exc=RuntimeError("no network")) + results = fetch_specs(sources, max_chars=100, session=session) + assert len(results) == len(sources) + for src in results: + assert src.text == "" + assert src.error != "" + + def test_session_exception_becomes_error(self): + session = _FakeSession(exc=ConnectionError("refused")) + results = fetch_specs(["https://example.com/spec.md"], max_chars=100, session=session) + assert results[0].error != "" + assert results[0].text == "" + + def test_output_order_matches_input(self): + results = fetch_specs( + ["https://example.com/a.md", "nope"], max_chars=10, session=_FakeSession( + _FakeResponse(body=b"ok") + ) + ) + assert [src.origin for src in results] == ["https://example.com/a.md", "nope"] + assert results[0].text == "ok" + assert "not a URL or path" in results[1].error + + +class TestDispatch: + def test_file(self, tmp_path): + path = tmp_path / "spec.md" + path.write_text("The API MUST return 200.", encoding="utf-8") + (src,) = fetch_specs([str(path)], max_chars=1000) + assert src.kind == "file" + assert src.error == "" + assert src.text == "The API MUST return 200." + + def test_directory(self, tmp_path): + (tmp_path / "a.md").write_text("alpha MUST hold", encoding="utf-8") + (tmp_path / "b.txt").write_text("beta SHOULD wait", encoding="utf-8") + (src,) = fetch_specs([str(tmp_path)], max_chars=1000) + assert src.kind == "dir" + assert "alpha MUST hold" in src.text + assert "beta SHOULD wait" in src.text + + def test_url(self): + session = _FakeSession(_FakeResponse(content_type="text/markdown", body=b"# Rules\n\nno MUST here")) + (src,) = fetch_specs(["https://example.com/rules.md"], max_chars=1000, session=session) + assert src.kind == "url" + assert src.error == "" + assert "# Rules" in src.text + + def test_garbage(self): + (src,) = fetch_specs(["definitely not real"], max_chars=100) + assert src.kind == "" + assert src.error.startswith("not a URL or path") + + +class TestJiraFetch: + _PAYLOAD = { + "fields": { + "summary": "Fix login flow", + "issuetype": {"name": "Bug"}, + "labels": ["auth", "urgent"], + "description": "Users MUST re-authenticate after password change.", + } + } + + def _jira_source(self) -> str: + return "https://jira.example.com/browse/AUTH-7" + + def test_authenticated_render(self): + session = _FakeSession(_FakeResponse(content_type="application/json", payload=self._PAYLOAD)) + (src,) = fetch_specs( + [self._jira_source()], + max_chars=1000, + jira_email="ops@example.com", + jira_api_token="secret-token", + session=session, + ) + assert src.kind == "jira" + assert src.error == "" + assert "Summary: Fix login flow" in src.text + assert "Type: Bug" in src.text + assert "Labels: auth, urgent" in src.text + assert "Users MUST re-authenticate" in src.text + url, kwargs = session.calls[0] + assert "/rest/api/2/issue/AUTH-7" in url + assert "fields=summary,description,issuetype,labels" in url + assert kwargs["auth"] == ("ops@example.com", "secret-token") + + def test_jira_base_url_override(self): + session = _FakeSession(_FakeResponse(content_type="application/json", payload=self._PAYLOAD)) + fetch_specs( + [self._jira_source()], + max_chars=1000, + jira_base_url="https://rest.internal", + session=session, + ) + assert session.calls[0][0].startswith("https://rest.internal/rest/api/2/issue/AUTH-7") + + def test_anonymous_when_no_credentials(self): + session = _FakeSession(_FakeResponse(content_type="application/json", payload=self._PAYLOAD)) + (src,) = fetch_specs([self._jira_source()], max_chars=1000, session=session) + assert src.kind == "jira" + assert src.error == "" + assert session.calls[0][1]["auth"] is None + + def test_401_without_credentials_names_env_vars(self): + session = _FakeSession(_FakeResponse(status_code=401, content_type="application/json")) + (src,) = fetch_specs([self._jira_source()], max_chars=1000, session=session) + assert src.text == "" + assert "401" in src.error + assert "PRXREF_JIRA_EMAIL" in src.error + assert "PRXREF_JIRA_API_TOKEN" in src.error + assert "PRXREF_JIRA_BASE_URL" in src.error + assert "secret" not in src.error.lower() + + def test_403_with_credentials_does_not_advise_env_vars(self): + session = _FakeSession(_FakeResponse(status_code=403, content_type="application/json")) + (src,) = fetch_specs( + [self._jira_source()], + max_chars=1000, + jira_email="ops@example.com", + jira_api_token="secret-token", + session=session, + ) + assert src.error != "" + assert "403" in src.error + assert "PRXREF_JIRA_EMAIL" not in src.error + + +class TestSizeCapAndTruncation: + def test_file_truncation_marker(self, tmp_path): + path = tmp_path / "big.md" + path.write_text("x" * 500, encoding="utf-8") + (src,) = fetch_specs([str(path)], max_chars=100) + assert src.text == "x" * 100 + SOURCE_TRUNCATION_MARKER.format(n=100) + + def test_url_stream_truncation(self): + session = _FakeSession( + _FakeResponse(content_type="text/plain", chunks=[b"a" * 60, b"b" * 60]) + ) + (src,) = fetch_specs(["https://example.com/stream.md"], max_chars=100, session=session) + assert src.text.startswith("a" * 60 + "b" * 40) + assert src.text.endswith(SOURCE_TRUNCATION_MARKER.format(n=100)) + + def test_under_cap_no_marker(self, tmp_path): + path = tmp_path / "small.md" + path.write_text("tiny", encoding="utf-8") + (src,) = fetch_specs([str(path)], max_chars=1000) + assert "truncated" not in src.text + + +class TestHtmlStripping: + def test_tags_and_scripts_stripped(self): + body = ( + b"" + b"" + b"

API Rules

Clients MUST retry.

" + ) + session = _FakeSession(_FakeResponse(content_type="text/html; charset=utf-8", body=body)) + (src,) = fetch_specs(["https://example.com/spec.html"], max_chars=5000, session=session) + assert src.error == "" + assert "API Rules" in src.text + assert "Clients MUST retry." in src.text + assert "

" not in src.text + assert "var tracking" not in src.text + assert "color: red" not in src.text + + def test_non_text_content_type_rejected(self): + session = _FakeSession(_FakeResponse(content_type="application/octet-stream", body=b"\x00\x01")) + (src,) = fetch_specs(["https://example.com/blob"], max_chars=100, session=session) + assert src.text == "" + assert "not a text content type" in src.error + + +class TestDirectoryCapAndSort: + def test_sorted_and_capped_at_20(self, tmp_path): + for i in range(SPEC_DIR_MAX_FILES + 5): + (tmp_path / f"f{i:02d}.md").write_text(f"marker-{i:02d} MUST hold", encoding="utf-8") + (src,) = fetch_specs([str(tmp_path)], max_chars=100_000) + assert src.kind == "dir" + assert "marker-00" in src.text + assert f"marker-{SPEC_DIR_MAX_FILES - 1:02d}" in src.text + assert f"marker-{SPEC_DIR_MAX_FILES:02d}" not in src.text + first = src.text.index("marker-00") + assert src.text.index(f"marker-{SPEC_DIR_MAX_FILES - 1:02d}") > first + + def test_sort_order_not_creation_order(self, tmp_path): + for name in ("zulu.md", "alpha.md", "mike.md"): + (tmp_path / name).write_text(f"content-{name}", encoding="utf-8") + (src,) = fetch_specs([str(tmp_path)], max_chars=10_000) + assert src.text.index("content-alpha.md") < src.text.index("content-mike.md") + assert src.text.index("content-mike.md") < src.text.index("content-zulu.md") + + def test_empty_directory_errors(self, tmp_path): + (src,) = fetch_specs([str(tmp_path)], max_chars=1000) + assert src.text == "" + assert src.error != "" + + +def _digest_sources() -> tuple[list[SpecSource], str]: + jira = SpecSource( + origin="https://jira.example.com/browse/SCOPE-1", + kind="jira", + text="Summary: Add widget export\nType: Story\nLabels: api\n\nWidgets MUST validate export size.", + error="", + ) + spec = SpecSource( + origin="/docs/widget-spec.md", + kind="file", + text=( + "# Widget rendering\n\n" + "The widget renderer MUST stream chunks.\n\n" + "The build cache MUST be purged weekly.\n\n" + "Docs MAY reference usage examples.\n" + ), + error="", + ) + diff = _added_file( + "src/widget_renderer.py", + ["def stream_chunks(self):", " return chunks"], + ) + return [jira, spec], diff + + +class TestBuildSpecDigest: + def test_determinism(self): + sources, diff = _digest_sources() + files = parse_unified_diff(diff) + first = build_spec_digest(sources, files, token_budget=3000) + second = build_spec_digest(sources, files, token_budget=3000) + assert first == second + assert first != "" + + def test_rank_ticket_then_relevant_then_unmatched_must_then_may(self): + sources, diff = _digest_sources() + digest = build_spec_digest(sources, parse_unified_diff(diff), token_budget=3000) + ticket = digest.index("[ticket:SCOPE-1]") + relevant = digest.index("widget renderer MUST stream chunks") + unmatched_must = digest.index("build cache MUST be purged") + unmatched_may = digest.index("Docs MAY reference") + assert ticket < relevant < unmatched_must < unmatched_may + + def test_budget_truncation_marker(self): + sources, diff = _digest_sources() + digest = build_spec_digest(sources, parse_unified_diff(diff), token_budget=10) + assert digest.endswith(TRUNCATION_MARKER) + assert "Widget rendering" not in digest + + def test_empty_source_explanatory_line(self): + sources, diff = _digest_sources() + empty = SpecSource(origin="/docs/unrelated.md", kind="file", text="Filler prose without keywords.", error="") + digest = build_spec_digest([*sources, empty], parse_unified_diff(diff), token_budget=3000) + assert "[spec:/docs/unrelated.md: nothing diff-relevant kept]" in digest + + def test_failed_source_explained(self): + failed = SpecSource(origin="https://example.com/gone.md", kind="url", text="", error="HTTP 404 fetching") + digest = build_spec_digest([failed], [], token_budget=3000) + assert "[spec:https://example.com/gone.md: nothing diff-relevant kept]" in digest + + def test_heading_scoping_line_accompanies_kept_constraint(self): + sources, diff = _digest_sources() + digest = build_spec_digest(sources, parse_unified_diff(diff), token_budget=3000) + assert "(heading)" in digest + assert digest.index("Widget rendering") < digest.index("widget renderer MUST stream chunks") + + def test_version_pin_kept(self): + src = SpecSource( + origin="/docs/protocol.md", + kind="file", + text="Protocol version 3.2 is REQUIRED for all clients.\n2026-07-28\n", + error="", + ) + digest = build_spec_digest([src], [], token_budget=3000) + assert "Protocol version 3.2 is REQUIRED" in digest + assert "2026-07-28" in digest + + def test_ticket_description_subbudget(self, monkeypatch): + import prxref.specs as specs + + monkeypatch.setattr(specs, "TICKET_DESC_BUDGET_CHARS", 40) + jira = SpecSource( + origin="https://jira.example.com/browse/BIG-1", + kind="jira", + text="Summary: Big ticket\n" + "\n".join(f"Detail line {i} text." for i in range(20)), + error="", + ) + digest = build_spec_digest([jira], [], token_budget=3000) + assert "Summary: Big ticket" in digest + assert "Detail line 19 text." not in digest + + def test_no_sources_no_files(self): + digest = build_spec_digest([], [], token_budget=3000) + assert isinstance(digest, str) + assert digest != "" + + +class TestRelevanceScoring: + def test_diff_token_overlap_ranks_first_among_specs(self): + widgets = SpecSource( + origin="/docs/widgets.md", + kind="file", + text="The widget renderer MUST stream chunks.\n", + error="", + ) + other = SpecSource( + origin="/docs/billing.md", + kind="file", + text="The invoice ledger MUST balance totals.\n", + error="", + ) + diff = _added_file("src/widget_renderer.py", ["chunks = stream()"]) + digest = build_spec_digest([widgets, other], parse_unified_diff(diff), token_budget=3000) + assert digest.index("widget renderer MUST stream") < digest.index("invoice ledger MUST balance") + + def test_unmatched_must_kept_without_overlap(self): + src = SpecSource( + origin="/docs/rules.md", + kind="file", + text="The deploy pipeline MUST gate on green tests.\n", + error="", + ) + diff = _added_file("src/unrelated.py", ["pass"]) + digest = build_spec_digest([src], parse_unified_diff(diff), token_budget=3000) + assert "deploy pipeline MUST gate on green tests" in digest + + +if __name__ == "__main__": + pytest.main([__file__]) From e6fa841cad21339a9b7c3a3ee682674a69a6f250 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:11:44 -0700 Subject: [PATCH 02/61] fix: pass the spec/Jira config keys from _run_review into orchestrate_review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --spec flag and PRXREF_SPEC_SOURCES were loaded into the config but never handed to the orchestrator, so every CLI and webhook review ran ungrounded. The TestSpecFlag assertions now observe the orchestrator kwargs instead of the llm factory's config, and a new test covers all six keys. β€” Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/cli.py | 6 ++++++ tests/test_cli.py | 35 ++++++++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/prxref/cli.py b/src/prxref/cli.py index a10fcaea..f3c3a8c6 100644 --- a/src/prxref/cli.py +++ b/src/prxref/cli.py @@ -360,6 +360,12 @@ def _run_review( post_verdict=cfg["post_verdict"], trace_file=cfg["trace_file"], trace_dir=cfg["trace_dir"], + spec_sources=cfg["spec_sources"], + spec_max_chars=cfg["spec_max_chars"], + spec_digest_tokens=cfg["spec_digest_tokens"], + jira_base_url=cfg["jira_base_url"], + jira_email=cfg["jira_email"], + jira_api_token=cfg["jira_api_token"], ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 02fb14df..a740b556 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -804,26 +804,27 @@ def test_the_flag_is_repeatable_and_defaults_to_none(self): assert args.spec == ["https://a/spec.md", "docs/specs"] def test_flag_values_reach_the_resolved_config(self, fake_runtime): - """The override is observed on the config the pipeline is actually - handed, not on the parser: fake_create_llm_client records it.""" + """The override is observed on the kwargs the orchestrator is actually + handed, not on the parser or the resolved config: a value that stops at + ``load_config`` grounds nothing.""" assert main([ "review", "--pr-url", self.URL, "--no-post", "--spec", "https://a/spec.md", "--spec", "docs/specs", ]) == 0 - assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + assert fake_runtime["orchestrate_calls"][0]["spec_sources"] == [ "https://a/spec.md", "docs/specs", ] def test_no_flag_and_no_env_leaves_the_default_empty(self, fake_runtime): assert main(["review", "--pr-url", self.URL, "--no-post"]) == 0 - assert fake_runtime["llm_calls"][0]["spec_sources"] == [] + assert fake_runtime["orchestrate_calls"][0]["spec_sources"] == [] def test_env_sources_load_through_the_normal_path( self, fake_runtime, monkeypatch ): monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://a/spec.md docs/specs") assert main(["review", "--pr-url", self.URL, "--no-post"]) == 0 - assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + assert fake_runtime["orchestrate_calls"][0]["spec_sources"] == [ "https://a/spec.md", "docs/specs", ] @@ -835,7 +836,7 @@ def test_the_flag_replaces_the_environment_without_merging( "review", "--pr-url", self.URL, "--no-post", "--spec", "https://flag/only.md", ]) == 0 - assert fake_runtime["llm_calls"][0]["spec_sources"] == [ + assert fake_runtime["orchestrate_calls"][0]["spec_sources"] == [ "https://flag/only.md" ] @@ -846,7 +847,27 @@ def test_the_webhook_daemon_gets_the_environment_sources( the only way it can ground a review β€” and it must reach the pipeline.""" monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://a/spec.md") cli._webhook_handler(self.URL) - assert fake_runtime["llm_calls"][0]["spec_sources"] == ["https://a/spec.md"] + assert fake_runtime["orchestrate_calls"][0]["spec_sources"] == ["https://a/spec.md"] + + def test_every_spec_key_reaches_the_orchestrator( + self, fake_runtime, monkeypatch + ): + """All six spec/Jira config keys ride into ``orchestrate_review``; the + daemon has no flags, so the environment is its only way in.""" + monkeypatch.setenv("PRXREF_SPEC_SOURCES", "https://a/spec.md") + monkeypatch.setenv("PRXREF_SPEC_MAX_CHARS", "5000") + monkeypatch.setenv("PRXREF_SPEC_DIGEST_TOKENS", "500") + monkeypatch.setenv("PRXREF_JIRA_BASE_URL", "https://jira.example.com") + monkeypatch.setenv("PRXREF_JIRA_EMAIL", "bot@example.com") + monkeypatch.setenv("PRXREF_JIRA_API_TOKEN", "t0ken") + cli._webhook_handler(self.URL) + kwargs = fake_runtime["orchestrate_calls"][0] + assert kwargs["spec_sources"] == ["https://a/spec.md"] + assert kwargs["spec_max_chars"] == 5000 + assert kwargs["spec_digest_tokens"] == 500 + assert kwargs["jira_base_url"] == "https://jira.example.com" + assert kwargs["jira_email"] == "bot@example.com" + assert kwargs["jira_api_token"] == "t0ken" class TestDryRun: From 5b09a6d28cac114357d4c5f392c8c636d25e39ed Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:14:33 -0700 Subject: [PATCH 03/61] fix: log spec grounding outcome so unposted runs are not silent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grounding note only reaches a posted summary, so a --no-post, dry-run, or inline-only run said nothing when a spec source failed. Each failed source now logs a redacted WARNING and the run logs one INFO line with the fetched/constraint counts. Also pins where the digest lands under --trace-dir: every unit's user.md, never its system.md. β€” Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/orchestrator.py | 12 ++++++++++++ tests/test_orchestrator.py | 21 +++++++++++++++++++++ tests/test_trace_dir.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+) diff --git a/src/prxref/orchestrator.py b/src/prxref/orchestrator.py index d7f269f7..07e9d297 100644 --- a/src/prxref/orchestrator.py +++ b/src/prxref/orchestrator.py @@ -524,6 +524,18 @@ def orchestrate_review( digest_chars=len(spec_digest), ) spec_note = _spec_note(fetched, spec_digest) + # The note only reaches a POSTED summary, so a --no-post, dry-run, + # or inline-only run would otherwise learn nothing about grounding. + for s in fetched: + if s.error: + logger.warning( + "spec source failed (review continues without it): %s", + redact_for_post(s.error), + ) + logger.info( + "spec grounding: %d/%d source(s) fetched, %d constraint(s) injected", + ok, len(fetched), _spec_constraint_count(spec_digest), + ) except Exception as e: # noqa: BLE001 logger.error("spec grounding failed (best-effort): %s", e) tracer.event("specs", "fail", error=e.__class__.__name__) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 91c069df..2013b439 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -2167,6 +2167,27 @@ def test_a_partial_failure_shows_the_note_and_the_failures( assert "> πŸ” Spec-grounded: 2 source(s) Β· 1 constraint(s) injected" in summary assert "> ⚠️ Spec fetch failed for 1 source(s)" in summary + def test_an_unposted_run_still_logs_each_failed_source( + self, monkeypatch, caplog, + ): + """The grounding note only reaches a POSTED summary; --no-post and + dry-run operators learn about a dead source from the log alone.""" + caplog.set_level(logging.INFO, logger="prxref") + fetched = self._fetched() + self._fetched(failed=True, origin="x/other.md") + forge, _res = self._run(monkeypatch, fetched=fetched, post=False) + assert forge.summaries == [] + warnings = [ + r.getMessage() for r in caplog.records + if r.levelno == logging.WARNING and "spec source failed" in r.getMessage() + ] + assert len(warnings) == 1 + assert "HTTP 404" in warnings[0] + assert "secret.example.invalid" not in warnings[0] + assert any( + "spec grounding: 1/2 source(s) fetched, 1 constraint(s) injected" + in r.getMessage() for r in caplog.records + ) + def test_fetch_failure_reasons_are_redacted(self, monkeypatch): leaked = SpecSource( origin="https://secret.example.invalid/browse/PROJ-9", diff --git a/tests/test_trace_dir.py b/tests/test_trace_dir.py index f14e344b..092adda6 100644 --- a/tests/test_trace_dir.py +++ b/tests/test_trace_dir.py @@ -192,6 +192,40 @@ def test_the_trace_directory_is_created_when_missing(self, tmp_path): assert (target / "sweep.meta.json").is_file() +class TestSpecDigestInTheTrace: + """The spec digest is per-run context, so it rides the USER half of every + unit's prompt: a trace reader finds it in ``.user.md`` for each + chunk and the sweep, and the static ``.system.md`` never changes + with the sources.""" + + CANARY = "Tools MUST carry the CANARY-4412 prefix." + + def test_the_digest_lands_in_every_user_file_and_no_system_file( + self, tmp_path, monkeypatch, + ): + from prxref import orchestrator + from prxref.specs import SpecSource + + monkeypatch.setattr( + orchestrator.specs, "fetch_specs", + lambda *a, **k: [SpecSource( + origin="docs/spec.md", kind="file", + text=f"## Rules\n\n{self.CANARY}\n", error="", + )], + ) + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + orchestrate_review( + forge, REF, FakeLLM(RAW_OK), post=False, trace_dir=str(tmp_path), + spec_sources=["docs/spec.md"], + ) + for unit in ("chunk0", "sweep"): + user = (tmp_path / f"{unit}.user.md").read_text(encoding="utf-8") + system = (tmp_path / f"{unit}.system.md").read_text(encoding="utf-8") + assert self.CANARY in user, unit + assert "(no specs provided for this review)" not in user, unit + assert self.CANARY not in system, unit + + class TestWriteFailureNeverAborts: def test_an_unwritable_trace_dir_is_a_warning_not_a_failure( self, tmp_path, caplog From be13ed15f51b95aa4b975d7e3c432de0ad78bdee Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 08:17:03 -0700 Subject: [PATCH 04/61] fix: the hedge gate no longer reads a spec finding's quoted constraint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec severity must quote its violated constraint verbatim, and normative text is conditional ("If a session already exists, the server MUST ..."), so the if-still / unless-already rules dropped legitimate spec findings as hedged. The Spec: "..." quote is now removed before the rules read the body; a hedge in the model's own words still drops. β€” Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- docs/quality.md | 2 +- src/prxref/quality.py | 12 +++++++++++- tests/test_quality.py | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/docs/quality.md b/docs/quality.md index aea8daf6..f4c44240 100644 --- a/docs/quality.md +++ b/docs/quality.md @@ -44,7 +44,7 @@ it (noted in the table). | 5 | `apply_settled_thread_suppression` | Drops a finding that re-litigates a subject a thread already argued out. Line-independent by design. A thread with no path β€” a general, unanchored PR comment β€” is ignored by this pass, since it cannot be "same path" as any finding. | | 6 | `apply_severity_consistency` | Rewrites only: findings sharing a normalized title are all raised to the group's maximum severity. | | 7 | `apply_removal_claim_check` | Drops a claim that a **named** path was removed when the post-image still carries it. The removal verb must **govern** that path (`removed src/app.py`, `src/app.py was removed`); a bare "removed" elsewhere in the body is not a removal claim. | -| 8 | `apply_hedge_gate` | Drops a finding whose own text conditions the defect on a precondition never established from the diff. | +| 8 | `apply_hedge_gate` | Drops a finding whose own text conditions the defect on a precondition never established from the diff. A `spec` finding's verbatim `Spec: "…"` quote is not read: a condition inside it belongs to the spec, not the model. | | 9 | `apply_quality_gate` | Severity vocabulary, confidence floor, per-review error cap. Returns its findings in content order. | | 10 | `apply_sweep_dedup` | Drops a sweep finding that restates a chunk finding which **survived** the gate. | | 11 | `apply_containment_note` | Decoration only: suffixes a throw/panic/crash finding that never named its containment boundary. | diff --git a/src/prxref/quality.py b/src/prxref/quality.py index 47a353de..4241bc4c 100644 --- a/src/prxref/quality.py +++ b/src/prxref/quality.py @@ -170,6 +170,12 @@ _HEDGE_SPAN_MAX: int = 80 +# The verbatim constraint a ``spec`` finding is required to quote. Normative +# text is conditional by nature ("If a session already exists, the server +# MUST reuse it"), so the quote is the spec's precondition, not the model's +# hedge, and is removed before the hedge rules read the body. +_SPEC_QUOTE_RE = re.compile(r"Spec:\s*[\"β€œ][^\"”\n]*[\"”]") + def active(findings: Sequence[Finding]) -> list[Finding]: """Return only the findings that survived every quality pass.""" @@ -1541,13 +1547,17 @@ def apply_hedge_gate(findings: Sequence[Finding]) -> list[Finding]: Pure and order-preserving: already-dropped findings pass through untouched, and a match sets ``drop_reason`` to ``hedged: ""`` naming the matched text so the drop is auditable in the run record. + A ``Spec: "..."`` quote is removed before matching: it is the verbatim + constraint the ``spec`` severity must cite, and its conditions belong to + the spec, not to the model's reasoning. """ out: list[Finding] = [] for f in findings: if f.drop_reason is not None: out.append(f) continue - span = _hedge_span(f.title or "") or _hedge_span(f.body or "") + body = _SPEC_QUOTE_RE.sub("", f.body or "") + span = _hedge_span(f.title or "") or _hedge_span(body) if span is None: out.append(f) continue diff --git a/tests/test_quality.py b/tests/test_quality.py index ee2fe9a6..fc780626 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -1733,6 +1733,26 @@ def test_hedged_finding_does_not_consume_error_cap(self): out = apply_quality_gate(hedged, max_errors=1) assert out[0].drop_reason.startswith("hedged:") + @pytest.mark.parametrize("quote", [ + '"If a session already exists, the server MUST reuse it."', + '"Clients MUST send clientInfo unless the session already exists."', + 'β€œIf the server is still initializing, the client MUST NOT send requests.”', + ]) + def test_a_quoted_spec_condition_is_not_a_hedge(self, quote): + """The spec severity must quote its constraint verbatim; a condition + inside that quote is the spec's, not the model's.""" + f = _f(severity="spec", body=f"Spec: {quote} The diff violates it.") + assert apply_hedge_gate([f])[0].drop_reason is None + + def test_a_hedge_outside_the_spec_quote_still_drops(self): + f = _f( + severity="spec", + body='Spec: "Tokens MUST NOT be logged." If the logger is still ' + "at debug level, the token leaks.", + ) + assert apply_hedge_gate([f])[0].drop_reason.startswith("hedged:") + + class TestApplyContainmentNote: """Issue #07: a throw-class finding with no named boundary gets flagged.""" From 41a9cf28e4ef5c04dbe35e6e303bbac9b3a64f4c Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:10:06 -0700 Subject: [PATCH 05/61] fix: make _run_worker's spec_digest keyword-only; pin the orchestrate kwargs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec branch passed spec_digest positionally after reader, the slot main uses for all_files. The rebase already converted every call site to a keyword; the `*` makes a future positional call a TypeError instead of a silently swapped argument. test_run_review_passes_only_real_orchestrate_kwargs captures the REAL orchestrate_review signature before fake_runtime swaps the module, and asserts every kwarg _run_review sends is a parameter. The fake accepts **kwargs, so a misspelt kwarg otherwise passes the suite while the real call raises TypeError, which `review` swallows to exit 0. Mutation-checked: an injected bogus kwarg fails it. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/orchestrator.py | 2 +- tests/test_cli.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/prxref/orchestrator.py b/src/prxref/orchestrator.py index 07e9d297..07f9ca78 100644 --- a/src/prxref/orchestrator.py +++ b/src/prxref/orchestrator.py @@ -1090,7 +1090,7 @@ def _run_worker( max_tokens: int | None = None, context_lines: int | None = None, tracer: Tracer | None = None, reader=None, all_files=None, trace_label: str = "", trace_dir: str | None = None, - spec_digest: str = "", + *, spec_digest: str = "", ) -> dict: tracer = tracer if tracer is not None else get_tracer() t0 = time.perf_counter() diff --git a/tests/test_cli.py b/tests/test_cli.py index a740b556..3e53b396 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,5 @@ """Tests for prxref.cli: review subcommand, serve daemon, --version, and non-blocking exits.""" +import inspect import json import logging import os @@ -10,6 +11,7 @@ import pytest from prxref import __version__, cli +from prxref import orchestrator as real_orchestrator from prxref.cli import main from prxref.forges.base import PRRef from prxref.llm import ConfigError @@ -870,6 +872,33 @@ def test_every_spec_key_reaches_the_orchestrator( assert kwargs["jira_api_token"] == "t0ken" +def test_run_review_passes_only_real_orchestrate_kwargs(fake_runtime, monkeypatch): + """Every kwarg ``_run_review`` hands the orchestrator is a real parameter. + + ``fake_runtime``'s double accepts ``**kwargs``, so a misspelt kwarg passes + every other test here while the real call raises ``TypeError`` β€” which + ``review`` swallows to exit 0, silently dropping the review. The signature + comes from the module imported at the top of this file, captured before the + fixture swapped ``sys.modules["prxref.orchestrator"]`` for the double. + """ + real = real_orchestrator.orchestrate_review + assert sys.modules["prxref.orchestrator"].orchestrate_review is not real + params = inspect.signature(real).parameters + assert not any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + ref = PRRef( + forge="github", host="github.com", owner="org", repo="repo", + number=7, url="https://github.com/org/repo/pull/7", + ) + monkeypatch.setattr("prxref.cli.detect_forge", lambda url: ref) + + assert main(["review", "--pr-url", ref.url, "--no-post", "--spec", "docs/specs"]) == 0 + + calls = fake_runtime["orchestrate_calls"] + assert len(calls) == 1 + assert calls[0], "the double recorded no kwargs, so the check below is vacuous" + assert sorted(set(calls[0]) - set(params)) == [] + + class TestDryRun: """PRXREF_DRY_RUN must reach BOTH review paths, daemon included. From 2c39c9516ff48461dccc63f420906e91761007f0 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:10:42 -0700 Subject: [PATCH 06/61] test: make the reviewer contract stubs an opt-in conftest fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The body of test_orchestrator.py's autouse _contract_stubs moves to tests/conftest.py as the non-autouse fixture `contract_stubs`. test_orchestrator.py keeps an autouse wrapper that requests it, so its 199 tests behave exactly as before (control: a wrapper that does not request it turns 45 of them red). Other modules now opt in with @pytest.mark.usefixtures("contract_stubs") instead of importing and re-patching the stubs, or leave it off to run the real reviewer, which the spec-grounding proof tests need. The fixture imports the stubs from tests.test_orchestrator at call time, so loading conftest never imports a test module. test_issue_06, test_issue_10 and test_issue_13 still define their own module-level `contract_stubs` (chunk + sweep only); pytest's module-over-conftest override keeps them unchanged. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- tests/conftest.py | 30 ++++++++++++++++++++++++++++++ tests/test_orchestrator.py | 19 ++++++++----------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 899cac26..42c45c29 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,3 +41,33 @@ def clear_prxref_env(monkeypatch: pytest.MonkeyPatch) -> list[str]: def _clear_prxref_env(monkeypatch): """No ambient PRXREF_* reaches any test; every test sets what it needs.""" clear_prxref_env(monkeypatch) + + +@pytest.fixture +def contract_stubs(monkeypatch): + """Pin the reviewer contract: chunk, sweep and summary-prompt stubs. Opt-in. + + ``tests/test_orchestrator.py`` requests it from an autouse wrapper, so every + test there runs against the stubs. Any other module opts in with + ``@pytest.mark.usefixtures("contract_stubs")``; a test that must prove the + real prompts end to end leaves it off and runs the real reviewer. + + The systemic sweep is stubbed to a clean no-findings success so + sweep-specific tests can monkeypatch their own doubles; chunk-count + assertions include the sweep unit. + + The stubs live in ``tests/test_orchestrator.py`` and are imported here, at + call time, so loading this conftest never imports a test module. + """ + from prxref import orchestrator + from tests.test_orchestrator import ( + _contract_load_prompt, + _contract_review_chunk, + _contract_review_systemic, + ) + + monkeypatch.setattr(orchestrator.reviewer, "review_chunk", _contract_review_chunk) + monkeypatch.setattr( + orchestrator.reviewer, "review_systemic", _contract_review_systemic, + ) + monkeypatch.setattr(orchestrator.reviewer, "load_prompt", _contract_load_prompt) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2013b439..190060ab 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -243,18 +243,15 @@ def list_threads(self, ref: PRRef) -> list[Thread]: @pytest.fixture(autouse=True) -def _contract_stubs(monkeypatch): - """Pin the reviewer contract. Env clearing lives in tests/conftest.py. - - The systemic sweep is stubbed to a clean no-findings success so the - sweep-specific classes below can monkeypatch their own doubles; the - chunk-count assertions in the older classes include the sweep unit. +def _contract_stubs(contract_stubs): + """Pin the reviewer contract for every test in this module. + + The stubs are installed by the opt-in ``contract_stubs`` fixture in + tests/conftest.py, which also holds env clearing. The systemic sweep is + stubbed to a clean no-findings success so the sweep-specific classes below + can monkeypatch their own doubles; the chunk-count assertions in the older + classes include the sweep unit. """ - monkeypatch.setattr(orchestrator.reviewer, "review_chunk", _contract_review_chunk) - monkeypatch.setattr( - orchestrator.reviewer, "review_systemic", _contract_review_systemic, - ) - monkeypatch.setattr(orchestrator.reviewer, "load_prompt", _contract_load_prompt) HAPPY_FINDINGS = { From 1c8805976bec0a14c6121cc3efa869c6ed5eb29e Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:12:44 -0700 Subject: [PATCH 07/61] chore: keep docs/issues/ out of git and the sdist, and test it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/issues/ holds local issue bundles written during triage; they are private planning notes and must never be published. Three guards, each proven alone: - .gitignore ignores docs/issues/. - [tool.hatch.build.targets.sdist] excludes docs/issues. Observed with `uv build --sdist` and an untracked probe file: either guard alone keeps it out of the tarball; with neither, the probe ships. - tests/test_repo_hygiene.py asserts `git ls-files docs/issues` is empty (control: a force-added file in a scratch repo turns it red), that the ignore rule matches, and that the sdist exclude is declared. The git checks skip without a git executable or outside a checkout of this project (an unpacked sdist), both observed. The test carries no deny list. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- .gitignore | 1 + pyproject.toml | 1 + tests/test_repo_hygiene.py | 61 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 tests/test_repo_hygiene.py diff --git a/.gitignore b/.gitignore index 746c1387..165d8608 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ dist/ docs/superpowers/ docs/configurability/ docs/release-hardening/ +docs/issues/ diff --git a/pyproject.toml b/pyproject.toml index e08960d8..8e968bd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ exclude = [ "docs/superpowers", "docs/configurability", "docs/release-hardening", + "docs/issues", ] [tool.ruff] diff --git a/tests/test_repo_hygiene.py b/tests/test_repo_hygiene.py new file mode 100644 index 00000000..6f055a38 --- /dev/null +++ b/tests/test_repo_hygiene.py @@ -0,0 +1,61 @@ +"""Repository hygiene: local issue bundles never enter the tracked tree or the sdist. + +``docs/issues/`` holds issue bundles written during local triage. Three guards +keep them out of anything published: ``.gitignore`` ignores the directory, the +sdist build config excludes it, and ``test_docs_issues_is_not_tracked`` fails if +a force-add tracked something there anyway. + +The git checks skip where there is no checkout to inspect: no ``git`` +executable, or a tree that is not a git checkout of this project (an unpacked +sdist, for example). +""" +from __future__ import annotations + +import shutil +import subprocess +import tomllib +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +ISSUES_DIR = "docs/issues" + + +def _git(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(ROOT), *args], + capture_output=True, text=True, check=False, + ) + + +@pytest.fixture +def checkout() -> Path: + """The project root when it is a git checkout of this project; skips otherwise.""" + if shutil.which("git") is None: + pytest.skip("git is not installed") + top = _git("rev-parse", "--show-toplevel") + if top.returncode != 0 or Path(top.stdout.strip()).resolve() != ROOT: + pytest.skip("not a git checkout of this project") + return ROOT + + +def test_docs_issues_is_not_tracked(checkout): + docs = _git("ls-files", "--", "docs") + assert docs.returncode == 0, docs.stderr + assert docs.stdout.strip(), "git ls-files lists no tracked docs, so the check below proves nothing" + + listed = _git("ls-files", "--", ISSUES_DIR) + assert listed.returncode == 0, listed.stderr + assert listed.stdout == "" + + +def test_docs_issues_is_gitignored(checkout): + assert _git("check-ignore", "-q", "--", f"{ISSUES_DIR}/any-bundle.md").returncode == 0 + assert _git("check-ignore", "-q", "--", "docs/any-page.md").returncode == 1 + + +def test_sdist_excludes_docs_issues(): + config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + sdist = config["tool"]["hatch"]["build"]["targets"]["sdist"] + assert ISSUES_DIR in sdist["exclude"] From de0ab9b667120724d6e6ef0af1a598253ad68566 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:21:04 -0700 Subject: [PATCH 08/61] docs: scrub dangling docs/issues/ and skill/session citations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docs/issues/ is private and gitignored as of 0.14.0, so the module docstrings and comments in the issue-03/05/12 regression tests that cited docs/issues/... paths pointed at nothing a reader can open. Replace each with a neutral "2026-09-04 inbox report, issue N (private, not tracked)" citation that keeps the issue identity without the dead path. Test logic and verbatim fixture strings are untouched. Also rewrite the live-instance-verification followup doc's Origin paragraph, which named a private `sharpen` skill and a session UUID, as a neutral reference to "a retrospective of the 2026-08-30/31 live-instance session" so the doc carries no private skill name or session identifier. πŸ€– Authored with Claude Code β€” Claude Sonnet 5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- .../followup-tasks-real-forge-fixtures.md | 9 ++++----- tests/test_issue_03_copy_header.py | 2 +- tests/test_issue_05_hedged_findings.py | 5 ++--- tests/test_issue_12_package_json_section.py | 6 +++--- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/live-instance-verification/followup-tasks-real-forge-fixtures.md b/docs/live-instance-verification/followup-tasks-real-forge-fixtures.md index 711a67cd..76d221b8 100644 --- a/docs/live-instance-verification/followup-tasks-real-forge-fixtures.md +++ b/docs/live-instance-verification/followup-tasks-real-forge-fixtures.md @@ -15,11 +15,10 @@ was transcribed from. ## Origin -Produced by the `sharpen` retro of session -`974bbdda-6362-4426-9971-fec77fabc1d9` (2026-08-30/31). That session stood up -Bitbucket Data Center 10.4.2 in Docker with an unattended timebomb-license -setup, seeded a repo with three planted bugs, opened a real PR, and ran -`prxref review` against it with posting enabled. +Produced by a retrospective of the 2026-08-30/31 live-instance session, which +stood up Bitbucket Data Center 10.4.2 in Docker with an unattended +timebomb-license setup, seeded a repo with three planted bugs, opened a real +PR, and ran `prxref review` against it with posting enabled. **Two genuine bugs fell out of one live run, neither of which any hand-written fixture had ever caught, for the life of the project:** diff --git a/tests/test_issue_03_copy_header.py b/tests/test_issue_03_copy_header.py index 679b8c01..e5528a01 100644 --- a/tests/test_issue_03_copy_header.py +++ b/tests/test_issue_03_copy_header.py @@ -1,6 +1,6 @@ """Regression tests for issue #03: copy/rename diff headers read as deletion. -docs/issues/inbox-2026-09-04/prxref-issues/03-rename-copy-header-read-as-deletion.md +The 2026-09-04 inbox report, issue 03 (private, not tracked) reports that a Bitbucket Server ``copy from``/``copy to`` diff header (a monorepo package copied to a new location) got the copy's SOURCE path reported as removed, even though the same diff contains a separate section diff --git a/tests/test_issue_05_hedged_findings.py b/tests/test_issue_05_hedged_findings.py index 4fce2d93..c9a269e4 100644 --- a/tests/test_issue_05_hedged_findings.py +++ b/tests/test_issue_05_hedged_findings.py @@ -56,8 +56,7 @@ def _diff_with_lines(path: str, lines: list[str]) -> str: DOCKERFILE, _DOCKER_LINES ) -# Verbatim from docs/issues/inbox-2026-09-04/prxref-issues/ -# 05-quality-gate-admits-self-hedged-findings.md +# Verbatim from the 2026-09-04 inbox report, issue 05 (private, not tracked) CANVAS_BODY = ( "The diff removes the close()/stream-wrapper lifecycle and returns " "createMcpHandler(serverFactory) directly, so nothing calls canvas cleanup " @@ -65,7 +64,7 @@ def _diff_with_lines(path: str, lines: list[str]) -> str: "lease is only released when the enclosing scope (service layer) closes, " "not per request." ) -# Verbatim from docs/issues/inbox-2026-09-04/prxref-issue-2026-09-03.md Β§1 +# Verbatim from the 2026-09-04 inbox report, issue 05 Β§1 (private, not tracked) NPM_CI_BODY = ( "The deps stage copies only api, worker, shared, " "config, auth and billing manifests, but this PR adds " diff --git a/tests/test_issue_12_package_json_section.py b/tests/test_issue_12_package_json_section.py index 858a9f48..304f2b43 100644 --- a/tests/test_issue_12_package_json_section.py +++ b/tests/test_issue_12_package_json_section.py @@ -1,7 +1,7 @@ """Issue #12 β€” package.json section misattribution: a devDependency reported as a runtime dependency. -docs/issues/inbox-2026-09-04/prxref-issue-2026-09-03.md Β§2: on ``acme-mcp`` +The 2026-09-04 inbox report, issue 12 Β§2 (private, not tracked): on ``acme-mcp`` a pull request, prxref flagged ``vitest`` as a production dependency at ``apps/server/package.json:20`` β€” line 20 is ``"@acme-mcp/gamma": "*",``, a ``dependencies`` entry, not the ``vitest`` line (which is line 37, inside @@ -115,8 +115,8 @@ + "\n".join(_HUNK_LINES) + "\n" ) -# Verbatim from docs/issues/inbox-2026-09-04/prxref-issue-2026-09-03.md Β§2, -# comment 1000001. +# Verbatim from the 2026-09-04 inbox report, issue 12 Β§2, comment 1000001 +# (private, not tracked). TITLE = "vitest added to runtime dependencies" BODY = ( "vitest is a test-only tool but is added under `dependencies` in " From 77dfed098dbcb5e4863cb3ac325f44d2c1cfa9cb Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:28:03 -0700 Subject: [PATCH 09/61] fix: send Jira credentials only to PRXREF_JIRA_BASE_URL; widen ticket URLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SEC-2: _fetch_jira built basic auth whenever the email and token were set, so any ticket-shaped URL (including plain http on a third-party host) received the operator's Jira credentials. Auth is now sent only when PRXREF_JIRA_BASE_URL, the email and the token are all set, which routes every ticket to the host the operator typed. Credentials without a base URL are withheld with a WARNING naming PRXREF_JIRA_BASE_URL; an http base URL is honoured with a WARNING; neither logs a value. The anonymous hint now also fires on 404 (Jira Cloud hides private issues behind 404), and explains that credentials only go to the base URL when they are set without one. COR-2: _TICKET_PATTERNS take a 0-2 segment context path on the /browse/ and REST shapes (so https://host/jira/browse/KEY-1 reaches https://host/jira/rest/api/2/issue/KEY-1 instead of being scraped as a page), recognize the Cloud team-managed issue view, and read a Cloud board's selectedIssue with parse_qs on /jira/ paths. Bitbucket Server file URLs (four segments before /browse/) still do not match. COR-4 (Type/Labels) and backlog 9b: the ticket text leaves out empty Summary/Type/Labels lines instead of rendering them bare, a 200 whose body is not JSON is a clean source error naming its content type, and a JSON body without a fields object is one too. Tests: the two tests/test_specs.py cases that asserted auth with an empty base now set one. NEW tests/test_specs_jira.py covers every URL shape and negative, where credentials go, the hints, the non-issue bodies, the rendered text, and an entry-point test that drives cli._run_review against a local recording server (no Authorization without a base URL; a control with the base URL set sees Basic auth). The new file has 53 tests; 29 of them fail against the pre-change specs.py, including the entry-point test. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/specs.py | 184 +++++++++++++++---- tests/test_specs.py | 3 + tests/test_specs_jira.py | 378 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 532 insertions(+), 33 deletions(-) create mode 100644 tests/test_specs_jira.py diff --git a/src/prxref/specs.py b/src/prxref/specs.py index 1e630b15..29d493fd 100644 --- a/src/prxref/specs.py +++ b/src/prxref/specs.py @@ -16,10 +16,16 @@ read-only retried session (GET/HEAD/OPTIONS only, the same policy the forge adapters use), streams at most ``max_chars`` per source with an explicit truncation marker, accepts only text-like content types, strips HTML to - text, and reads local files as UTF-8. Jira tickets are fetched over REST - with basic auth when credentials are configured, anonymously otherwise; - a 401/403 without credentials names the ``PRXREF_JIRA_*`` variables β€” the - fix an operator can act on β€” never their values. + text, and reads local files as UTF-8. Jira tickets (a ``/browse/`` or REST + URL under a context path of up to two segments, a Cloud issue view, or a + Cloud board's ``selectedIssue``) are fetched over REST. Credentials are + only ever sent to ``PRXREF_JIRA_BASE_URL``: basic auth goes out only when + it, ``PRXREF_JIRA_EMAIL`` and ``PRXREF_JIRA_API_TOKEN`` are all set, and + every other fetch is anonymous. Credentials set without the base URL are + withheld with a warning, and a plain-http base URL is used with a warning. + An anonymous 401, 403 or 404 β€” Jira Cloud hides a private issue behind a + 404 β€” names the variables that would fix it, never their values, and a + 200 that is not a JSON issue fails its source cleanly. - The digest (:func:`build_spec_digest`) extracts constraints per source in document order β€” headings, RFC-2119 normative statements, version pins, and @@ -32,11 +38,12 @@ import codecs import json +import logging import os import re from dataclasses import dataclass from html.parser import HTMLParser -from urllib.parse import urlparse +from urllib.parse import parse_qs, urlparse import requests from requests.adapters import HTTPAdapter @@ -45,6 +52,8 @@ from .retry_logging import LoggingRetry from .triage import FileDiff +logger = logging.getLogger(__name__) + SPEC_FETCH_TIMEOUT_S = 15 SPEC_DIR_MAX_FILES = 20 @@ -61,14 +70,20 @@ _KEY = r"[A-Z][A-Z0-9_]*-\d+" +_KEY_RE = re.compile(_KEY) + +_CTX = r"(?:/[^/?#]+){0,2}" + _TICKET_PATTERNS: tuple[re.Pattern[str], ...] = ( - re.compile(rf"^(?Phttps?://[^/]+)/browse/(?P{_KEY})(?:[/?#]|$)"), - re.compile(rf"^(?Phttps?://[^/]+)/rest/api/(?:2|3)/issue/(?P{_KEY})(?:[/?#]|$)"), + re.compile(rf"^(?Phttps?://[^/?#]+{_CTX})/browse/(?P{_KEY})(?:[/?#]|$)"), + re.compile(rf"^(?Phttps?://[^/?#]+{_CTX})/rest/api/(?:2|3)/issue/(?P{_KEY})(?:[/?#]|$)"), re.compile( - rf"^(?Phttps?://[^/]+)/jira/software/c/projects/[A-Z][A-Z0-9_]*/issues/(?P{_KEY})(?:[/?#]|$)" + rf"^(?Phttps?://[^/?#]+)/jira/software/(?:c/)?projects/[^/?#]+/issues/(?P{_KEY})(?:[/?#]|$)" ), ) +_JIRA_ENV_HINT = "PRXREF_JIRA_BASE_URL, PRXREF_JIRA_EMAIL and PRXREF_JIRA_API_TOKEN" + _MD_HEADING_RE = re.compile(r"^#{1,6}\s+(.+?)\s*#*\s*$") _SETEXT_UNDER_RE = re.compile(r"^[-=]{3,}\s*$") @@ -127,19 +142,46 @@ class TicketRef: def parse_ticket_url(url: str) -> TicketRef | None: """Recognize a Jira ticket URL, returning its REST base and key. - Three shapes are recognized: ``{host}/browse/{KEY}-{n}`` (Jira Cloud and - Server classic), ``{host}/rest/api/{2|3}/issue/{KEY}-{n}`` (raw REST - links), and ``{host}/jira/software/c/projects/{KEY}/issues/{KEY}-{n}`` - (Cloud new UI). Project keys are uppercase letters, digits, and - underscores; the numeric suffix is required. Anything else returns - ``None``. + Four shapes are recognized: + + - ``{base}/browse/{KEY}-{n}`` (Jira Cloud and Server classic) and + ``{base}/rest/api/{2|3}/issue/{KEY}-{n}`` (raw REST links), where + ``{base}`` is ``scheme://host`` plus a context path of zero to two + segments (``https://issues.apache.org/jira``, + ``https://acme.com/tools/jira``); the base keeps that context path, + because Server serves its REST API under it. + - ``{host}/jira/software/projects/{P}/issues/{KEY}-{n}`` and its ``/c/`` + form (Cloud team- and company-managed issue views). + - A Cloud board or backlog URL on a ``/jira/`` path carrying the ticket + in its ``selectedIssue`` query parameter, read wherever it sits in the + query string; the base is ``scheme://host``. + + Project keys are uppercase letters, digits, and underscores; the numeric + suffix is required. The context-path bound is what keeps a Bitbucket + Server file URL (``/projects/P/repos/R/browse/…``, four segments deep) + from matching. A non-Jira URL with at most two path segments before + ``/browse/{KEY}-{n}`` does match, and is fetched from Jira REST on that + host anonymously, or looked up by key on ``PRXREF_JIRA_BASE_URL`` when + that is set. Anything else returns ``None``. """ text = (url or "").strip() for pattern in _TICKET_PATTERNS: m = pattern.match(text) if m: return TicketRef(base_url=m.group("base"), key=m.group("key"), url=text) - return None + return _board_ticket(text) + + +def _board_ticket(text: str) -> TicketRef | None: + parsed = urlparse(text) + if parsed.scheme not in ("http", "https") or not parsed.netloc: + return None + if not parsed.path.startswith("/jira/"): + return None + selected = parse_qs(parsed.query).get("selectedIssue") + if not selected or not _KEY_RE.fullmatch(selected[0]): + return None + return TicketRef(base_url=f"{parsed.scheme}://{parsed.netloc}", key=selected[0], url=text) def _create_default_session() -> requests.Session: @@ -296,31 +338,102 @@ def _fetch_jira( jira_api_token: str, session: requests.Session, ) -> None: - base = (jira_base_url or ref.base_url).rstrip("/") - url = f"{base}/rest/api/2/issue/{ref.key}?fields=summary,description,issuetype,labels" - auth = (jira_email, jira_api_token) if jira_email and jira_api_token else None + url, auth = _jira_request(ref, jira_base_url, jira_email, jira_api_token) resp = session.get(url, timeout=SPEC_FETCH_TIMEOUT_S, auth=auth) - if resp.status_code in (401, 403) and auth is None: - src.error = ( - f"Jira returned {resp.status_code} for {ref.key} without credentials; set " - "PRXREF_JIRA_BASE_URL, PRXREF_JIRA_EMAIL and PRXREF_JIRA_API_TOKEN to authenticate." + if resp.status_code != 200: + src.error = _jira_status_error( + resp.status_code, ref.key, auth=auth, credentials_set=bool(jira_email and jira_api_token) ) return - if resp.status_code != 200: - src.error = f"Jira returned {resp.status_code} for {ref.key}" + try: + payload = resp.json() + except ValueError: + content_type = (resp.headers.get("Content-Type") or "").split(";")[0].strip().lower() + src.error = f"Jira returned a non-JSON body for {ref.key} ({content_type or 'no content type'})" + return + text = _jira_ticket_text(payload) + if text is None: + src.error = f"Jira returned no issue fields for {ref.key}" return - payload = resp.json() - fields = payload.get("fields") or {} if isinstance(payload, dict) else {} - summary = fields.get("summary") or "" + src.text = text + + +def _jira_request( + ref: TicketRef, jira_base_url: str, jira_email: str, jira_api_token: str +) -> tuple[str, tuple[str, str] | None]: + """Resolve a ticket's REST URL and the basic auth to send with it. + + Credentials are only ever sent to ``jira_base_url``: without it the fetch + is anonymous even when the email and token are set, because the request + host then comes from the ticket URL, which is not the host the operator + trusted with them. Both warnings name variables, never their values. + """ + base = (jira_base_url or ref.base_url).rstrip("/") + url = f"{base}/rest/api/2/issue/{ref.key}?fields=summary,description,issuetype,labels" + if not (jira_email and jira_api_token): + return url, None + if not jira_base_url: + logger.warning( + "PRXREF_JIRA_EMAIL and PRXREF_JIRA_API_TOKEN are set but PRXREF_JIRA_BASE_URL is " + "empty; Jira credentials are only sent to PRXREF_JIRA_BASE_URL, so %s is fetched " + "anonymously", + ref.key, + ) + return url, None + if urlparse(jira_base_url).scheme.lower() == "http": + logger.warning( + "PRXREF_JIRA_BASE_URL is plain http; the Jira credentials for %s cross the network " + "unencrypted", + ref.key, + ) + return url, (jira_email, jira_api_token) + + +def _jira_status_error( + status: int, key: str, *, auth: tuple[str, str] | None, credentials_set: bool +) -> str: + """Explain a non-200 Jira answer, naming the variables that would fix it. + + An anonymous 401 or 403 is a missing login, and so is an anonymous 404: + Jira Cloud answers 404, not 403, for a private issue it will not show + anonymously. + """ + if auth is not None or status not in (401, 403, 404): + return f"Jira returned {status} for {key}" + if credentials_set: + return ( + f"Jira returned {status} for {key} without credentials: PRXREF_JIRA_EMAIL and " + "PRXREF_JIRA_API_TOKEN are set, but credentials are only sent to " + "PRXREF_JIRA_BASE_URL, which is empty; set it to this Jira's base URL to authenticate." + ) + reason = " (Jira hides a private issue from anonymous readers as 404)" if status == 404 else "" + return f"Jira returned {status} for {key} without credentials{reason}; set {_JIRA_ENV_HINT} to authenticate." + + +def _jira_ticket_text(payload: object) -> str | None: + """Render an issue payload as ``Summary:``/``Type:``/``Labels:`` lines plus the description. + + A header line whose value is empty is left out rather than rendered bare, + since every non-blank ticket line becomes a digest constraint. Returns + ``None`` when the payload carries no ``fields`` object. + """ + fields = payload.get("fields") if isinstance(payload, dict) else None + if not isinstance(fields, dict): + return None issuetype = fields.get("issuetype") - issue_type = issuetype.get("name", "") if isinstance(issuetype, dict) else "" - labels = ", ".join(str(label) for label in fields.get("labels") or []) + raw_labels = fields.get("labels") + header = [ + ("Summary", str(fields.get("summary") or "").strip()), + ("Type", str(issuetype.get("name") or "").strip() if isinstance(issuetype, dict) else ""), + ("Labels", ", ".join(str(label) for label in raw_labels) if isinstance(raw_labels, list) else ""), + ] description = fields.get("description") if description is None: description = "" elif not isinstance(description, str): description = json.dumps(description) - src.text = "\n".join([f"Summary: {summary}", f"Type: {issue_type}", f"Labels: {labels}", "", description]).strip() + lines = [f"{name}: {value}" for name, value in header if value] + return "\n".join([*lines, "", description]).strip() def _dispatch( @@ -371,9 +484,14 @@ def fetch_specs( ``not a URL or path`` error. Every exception β€” network, decode, bad credentials β€” becomes that source's ``error`` string with empty ``text``; a source never aborts the run. ``jira_base_url`` overrides the ticket - URL's own host when non-empty (self-hosted boards behind a different REST - host); Jira authenticates with basic auth only when both ``jira_email`` - and ``jira_api_token`` are non-empty, anonymously otherwise. + URL's own base when non-empty (self-hosted boards behind a different REST + host), so every ticket is looked up there by key. Credentials are only + ever sent to ``jira_base_url`` (``PRXREF_JIRA_BASE_URL``): Jira + authenticates with basic auth only when ``jira_base_url``, ``jira_email`` + and ``jira_api_token`` are all non-empty, and anonymously otherwise. An + email and token without a base URL are withheld and a warning names + ``PRXREF_JIRA_BASE_URL``; a plain-http base URL is honoured with a + warning. """ active = session if session is not None else _create_default_session() out: list[SpecSource] = [] diff --git a/tests/test_specs.py b/tests/test_specs.py index 95539f20..116409c0 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -169,6 +169,7 @@ def test_authenticated_render(self): (src,) = fetch_specs( [self._jira_source()], max_chars=1000, + jira_base_url="https://jira.example.com", jira_email="ops@example.com", jira_api_token="secret-token", session=session, @@ -216,10 +217,12 @@ def test_403_with_credentials_does_not_advise_env_vars(self): (src,) = fetch_specs( [self._jira_source()], max_chars=1000, + jira_base_url="https://jira.example.com", jira_email="ops@example.com", jira_api_token="secret-token", session=session, ) + assert session.calls[0][1]["auth"] == ("ops@example.com", "secret-token") assert src.error != "" assert "403" in src.error assert "PRXREF_JIRA_EMAIL" not in src.error diff --git a/tests/test_specs_jira.py b/tests/test_specs_jira.py new file mode 100644 index 00000000..be96aa00 --- /dev/null +++ b/tests/test_specs_jira.py @@ -0,0 +1,378 @@ +"""Jira ticket sources: URL shapes, where credentials go, the fetch hints, the rendered text. + +SEC-2: Jira credentials are only ever sent to ``PRXREF_JIRA_BASE_URL``. With an +email and token but no base URL the fetch is anonymous and a warning names the +missing variable; a plain-http base URL is honoured with a warning. The last +class drives ``cli._run_review`` against a local recording server, which is +the only place the config wiring and the credential rule meet. + +COR-2: a ticket URL under a Jira context path, a Cloud issue view, and a Cloud +board's ``selectedIssue`` are recognized; a Bitbucket Server file URL is not. + +COR-4: an empty ``Type:`` or ``Labels:`` value is left out rather than +rendered as a bare line, since every non-blank ticket line is a constraint. + +Backlog 9b: a 200 whose body is not a JSON issue fails its source cleanly. +""" +from __future__ import annotations + +import json +import logging +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import pytest +import requests + +from prxref import cli +from prxref.specs import SpecSource, build_spec_digest, fetch_specs, parse_ticket_url +from tests.test_orchestrator import REF, FakeForge, FakeLLM, _added_file_diff +from tests.test_specs import _FakeResponse, _FakeSession + +EMAIL = "ops@example.com" +TOKEN = "secret-token-value" +TICKET = "https://jira.example.com/browse/AUTH-7" +FIELDS_QUERY = "?fields=summary,description,issuetype,labels" +PAYLOAD = { + "fields": { + "summary": "Fix login flow", + "issuetype": {"name": "Bug"}, + "labels": ["auth", "urgent"], + "description": "Users MUST re-authenticate after password change.", + } +} + + +def _json_session(status_code: int = 200, payload: object = PAYLOAD) -> _FakeSession: + return _FakeSession(_FakeResponse(status_code=status_code, content_type="application/json", payload=payload)) + + +def _real_response(body: bytes, content_type: str, status_code: int = 200) -> requests.Response: + resp = requests.models.Response() + resp.status_code = status_code + resp._content = body + resp.headers["Content-Type"] = content_type + return resp + + +def _spec_warnings(caplog) -> list[logging.LogRecord]: + return [r for r in caplog.records if r.name == "prxref.specs" and r.levelno == logging.WARNING] + + +class TestTicketUrlShapes: + @pytest.mark.parametrize( + ("url", "base", "key"), + [ + ("https://jira.acme.com/browse/PROJ-12?focusedCommentId=5", "https://jira.acme.com", "PROJ-12"), + ("https://issues.apache.org/jira/browse/KAFKA-1", "https://issues.apache.org/jira", "KAFKA-1"), + ("https://acme.com/tools/jira/browse/OPS-1", "https://acme.com/tools/jira", "OPS-1"), + ("https://acme.com/jira/rest/api/2/issue/OPS-2", "https://acme.com/jira", "OPS-2"), + ("https://acme.com/tools/jira/rest/api/3/issue/OPS-3", "https://acme.com/tools/jira", "OPS-3"), + ( + "https://acme.atlassian.net/jira/software/projects/ENG/issues/ENG-9", + "https://acme.atlassian.net", + "ENG-9", + ), + ( + "https://acme.atlassian.net/jira/software/c/projects/ENG/issues/ENG-10?jql=all", + "https://acme.atlassian.net", + "ENG-10", + ), + ( + "https://acme.atlassian.net/jira/software/c/projects/ENG/boards/12?a=1&selectedIssue=ENG-42", + "https://acme.atlassian.net", + "ENG-42", + ), + ( + "https://acme.atlassian.net/jira/software/projects/ENG/boards/3/backlog?selectedIssue=ENG-7", + "https://acme.atlassian.net", + "ENG-7", + ), + ], + ) + def test_recognized(self, url, base, key): + ref = parse_ticket_url(url) + assert ref is not None + assert (ref.base_url, ref.key, ref.url) == (base, key, url) + + @pytest.mark.parametrize( + "url", + [ + "https://bitbucket.acme.com/projects/ARCH/repos/adrs/browse/ADR-0012", + "https://acme.com/bitbucket/projects/ARCH/repos/adrs/browse/ADR-0012", + "https://bitbucket.acme.com/users/jdoe/repos/notes/browse/ADR-0012", + "https://acme.com/a/b/c/browse/OPS-1", + "https://acme.com/a/b/c/rest/api/2/issue/OPS-1", + "https://jira.acme.com?x=/browse/ABC-1", + "https://acme.atlassian.net/jira/software/c/projects/ENG/boards/12", + "https://acme.atlassian.net/jira/software/c/projects/ENG/boards/12?selectedIssue=eng-1", + "https://acme.atlassian.net/jira/software/c/projects/ENG/boards/12?selectedIssue=ENG-1x", + "https://example.com/boards?selectedIssue=ENG-1", + "https://example.com/wiki/jira?selectedIssue=ENG-1", + "ftp://acme.com/jira/boards?selectedIssue=ENG-1", + ], + ) + def test_not_a_ticket(self, url): + assert parse_ticket_url(url) is None + + def test_documented_residual_short_browse_path_matches(self): + ref = parse_ticket_url("https://git.acme.com/scm/browse/ADR-1") + assert ref is not None + assert (ref.base_url, ref.key) == ("https://git.acme.com/scm", "ADR-1") + + +class TestContextPathDispatch: + def test_context_path_ticket_reaches_jira_rest_under_the_context_path(self): + session = _json_session() + (src,) = fetch_specs(["https://acme.com/jira/browse/OPS-1"], max_chars=1000, session=session) + assert src.kind == "jira" + assert src.error == "" + assert session.calls[0][0] == "https://acme.com/jira/rest/api/2/issue/OPS-1" + FIELDS_QUERY + + def test_board_ticket_reaches_jira_rest_on_the_host(self): + session = _json_session() + url = "https://acme.atlassian.net/jira/software/c/projects/ENG/boards/12?a=1&selectedIssue=ENG-42" + (src,) = fetch_specs([url], max_chars=1000, session=session) + assert src.kind == "jira" + assert session.calls[0][0] == "https://acme.atlassian.net/rest/api/2/issue/ENG-42" + FIELDS_QUERY + + def test_bitbucket_server_browse_url_is_fetched_as_a_page(self): + url = "https://bitbucket.acme.com/projects/ARCH/repos/adrs/browse/ADR-0012" + session = _FakeSession(_FakeResponse(content_type="text/plain", body=b"ADRs MUST be numbered.")) + (src,) = fetch_specs([url], max_chars=1000, session=session) + assert src.kind == "url" + assert session.calls[0][0] == url + + +class TestCredentialsOnlyGoToTheBaseUrl: + def test_credentials_without_base_fetch_anonymously_and_warn(self, caplog): + caplog.set_level(logging.WARNING, logger="prxref.specs") + session = _json_session() + (src,) = fetch_specs([TICKET], max_chars=1000, jira_email=EMAIL, jira_api_token=TOKEN, session=session) + assert src.error == "" + url, kwargs = session.calls[0] + assert kwargs["auth"] is None + assert url.startswith("https://jira.example.com/rest/api/2/issue/AUTH-7") + (warning,) = _spec_warnings(caplog) + assert "PRXREF_JIRA_BASE_URL" in warning.getMessage() + assert TOKEN not in caplog.text + assert EMAIL not in caplog.text + + def test_credentials_reach_a_configured_https_base(self, caplog): + caplog.set_level(logging.WARNING, logger="prxref.specs") + session = _json_session() + fetch_specs( + [TICKET], max_chars=1000, jira_base_url="https://rest.example.com/jira/", + jira_email=EMAIL, jira_api_token=TOKEN, session=session, + ) + url, kwargs = session.calls[0] + assert kwargs["auth"] == (EMAIL, TOKEN) + assert url == "https://rest.example.com/jira/rest/api/2/issue/AUTH-7" + FIELDS_QUERY + assert _spec_warnings(caplog) == [] + + def test_a_foreign_ticket_host_never_receives_the_credentials(self): + session = _json_session() + fetch_specs( + ["https://tickets.other.example/browse/AUTH-7"], max_chars=1000, + jira_base_url="https://jira.example.com", jira_email=EMAIL, jira_api_token=TOKEN, session=session, + ) + url, kwargs = session.calls[0] + assert url.startswith("https://jira.example.com/rest/api/2/issue/AUTH-7") + assert kwargs["auth"] == (EMAIL, TOKEN) + + def test_plain_http_base_is_honoured_with_a_warning(self, caplog): + caplog.set_level(logging.WARNING, logger="prxref.specs") + session = _json_session() + (src,) = fetch_specs( + [TICKET], max_chars=1000, jira_base_url="http://jira.internal", + jira_email=EMAIL, jira_api_token=TOKEN, session=session, + ) + assert src.error == "" + assert session.calls[0][1]["auth"] == (EMAIL, TOKEN) + (warning,) = _spec_warnings(caplog) + assert "PRXREF_JIRA_BASE_URL" in warning.getMessage() + assert "http" in warning.getMessage() + assert TOKEN not in caplog.text + + @pytest.mark.parametrize( + ("base", "email", "token"), + [("", "", ""), ("https://jira.example.com", "", ""), ("https://jira.example.com", EMAIL, "")], + ) + def test_incomplete_credentials_are_anonymous_and_silent(self, caplog, base, email, token): + caplog.set_level(logging.WARNING, logger="prxref.specs") + session = _json_session() + fetch_specs([TICKET], max_chars=1000, jira_base_url=base, jira_email=email, jira_api_token=token, + session=session) + assert session.calls[0][1]["auth"] is None + assert _spec_warnings(caplog) == [] + + +class TestStatusHints: + def test_anonymous_404_without_credentials_names_every_variable(self): + (src,) = fetch_specs([TICKET], max_chars=1000, session=_json_session(status_code=404)) + assert src.text == "" + assert src.error.startswith("Jira returned 404 for AUTH-7 without credentials") + assert "private issue" in src.error + for name in ("PRXREF_JIRA_BASE_URL", "PRXREF_JIRA_EMAIL", "PRXREF_JIRA_API_TOKEN"): + assert name in src.error + + @pytest.mark.parametrize("status", [401, 403, 404]) + def test_credentials_without_base_explain_where_credentials_go(self, status): + (src,) = fetch_specs( + [TICKET], max_chars=1000, jira_email=EMAIL, jira_api_token=TOKEN, + session=_json_session(status_code=status), + ) + assert src.error.startswith(f"Jira returned {status} for AUTH-7 without credentials") + assert "only sent to PRXREF_JIRA_BASE_URL" in src.error + assert TOKEN not in src.error + assert EMAIL not in src.error + + @pytest.mark.parametrize("status", [401, 404]) + def test_authenticated_failure_gives_no_env_advice(self, status): + (src,) = fetch_specs( + [TICKET], max_chars=1000, jira_base_url="https://jira.example.com", + jira_email=EMAIL, jira_api_token=TOKEN, session=_json_session(status_code=status), + ) + assert src.error == f"Jira returned {status} for AUTH-7" + + def test_anonymous_server_error_gives_no_env_advice(self): + (src,) = fetch_specs([TICKET], max_chars=1000, session=_json_session(status_code=500)) + assert src.error == "Jira returned 500 for AUTH-7" + + +class TestNonIssueBodies: + def test_non_json_200_is_a_clean_source_error(self): + session = _FakeSession(_real_response(b"Log in to continue", "text/html; charset=utf-8")) + (src,) = fetch_specs([TICKET], max_chars=1000, session=session) + assert src.kind == "jira" + assert src.text == "" + assert src.error == "Jira returned a non-JSON body for AUTH-7 (text/html)" + + def test_non_json_200_without_content_type(self): + session = _FakeSession(_real_response(b"", "")) + (src,) = fetch_specs([TICKET], max_chars=1000, session=session) + assert src.error == "Jira returned a non-JSON body for AUTH-7 (no content type)" + + @pytest.mark.parametrize("payload", [[], {"errorMessages": ["nope"]}, {"fields": None}, {"fields": "x"}]) + def test_json_without_issue_fields_is_a_source_error(self, payload): + session = _FakeSession(_real_response(json.dumps(payload).encode(), "application/json")) + (src,) = fetch_specs([TICKET], max_chars=1000, session=session) + assert src.text == "" + assert src.error == "Jira returned no issue fields for AUTH-7" + + +class TestTicketText: + def _text(self, fields: dict) -> str: + (src,) = fetch_specs([TICKET], max_chars=1000, session=_json_session(payload={"fields": fields})) + assert src.error == "" + return src.text + + def test_full_payload_renders_header_then_description(self): + assert self._text(PAYLOAD["fields"]) == ( + "Summary: Fix login flow\nType: Bug\nLabels: auth, urgent\n\n" + "Users MUST re-authenticate after password change." + ) + + def test_missing_type_and_labels_are_dropped(self): + text = self._text({"summary": "Fix login flow", "description": "Tokens MUST expire."}) + assert text == "Summary: Fix login flow\n\nTokens MUST expire." + + def test_empty_type_and_labels_are_dropped(self): + text = self._text( + {"summary": "Fix login flow", "issuetype": {"name": None}, "labels": [], "description": "D."} + ) + assert "Type:" not in text + assert "Labels:" not in text + assert text == "Summary: Fix login flow\n\nD." + + def test_non_list_labels_are_not_split_into_characters(self): + text = self._text({"summary": "S", "labels": "auth"}) + assert text == "Summary: S" + + def test_structured_description_is_serialized(self): + doc = {"type": "doc", "content": [{"type": "text", "text": "Hello"}]} + text = self._text({"summary": "S", "description": doc}) + assert text == "Summary: S\n\n" + json.dumps(doc) + + def test_every_digest_ticket_line_carries_content(self): + text = self._text({"summary": "Rotate keys", "description": "Keys MUST rotate every 90 days."}) + src = SpecSource(origin=TICKET, kind="jira", text=text, error="") + digest = build_spec_digest([src], [], token_budget=3000) + ticket_lines = [ln for ln in digest.splitlines() if ln.startswith("[ticket:AUTH-7]")] + assert ticket_lines == [ + "[ticket:AUTH-7] Summary: Rotate keys", + "[ticket:AUTH-7] Keys MUST rotate every 90 days.", + ] + + +class _RecordingJira: + """A local Jira stand-in that records each request's path and Authorization header.""" + + def __init__(self) -> None: + self.seen: list[dict[str, str | None]] = [] + seen = self.seen + body = json.dumps( + {"fields": {"summary": "Rotate keys", "description": "Keys MUST rotate every 90 days."}} + ).encode() + + class _Handler(BaseHTTPRequestHandler): + def do_GET(self): + seen.append({"path": self.path, "authorization": self.headers.get("Authorization")}) + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format, *args): + pass + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + self.port = self.server.server_address[1] + self._thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +class TestRunReviewSendsCredentialsOnlyToTheBase: + """``cli._run_review`` β†’ load_config β†’ orchestrate_review β†’ fetch_specs, against a real socket.""" + + @pytest.fixture + def jira(self, monkeypatch): + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("no_proxy", "127.0.0.1,localhost") + server = _RecordingJira() + forge = FakeForge(diff=_added_file_diff("src/keys.py", 3)) + monkeypatch.setattr("prxref.cli.detect_forge", lambda url: REF) + monkeypatch.setattr("prxref.cli.make_forge", lambda ref: forge) + monkeypatch.setattr("prxref.llm_backends.create_llm_client", lambda cfg: FakeLLM({})) + monkeypatch.setenv("PRXREF_SPEC_SOURCES", f"http://127.0.0.1:{server.port}/browse/ABC-1") + monkeypatch.setenv("PRXREF_JIRA_EMAIL", EMAIL) + monkeypatch.setenv("PRXREF_JIRA_API_TOKEN", TOKEN) + yield server + server.close() + + @pytest.mark.usefixtures("contract_stubs") + def test_no_base_url_means_no_authorization_header(self, jira, caplog): + caplog.set_level(logging.WARNING, logger="prxref.specs") + result = cli._run_review(REF.url, post=False) + assert isinstance(result, dict) + assert len(jira.seen) == 1 + assert jira.seen[0]["path"].startswith("/rest/api/2/issue/ABC-1?") + assert jira.seen[0]["authorization"] is None + assert any("PRXREF_JIRA_BASE_URL" in r.getMessage() for r in _spec_warnings(caplog)) + assert TOKEN not in caplog.text + + @pytest.mark.usefixtures("contract_stubs") + def test_control_with_base_url_sends_basic_auth(self, jira, monkeypatch): + monkeypatch.setenv("PRXREF_JIRA_BASE_URL", f"http://127.0.0.1:{jira.port}") + result = cli._run_review(REF.url, post=False) + assert isinstance(result, dict) + assert len(jira.seen) == 1 + assert jira.seen[0]["path"].startswith("/rest/api/2/issue/ABC-1?") + assert (jira.seen[0]["authorization"] or "").startswith("Basic ") From af9a0dcc156b659dec03d6208149d72bb67ed8ae Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:29:10 -0700 Subject: [PATCH 10/61] feat: LLM factory foundation for #61, #66 and #67 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lands the LLM-layer foundation for 0.14.0 (seat F1-LLM). Every signature and constant the #66/#67 feature seats build on is in place; the feature bodies come later. #61: litellm no longer needs PRXREF_LLM_BASE_URL. The base-URL check now runs only for openai-compat/ferry/http. It still runs before the models check, so a run with both values unset still names the endpoint first. When the base URL is set on any other backend, it is ignored with one INFO line and never forwarded. So a litellm deployment that set a placeholder URL to get past the old check keeps its routing after upgrading, and there is no api_base pass-through (G4 L4). docs/llm.md gets the matching #61 sentences. #66 (factory): three constants, OPENAI_COMPAT_BACKENDS, CLI_BACKENDS and BACKENDS, plus DEFAULT_CLI_CONCURRENCY. An unknown PRXREF_LLM_BACKEND is now a ConfigError that names the variable and lists the six accepted values. The check is the factory's first step, so it gives exit 2 instead of the old LLMError, a failed review that exited 0. claude-cli and kiro-cli go to llm_cli_backends.build_cli_client, which the factory imports lazily. It passes the models, timeout, reasoning effort, PRXREF_LLM_CLI_PATH and PRXREF_LLM_CLI_CONCURRENCY. The concurrency value is re-checked here as an integer >= 1, defaulting to 2. An explicitly set temperature or seed logs one WARNING saying it is not applied. The new llm_cli_backends module is a fail-closed stub: both entry points raise "PRXREF_LLM_BACKEND: is not wired in this build", which exits 2. #67 (fields): InvokeResult gains cost_usd (None = not reported, never 0.0) and cost_source, after finish_reason. The module and factory docstrings are written in their final form for all three issues. The existing unknown-backend test now expects a ConfigError, and test_litellm_selection unsets the base URL. The new tests/test_llm_factory.py covers the vocabulary, the gated base URL, the CLI wiring through a recorder, the stub contract (signatures, fail-closed, stdlib-only) and exit 2 through cli.main. It asserts only what still holds once the real CLI clients land. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- docs/llm.md | 5 +- src/prxref/llm.py | 24 +- src/prxref/llm_backends.py | 133 ++++++-- src/prxref/llm_cli_backends.py | 67 ++++ tests/test_llm_backends.py | 8 +- tests/test_llm_factory.py | 553 +++++++++++++++++++++++++++++++++ 6 files changed, 762 insertions(+), 28 deletions(-) create mode 100644 src/prxref/llm_cli_backends.py create mode 100644 tests/test_llm_factory.py diff --git a/docs/llm.md b/docs/llm.md index 92e400dd..fa58a01d 100644 --- a/docs/llm.md +++ b/docs/llm.md @@ -1,6 +1,6 @@ # LLM Backends & Failover Architecture -`prxref` connects to LLM inference endpoints using two interchangeable backends: a lightweight OpenAI-compatible plain-HTTP client or an optional in-process `litellm` wrapper. There is no default endpoint and no default model chain β€” `PRXREF_LLM_BASE_URL` and `PRXREF_LLM_MODELS` are required, and leaving either unset raises `ConfigError` (`prxref review` exits `2`). +`prxref` connects to LLM inference endpoints using two interchangeable backends: a lightweight OpenAI-compatible plain-HTTP client or an optional in-process `litellm` wrapper. There is no default model chain on any backend β€” `PRXREF_LLM_MODELS` is required, and leaving it unset raises `ConfigError` (`prxref review` exits `2`). There is no default endpoint either: `PRXREF_LLM_BASE_URL` is required by the `openai-compat` backend (and its `ferry`/`http` aliases), with the same exit `2` when unset, and is not used by any other backend (see [Optional Backend: litellm](#optional-backend-litellm)). ## Key Architectural Principles @@ -14,7 +14,7 @@ The primary and default backend communicates via plain HTTP requests with any OpenAI-compatible `/v1/chat/completions` server β€” a hosted router (OpenRouter, Together, Groq), a self-hosted gateway such as `llm-ferry`, or a local runtime such as vLLM or Ollama. - **Default Backend Alias:** `PRXREF_LLM_BACKEND=openai-compat` (aliases: `ferry`, `http`). -- **Endpoint URL:** `PRXREF_LLM_BASE_URL=https://llm.example.com/v1`. Required; there is no default. +- **Endpoint URL:** `PRXREF_LLM_BASE_URL=https://llm.example.com/v1`. Required for this backend; there is no default. - **API Key:** `PRXREF_LLM_API_KEY` (sent as `Authorization: Bearer `). Optional β€” leave empty for a local no-auth server. - **Models:** Model names are whatever the endpoint accepts, listed cheapest first. Required; there is no default. @@ -68,6 +68,7 @@ For environments running without a centralized inference gateway, `prxref` suppo - **Backend Setting:** `PRXREF_LLM_BACKEND=litellm` - **Installation:** `pip install 'prxref[litellm]'` +- **Endpoint URL: not used.** litellm resolves each model's own provider endpoint and reads that provider's credential (for example `OPENROUTER_API_KEY`) from its own environment, so `PRXREF_LLM_BASE_URL` is not required here and neither it nor `PRXREF_LLM_API_KEY` is ever passed to litellm. A set `PRXREF_LLM_BASE_URL` is ignored with one INFO line (`PRXREF_LLM_BASE_URL is set but not used by the litellm backend; ignoring it`), so a deployment that set a placeholder URL to get past the check older releases applied to every backend keeps working unchanged. To route through a LiteLLM **proxy**, which speaks the OpenAI API, use the `openai-compat` backend with `PRXREF_LLM_BASE_URL` pointing at the proxy. - **Shared settings:** `PRXREF_LLM_MAX_TOKENS`, `PRXREF_LLM_TIMEOUT`, `PRXREF_LLM_TEMPERATURE`, and `PRXREF_LLM_SEED` apply here too β€” temperature resolves to the same `0.0` default when unset, and a configured seed is passed as `seed=` to `litellm.completion`. `PRXREF_LLM_REASONING_EFFORT` is openai-compat only. ### Configuration Example diff --git a/src/prxref/llm.py b/src/prxref/llm.py index 610f9415..0dca840e 100644 --- a/src/prxref/llm.py +++ b/src/prxref/llm.py @@ -1,10 +1,12 @@ """LLM contract: protocol + fallback-chain factory. -Backends live in llm_backends.py (ferry / litellm / http). This module freezes -the interface the pipeline codes against. NO provider-specific keys are read -here; backend selection is PRXREF_LLM_BACKEND=ferry|litellm|http and the model -fallback chain is PRXREF_LLM_MODELS="model1,model2,..." (first that answers -within timeout wins; failures fail over fast). +Backends live in llm_backends.py (openai-compat with its ferry / http aliases, +and litellm) and llm_cli_backends.py (claude-cli, kiro-cli). This module +freezes the interface the pipeline codes against. NO provider-specific keys +are read here; backend selection is +PRXREF_LLM_BACKEND=openai-compat|ferry|http|litellm|claude-cli|kiro-cli and +the model fallback chain is PRXREF_LLM_MODELS="model1,model2,..." (first that +answers within timeout wins; failures fail over fast). """ from __future__ import annotations @@ -33,6 +35,16 @@ class InvokeResult: reviewer can tell an operator to raise ``PRXREF_LLM_MAX_TOKENS`` instead of handing them a bare ``JSONDecodeError``. A backend that does not report one leaves it ``""`` β€” absent, never guessed. + + ``cost_usd`` is the dollar amount the backend REPORTED for this call. When + the backend's fallback chain moved past truncated completions, those were + billed too and are included. ``None`` means no figure was reported: it + never means free, and no backend may default it to ``0.0``. ``cost_source`` + names where the figure came from (``"usage.cost"``, + ``"x-litellm-response-cost"``, ``"litellm"`` or ``"claude-cli"``), and is + ``""`` whenever ``cost_usd`` is ``None``. A price-table estimate never + appears here: backends only report, and :mod:`prxref.costs` estimates + once, over the whole run. """ text: str @@ -42,6 +54,8 @@ class InvokeResult: backend: str = "" elapsed_ms: int = 0 finish_reason: str = "" + cost_usd: float | None = None + cost_source: str = "" class LLMClient(Protocol): diff --git a/src/prxref/llm_backends.py b/src/prxref/llm_backends.py index fca7caf5..c2692c74 100644 --- a/src/prxref/llm_backends.py +++ b/src/prxref/llm_backends.py @@ -1,9 +1,16 @@ -"""LLM backends: an OpenAI-compatible plain-HTTP client and an optional litellm wrapper. +"""LLM backends: OpenAI-compatible HTTP, optional litellm, and subscription CLIs (``llm_cli_backends``). The primary backend speaks plain HTTP to any OpenAI-compatible -``/chat/completions`` endpoint. There is no default endpoint and no default -model chain: ``PRXREF_LLM_BASE_URL`` and ``PRXREF_LLM_MODELS`` are required, -and an unset one raises ``ConfigError`` rather than guessing a host. +``/chat/completions`` endpoint. There is no default model chain on any +backend: ``PRXREF_LLM_MODELS`` is required, and an unset one raises +``ConfigError``. There is no default endpoint either: +``PRXREF_LLM_BASE_URL`` is required by the openai-compat backend (and its +``ferry``/``http`` aliases) and raises ``ConfigError`` when unset rather than +guessing a host. The other backends do not use it: litellm resolves each +model's own provider endpoint, and the CLI backends talk to their CLI. A set +value is ignored there with one INFO line, never forwarded, so a deployment +that set a placeholder URL to get past the old unconditional check keeps its +routing on upgrade. Fallback is a caller-side loop over the model chain: a model that answers with HTTP >= 500, HTTP 429, a connection error, a timeout, a malformed @@ -17,9 +24,28 @@ mechanism (which advances on errors only β€” a truncated litellm answer still returns as success). -Tenet: no provider credential is ever read and no env name is -provider-specific β€” provider keys live behind the configured endpoint, -never here. +The ``claude-cli`` and ``kiro-cli`` backends (``llm_cli_backends``) run the +user's own installed, logged-in CLI as a subprocess, one process per call, +with the same caller-side model chain. They take the model chain, the +timeout, a process-count cap (``PRXREF_LLM_CLI_CONCURRENCY``) and an +optional binary path (``PRXREF_LLM_CLI_PATH``); the base URL, the API key, +``max_tokens``, temperature and seed are not applied. The factory imports +that module lazily, so the HTTP backends never load it. + +Cost: a backend reports the dollar figure its provider returned and never +estimates one. The openai-compat client reads the body's ``usage.cost`` +first, then a LiteLLM-based gateway's ``x-litellm-response-cost`` response +header; the litellm client reads ``_hidden_params["response_cost"]``. No +figure leaves ``InvokeResult.cost_usd`` as ``None``, never ``0.0``. The +price-table estimate is made once per run, in :mod:`prxref.costs`. + +Tenet: prxref never reads, stores, or forwards a provider credential, and +its own settings are provider-neutral ``PRXREF_*`` names. A provider key +lives behind the configured endpoint (openai-compat), in the provider SDK's +own environment (litellm), or inside the user's own logged-in CLI +(claude-cli, kiro-cli). The CLI backends remove a fixed list of +credential-routing variable NAMES from the child process environment so the +CLI falls back to its subscription login; the values are never read. """ from __future__ import annotations @@ -43,6 +69,10 @@ # actually reaches the wire. Resolved by create_llm_client when the operator # left PRXREF_LLM_TEMPERATURE unset or empty. DEFAULT_TEMPERATURE = 0.0 +DEFAULT_CLI_CONCURRENCY = 2 +OPENAI_COMPAT_BACKENDS = ("openai-compat", "ferry", "http") +CLI_BACKENDS = ("claude-cli", "kiro-cli") +BACKENDS = (*OPENAI_COMPAT_BACKENDS, "litellm", *CLI_BACKENDS) logger = logging.getLogger(__name__) # Connecting is not generating: a reachable endpoint answers the TCP/TLS # handshake in well under this, so a separate, much smaller connect budget @@ -592,16 +622,28 @@ def create_llm_client( """Build the configured client from ``cfg`` overrides then PRXREF_LLM_* env. ``cfg`` keys (LLM_BACKEND, LLM_BASE_URL, LLM_API_KEY, LLM_MODELS, - LLM_REASONING_EFFORT) win over env; env never includes provider - credentials. PRXREF_LLM_BACKEND selects ``openai-compat`` (default) - with ``ferry`` as an alias, or ``litellm``. PRXREF_LLM_BASE_URL and - PRXREF_LLM_MODELS are required and have no defaults β€” an unset one - raises :class:`~prxref.llm.ConfigError`. PRXREF_LLM_API_KEY is - optional and may be empty for a local no-auth server. - PRXREF_LLM_MODELS (comma list, cheap first) feeds litellm too. + LLM_REASONING_EFFORT, LLM_TIMEOUT, LLM_TEMPERATURE, LLM_SEED, + LLM_CLI_PATH, LLM_CLI_CONCURRENCY, in either case) win over env; env + never includes provider credentials. PRXREF_LLM_BACKEND is read + case-insensitively and selects ``openai-compat`` (the default, with + ``ferry`` and ``http`` as aliases), ``litellm``, ``claude-cli`` or + ``kiro-cli``. Any other value raises :class:`~prxref.llm.ConfigError` + naming PRXREF_LLM_BACKEND, before any other setting is looked at, so a + typo is reported as itself (exit 2) rather than as a missing endpoint or + a failed review. + PRXREF_LLM_MODELS (comma list, cheap first) is required by every + backend and has no default. PRXREF_LLM_BASE_URL has no default and is + required by the openai-compat family only; it is checked before the + models, so a run with both unset still names the endpoint first. The + other backends do not use it: when it is set anyway it is ignored with + one INFO line and never forwarded (litellm resolves each model's own + provider endpoint; a LiteLLM proxy is OpenAI-compatible and belongs on + ``openai-compat``). PRXREF_LLM_API_KEY is openai-compat only, optional, + and may be empty for a local no-auth server. PRXREF_LLM_REASONING_EFFORT is passed through unvalidated to the openai-compat client for models that cannot disable reasoning - (e.g. GLM-5.3-Flash's ``low``/``high``/``max``); empty omits it. + (e.g. GLM-5.3-Flash's ``low``/``high``/``max``) and to claude-cli as its + effort setting; empty omits it, and litellm and kiro-cli ignore it. PRXREF_LLM_TIMEOUT (seconds, default 45.0, must be > 0) becomes the client's ``default_timeout``. PRXREF_LLM_TEMPERATURE is parsed to a float (finite, >= 0 β€” no upper bound, since the maximum is @@ -609,7 +651,8 @@ def create_llm_client( ``DEFAULT_TEMPERATURE`` (0.0), which IS sent β€” temperature 0 keeps reviews reproducible by default, and an operator-set value wins. PRXREF_LLM_SEED (integer >= 0, where 0 is a valid seed) is passed to - both backends as a top-level ``seed`` and always wins when set. Unset + the openai-compat and litellm backends as a top-level ``seed`` and + always wins when set. Unset or empty does NOT omit the field: temperature 0 alone cannot pin hosted inference, so the factory derives ONE random seed per process (:func:`_auto_run_seed`) and stamps it on every client it builds β€” @@ -621,6 +664,19 @@ def create_llm_client( ``PRXREF_LLM_MAX_TOKENS`` is deliberately NOT read here: it is a per-call budget threaded cfg -> orchestrator -> reviewer -> ``invoke``, so a client-level copy could never win and would be dead config. + + The CLI backends (``claude-cli``, ``kiro-cli``) are built by + :func:`prxref.llm_cli_backends.build_cli_client`, imported lazily. + PRXREF_LLM_CLI_PATH overrides the binary (empty = ``claude`` or + ``kiro-cli`` on ``PATH``; one that cannot be found is a ConfigError + naming it). PRXREF_LLM_CLI_CONCURRENCY caps the CLI processes one client + runs at once (integer >= 1, default ``DEFAULT_CLI_CONCURRENCY``), and is + re-checked here for callers that bypass ``config.load_config``. Neither + CLI has a temperature or seed option, so both are still parsed (a + malformed value still exits 2) but are not applied, and an explicitly + set one logs one WARNING saying so; the client's ``temperature`` and + ``seed`` attributes are ``None``, which the run record's ``sampling`` + reports truthfully. """ cfg = cfg or {} @@ -634,10 +690,15 @@ def _get(key: str, env: str, default: str | None = None) -> str | None: return os.environ.get(env, default) backend = (_get("LLM_BACKEND", "PRXREF_LLM_BACKEND", "openai-compat") or "").strip().lower() or "openai-compat" + if backend not in BACKENDS: + raise ConfigError( + f"PRXREF_LLM_BACKEND: must be one of {', '.join(BACKENDS)} " + f"(case-insensitive), got {backend!r}" + ) raw_models = _get("LLM_MODELS", "PRXREF_LLM_MODELS", DEFAULT_MODELS) or "" models = [m.strip() for m in raw_models.split(",") if m.strip()] base_url = _get("LLM_BASE_URL", "PRXREF_LLM_BASE_URL", DEFAULT_BASE_URL) or "" - if not base_url.strip(): + if backend in OPENAI_COMPAT_BACKENDS and not base_url.strip(): raise ConfigError( "no LLM endpoint configured. Set PRXREF_LLM_BASE_URL to an " "OpenAI-compatible /chat/completions endpoint " @@ -669,7 +730,7 @@ def _get(key: str, env: str, default: str | None = None) -> str | None: ) if seed is None: seed = _auto_run_seed() - if backend in ("openai-compat", "ferry", "http"): + if backend in OPENAI_COMPAT_BACKENDS: return OpenAICompatClient( base_url=base_url, api_key=_get("LLM_API_KEY", "PRXREF_LLM_API_KEY", DEFAULT_API_KEY) or DEFAULT_API_KEY, @@ -680,8 +741,42 @@ def _get(key: str, env: str, default: str | None = None) -> str | None: temperature=temperature, seed=seed, ) + if base_url.strip(): + logger.info( + "PRXREF_LLM_BASE_URL is set but not used by the %s backend; ignoring it", + backend, + ) if backend == "litellm": return LiteLLMClient( models=models, default_timeout=timeout, temperature=temperature, seed=seed ) - raise LLMError(f"unknown PRXREF_LLM_BACKEND {backend!r}; expected openai-compat|ferry|http|litellm") + unapplied = [ + env + for key, env in ( + ("LLM_TEMPERATURE", "PRXREF_LLM_TEMPERATURE"), + ("LLM_SEED", "PRXREF_LLM_SEED"), + ) + if (_get(key, env) or "").strip() + ] + if unapplied: + logger.warning( + "%s %s not applied by %s (the CLI has no such option)", + " / ".join(unapplied), + "is" if len(unapplied) == 1 else "are", + backend, + ) + concurrency = _int_setting( + _get("LLM_CLI_CONCURRENCY", "PRXREF_LLM_CLI_CONCURRENCY"), + "PRXREF_LLM_CLI_CONCURRENCY", + minimum=1, + ) + from .llm_cli_backends import build_cli_client + + return build_cli_client( + backend, + models=models, + default_timeout=timeout, + reasoning_effort=_get("LLM_REASONING_EFFORT", "PRXREF_LLM_REASONING_EFFORT") or None, + cli_path=_get("LLM_CLI_PATH", "PRXREF_LLM_CLI_PATH") or "", + concurrency=DEFAULT_CLI_CONCURRENCY if concurrency is None else concurrency, + ) diff --git a/src/prxref/llm_cli_backends.py b/src/prxref/llm_cli_backends.py new file mode 100644 index 00000000..45f3be4c --- /dev/null +++ b/src/prxref/llm_cli_backends.py @@ -0,0 +1,67 @@ +"""Subscription CLI backends: ``claude-cli`` and ``kiro-cli``. + +These backends run the user's own installed, logged-in CLI as a subprocess +(argv list, never a shell; the user message on stdin; a fresh temporary +working directory) and walk ``PRXREF_LLM_MODELS`` as a caller-side chain, +exactly like the HTTP backends. The module is stdlib-only and is imported +lazily by :func:`prxref.llm_backends.create_llm_client`, so the HTTP +backends never load it. The backend names live in +``prxref.llm_backends.CLI_BACKENDS``. + +This build does not wire either backend yet: both entry points fail closed +with a :class:`~prxref.llm.ConfigError` naming ``PRXREF_LLM_BACKEND``, which +``prxref review`` reports as a configuration error (exit 2) before any forge +or model call, rather than silently reviewing with nothing. +""" +from __future__ import annotations + +import shutil +import subprocess +from collections.abc import Sequence + +from .llm import ConfigError, LLMClient + + +def _not_wired(backend: str) -> ConfigError: + """The fail-closed error for a CLI backend this build cannot run.""" + return ConfigError(f"PRXREF_LLM_BACKEND: {backend} is not wired in this build") + + +def resolve_cli_binary(backend: str, cli_path: str, *, which=shutil.which) -> str: + """Resolve the CLI binary for ``backend`` to an executable path. + + ``cli_path`` is ``PRXREF_LLM_CLI_PATH``: empty means the backend's default + binary name on ``PATH``, otherwise a ``~``-expanded path or name looked up + through ``which``. A binary that cannot be found raises + :class:`~prxref.llm.ConfigError` naming ``PRXREF_LLM_CLI_PATH`` (or + ``PRXREF_LLM_BACKEND`` when no override was given), so a missing CLI + exits 2 before any network call. + + In this build every call raises the fail-closed ``ConfigError``. + """ + raise _not_wired(backend) + + +def build_cli_client( + backend: str, + *, + models: Sequence[str], + default_timeout: float, + reasoning_effort: str | None, + cli_path: str, + concurrency: int, + which=shutil.which, + runner=subprocess.Popen, +) -> LLMClient: + """Build the client for ``backend`` (one of ``llm_backends.CLI_BACKENDS``). + + ``models`` is the chain walked in order; ``default_timeout`` is the + per-model deadline in seconds; ``reasoning_effort`` feeds claude's effort + setting and is ignored by kiro; ``cli_path`` is passed to + :func:`resolve_cli_binary`; ``concurrency`` caps the CLI processes this + client runs at once. ``which`` and ``runner`` are the binary lookup and + the process launcher, injectable for tests. + + In this build every call raises the fail-closed ``ConfigError``. + """ + raise _not_wired(backend) diff --git a/tests/test_llm_backends.py b/tests/test_llm_backends.py index 311d3b74..04dc2600 100644 --- a/tests/test_llm_backends.py +++ b/tests/test_llm_backends.py @@ -332,19 +332,23 @@ def test_ferry_alias_maps_to_openai_compat(self, monkeypatch): assert isinstance(create_llm_client(), OpenAICompatClient) def test_litellm_selection(self, monkeypatch): + """litellm needs no endpoint: PRXREF_LLM_BASE_URL is unset (#61).""" fake = types.SimpleNamespace(completion=lambda **kw: None) monkeypatch.setitem(sys.modules, "litellm", fake) monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") - monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) monkeypatch.setenv("PRXREF_LLM_MODELS", "a,b") assert isinstance(create_llm_client(), LiteLLMClient) def test_unknown_backend_raises(self, monkeypatch): + """A value outside the vocabulary is a configuration error (exit 2), not a review failure.""" monkeypatch.setenv("PRXREF_LLM_BACKEND", "skynet") monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") monkeypatch.setenv("PRXREF_LLM_MODELS", "a,b") - with pytest.raises(LLMError): + with pytest.raises(ConfigError, match="PRXREF_LLM_BACKEND") as exc: create_llm_client() + assert "claude-cli" in str(exc.value) + assert "'skynet'" in str(exc.value) def test_env_overrides(self, monkeypatch): monkeypatch.delenv("PRXREF_LLM_BACKEND", raising=False) diff --git a/tests/test_llm_factory.py b/tests/test_llm_factory.py new file mode 100644 index 00000000..079b4617 --- /dev/null +++ b/tests/test_llm_factory.py @@ -0,0 +1,553 @@ +"""Tests for prxref.llm_backends.create_llm_client: backend vocabulary (#66 D11), +the openai-compat-only base URL (#61), the CLI-backend wiring (#66), the +InvokeResult cost fields (#67), and the llm_cli_backends entry-point contract. + +The CLI-backend tests assert only what holds both before and after the real +claude-cli / kiro-cli clients land: the factory wiring is observed through a +recorder, and every "cannot run" case points ``PRXREF_LLM_CLI_PATH`` at a path +that does not exist, so nothing here depends on what is installed on ``PATH``. +""" +from __future__ import annotations + +import ast +import dataclasses +import inspect +import json +import logging +import pathlib +import shutil +import subprocess +import sys +import types +from types import SimpleNamespace + +import pytest + +import prxref.llm_backends +import prxref.llm_cli_backends +from prxref import cli +from prxref import orchestrator as real_orchestrator +from prxref.forges.base import PRRef +from prxref.llm import ConfigError, InvokeResult +from prxref.llm_backends import ( + BACKENDS, + CLI_BACKENDS, + DEFAULT_CLI_CONCURRENCY, + OPENAI_COMPAT_BACKENDS, + LiteLLMClient, + OpenAICompatClient, + create_llm_client, +) + +_IGNORED_LINE = "PRXREF_LLM_BASE_URL is set but not used by the {} backend; ignoring it" +_LOGGER = "prxref.llm_backends" + + +def _fake_litellm(monkeypatch) -> list[dict]: + """Install a fake ``litellm`` module; returns the kwargs of every completion call.""" + captured: list[dict] = [] + + def completion(**kwargs): + captured.append(kwargs) + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content="ok"))], + usage=None, + model=kwargs["model"], + ) + + monkeypatch.setitem(sys.modules, "litellm", types.SimpleNamespace(completion=completion)) + return captured + + +def _ignored_lines(caplog) -> list[str]: + return [ + r.getMessage() + for r in caplog.records + if r.name == _LOGGER and "is set but not used by the" in r.getMessage() + ] + + +@pytest.fixture +def recorded_cli_builds(monkeypatch): + """Replace ``build_cli_client`` with a recorder; returns the recorded calls. + + The factory imports it lazily from ``prxref.llm_cli_backends`` at call time, + so patching the module attribute is exactly what the factory sees. + """ + calls: list[tuple[tuple, dict]] = [] + sentinel = object() + + def fake_build(*args, **kwargs): + calls.append((args, kwargs)) + return sentinel + + monkeypatch.setattr(prxref.llm_cli_backends, "build_cli_client", fake_build) + monkeypatch.delenv("PRXREF_LLM_CLI_PATH", raising=False) + monkeypatch.delenv("PRXREF_LLM_CLI_CONCURRENCY", raising=False) + return SimpleNamespace(calls=calls, sentinel=sentinel) + + +class TestBackendVocabulary: + def test_the_constants_name_the_six_backends(self): + assert OPENAI_COMPAT_BACKENDS == ("openai-compat", "ferry", "http") + assert CLI_BACKENDS == ("claude-cli", "kiro-cli") + assert BACKENDS == (*OPENAI_COMPAT_BACKENDS, "litellm", *CLI_BACKENDS) + assert len(set(BACKENDS)) == 6 + assert DEFAULT_CLI_CONCURRENCY == 2 + + def test_unknown_backend_is_named_before_missing_models(self, monkeypatch): + """A typo is reported as itself, not as the missing endpoint or chain it + would otherwise trip over first.""" + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude_cli") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + monkeypatch.delenv("PRXREF_LLM_MODELS", raising=False) + with pytest.raises(ConfigError) as exc: + create_llm_client() + message = str(exc.value) + assert message.startswith("PRXREF_LLM_BACKEND:") + assert "'claude_cli'" in message + assert "PRXREF_LLM_MODELS" not in message + assert "PRXREF_LLM_BASE_URL" not in message + + def test_the_error_lists_every_accepted_name(self, monkeypatch): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude") + with pytest.raises(ConfigError) as exc: + create_llm_client() + for name in BACKENDS: + assert name in str(exc.value) + assert "case-insensitive" in str(exc.value) + + def test_an_unknown_backend_from_cfg_is_a_config_error_too(self, monkeypatch): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + with pytest.raises(ConfigError, match=r"^PRXREF_LLM_BACKEND: must be one of .*got 'bedrock'$"): + create_llm_client({"llm_backend": "bedrock"}) + + @pytest.mark.parametrize( + ("raw", "cls"), + [(" LiteLLM ", LiteLLMClient), ("FERRY", OpenAICompatClient), ("Http", OpenAICompatClient)], + ) + def test_backend_name_is_case_insensitive(self, monkeypatch, raw, cls): + _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", raw) + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + monkeypatch.setenv("PRXREF_LLM_MODELS", "a") + assert isinstance(create_llm_client(), cls) + + def test_a_blank_backend_means_the_default(self, monkeypatch): + monkeypatch.setenv("PRXREF_LLM_BACKEND", " ") + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + monkeypatch.setenv("PRXREF_LLM_MODELS", "a") + assert isinstance(create_llm_client(), OpenAICompatClient) + + +class TestBaseUrlIsOpenAICompatOnly: + """#61: PRXREF_LLM_BASE_URL is required by openai-compat/ferry/http only.""" + + def test_unset_endpoint_names_base_url_before_models_when_both_missing(self, monkeypatch): + """Precedence guard: gating the check must not move it after the models check.""" + monkeypatch.delenv("PRXREF_LLM_BACKEND", raising=False) + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + monkeypatch.delenv("PRXREF_LLM_MODELS", raising=False) + with pytest.raises(ConfigError) as exc: + create_llm_client() + assert "PRXREF_LLM_BASE_URL" in str(exc.value) + assert "PRXREF_LLM_MODELS" not in str(exc.value) + + @pytest.mark.parametrize("backend", OPENAI_COMPAT_BACKENDS) + @pytest.mark.parametrize("base_url", [None, " "]) + def test_openai_compat_backend_still_requires_base_url(self, monkeypatch, backend, base_url): + monkeypatch.setenv("PRXREF_LLM_BACKEND", backend) + monkeypatch.setenv("PRXREF_LLM_MODELS", "a") + if base_url is None: + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + else: + monkeypatch.setenv("PRXREF_LLM_BASE_URL", base_url) + with pytest.raises(ConfigError, match="PRXREF_LLM_BASE_URL"): + create_llm_client() + + def test_litellm_backend_does_not_require_base_url(self, monkeypatch): + """The issue's own repro: litellm, a model chain, and no endpoint.""" + _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.setenv("PRXREF_LLM_MODELS", "openrouter/openai/gpt-oss-20b") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + client = create_llm_client() + assert isinstance(client, LiteLLMClient) + assert client.models == ["openrouter/openai/gpt-oss-20b"] + + def test_litellm_builds_with_no_base_url_and_no_models_still_raises(self, monkeypatch): + _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + monkeypatch.delenv("PRXREF_LLM_MODELS", raising=False) + with pytest.raises(ConfigError) as exc: + create_llm_client() + assert "PRXREF_LLM_MODELS" in str(exc.value) + assert "PRXREF_LLM_BASE_URL" not in str(exc.value) + + def test_litellm_ignores_a_set_base_url(self, monkeypatch, caplog): + """Never forwarded: every pre-0.14 litellm deployment carries a dummy URL.""" + captured = _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.setenv("PRXREF_LLM_MODELS", "m1,m2") + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "http://127.0.0.1:9/v1") + monkeypatch.setenv("PRXREF_LLM_API_KEY", "not-a-real-key") + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client().invoke("sys", "usr") + assert _ignored_lines(caplog) == [_IGNORED_LINE.format("litellm")] + kwargs = captured[0] + assert kwargs["model"] == "m1" + assert kwargs["fallbacks"] == ["m2"] + for leaked in ("api_base", "base_url", "api_key"): + assert leaked not in kwargs + assert "127.0.0.1" not in json.dumps(kwargs, default=str) + + def test_a_base_url_from_cfg_is_ignored_the_same_way(self, monkeypatch, caplog): + captured = _fake_litellm(monkeypatch) + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client( + {"llm_backend": "litellm", "llm_models": ["m1"], "llm_base_url": "https://gw.test/v1"} + ).invoke("sys", "usr") + assert _ignored_lines(caplog) == [_IGNORED_LINE.format("litellm")] + assert "api_base" not in captured[0] + + def test_litellm_without_a_base_url_logs_nothing_about_it(self, monkeypatch, caplog): + _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.setenv("PRXREF_LLM_MODELS", "m1") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client() + assert _ignored_lines(caplog) == [] + + @pytest.mark.parametrize("backend", OPENAI_COMPAT_BACKENDS) + def test_openai_compat_uses_the_base_url_and_never_logs_it_as_ignored( + self, monkeypatch, caplog, backend + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", backend) + monkeypatch.setenv("PRXREF_LLM_MODELS", "m1") + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1/") + caplog.set_level(logging.INFO, logger=_LOGGER) + client = create_llm_client() + assert client.base_url == "https://llm.test/v1" + assert _ignored_lines(caplog) == [] + + +class TestCliBackendWiring: + """#66: the factory side of claude-cli / kiro-cli, observed through a recorder.""" + + @pytest.mark.parametrize("backend", CLI_BACKENDS) + def test_a_cli_backend_builds_without_base_url_and_gets_the_defaults( + self, monkeypatch, recorded_cli_builds, backend + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", backend) + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet, haiku") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + assert create_llm_client() is recorded_cli_builds.sentinel + [(args, kwargs)] = recorded_cli_builds.calls + assert args == (backend,) + assert kwargs == { + "models": ["sonnet", "haiku"], + "default_timeout": 45.0, + "reasoning_effort": None, + "cli_path": "", + "concurrency": DEFAULT_CLI_CONCURRENCY, + } + + def test_env_settings_reach_the_builder(self, monkeypatch, recorded_cli_builds): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + monkeypatch.setenv("PRXREF_LLM_TIMEOUT", "180") + monkeypatch.setenv("PRXREF_LLM_REASONING_EFFORT", "low") + monkeypatch.setenv("PRXREF_LLM_CLI_PATH", "~/bin/claude") + monkeypatch.setenv("PRXREF_LLM_CLI_CONCURRENCY", "3") + create_llm_client() + [(_, kwargs)] = recorded_cli_builds.calls + assert kwargs["default_timeout"] == 180.0 + assert kwargs["reasoning_effort"] == "low" + assert kwargs["cli_path"] == "~/bin/claude" + assert kwargs["concurrency"] == 3 + + def test_cfg_settings_win_over_env(self, monkeypatch, recorded_cli_builds): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.setenv("PRXREF_LLM_CLI_PATH", "/env/claude") + monkeypatch.setenv("PRXREF_LLM_CLI_CONCURRENCY", "3") + create_llm_client({ + "llm_backend": "kiro-cli", + "llm_models": ["claude-haiku-4.5"], + "llm_cli_path": "/cfg/kiro-cli", + "llm_cli_concurrency": 5, + }) + [(args, kwargs)] = recorded_cli_builds.calls + assert args == ("kiro-cli",) + assert kwargs["cli_path"] == "/cfg/kiro-cli" + assert kwargs["concurrency"] == 5 + + def test_backend_name_is_case_insensitive_for_cli_backends( + self, monkeypatch, recorded_cli_builds + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", " Claude-CLI ") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + create_llm_client() + assert recorded_cli_builds.calls[0][0] == ("claude-cli",) + + def test_a_cli_backend_still_requires_models(self, monkeypatch, recorded_cli_builds): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.delenv("PRXREF_LLM_MODELS", raising=False) + with pytest.raises(ConfigError, match="PRXREF_LLM_MODELS"): + create_llm_client() + assert recorded_cli_builds.calls == [] + + @pytest.mark.parametrize("raw", ["0", "-1", "two", "1.5"]) + def test_a_bad_cli_concurrency_names_the_variable(self, monkeypatch, recorded_cli_builds, raw): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + monkeypatch.setenv("PRXREF_LLM_CLI_CONCURRENCY", raw) + with pytest.raises(ConfigError, match="PRXREF_LLM_CLI_CONCURRENCY"): + create_llm_client() + assert recorded_cli_builds.calls == [] + + def test_a_blank_cli_concurrency_means_the_default(self, monkeypatch, recorded_cli_builds): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "kiro-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "claude-haiku-4.5") + monkeypatch.setenv("PRXREF_LLM_CLI_CONCURRENCY", " ") + create_llm_client() + assert recorded_cli_builds.calls[0][1]["concurrency"] == DEFAULT_CLI_CONCURRENCY + + def test_cli_concurrency_is_not_read_by_the_http_backends(self, monkeypatch): + """Only a CLI backend parses it; an HTTP run is not failed by a knob it never uses.""" + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + monkeypatch.setenv("PRXREF_LLM_MODELS", "a") + monkeypatch.setenv("PRXREF_LLM_CLI_CONCURRENCY", "zero") + assert isinstance(create_llm_client(), OpenAICompatClient) + + @pytest.mark.parametrize("backend", CLI_BACKENDS) + def test_a_set_base_url_is_ignored_with_an_info_line( + self, monkeypatch, caplog, recorded_cli_builds, backend + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", backend) + monkeypatch.setenv("PRXREF_LLM_MODELS", "m") + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client() + assert _ignored_lines(caplog) == [_IGNORED_LINE.format(backend)] + assert "https://llm.test/v1" not in repr(recorded_cli_builds.calls) + + @pytest.mark.parametrize( + ("env", "expected"), + [ + ({"PRXREF_LLM_TEMPERATURE": "0.2"}, "PRXREF_LLM_TEMPERATURE is not applied by claude-cli"), + ({"PRXREF_LLM_SEED": "0"}, "PRXREF_LLM_SEED is not applied by claude-cli"), + ( + {"PRXREF_LLM_TEMPERATURE": "0.2", "PRXREF_LLM_SEED": "7"}, + "PRXREF_LLM_TEMPERATURE / PRXREF_LLM_SEED are not applied by claude-cli", + ), + ], + ) + def test_temperature_or_seed_set_with_a_cli_backend_warns_not_applied( + self, monkeypatch, caplog, recorded_cli_builds, env, expected + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + for name, value in env.items(): + monkeypatch.setenv(name, value) + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client() + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert warnings == [f"{expected} (the CLI has no such option)"] + [(_, kwargs)] = recorded_cli_builds.calls + assert "temperature" not in kwargs + assert "seed" not in kwargs + + def test_a_seed_from_cfg_warns_too(self, monkeypatch, caplog, recorded_cli_builds): + caplog.set_level(logging.WARNING, logger=_LOGGER) + create_llm_client({"llm_backend": "kiro-cli", "llm_models": ["m"], "llm_seed": 3}) + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert warnings == ["PRXREF_LLM_SEED is not applied by kiro-cli (the CLI has no such option)"] + + def test_no_warning_when_temperature_and_seed_are_unset( + self, monkeypatch, caplog, recorded_cli_builds + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + monkeypatch.setenv("PRXREF_LLM_TEMPERATURE", " ") + caplog.set_level(logging.INFO, logger=_LOGGER) + create_llm_client() + assert [r for r in caplog.records if r.levelno >= logging.WARNING] == [] + + def test_a_malformed_temperature_still_exits_2_on_a_cli_backend( + self, monkeypatch, recorded_cli_builds + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude-cli") + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + monkeypatch.setenv("PRXREF_LLM_TEMPERATURE", "hot") + with pytest.raises(ConfigError, match="PRXREF_LLM_TEMPERATURE"): + create_llm_client() + assert recorded_cli_builds.calls == [] + + +class TestCliBackendModuleContract: + """The llm_cli_backends entry points every later seat codes against.""" + + def test_resolve_cli_binary_signature(self): + sig = inspect.signature(prxref.llm_cli_backends.resolve_cli_binary) + assert list(sig.parameters) == ["backend", "cli_path", "which"] + assert sig.parameters["which"].kind is inspect.Parameter.KEYWORD_ONLY + assert sig.parameters["which"].default is shutil.which + + def test_build_cli_client_signature(self): + sig = inspect.signature(prxref.llm_cli_backends.build_cli_client) + assert list(sig.parameters) == [ + "backend", "models", "default_timeout", "reasoning_effort", + "cli_path", "concurrency", "which", "runner", + ] + params = list(sig.parameters.values()) + assert params[0].kind is inspect.Parameter.POSITIONAL_OR_KEYWORD + assert all(p.kind is inspect.Parameter.KEYWORD_ONLY for p in params[1:]) + assert sig.parameters["which"].default is shutil.which + assert sig.parameters["runner"].default is subprocess.Popen + + @pytest.mark.parametrize("backend", CLI_BACKENDS) + def test_an_unresolvable_cli_is_a_config_error_naming_a_variable(self, tmp_path, backend): + with pytest.raises(ConfigError) as exc: + prxref.llm_cli_backends.resolve_cli_binary( + backend, str(tmp_path / "missing"), which=lambda name: None + ) + assert "PRXREF_LLM_" in str(exc.value) + + @pytest.mark.parametrize("backend", CLI_BACKENDS) + def test_a_client_that_cannot_run_fails_closed_before_launching(self, tmp_path, backend): + launched: list[object] = [] + + def runner(*args, **kwargs): + launched.append(args) + raise AssertionError("no process may start when the CLI cannot be resolved") + + with pytest.raises(ConfigError) as exc: + prxref.llm_cli_backends.build_cli_client( + backend, + models=["m"], + default_timeout=45.0, + reasoning_effort=None, + cli_path=str(tmp_path / "missing"), + concurrency=2, + which=lambda name: None, + runner=runner, + ) + assert "PRXREF_LLM_" in str(exc.value) + assert launched == [] + + def test_the_module_is_stdlib_only(self): + source = pathlib.Path(inspect.getfile(prxref.llm_cli_backends)).read_text() + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + roots = [alias.name.split(".")[0] for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + roots = [(node.module or "").split(".")[0]] + else: + continue + for root in roots: + assert root in sys.stdlib_module_names or root == "__future__", root + + def test_the_module_names_no_private_host_or_lane(self): + source = pathlib.Path(inspect.getfile(prxref.llm_cli_backends)).read_text() + for token in ("8090", "flash,orch", "llm-ferry"): + assert token not in source, f"private default {token!r} in source" + + def test_the_http_backends_do_not_import_it(self): + source = pathlib.Path(inspect.getfile(prxref.llm_backends)).read_text() + top_level = [ + node for node in ast.parse(source).body + if isinstance(node, (ast.Import, ast.ImportFrom)) + ] + assert not any( + isinstance(node, ast.ImportFrom) and node.module == "llm_cli_backends" + for node in top_level + ) + + +class TestInvokeResultCostFields: + """#67: the two fields every backend fills in, defined once.""" + + def test_defaults_mean_not_reported(self): + result = InvokeResult(text="ok") + assert result.cost_usd is None + assert result.cost_source == "" + + def test_the_fields_follow_finish_reason(self): + names = [f.name for f in dataclasses.fields(InvokeResult)] + assert names[-3:] == ["finish_reason", "cost_usd", "cost_source"] + + def test_existing_positional_constructions_keep_working(self): + result = InvokeResult("t", 1, 2, "m", "b", 3, "stop") + assert (result.finish_reason, result.cost_usd, result.cost_source) == ("stop", None, "") + + def test_a_reported_cost_is_carried(self): + result = InvokeResult(text="ok", cost_usd=0.0021, cost_source="usage.cost") + assert (result.cost_usd, result.cost_source) == (0.0021, "usage.cost") + + +class TestExitCodesThroughTheCli: + """The factory's ConfigErrors surface as exit 2 through the real ``review`` entry point.""" + + URL = "https://github.com/org/repo/pull/7" + + @pytest.fixture + def runtime(self, monkeypatch): + ref = PRRef(forge="github", host="github.com", owner="org", repo="repo", number=7, url=self.URL) + orchestrate_calls: list[dict] = [] + + def fake_orchestrate(**kwargs): + orchestrate_calls.append(kwargs) + return { + "verdict": "commented", + "findings_active": [], + "findings_dropped": [], + "input_tokens": 0, + "output_tokens": 0, + } + + monkeypatch.setattr(cli, "detect_forge", lambda url: ref) + monkeypatch.setattr(cli, "make_forge", lambda r: object()) + monkeypatch.setattr(real_orchestrator, "orchestrate_review", fake_orchestrate) + monkeypatch.delenv("PRXREF_LLM_CLI_PATH", raising=False) + monkeypatch.delenv("PRXREF_LLM_CLI_CONCURRENCY", raising=False) + return SimpleNamespace(orchestrate_calls=orchestrate_calls) + + def _review(self) -> int: + return cli.main(["review", "--pr-url", self.URL, "--no-post", "--format", "json"]) + + def test_review_exits_2_on_unknown_backend(self, monkeypatch, capsys, runtime): + """Before 0.14 this was an LLMError, i.e. "review failed" and exit 0.""" + monkeypatch.setenv("PRXREF_LLM_BACKEND", "claude") + monkeypatch.setenv("PRXREF_LLM_BASE_URL", "https://llm.test/v1") + monkeypatch.setenv("PRXREF_LLM_MODELS", "m") + assert self._review() == 2 + err = capsys.readouterr().err + assert "configuration error: PRXREF_LLM_BACKEND: must be one of" in err + assert runtime.orchestrate_calls == [] + + @pytest.mark.parametrize("backend", CLI_BACKENDS) + def test_review_exits_2_when_a_cli_backend_cannot_run( + self, monkeypatch, capsys, tmp_path, runtime, backend + ): + monkeypatch.setenv("PRXREF_LLM_BACKEND", backend) + monkeypatch.setenv("PRXREF_LLM_MODELS", "sonnet") + monkeypatch.setenv("PRXREF_LLM_CLI_PATH", str(tmp_path / "missing-cli")) + assert self._review() == 2 + err = capsys.readouterr().err + assert "configuration error: PRXREF_LLM_" in err + assert runtime.orchestrate_calls == [] + + def test_review_on_litellm_without_base_url_reaches_the_orchestrator( + self, monkeypatch, capsys, runtime + ): + """#61 through the real entry point; also the control for the two exit-2 cases.""" + _fake_litellm(monkeypatch) + monkeypatch.setenv("PRXREF_LLM_BACKEND", "litellm") + monkeypatch.setenv("PRXREF_LLM_MODELS", "openrouter/openai/gpt-oss-20b") + monkeypatch.delenv("PRXREF_LLM_BASE_URL", raising=False) + assert self._review() == 0 + [call] = runtime.orchestrate_calls + assert isinstance(call["llm"], LiteLLMClient) + assert "configuration error" not in capsys.readouterr().err From b10b194651963d049508730aa5b98d26c409841a Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:27:36 -0700 Subject: [PATCH 11/61] fix: spec digest counts ticket lines, goes empty when nothing was kept, leaks no origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec digest and the grounding note disagreed with each other and with the prompt about what a grounded run is. Five fixes, one contract: - specs.constraint_count (new, public) replaces orchestrator's private _SPEC_CONSTRAINT_RE/_spec_constraint_count. The regex is regrouped so the (MUST|SHOULD|MAY) label binds to spec lines only: [ticket:KEY] lines carry no label, so a Jira-only run counted 0 constraints (LIVE-5, INT-5, COR-4). Exact counts now: Jira-only 4, file-only 2, both 6. _spec_note and the INFO grounding log line call it; the log line's wording is untouched. - build_spec_digest returns "" when sources were given but no unit was extracted from any of them (every source failed, or none held a constraint), so the prompt shows its no-specs text (LIVE-4). The test runs before the budget cut, so a too-small budget still yields the intro plus the truncation marker, and no sources still yields the intro. - Failed sources get no "nothing diff-relevant kept" line, that line names the source by _origin_short, and _origin_short falls back to the bare hostname (never netloc) and strips a trailing slash from paths, so no URL query, userinfo, port, or full local path reaches the LLM (SEC-4). - Ranking interleaves sections, so the digest re-emits a heading whenever the open section changes and writes "[spec:{short}] (heading) (no section)" before a headingless unit that follows a headed one; HTML elements become "#"*N heading lines, ignoring block tags nested inside the heading, dropping zero-width spaces and pilcrow permalinks, and falling back to plain text for a heading never closed (COR-3). - _spec_note labels each failed source "source N (kind)", or "source N" when the kind is unknown, ahead of its redacted reason (SEC-5, note half). The "Spec fetch failed for N source(s):" prefix is unchanged. tests/test_specs.py flips test_failed_source_explained (all-failed digest is "") and test_empty_source_explanatory_line (short origin, no /docs/). New tests/test_specs_digest.py covers the counts, the exclusions, the empty digest, the no-leak digest and every real-reviewer prompt, the section scoping and HTML headings, and the note labels. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/orchestrator.py | 49 ++--- src/prxref/specs.py | 139 ++++++++++--- tests/test_specs.py | 5 +- tests/test_specs_digest.py | 411 +++++++++++++++++++++++++++++++++++++ 4 files changed, 547 insertions(+), 57 deletions(-) create mode 100644 tests/test_specs_digest.py diff --git a/src/prxref/orchestrator.py b/src/prxref/orchestrator.py index 07f9ca78..018aef00 100644 --- a/src/prxref/orchestrator.py +++ b/src/prxref/orchestrator.py @@ -534,7 +534,7 @@ def orchestrate_review( ) logger.info( "spec grounding: %d/%d source(s) fetched, %d constraint(s) injected", - ok, len(fetched), _spec_constraint_count(spec_digest), + ok, len(fetched), specs.constraint_count(spec_digest), ) except Exception as e: # noqa: BLE001 logger.error("spec grounding failed (best-effort): %s", e) @@ -1346,47 +1346,38 @@ def _render_summary( return rendered -# A digest line that IS a constraint: a spec unit -# (``[spec:origin#anchor] (MUST|SHOULD|MAY) statement``) or a ticket line -# (``[ticket:KEY] statement``). Heading lines, the truncation marker, and -# the ``nothing diff-relevant kept`` lines are scoping or bookkeeping, not -# injected constraints, and never match. -_SPEC_CONSTRAINT_RE = re.compile( - r"^\[(?:spec:[^\]]*#\S+|ticket:[^\]]+)\] \((?:MUST|SHOULD|MAY)\) " -) - - -def _spec_constraint_count(digest: str) -> int: - """The number of constraint lines the digest actually injects.""" - return sum( - 1 for line in digest.splitlines() if _SPEC_CONSTRAINT_RE.match(line) - ) - - def _spec_note(sources: Sequence[Any], digest: str) -> str: """Render the summary's grounding note, ``""`` when nothing was requested. - One blockquote line counts what was injected; one names every failed - source, each reason through :func:`redact_for_post` first, because this - text is posted. A run whose every source failed renders ONLY the failure - line β€” the review was un-grounded, and the note must not dress it up as - grounded. The note rides the ``{spec_note}`` placeholder on its own line - between the counts and the findings, and a non-empty note carries its own - trailing newline, so an empty return leaves the summary byte-identical to - an ungrounded run's. + One blockquote line counts what was injected + (:func:`prxref.specs.constraint_count`); one lists every failed source. + A failure is labelled by its 1-based position in the configured source + list and its kind, ``source 2 (url)``, or ``source 2`` when the kind was + never determined, never by its origin: a local path or a URL's query is + not the PR audience's business, and the operator can map the ordinal + back to the list. Each reason goes through :func:`redact_for_post` + first, because this text is posted. A run whose every source failed + renders ONLY the failure line β€” the review was un-grounded, and the note + must not dress it up as grounded. The note rides the ``{spec_note}`` + placeholder on its own line between the counts and the findings, and a + non-empty note carries its own trailing newline, so an empty return + leaves the summary byte-identical to an ungrounded run's. """ if not sources: return "" total = len(sources) - failed = [s for s in sources if s.error] + failed = [(i, s) for i, s in enumerate(sources, start=1) if s.error] lines: list[str] = [] if len(failed) < total: lines.append( f"> πŸ” Spec-grounded: {total} source(s) Β· " - f"{_spec_constraint_count(digest)} constraint(s) injected" + f"{specs.constraint_count(digest)} constraint(s) injected" ) if failed: - reasons = "; ".join(redact_for_post(s.error) for s in failed) + reasons = "; ".join( + f"source {i}{f' ({s.kind})' if s.kind else ''}: {redact_for_post(s.error)}" + for i, s in failed + ) lines.append( f"> ⚠️ Spec fetch failed for {len(failed)} source(s): {reasons}" ) diff --git a/src/prxref/specs.py b/src/prxref/specs.py index 1e630b15..68c292be 100644 --- a/src/prxref/specs.py +++ b/src/prxref/specs.py @@ -191,7 +191,16 @@ def _read_stream(resp: requests.Response, max_chars: int) -> str: class _HTMLTextExtractor(HTMLParser): - """Strip tags to text: block tags become line breaks, script/style drop.""" + """Strip tags to text: block tags become line breaks, script/style drop. + + An ```` element becomes one markdown heading line, ``"#" * N + " " + + text``, so HTML sources scope their constraints the way markdown sources + do. Its text is every data run inside it with the block tags nested there + ignored (doc sites wrap a permalink ``

`` inside each heading), + zero-width spaces and pilcrow permalinks removed, whitespace collapsed. A + heading never closed by any ```` falls back to plain text, so a + malformed page loses no content. + """ _BLOCK = frozenset( { @@ -202,29 +211,64 @@ class _HTMLTextExtractor(HTMLParser): } ) _DROP = frozenset({"script", "style", "template"}) + _HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"}) + _HEADING_NOISE = str.maketrans("", "", "​¢") def __init__(self) -> None: super().__init__(convert_charrefs=True) self._parts: list[str] = [] self._skip = 0 + self._heading: str | None = None + self._heading_text: list[str] = [] + self._heading_raw: list[str] = [] def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: if tag in self._DROP: self._skip += 1 + elif self._heading is not None: + if tag in self._BLOCK: + self._heading_raw.append("\n") + elif tag in self._HEADINGS and not self._skip: + self._heading = tag + self._heading_text = [] + self._heading_raw = ["\n"] elif tag in self._BLOCK: self._parts.append("\n") def handle_endtag(self, tag: str) -> None: if tag in self._DROP: self._skip = max(0, self._skip - 1) + elif self._heading is not None: + if tag in self._HEADINGS: + self._close_heading() + elif tag in self._BLOCK: + self._heading_raw.append("\n") elif tag in self._BLOCK: self._parts.append("\n") def handle_data(self, data: str) -> None: - if not self._skip: + if self._skip: + return + if self._heading is not None: + self._heading_text.append(data) + self._heading_raw.append(data) + else: self._parts.append(data) + def _close_heading(self) -> None: + level = int((self._heading or "h1")[1]) + text = " ".join("".join(self._heading_text).translate(self._HEADING_NOISE).split()) + self._parts.append(f"\n{'#' * level} {text}\n" if text else "\n") + self._heading = None + self._heading_text = [] + self._heading_raw = [] + def text(self) -> str: + if self._heading is not None: + self._parts.extend(self._heading_raw) + self._heading = None + self._heading_text = [] + self._heading_raw = [] collapsed: list[str] = [] blank = True for line in "".join(self._parts).splitlines(): @@ -405,11 +449,17 @@ class _Unit: def _origin_short(origin: str) -> str: + """The short name a source goes by in the digest, which the LLM sees. + + A URL keeps only its last path segment, falling back to the bare host, so + no query, fragment, userinfo, or port reaches the prompt; a path keeps its + last component. A credential that IS the last path segment survives. + """ parsed = urlparse(origin) if parsed.scheme: tail = parsed.path.rstrip("/").rsplit("/", 1)[-1] - return tail or parsed.netloc - return origin.rsplit("/", 1)[-1] or origin + return tail or parsed.hostname or parsed.scheme + return origin.rstrip("/").rsplit("/", 1)[-1] or origin def _heading_slug(text: str) -> str: @@ -561,20 +611,33 @@ def build_spec_digest(sources: list[SpecSource], files: list[FileDiff], token_bu Per source, in document order, headings, RFC-2119 normative statements, version pins, and naming/shape rules are extracted; each kept unit - renders as one ``[spec:{origin}#anchor] (STRENGTH) statement`` line, its - nearest preceding heading riding along as a scoping line. Units are - ranked: ticket constraints first (scope beats relevance), then units - sharing at least one content token with the diff's token set (file paths - plus changed lines, compound-split, the same evidence vocabulary - :mod:`prxref.quality` uses β€” higher overlap first), then unmatched - MUST-level rules in source order, then unmatched SHOULD- and MAY-level - rules, which the budget exhausts first. The walk stops at - ``token_budget * 4`` chars with :data:`TRUNCATION_MARKER`. Every source - that contributed nothing after pruning gets an explicit - ``[spec:{origin}: nothing diff-relevant kept]`` line, so silence is - explained. A ``token_budget`` below 1 degrades to a truncated one-line - digest rather than raising. + renders as one ``[spec:{short}#anchor] (STRENGTH) statement`` line, where + ``{short}`` is the source's last path segment (or bare host), never its + full origin. Units are ranked: ticket constraints first (scope beats + relevance), then units sharing at least one content token with the + diff's token set (file paths plus changed lines, compound-split, the same + evidence vocabulary :mod:`prxref.quality` uses β€” higher overlap first), + then unmatched MUST-level rules in source order, then unmatched SHOULD- + and MAY-level rules, which the budget exhausts first. Ranking interleaves + sections, so a constraint's heading line is re-emitted whenever the open + section changes, and a unit with no heading after one that had one is + preceded by ``[spec:{short}] (heading) (no section)``: every constraint + sits under its own section. The walk stops at ``token_budget * 4`` chars + with :data:`TRUNCATION_MARKER`. A source that fetched fine but + contributed nothing gets an explicit + ``[spec:{short}: nothing diff-relevant kept]`` line, so silence is + explained; a failed source gets no line (the grounding note reports it). + + Returns ``""`` when sources were given but no unit was extracted from any + of them β€” every source failed, or none held a constraint β€” so the prompt + shows its no-specs text and :func:`constraint_count` is 0. The test runs + before the budget cut: a budget too small for any unit, a ``token_budget`` + below 1 included, yields the intro plus :data:`TRUNCATION_MARKER` rather + than raising. No sources at all yields the intro alone. """ + units = _rank_units(sources, _diff_tokens(files)) + if sources and not units: + return "" budget = max(1, token_budget) * CHARS_PER_TOKEN intro = ( "Spec constraints ranked for this diff: ticket scope first, then " @@ -583,15 +646,18 @@ def build_spec_digest(sources: list[SpecSource], files: list[FileDiff], token_bu out = [intro] used = len(intro) contributed: set[str] = set() - emitted_headings: set[tuple[int, str]] = set() + open_scope: tuple[int, str | None] | None = None truncated = False - for unit in _rank_units(sources, _diff_tokens(files)): + for unit in units: parts: list[str] = [] - if unit.heading_render is not None: - key = (unit.source_idx, unit.heading_key or "") - if key not in emitted_headings: - emitted_headings.add(key) - parts.append(unit.heading_render) + if not unit.is_ticket: + key = (unit.source_idx, unit.heading_key) if unit.heading_render is not None else None + if key != open_scope: + parts.append( + unit.heading_render + or f"[spec:{_origin_short(unit.origin)}] (heading) (no section)" + ) + open_scope = key parts.append(unit.render) block = "\n".join(parts) if used + len(block) + 1 > budget: @@ -603,12 +669,33 @@ def build_spec_digest(sources: list[SpecSource], files: list[FileDiff], token_bu contributed.add(unit.origin) if not truncated: for src in sources: - if src.origin in contributed: + if src.error or src.origin in contributed: continue - line = f"[spec:{src.origin}: nothing diff-relevant kept]" + line = f"[spec:{_origin_short(src.origin)}: nothing diff-relevant kept]" if used + len(line) + 1 > budget: out.append(TRUNCATION_MARKER) break out.append(line) used += len(line) + 1 return "\n".join(out) + + +_CONSTRAINT_LINE_RE = re.compile( + r"^\[(?:spec:[^\]]*#\S+\] \((?:MUST|SHOULD|MAY)\) |ticket:[^\]]+\] )", re.M +) + + +def constraint_count(digest: str) -> int: + """The number of constraint lines a :func:`build_spec_digest` digest injects. + + A constraint line is a spec unit, ``[spec:{short}#anchor] (MUST|SHOULD|MAY) + statement``, or a ticket line, ``[ticket:KEY] statement``; the strength + label binds to the spec form only, because ticket lines carry none. The + intro, heading lines (``(heading)``, including ``(heading) (no + section)``), the truncation markers, and the bracket-closed bookkeeping + lines such as ``[spec:{short}: nothing diff-relevant kept]`` are scoping + or bookkeeping and never count. An empty digest counts 0. The grounding + note and every decision about whether a run is spec-grounded use this + count, so it and the render f-strings live in one module. + """ + return len(_CONSTRAINT_LINE_RE.findall(digest)) diff --git a/tests/test_specs.py b/tests/test_specs.py index 95539f20..4bdff8e7 100644 --- a/tests/test_specs.py +++ b/tests/test_specs.py @@ -348,12 +348,13 @@ def test_empty_source_explanatory_line(self): sources, diff = _digest_sources() empty = SpecSource(origin="/docs/unrelated.md", kind="file", text="Filler prose without keywords.", error="") digest = build_spec_digest([*sources, empty], parse_unified_diff(diff), token_budget=3000) - assert "[spec:/docs/unrelated.md: nothing diff-relevant kept]" in digest + assert "[spec:unrelated.md: nothing diff-relevant kept]" in digest + assert "/docs/" not in digest def test_failed_source_explained(self): failed = SpecSource(origin="https://example.com/gone.md", kind="url", text="", error="HTTP 404 fetching") digest = build_spec_digest([failed], [], token_budget=3000) - assert "[spec:https://example.com/gone.md: nothing diff-relevant kept]" in digest + assert digest == "" def test_heading_scoping_line_accompanies_kept_constraint(self): sources, diff = _digest_sources() diff --git a/tests/test_specs_digest.py b/tests/test_specs_digest.py new file mode 100644 index 00000000..372e3870 --- /dev/null +++ b/tests/test_specs_digest.py @@ -0,0 +1,411 @@ +"""Spec digest contract: the constraint count, the empty digest, no origin leak, section scoping. + +Covers ``specs.constraint_count`` (COR-4/INT-5/LIVE-5: ticket lines count, the +strength label binds to spec lines only), ``build_spec_digest`` returning +``""`` when nothing was extracted (LIVE-4), the short origin in every digest +line (SEC-4), heading re-emission and HTML headings (COR-3), and the grounding +note's ``source N (kind)`` failure labels (SEC-5). + +The end-to-end cases run the REAL reviewer (this module does not request +``contract_stubs``), so what they assert is what the LLM is actually sent. +""" +from __future__ import annotations + +import re + +import pytest + +from prxref import orchestrator +from prxref.llm import InvokeResult +from prxref.orchestrator import _spec_note, orchestrate_review +from prxref.reviewer import _NO_SPECS_TEXT as NO_SPECS +from prxref.specs import ( + SOURCE_TRUNCATION_MARKER, + TRUNCATION_MARKER, + SpecSource, + _origin_short, + _strip_html, + build_spec_digest, + constraint_count, + fetch_specs, +) +from prxref.triage import parse_unified_diff +from tests.test_orchestrator import REF, FakeForge, _added_file_diff +from tests.test_specs import _added_file, _FakeResponse, _FakeSession + +AUTH_DIFF = """diff --git a/app.py b/app.py +--- a/app.py ++++ b/app.py +@@ -1,1 +1,2 @@ + x = 1 ++token = request.headers["X-Auth"] +""" + + +def _jira() -> SpecSource: + return SpecSource( + origin="https://acme.example.invalid/browse/PROJ-9", + kind="jira", + text=( + "Summary: auth header\nType: Bug\nLabels: auth\n" + "Users MUST re-authenticate after password change." + ), + error="", + ) + + +def _doc() -> SpecSource: + return SpecSource( + origin="/tmp/spec.md", + kind="file", + text="# Auth\nThe client MUST send the X-Auth header.\nServers MAY cache tokens.\n", + error="", + ) + + +def _failed(origin: str = "/nonexistent/spec.md", kind: str = "file") -> SpecSource: + return SpecSource(origin=origin, kind=kind, text="", error="not a URL or path") + + +def _keywordless(origin: str = "/docs/prose.md") -> SpecSource: + return SpecSource(origin=origin, kind="file", text="Filler prose without keywords.", error="") + + +def _digest(sources: list[SpecSource], budget: int = 3000) -> str: + return build_spec_digest(sources, parse_unified_diff(AUTH_DIFF), budget) + + +class TestConstraintCount: + """Exact counts, never ``> 0``: the count is the grounding predicate.""" + + def test_jira_only_counts_every_ticket_line(self): + digest = _digest([_jira()]) + assert constraint_count(digest) == 4 + + def test_file_only_counts_its_units(self): + digest = _digest([_doc()]) + assert constraint_count(digest) == 2 + + def test_jira_plus_file_counts_both(self): + digest = _digest([_jira(), _doc()]) + assert constraint_count(digest) == 6 + + def test_scoping_and_bookkeeping_lines_never_count(self): + digest = "\n".join( + [ + "Spec constraints ranked for this diff: ticket scope first, then " + "constraints sharing tokens with the diff, then unmatched MUST-level rules.", + "[spec:spec.md#auth] (heading) Auth", + "[spec:spec.md] (heading) (no section)", + "[spec:spec.md#L2] (MUST) The client MUST send the X-Auth header.", + "[ticket:PROJ-9] Summary: auth header", + TRUNCATION_MARKER, + SOURCE_TRUNCATION_MARKER.format(n=100), + "[spec:unrelated.md: nothing diff-relevant kept]", + '[ticket:PROJ-9: shown in full under "Ticket context" above]', + "", + ] + ) + assert constraint_count(digest) == 2 + + def test_the_strength_label_is_one_of_three(self): + assert constraint_count("[spec:a.md#L1] (MUST) x") == 1 + assert constraint_count("[spec:a.md#L1] (SHOULD) x") == 1 + assert constraint_count("[spec:a.md#L1] (MAY) x") == 1 + assert constraint_count("[spec:a.md#L1] (heading) x") == 0 + assert constraint_count("[spec:a.md#L1] (must) x") == 0 + assert constraint_count("[spec:a.md] (MUST) x") == 0 + + def test_a_ticket_line_counts_once_labelled_or_not(self): + assert constraint_count("[ticket:K-1] plain statement") == 1 + assert constraint_count("[ticket:K-1] (MUST) labelled statement") == 1 + + def test_only_line_starts_count(self): + assert constraint_count("prose [ticket:K-1] inline mention") == 0 + assert constraint_count("a\n[ticket:K-1] b\n[spec:x#L1] (MAY) c\n") == 2 + + def test_empty_digest_counts_zero(self): + assert constraint_count("") == 0 + + def test_the_no_section_line_leaves_the_count_at_the_unit_count(self): + pre = ( + "Implementations MUST use UTF-8 encoding.\n\n## Client requirements\n\n" + "Clients MUST send the session header.\n" + ) + src = SpecSource(origin="/docs/pre.md", kind="file", text=pre, error="") + diff = _added_file("src/c.py", ["session_header = 1"]) + digest = build_spec_digest([src], parse_unified_diff(diff), token_budget=3000) + assert "(heading) (no section)" in digest + assert constraint_count(digest) == 2 + + +class TestEmptyDigest: + """``""`` when sources were given and no unit came out of any of them.""" + + def test_every_source_failed_is_empty(self): + assert _digest([_failed(), _failed("not-a-url", kind="")]) == "" + + def test_only_keywordless_sources_is_empty(self): + assert _digest([_keywordless()]) == "" + + def test_failed_plus_keywordless_is_empty(self): + assert _digest([_failed(), _keywordless()]) == "" + + def test_no_sources_keeps_the_intro(self): + digest = _digest([]) + assert digest.startswith("Spec constraints ranked for this diff") + assert constraint_count(digest) == 0 + + def test_a_budget_too_small_for_any_unit_is_not_empty(self): + digest = _digest([_doc()], budget=10) + assert digest.endswith(TRUNCATION_MARKER) + assert constraint_count(digest) == 0 + + def test_partial_failure_drops_the_failed_origin(self): + digest = _digest([_doc(), _failed()]) + assert constraint_count(digest) == 2 + assert "nonexistent" not in digest + assert "nothing diff-relevant kept" not in digest + + def test_a_keywordless_sibling_is_explained_by_short_name(self): + digest = _digest([_doc(), _keywordless("/home/someone/docs/prose.md")]) + assert "[spec:prose.md: nothing diff-relevant kept]" in digest + assert "/home/someone" not in digest + + +class TestOriginShort: + @pytest.mark.parametrize( + ("origin", "short"), + [ + ("https://user:tok@wiki.example.com/", "wiki.example.com"), + ("https://user:tok@wiki.example.com", "wiki.example.com"), + ("https://wiki.example.com:8443/", "wiki.example.com"), + ("https://git.example.com/raw/spec.md?private_token=abc#frag", "spec.md"), + ("https://git.example.com/raw/specs/", "specs"), + ("/abs/path/spec.md", "spec.md"), + ("docs/specs/", "specs"), + ("spec.md", "spec.md"), + ], + ) + def test_short_names(self, origin, short): + assert _origin_short(origin) == short + + +SECRETS = ("SECRETPT0123", "SECRETSIG0123", "SECRETTOK0123", "user:") + + +def _leaky_sources() -> list[SpecSource]: + return [ + SpecSource( + origin="https://git.example.com/raw/spec.md?private_token=SECRETPT0123", + kind="url", text="", error="HTTP 404", + ), + SpecSource( + origin="https://files.example.com/doc.md?sig=SECRETSIG0123", + kind="url", text="Filler prose without keywords.", error="", + ), + SpecSource( + origin="https://user:SECRETTOK0123@wiki.example.com/", + kind="url", text="Every handler MUST validate the data payload.\n", error="", + ), + ] + + +class _RecordingLLM: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + + def invoke(self, system, user, *, max_tokens=4096, json_mode=False, timeout_s=60.0): + self.calls.append((system, user)) + return InvokeResult( + text='{"findings": []}', input_tokens=1, output_tokens=1, + model="test-model-1", backend="fake", elapsed_ms=1, + ) + + +def _run(monkeypatch, fetched: list[SpecSource], *, post: bool = False): + monkeypatch.setattr(orchestrator.specs, "fetch_specs", lambda *a, **k: fetched) + forge = FakeForge(diff=_added_file_diff("src/app.py", 20)) + llm = _RecordingLLM() + orchestrate_review(forge, REF, llm, post=post, spec_sources=["configured"]) + assert len(llm.calls) == 2, "one chunk plus the sweep" + return forge, llm + + +class TestNoOriginLeak: + def test_the_digest_carries_no_query_userinfo_or_failed_origin(self): + digest = build_spec_digest(_leaky_sources(), [], token_budget=3000) + assert "Every handler MUST validate the data payload." in digest + assert "[spec:doc.md: nothing diff-relevant kept]" in digest + for secret in SECRETS: + assert secret not in digest + assert "git.example.com" not in digest + + def test_no_prompt_the_llm_is_sent_carries_them(self, monkeypatch): + _forge, llm = _run(monkeypatch, _leaky_sources()) + for system, user in llm.calls: + assert "Every handler MUST validate the data payload." in user + for secret in SECRETS: + assert secret not in system + assert secret not in user + + +class TestPromptPlaceholder: + def test_every_source_failed_leaves_every_prompt_on_the_no_specs_text(self, monkeypatch): + _forge, llm = _run(monkeypatch, [_failed(), _failed("https://gone.example.com/x.md", "url")]) + for system, user in llm.calls: + assert NO_SPECS in user + assert "nonexistent" not in system + user + assert "gone.example.com" not in system + user + + def test_a_grounded_run_replaces_the_no_specs_text(self, monkeypatch): + _forge, llm = _run(monkeypatch, [_doc()]) + for _system, user in llm.calls: + assert NO_SPECS not in user + assert "(MUST) The client MUST send the X-Auth header." in user + + +_HEADING = re.compile(r"\(heading\) (.*)$") + + +def _scope_of(digest: str, needle: str) -> str | None: + scope = None + for line in digest.splitlines(): + m = _HEADING.search(line) + if m: + scope = m.group(1) + elif needle in line: + return scope + raise AssertionError(f"{needle!r} not in digest") + + +def _html_source(body: bytes) -> SpecSource: + session = _FakeSession(_FakeResponse(content_type="text/html; charset=utf-8", body=body)) + (src,) = fetch_specs(["https://example.com/spec"], max_chars=50_000, session=session) + assert src.error == "" + return src + + +SESSION_SPEC = ( + "## Client requirements\n\nClients MUST send the session header on every request.\n\n" + "Clients MUST cache the protocol handshake.\n\n## Server requirements\n\n" + "Servers MUST reject unknown session ids.\n" +) +SESSION_DIFF = _added_file("src/server.py", ["def check(header, session):"]) + + +class TestSectionScoping: + def test_ranked_interleaving_keeps_each_constraint_under_its_own_heading(self): + src = SpecSource(origin="/docs/spec.md", kind="file", text=SESSION_SPEC, error="") + digest = build_spec_digest([src], parse_unified_diff(SESSION_DIFF), token_budget=3000) + assert _scope_of(digest, "Clients MUST send the session header") == "Client requirements" + assert _scope_of(digest, "Servers MUST reject unknown session ids") == "Server requirements" + assert _scope_of(digest, "Clients MUST cache the protocol handshake") == "Client requirements" + assert digest.count("(heading) Client requirements") == 2 + + def test_a_same_section_run_emits_its_heading_once(self): + src = SpecSource(origin="/docs/spec.md", kind="file", text=SESSION_SPEC, error="") + digest = build_spec_digest([src], [], token_budget=3000) + assert digest.count("(heading) Client requirements") == 1 + assert digest.count("(heading) Server requirements") == 1 + + def test_headingless_preamble_unit_is_not_scoped_under_a_later_heading(self): + pre = ( + "Implementations MUST use UTF-8 encoding.\n\n## Client requirements\n\n" + "Clients MUST send the session header.\n" + ) + src = SpecSource(origin="/docs/pre.md", kind="file", text=pre, error="") + diff = _added_file("src/c.py", ["session_header = 1"]) + digest = build_spec_digest([src], parse_unified_diff(diff), token_budget=3000) + assert _scope_of(digest, "Implementations MUST use UTF-8 encoding") == "(no section)" + assert "[spec:pre.md] (heading) (no section)" in digest + + def test_a_headingless_second_source_is_not_scoped_under_the_first(self): + headed = SpecSource( + origin="/docs/a.md", kind="file", + text="## Alpha rules\n\nAlpha widgets MUST stream.\n", error="", + ) + bare = SpecSource( + origin="/docs/b.md", kind="file", text="Beta gadgets MUST batch.\n", error="", + ) + digest = build_spec_digest([headed, bare], [], token_budget=3000) + assert _scope_of(digest, "Beta gadgets MUST batch") == "(no section)" + + def test_html_headings_become_scoping_lines(self): + html = ( + b"

Client requirements

" + b"

Clients MUST send the session header on every request.

" + b"

Clients MUST cache the protocol handshake.

" + b"

Server requirements

Servers MUST reject unknown session ids.

" + ) + digest = build_spec_digest( + [_html_source(html)], parse_unified_diff(SESSION_DIFF), token_budget=3000, + ) + assert _scope_of(digest, "Servers MUST reject unknown session ids") == "Server requirements" + assert _scope_of(digest, "Clients MUST cache the protocol handshake") == "Client requirements" + + def test_html_heading_with_block_child_anchor_is_one_heading(self): + html = ( + '

' + "​
Initialization

" + "

The client MUST send an initialize request.

" + ).encode() + digest = build_spec_digest([_html_source(html)], [], token_budget=3000) + assert _scope_of(digest, "The client MUST send an initialize request") == "Initialization" + + +class TestHtmlHeadings: + def test_each_level_becomes_its_markdown_heading(self): + text = _strip_html("

One

a

Four

b

") + assert text.splitlines() == ["# One", "", "a", "", "#### Four", "", "b"] + + def test_permalink_glyphs_and_whitespace_are_dropped(self): + text = _strip_html('

\n Error handlingΒΆ

') + assert text == "## Error handling" + + def test_a_heading_inside_a_dropped_tag_stays_dropped(self): + text = _strip_html("

Shown

") + assert "Hidden" not in text + assert text == "Shown" + + def test_an_empty_heading_is_just_a_break(self): + assert _strip_html("

a

b

") == "a\n\nb" + + def test_an_unclosed_heading_loses_no_content(self): + text = _strip_html("

Title

Clients MUST retry.

Servers MAY cache.

") + assert "Clients MUST retry." in text.splitlines() + assert "Servers MAY cache." in text.splitlines() + assert "Title" in text + + +class TestSpecNoteLabels: + def test_failures_are_labelled_by_ordinal_and_kind_never_origin(self): + sources = [ + _doc(), + SpecSource(origin="https://host.example.com/x.md?t=1", kind="url", text="", error="HTTP 404"), + SpecSource(origin="/Users/alice/Google Drive/specs", kind="", text="", error="not a URL or path"), + ] + note = _spec_note(sources, _digest(sources)) + assert note == ( + "> πŸ” Spec-grounded: 3 source(s) Β· 2 constraint(s) injected\n" + "> ⚠️ Spec fetch failed for 2 source(s): " + "source 2 (url): HTTP 404; source 3: not a URL or path\n" + ) + + def test_reasons_are_still_redacted(self): + failed = SpecSource( + origin="https://secret.example.invalid/spec.md", kind="url", text="", + error="HTTP 404 fetching https://secret.example.invalid/spec.md", + ) + note = _spec_note([failed], "") + assert note.startswith("> ⚠️ Spec fetch failed for 1 source(s): source 1 (url): HTTP 404") + assert "secret.example.invalid" not in note + assert "Spec-grounded" not in note + + def test_a_jira_only_run_posts_the_exact_ticket_count(self, monkeypatch): + forge, _llm = _run(monkeypatch, [_jira()], post=True) + assert "> πŸ” Spec-grounded: 1 source(s) Β· 4 constraint(s) injected" in forge.summaries[0] + + def test_a_jira_plus_file_run_posts_the_exact_total(self, monkeypatch): + forge, _llm = _run(monkeypatch, [_jira(), _doc()], post=True) + assert "> πŸ” Spec-grounded: 2 source(s) Β· 6 constraint(s) injected" in forge.summaries[0] From 743ea1558fa82e9ebee3a7a0b118fe8875ca94d5 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:30:25 -0700 Subject: [PATCH 12/61] test: let the real-Response Jira fixture serve its body through iter_content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _real_response sets _content_consumed so a streamed read of the fixture (iter_content, the fallback a later streaming reader takes when raw has no read1) yields the same bytes that .json() parses. Without it the non-JSON and no-fields cases would fail on a None raw stream once the Jira fetch reads its body by streaming, which is a fixture failure and not the clean source error those tests pin. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- tests/test_specs_jira.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_specs_jira.py b/tests/test_specs_jira.py index be96aa00..e76f4543 100644 --- a/tests/test_specs_jira.py +++ b/tests/test_specs_jira.py @@ -51,6 +51,7 @@ def _real_response(body: bytes, content_type: str, status_code: int = 200) -> re resp = requests.models.Response() resp.status_code = status_code resp._content = body + resp._content_consumed = True resp.headers["Content-Type"] = content_type return resp From e16cd3ccf6f08411888892bf999e51b5fee433a8 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:35:10 -0700 Subject: [PATCH 13/61] feat: shared text loader, cost helpers, glyph table and scope vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leaf modules for 0.14.0 that the later seats build on (F1-LEAF): - text_inputs: bounded, fingerprinted reads of operator-named text files. Streams in 64 KiB chunks, keeps at most max_chars, hashes the raw bytes (equals shasum -a 256), validates strict UTF-8 across the whole file, folds CRLF/CR to LF, refuses non-regular files before opening, and refuses a path under the cwd whose symlinks resolve outside it (SEC-3 loader half). Plain OSError-family errors with separate strerror and filename so callers can report a path-free reason. - costs: price-table parsing (inline JSON, a JSON file, or a mapping; strict validation with ConfigError messages), reported / estimated / unknown per-unit and per-run cost (unknown is None, never 0 or a partial sum), and the USD formatter and cost label. - markers: the one read-only severity glyph table (outofscope is now the white square; the blue square is only the out-of-ticket prefix), the fallback marker, scope labels and marker_for. - triage: SCOPE_IN / SCOPE_OUT / SCOPE_UNKNOWN, SCOPES and a strict normalize_scope with no synonym table. Tests: tests/test_text_inputs.py, tests/test_costs.py and tests/test_markers.py, including AST checks that each module stays a leaf. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/costs.py | 370 ++++++++++++++++++++++++++++++++++ src/prxref/markers.py | 54 +++++ src/prxref/text_inputs.py | 216 ++++++++++++++++++++ src/prxref/triage.py | 22 +++ tests/test_costs.py | 406 ++++++++++++++++++++++++++++++++++++++ tests/test_markers.py | 135 +++++++++++++ tests/test_text_inputs.py | 400 +++++++++++++++++++++++++++++++++++++ 7 files changed, 1603 insertions(+) create mode 100644 src/prxref/costs.py create mode 100644 src/prxref/markers.py create mode 100644 src/prxref/text_inputs.py create mode 100644 tests/test_costs.py create mode 100644 tests/test_markers.py create mode 100644 tests/test_text_inputs.py diff --git a/src/prxref/costs.py b/src/prxref/costs.py new file mode 100644 index 00000000..b0a25eef --- /dev/null +++ b/src/prxref/costs.py @@ -0,0 +1,370 @@ +"""Dollar cost of a review run: reported, estimated, or unknown. + +A run's cost is always in exactly one of three states, and they are never +blended into a number that looks more certain than it is: + +- **Reported.** The backend returned a dollar figure for the call: + OpenRouter's body ``usage.cost``, a LiteLLM gateway's (or llm-ferry's) + ``x-litellm-response-cost`` header, the litellm SDK's ``response_cost``, or + claude-cli's ``total_cost_usd``. A reported figure always wins. +- **Estimated.** No figure came back, but ``PRXREF_PRICE_TABLE`` prices the + exact model name the call reported. The run is flagged ``cost_estimated``. +- **Unknown.** Neither. The run's ``cost_usd`` is ``None``: never ``0`` and + never a partial sum of the units that were priced. + +Backends only report (:func:`valid_usd`, :func:`combine_reported`); nothing +below the orchestrator ever sees the price table. Estimation happens once, +over the finished run, in :func:`run_cost`. A run that made no LLM request at +all never calls it: its cost is a known ``0.0``. +""" +from __future__ import annotations + +import json +import math +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any, NamedTuple + +from .llm import ConfigError + +_TOKENS_PER_PRICE_UNIT = 1_000_000 +_PRICE_FIELDS = ("input", "output") +_TABLE_SHAPE = 'a JSON object mapping model name to {"input": USD, "output": USD} per million tokens' +_ENTRY_EXAMPLE = '{"input": 0.15, "output": 0.60}' +_UNKNOWN_MODEL = "" +_SMALLEST_SHOWN = 0.0001 + + +class ModelPrice(NamedTuple): + """List price of one model in USD per MILLION tokens.""" + + input: float + output: float + + +class _DuplicateKeyError(ValueError): + def __init__(self, key: str) -> None: + super().__init__(key) + self.key = key + + +def parse_price_table( + raw: str | Mapping[str, Any] | None, source: str = "PRXREF_PRICE_TABLE" +) -> dict[str, ModelPrice]: + """Parse and strictly validate a price table. + + ``raw`` is one of: + + - ``None``, ``""`` or whitespace: no table, so ``{}``. + - a string whose first non-space character is ``{``: inline JSON. + - any other string: a path to a JSON file (``~`` expanded, read as UTF-8). + - a mapping: an already-decoded table from a library caller, validated + exactly like decoded JSON. + + The table maps a model name to ``{"input": n, "output": n}``, each ``n`` a + finite number >= 0 in USD per million tokens. Model names are stripped + and must be non-empty and unique. Invalid JSON, an unreadable file, a + non-object at any level, a missing or unknown field (``"ouput"``), a + duplicate key, a bool, a numeric string, ``NaN``, ``Infinity`` or a + negative price each raise ``ConfigError`` whose message starts with + ``source``, so the CLI names the env var or flag that supplied it. Zero + prices are legal, for local or free models. Returns a new dict. + """ + if raw is None: + return {} + if isinstance(raw, str): + text = raw.strip() + if not text: + return {} + data = _load_json(text, source, "") if text.startswith("{") else _load_json_file(text, source) + elif isinstance(raw, Mapping): + data = raw + else: + raise ConfigError(f"{source}: must be inline JSON or a path to a JSON file, got {type(raw).__name__}") + return _validate_table(data, source) + + +def estimate_usd( + table: Mapping[str, ModelPrice], model: str, input_tokens: int, output_tokens: int +) -> float | None: + """Price one call from ``table``, or ``None`` when the model has no entry. + + The lookup is on the exact model name, never a prefix or a pattern: + a wrong estimate is worse than an unknown one. + """ + price = table.get(model) if table and isinstance(model, str) else None + if price is None: + return None + return ( + _count(input_tokens) * price.input / _TOKENS_PER_PRICE_UNIT + + _count(output_tokens) * price.output / _TOKENS_PER_PRICE_UNIT + ) + + +def valid_usd(value: object) -> float | None: + """Return ``value`` as a reported dollar amount, or ``None`` if it is not one. + + Accepted: a finite real number >= 0 (not a bool), or a string that + parses to one, because response headers are strings. Everything else, + including ``NaN``, ``inf``, negatives and ``""``, is ``None``: an + unusable figure is no figure. + """ + if isinstance(value, bool): + return None + if isinstance(value, str): + try: + number = float(value.strip()) + except ValueError: + return None + elif isinstance(value, int | float): + try: + number = float(value) + except OverflowError: + return None + else: + return None + if not math.isfinite(number) or number < 0: + return None + return 0.0 if number == 0 else number + + +def combine_reported(parts: Sequence[tuple[float | None, str]]) -> tuple[float | None, str]: + """Fold the reported costs of every attempt received inside ONE invoke. + + Each part is ``(cost_usd, cost_source)``. Every attempt that came back + was billed, including truncated ones a fallback chain moved past, so the + figures are summed and the last part's source is kept. No parts, or any + part without a figure, gives ``(None, "")``: one unpriced attempt makes + the whole call's cost unknown. + """ + parts = list(parts) + if not parts: + return None, "" + costs: list[float] = [] + for cost, _source in parts: + if cost is None: + return None, "" + costs.append(cost) + return math.fsum(costs), parts[-1][1] + + +def unit_cost(unit: Mapping[str, Any]) -> tuple[bool, float | None, str]: + """Read ``(received, cost_usd, cost_source)`` off one review unit. + + ``unit`` is an orchestrator worker or sweep result, or a reviewer meta + dict. ``received`` is true when a completion came back: the unit names a + model or counts any tokens. A unit whose request raised has model ``""`` + and zero tokens, so it was never received. The cost goes through + :func:`valid_usd`; a unit without cost keys (an older stub) reads as + ``(…, None, "")``. The source is ``""`` whenever the cost is ``None``. + """ + received = ( + bool(unit.get("model")) + or _count(unit.get("input_tokens")) > 0 + or _count(unit.get("output_tokens")) > 0 + ) + cost = valid_usd(unit.get("cost_usd")) + source = str(unit.get("cost_source") or "") if cost is not None else "" + return received, cost, source + + +def run_cost( + units: Iterable[Mapping[str, Any]], table: Mapping[str, ModelPrice] | None +) -> tuple[float | None, bool, list[str]]: + """Total a run's cost: ``(cost_usd, cost_estimated, unpriced_models)``. + + ``units`` are the orchestrator's chunk results plus the sweep; ``table`` + is the parsed ``PRXREF_PRICE_TABLE`` (``None`` or ``{}`` estimates + nothing). Rules: + + 1. A unit that was never received is skipped: its request raised, so no + completion came back to read a cost from. + 2. A received unit's reported cost is used whenever there is one, even if + the table also prices its model. + 3. Otherwise the table prices it, if the model has an exact entry and the + unit counted input tokens (every prompt has a system prompt, so zero + input tokens means the backend reported no usage and an estimate would + be a fake ``0``). Such a unit makes the run ``cost_estimated``. + 4. Otherwise the unit is unknown, and its model is named in + ``unpriced_models``. + + No unit received gives ``(None, False, [])``: requests went out and + nothing came back to price. Any unknown unit gives ``(None, False, + sorted_models)``: unknown is ``None``, never ``0`` and never a partial + sum. Otherwise the total is rounded to 10 decimals, which removes float + noise and nothing else. + + Boundaries: tokens and cost cover the same units, so a unit that failed + after its response arrived (a parse failure, a truncation) was billed and + is counted. A request abandoned at the deadline returned nothing and adds + nothing, so a provider that bills abandoned generations may charge more + than this reports. An estimate prices every input token at the list rate, + so it ignores prompt-cache discounts a provider's own figure reflects. + """ + parts: list[float] = [] + unpriced: set[str] = set() + estimated = False + received_any = False + for unit in units: + received, reported, _source = unit_cost(unit) + if not received: + continue + received_any = True + if reported is not None: + parts.append(reported) + continue + model = unit.get("model") + model = model if isinstance(model, str) else "" + input_tokens = _count(unit.get("input_tokens")) + estimate = None + if table and model and input_tokens > 0: + estimate = estimate_usd(table, model, input_tokens, _count(unit.get("output_tokens"))) + if estimate is None: + unpriced.add(model or _UNKNOWN_MODEL) + continue + parts.append(estimate) + estimated = True + if not received_any: + return None, False, [] + if unpriced: + return None, False, sorted(unpriced) + return round(math.fsum(parts), 10), estimated, [] + + +def format_usd(value: float) -> str: + """Render a dollar amount for people. + + ``0`` is ``$0.00``; anything above zero but below $0.0001 is + ``<$0.0001``; below $1 shows four decimals (``$0.0007``); $1 and up shows + two (``$1.23``). A nonzero cost never renders as ``$0.00``. A value that + is not a finite number >= 0 raises ``ValueError``. + """ + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"not a dollar amount: {value!r}") + try: + number = float(value) + except OverflowError as e: + raise ValueError(f"not a dollar amount: {value!r}") from e + if not math.isfinite(number) or number < 0: + raise ValueError(f"not a dollar amount: {value!r}") + if number == 0: + return "$0.00" + if number < _SMALLEST_SHOWN: + return "<$0.0001" + if number < 1: + four = f"{number:.4f}" + if four != "1.0000": + return f"${four}" + return f"${number:.2f}" + + +def cost_label(cost_usd: float | None, estimated: bool) -> str: + """Return the label a run's cost is shown with. + + ``"cost unknown"`` when there is no usable figure; ``"~$0.0007 (est.)"`` + when it was estimated; ``"$0.0007"`` when it was reported (or a known + zero). The same label goes on the attribution line and the CLI. + """ + value = valid_usd(cost_usd) + if value is None: + return "cost unknown" + text = format_usd(value) + return f"~{text} (est.)" if estimated else text + + +def _count(value: object) -> int: + if isinstance(value, bool) or not isinstance(value, int): + return 0 + return max(value, 0) + + +def _load_json(text: str, source: str, where: str) -> Any: + try: + return json.loads(text, object_pairs_hook=_reject_duplicate_keys) + except json.JSONDecodeError as e: + raise ConfigError(f"{source}: not valid JSON ({e.msg} at line {e.lineno} column {e.colno}){where}") from e + except _DuplicateKeyError as e: + raise ConfigError( + f"{source}: duplicate key {e.key!r}{where}; each model name, and each field of an entry, may appear once" + ) from e + + +def _load_json_file(text: str, source: str) -> Any: + path = Path(text).expanduser() + try: + data = path.read_bytes() + except OSError as e: + raise ConfigError( + f"{source}: cannot read price table file '{path}': {e.strerror or e} (inline JSON must start with '{{')" + ) from e + try: + body = data.decode("utf-8-sig") + except UnicodeDecodeError as e: + raise ConfigError(f"{source}: price table file '{path}' is not valid UTF-8") from e + return _load_json(body, source, f" in '{path}'") + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _DuplicateKeyError(key) + result[key] = value + return result + + +def _validate_table(data: object, source: str) -> dict[str, ModelPrice]: + if not isinstance(data, Mapping): + raise ConfigError(f"{source}: must be {_TABLE_SHAPE}, got {_json_kind(data)}") + table: dict[str, ModelPrice] = {} + for name, entry in data.items(): + if not isinstance(name, str) or not name.strip(): + raise ConfigError(f"{source}: every model name must be a non-empty string, got {name!r}") + model = name.strip() + if model in table: + raise ConfigError(f"{source}: duplicate model name {model!r}") + table[model] = _validate_entry(model, entry, source) + return table + + +def _validate_entry(model: str, entry: object, source: str) -> ModelPrice: + if not isinstance(entry, Mapping): + raise ConfigError(f"{source}: the entry for {model!r} must be an object like {_ENTRY_EXAMPLE}, " + f"got {_json_kind(entry)}") + unknown = sorted(repr(key) for key in entry if key not in _PRICE_FIELDS) + if unknown: + raise ConfigError(f"{source}: the entry for {model!r} has unknown key(s) {', '.join(unknown)}; " + "only 'input' and 'output' are allowed") + missing = [f"'{field}'" for field in _PRICE_FIELDS if field not in entry] + if missing: + raise ConfigError(f"{source}: the entry for {model!r} is missing {' and '.join(missing)}") + return ModelPrice(*(_price(model, field, entry[field], source) for field in _PRICE_FIELDS)) + + +def _price(model: str, field: str, value: object, source: str) -> float: + problem = f"{source}: {model!r} {field} must be a finite number >= 0 (USD per million tokens), got " + if isinstance(value, bool) or not isinstance(value, int | float): + raise ConfigError(problem + _json_kind(value)) + try: + number = float(value) + except OverflowError as e: + raise ConfigError(problem + "a number too large to represent") from e + if not math.isfinite(number) or number < 0: + raise ConfigError(problem + repr(value)) + return number + + +def _json_kind(value: object) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return f"the boolean {str(value).lower()}" + if isinstance(value, int | float): + return f"the number {value!r}" + if isinstance(value, str): + return f"the string {value!r}" + if isinstance(value, Mapping): + return "an object" + if isinstance(value, list | tuple): + return "an array" + return type(value).__name__ diff --git a/src/prxref/markers.py b/src/prxref/markers.py new file mode 100644 index 00000000..4bf85514 --- /dev/null +++ b/src/prxref/markers.py @@ -0,0 +1,54 @@ +"""The one table of finding glyphs every rendering surface draws from. + +A severity maps to exactly one glyph, and that glyph is the same in the +summary counts line, the summary findings list, an inline comment header and +the CLI. A finding outside the ticket's scope keeps its severity glyph and +gains the out-of-ticket prefix in front of it; that prefix is never a severity +glyph itself, so scope and severity stay readable independently. + +The summary templates (``prompts/summary.md`` and the two fallback templates) +keep their glyphs as literals because they are the readable source of the +layout; a parity test holds each of them to this table. Changing a glyph or +adding a severity is one edit here. +""" +from __future__ import annotations + +from collections.abc import Mapping +from types import MappingProxyType + +from .triage import SCOPE_OUT + +SEVERITY_MARKERS: Mapping[str, str] = MappingProxyType({ + "error": "πŸŸ₯", + "warning": "🟧", + "spec": "πŸ”", + "outofscope": "⬜", +}) + +# An unrecognised severity renders as the minor class, matching the way the +# library formatter folds an unknown severity into ``outofscope``. +FALLBACK_MARKER: str = SEVERITY_MARKERS["outofscope"] + +OUT_OF_TICKET_MARKER: str = "🟦" + +SCOPE_LABELS: Mapping[str, str] = MappingProxyType({SCOPE_OUT: "OUTSIDE TICKET"}) + + +def severity_marker(severity: str) -> str: + """Return the glyph for ``severity``, or :data:`FALLBACK_MARKER` if unknown. + + The lookup is exact: severities are normalised by the quality gate before + anything renders them. + """ + return SEVERITY_MARKERS.get(severity, FALLBACK_MARKER) + + +def marker_for(severity: str, scope: str) -> str: + """Return the full marker for a finding: its severity glyph, prefixed by + :data:`OUT_OF_TICKET_MARKER` and a space when ``scope`` is ``"out"``. + + Scope ``"in"`` and ``"unknown"`` add nothing, so a run without a ticket + renders exactly the severity glyph. + """ + marker = severity_marker(severity) + return f"{OUT_OF_TICKET_MARKER} {marker}" if scope == SCOPE_OUT else marker diff --git a/src/prxref/text_inputs.py b/src/prxref/text_inputs.py new file mode 100644 index 00000000..d0f9b537 --- /dev/null +++ b/src/prxref/text_inputs.py @@ -0,0 +1,216 @@ +"""Bounded, fingerprinted reads of operator-named text files. + +Team review rules, ticket context and local spec sources all put text that an +operator named by path into a prompt, and all three need the same four +guarantees, so they share this loader: + +- **Bounded memory.** :func:`read_capped_file` streams the file. It keeps at + most ``max_chars`` decoded characters and reads the rest in fixed-size + chunks only to hash, count and validate it, so a huge file or a mistyped + path to one costs a scan, not the file's size in memory. +- **A stable fingerprint.** ``sha256`` covers the raw file bytes, before + decoding and capping. It equals ``shasum -a 256 FILE``, it does not change + when the cap changes, and it changes when any byte does. +- **Strict text.** UTF-8 only, a leading BOM dropped, CRLF and lone CR folded + to LF. Invalid UTF-8 anywhere in the file is an error, even past the cap. +- **No symlink escape out of the working directory.** A path that sits under + the working directory may be inside a PR checkout, where the PR author + controls every file and every symlink. Such a path must still resolve + under the working directory once its symlinks are followed, so a committed + ``docs/SPEC.md -> ~/.ssh/id_rsa`` is refused. An absolute path outside the + working directory (``$RUNNER_TEMP/rules.md``, a CI file variable) is the + operator's own choice and is read as given. + +Only regular files are read: a directory, FIFO or device is refused before +anything is opened, so a named pipe can never block a review. + +The module raises plain exceptions and never a ``ConfigError``: +``FileNotFoundError``, ``IsADirectoryError``, ``PermissionError`` (including +a confinement refusal), ``OSError`` (not a regular file), ``UnicodeDecodeError`` +and ``ValueError`` (a cap below 1). Each ``OSError`` carries ``strerror`` and +``filename`` separately, so a caller can report the reason without the path. +The caller decides what a failure means: a spec source records it as that +source's error, while rules and ticket context turn it into a ``ConfigError`` +naming the env var or flag. Truncation markers are also the caller's; the +kept text never contains one. +""" +from __future__ import annotations + +import codecs +import errno +import hashlib +import io +import os +import stat +from dataclasses import dataclass + +_READ_CHUNK_BYTES = 64 * 1024 + + +@dataclass(frozen=True) +class CappedText: + """Operator-supplied text, capped for a prompt and fingerprinted for the run record. + + ``text`` is exactly the first ``max_chars`` characters of the decoded, + newline-normalised text, with no marker. ``chars`` is the length of the + whole text the cap was applied to, so ``truncated`` is ``chars > + max_chars`` (a text exactly ``max_chars`` long is not truncated). + ``sha256`` is the lowercase hex digest of the source: the raw bytes for a + file. + """ + + text: str + sha256: str + chars: int + truncated: bool + max_chars: int + + def record(self) -> dict[str, object]: + """Return the run-record fingerprint: ``sha256``, ``chars``, + ``max_chars`` and ``truncated``, JSON-native values only and never the + text itself. + """ + return { + "sha256": self.sha256, + "chars": self.chars, + "max_chars": self.max_chars, + "truncated": self.truncated, + } + + +def confine_to_cwd(path: str | os.PathLike[str]) -> str: + """Return the resolved path, refusing one that symlinks out of the cwd. + + A path counts as inside the working directory when its absolute form is + lexically under the cwd (as ``os.getcwd()`` spells it, or as its + ``realpath``), or when one of its parent directories resolves under the + cwd: the cwd reached through another spelling, such as ``/tmp/x`` for + ``/private/tmp/x``, or through a symlink into it. Such a path must + resolve, all symlinks followed, to a location under the cwd's + ``realpath``; otherwise ``PermissionError`` is raised. A path outside the + cwd is returned resolved and unchecked. ``..`` segments are folded + lexically first, so a path the operator writes to climb out of the cwd + counts as outside. Works for files and directories alike, and does not + require the path to exist. Comparisons use ``os.path.commonpath``, never + string prefixes. + """ + raw = os.fspath(path) + cwd = os.getcwd() + real_cwd = os.path.realpath(cwd) + resolved = os.path.realpath(raw) + if _is_within(resolved, real_cwd): + return resolved + if _lexically_inside(os.path.abspath(raw), (os.path.abspath(cwd), real_cwd), real_cwd): + raise PermissionError(errno.EACCES, "resolves outside the working directory", raw) + return resolved + + +def check_readable_path(path: str | os.PathLike[str], *, confine: bool = True) -> str: + """Return the resolved path of a regular file that is safe to open. + + With ``confine`` (the default) the path first goes through + :func:`confine_to_cwd`. Then ``os.stat`` must report a regular file, so a + directory raises ``IsADirectoryError`` and a FIFO or device raises + ``OSError`` before anything is opened. A missing file raises + ``FileNotFoundError`` naming the path as given. + """ + raw = os.fspath(path) + resolved = confine_to_cwd(raw) if confine else os.path.realpath(raw) + _require_regular_file(os.stat(raw), raw) + return resolved + + +def decode_text(raw: bytes) -> str: + """Decode strict UTF-8, drop a leading BOM, and fold CRLF and lone CR to LF. + + Raises ``UnicodeDecodeError`` (a ``ValueError``) on invalid UTF-8. The + result is identical to what :func:`read_capped_file` decodes from the same + bytes. + """ + return raw.decode("utf-8-sig").replace("\r\n", "\n").replace("\r", "\n") + + +def cap_text(text: str, max_chars: int, *, sha256: str | None = None) -> CappedText: + """Cap ``text`` at ``max_chars`` characters. + + ``sha256`` is the fingerprint to record; it defaults to the digest of the + whole ``text`` encoded as UTF-8, taken before the cap. A caller that read + the text from a file passes the digest of the raw file bytes instead. + ``max_chars`` below 1 raises ``ValueError``. + """ + _require_cap(max_chars) + if sha256 is None: + sha256 = hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest() + chars = len(text) + return CappedText(text[:max_chars], sha256, chars, chars > max_chars, max_chars) + + +def read_capped_file( + path: str | os.PathLike[str], max_chars: int, *, confine: bool = True +) -> CappedText: + """Stream a text file into a :class:`CappedText` in bounded memory. + + The path is checked with :func:`check_readable_path` first (``confine`` + is passed through). The file is read once in 64 KiB chunks: every byte is + hashed and decoded as in :func:`decode_text`, the first ``max_chars`` + characters are kept, and the rest are only counted. So the whole file is + validated, ``sha256`` matches ``shasum -a 256``, and ``chars`` counts the + full text even when it was truncated. ``max_chars`` below 1 raises + ``ValueError`` before the filesystem is touched. + """ + _require_cap(max_chars) + raw_path = os.fspath(path) + resolved = check_readable_path(raw_path, confine=confine) + digest = hashlib.sha256() + decoder = io.IncrementalNewlineDecoder(codecs.getincrementaldecoder("utf-8-sig")("strict"), True) + kept: list[str] = [] + kept_chars = 0 + total_chars = 0 + with open(resolved, "rb") as fh: + _require_regular_file(os.fstat(fh.fileno()), raw_path) + while True: + chunk = fh.read(_READ_CHUNK_BYTES) + final = not chunk + digest.update(chunk) + piece = decoder.decode(chunk, final=final) + total_chars += len(piece) + if piece and kept_chars < max_chars: + take = piece[: max_chars - kept_chars] + kept.append(take) + kept_chars += len(take) + if final: + break + return CappedText("".join(kept), digest.hexdigest(), total_chars, total_chars > max_chars, max_chars) + + +def _require_cap(max_chars: int) -> None: + if max_chars < 1: + raise ValueError(f"max_chars must be at least 1, got {max_chars!r}") + + +def _require_regular_file(st: os.stat_result, raw: str) -> None: + if stat.S_ISREG(st.st_mode): + return + if stat.S_ISDIR(st.st_mode): + raise IsADirectoryError(errno.EISDIR, os.strerror(errno.EISDIR), raw) + raise OSError(errno.EINVAL, "not a regular file", raw) + + +def _is_within(path: str, root: str) -> bool: + try: + return os.path.commonpath([path, root]) == root + except ValueError: + return False + + +def _lexically_inside(lexical: str, roots: tuple[str, ...], real_cwd: str) -> bool: + if any(_is_within(lexical, root) for root in roots): + return True + current = os.path.dirname(lexical) + while True: + if _is_within(os.path.realpath(current), real_cwd): + return True + parent = os.path.dirname(current) + if parent == current: + return False + current = parent diff --git a/src/prxref/triage.py b/src/prxref/triage.py index eb6d5a9d..838b114c 100644 --- a/src/prxref/triage.py +++ b/src/prxref/triage.py @@ -27,6 +27,28 @@ # adds what it did not. DEFAULT_CONTEXT_LINES: int = 3 +# Where a finding sits relative to the ticket the PR is meant to implement. +# Orthogonal to severity: scope never feeds dedup, the error cap, the verdict +# or the fail-on exit. "unknown" is both the no-ticket state and the answer +# for anything the model returned that is not exactly one of these words. +SCOPE_IN, SCOPE_OUT, SCOPE_UNKNOWN = "in", "out", "unknown" +SCOPES: tuple[str, ...] = (SCOPE_IN, SCOPE_OUT, SCOPE_UNKNOWN) + + +def normalize_scope(raw: object) -> str: + """Map a model-supplied ``scope`` value onto :data:`SCOPES`. + + Only a string that is exactly ``"in"``, ``"out"`` or ``"unknown"`` after + ``strip().casefold()`` keeps its meaning; everything else (``None``, a + bool, ``"In scope"``, ``"yes"``) is ``"unknown"``. There is deliberately + no synonym table: a lenient mapping would turn a malformed answer into a + confident ``in`` or ``out``. + """ + if not isinstance(raw, str): + return SCOPE_UNKNOWN + value = raw.strip().casefold() + return value if value in SCOPES else SCOPE_UNKNOWN + @dataclass class Finding: diff --git a/tests/test_costs.py b/tests/test_costs.py new file mode 100644 index 00000000..e96b4280 --- /dev/null +++ b/tests/test_costs.py @@ -0,0 +1,406 @@ +"""Tests for prxref.costs: price-table parsing, reported-cost validation, the +run total and its null rule, and the dollar labels. + +The issue #67 acceptance cases are the three ``run_cost`` states: passthrough +(reported), estimate (flagged), and unknown (``None``, never ``0``). +""" +from __future__ import annotations + +import ast +import math +import sys +from pathlib import Path + +import pytest + +from prxref import costs +from prxref.costs import ( + ModelPrice, + combine_reported, + cost_label, + estimate_usd, + format_usd, + parse_price_table, + run_cost, + unit_cost, + valid_usd, +) +from prxref.llm import ConfigError + +MINI = "openai/gpt-4o-mini" +TABLE = {MINI: ModelPrice(0.15, 0.60)} + + +def _unit(model: str = MINI, input_tokens: int = 1000, output_tokens: int = 100, **extra) -> dict: + """An orchestrator unit dict shaped like a finished worker or sweep result.""" + unit = { + "findings": [], + "error": None, + "model": model, + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "elapsed_ms": 10, + } + unit.update(extra) + return unit + + +def _raised_unit() -> dict: + """A unit whose request raised: no model, no tokens, an error.""" + return _unit(model="", input_tokens=0, output_tokens=0, error="LLMError: deadline", cost_usd=None, cost_source="") + + +class TestParsePriceTable: + @pytest.mark.parametrize("raw", [None, "", " ", "\n\t"]) + def test_unset_empty_and_whitespace_mean_no_table(self, raw): + assert parse_price_table(raw) == {} + + def test_inline_json_object_is_parsed_into_model_prices(self): + table = parse_price_table( + ' {"openai/gpt-4o-mini": {"input": 0.15, "output": 0.60}, "m2": {"input": 3, "output": 15}}' + ) + assert table == {MINI: ModelPrice(0.15, 0.60), "m2": ModelPrice(3.0, 15.0)} + assert all(isinstance(p, ModelPrice) for p in table.values()) + assert isinstance(table["m2"].input, float) + + def test_a_path_is_read_as_a_json_file(self, tmp_path): + path = tmp_path / "prices.json" + path.write_text('{"m": {"input": 1.5, "output": 2}}', encoding="utf-8") + assert parse_price_table(str(path)) == {"m": ModelPrice(1.5, 2.0)} + + def test_a_json_file_with_a_bom_is_read(self, tmp_path): + path = tmp_path / "prices.json" + path.write_bytes(b'\xef\xbb\xbf{"m": {"input": 1, "output": 2}}') + assert parse_price_table(str(path)) == {"m": ModelPrice(1.0, 2.0)} + + def test_tilde_in_the_path_is_expanded(self, tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + (tmp_path / "prices.json").write_text('{"m": {"input": 1, "output": 2}}', encoding="utf-8") + assert parse_price_table("~/prices.json") == {"m": ModelPrice(1.0, 2.0)} + + def test_a_mapping_override_is_validated_like_a_string(self): + assert parse_price_table({" m ": {"input": 1, "output": 0}}) == {"m": ModelPrice(1.0, 0.0)} + with pytest.raises(ConfigError, match="PRXREF_PRICE_TABLE: 'm' input must be a finite number"): + parse_price_table({"m": {"input": -1, "output": 0}}) + + def test_the_result_is_a_new_dict(self): + given = {"m": {"input": 1, "output": 2}} + table = parse_price_table(given) + table["other"] = ModelPrice(0, 0) + assert "other" not in given + + def test_invalid_json_is_a_config_error_naming_the_variable(self): + with pytest.raises(ConfigError, match=r"^PRXREF_PRICE_TABLE: not valid JSON \(.* at line 1 column \d+\)$"): + parse_price_table('{"m": {"input": 1, "output": 2}') + + def test_invalid_json_in_a_file_names_the_file(self, tmp_path): + path = tmp_path / "prices.json" + path.write_text("{nope", encoding="utf-8") + with pytest.raises(ConfigError, match=r"not valid JSON .* in '.*prices\.json'"): + parse_price_table(str(path)) + + def test_an_unreadable_path_is_a_config_error_naming_variable_and_path(self, tmp_path): + missing = tmp_path / "nope.json" + with pytest.raises(ConfigError) as info: + parse_price_table(str(missing)) + message = str(info.value) + assert message.startswith("PRXREF_PRICE_TABLE: cannot read price table file") + assert str(missing) in message + assert "No such file or directory" in message + assert "inline JSON must start with '{'" in message + + def test_a_directory_path_is_a_config_error(self, tmp_path): + with pytest.raises(ConfigError, match="cannot read price table file"): + parse_price_table(str(tmp_path)) + + def test_a_file_that_is_not_utf8_is_a_config_error(self, tmp_path): + path = tmp_path / "prices.json" + path.write_bytes(b'{"m\xff": {"input": 1, "output": 2}}') + with pytest.raises(ConfigError, match="is not valid UTF-8"): + parse_price_table(str(path)) + + @pytest.mark.parametrize( + ("raw", "kind"), + [("[1, 2]", "an array"), ('"text"', "the string 'text'"), ("3", "the number 3"), ("null", "null")], + ) + def test_top_level_must_be_an_object(self, tmp_path, raw, kind): + path = tmp_path / "prices.json" + path.write_text(raw, encoding="utf-8") + with pytest.raises(ConfigError, match="must be a JSON object mapping model name") as info: + parse_price_table(str(path)) + assert str(info.value).endswith(f"got {kind}") + + @pytest.mark.parametrize( + ("entry", "fragment"), + [ + ('{"input": 0.15}', "is missing 'output'"), + ("{}", "is missing 'input' and 'output'"), + ('{"input": 0.15, "ouput": 0.6}', "unknown key(s) 'ouput'"), + ('{"input": -0.01, "output": 0.6}', "input must be a finite number >= 0"), + ('{"input": NaN, "output": 0.6}', "got nan"), + ('{"input": 0.15, "output": Infinity}', "got inf"), + ('{"input": 0.15, "output": -Infinity}', "got -inf"), + ('{"input": true, "output": 0.6}', "got the boolean true"), + ('{"input": "0.15", "output": 0.6}', "got the string '0.15'"), + ('{"input": null, "output": 0.6}', "got null"), + ('{"input": [0.15], "output": 0.6}', "got an array"), + ('{"input": 1' + "0" * 400 + ', "output": 0.6}', "too large to represent"), + ("[0.15, 0.6]", "must be an object like"), + ("0.15", "must be an object like"), + ], + ) + def test_bad_entry_is_rejected(self, entry, fragment): + with pytest.raises(ConfigError) as info: + parse_price_table('{"m": ' + entry + "}") + assert str(info.value).startswith("PRXREF_PRICE_TABLE: ") + assert fragment in str(info.value) + + @pytest.mark.parametrize("name", ['""', '" "']) + def test_empty_model_name_is_rejected(self, name): + with pytest.raises(ConfigError, match="every model name must be a non-empty string"): + parse_price_table("{" + name + ': {"input": 1, "output": 2}}') + + def test_non_string_model_name_in_a_mapping_is_rejected(self): + with pytest.raises(ConfigError, match="every model name must be a non-empty string, got 5"): + parse_price_table({5: {"input": 1, "output": 2}}) + + def test_duplicate_model_names_are_rejected(self): + with pytest.raises(ConfigError, match="duplicate key 'm'"): + parse_price_table('{"m": {"input": 1, "output": 2}, "m": {"input": 3, "output": 4}}') + + def test_names_that_collide_after_stripping_are_duplicates(self): + with pytest.raises(ConfigError, match="duplicate model name 'm'"): + parse_price_table('{"m": {"input": 1, "output": 2}, " m": {"input": 3, "output": 4}}') + + def test_a_duplicate_field_inside_an_entry_is_rejected(self): + with pytest.raises(ConfigError, match="duplicate key 'input'"): + parse_price_table('{"m": {"input": 1, "input": 2, "output": 3}}') + + def test_the_error_names_the_source_label_it_was_given(self): + with pytest.raises(ConfigError, match=r"^price_table: not valid JSON"): + parse_price_table("{bad", source="price_table") + + def test_a_non_string_non_mapping_is_rejected(self): + with pytest.raises(ConfigError, match="PRXREF_PRICE_TABLE: must be inline JSON or a path"): + parse_price_table(42) + + def test_zero_prices_are_legal(self): + table = parse_price_table('{"local/llama": {"input": 0, "output": 0.0}}') + assert table == {"local/llama": ModelPrice(0.0, 0.0)} + + def test_config_error_is_a_value_error(self): + with pytest.raises(ValueError): + parse_price_table("{bad") + + +class TestEstimate: + def test_estimate_multiplies_per_million_prices(self): + assert estimate_usd(TABLE, MINI, 9, 2) == pytest.approx(2.55e-06, rel=1e-12) + + def test_estimate_is_none_on_a_model_miss(self): + assert estimate_usd(TABLE, "openai/gpt-4o", 9, 2) is None + assert estimate_usd({}, MINI, 9, 2) is None + + def test_lookup_is_exact_not_prefix(self): + assert estimate_usd(TABLE, "openai/gpt-4o-mini-2024-07-18", 9, 2) is None + assert estimate_usd(TABLE, "gpt-4o-mini", 9, 2) is None + assert estimate_usd(TABLE, "OPENAI/GPT-4O-MINI", 9, 2) is None + + +class TestValidUsd: + @pytest.mark.parametrize( + ("value", "expected"), + [(0, 0.0), (0.0, 0.0), (1e-6, 1e-6), ("0.0021", 0.0021), (" 0.5 ", 0.5), (3, 3.0), (-0.0, 0.0)], + ) + def test_accepts_finite_non_negative_numbers_and_numeric_strings(self, value, expected): + result = valid_usd(value) + assert result == expected + assert isinstance(result, float) + assert math.copysign(1.0, result) == 1.0 + + @pytest.mark.parametrize( + "value", + [-1, -1e-9, float("nan"), float("inf"), "nan", "inf", True, False, "abc", None, [], {}, "", " ", 10**400], + ) + def test_rejects_everything_else(self, value): + assert valid_usd(value) is None + + +class TestCombineReported: + def test_empty_is_unknown(self): + assert combine_reported([]) == (None, "") + + def test_every_received_attempt_is_summed_and_the_last_source_kept(self): + total, source = combine_reported([(0.001, "usage.cost"), (0.002, "x-litellm-response-cost")]) + assert total == pytest.approx(0.003) + assert source == "x-litellm-response-cost" + + def test_any_attempt_without_a_figure_makes_the_call_unknown(self): + assert combine_reported([(0.001, "usage.cost"), (None, ""), (0.002, "usage.cost")]) == (None, "") + + def test_a_reported_zero_is_kept(self): + assert combine_reported([(0.0, "claude-cli")]) == (0.0, "claude-cli") + + def test_accepts_any_iterable(self): + assert combine_reported(iter([(0.5, "litellm")])) == (0.5, "litellm") + + +class TestUnitCost: + def test_a_received_unit_with_a_reported_cost(self): + assert unit_cost(_unit(cost_usd=0.002, cost_source="usage.cost")) == (True, 0.002, "usage.cost") + + def test_a_unit_that_raised_was_never_received(self): + assert unit_cost(_raised_unit()) == (False, None, "") + + def test_tokens_alone_mean_received(self): + assert unit_cost(_unit(model="", input_tokens=0, output_tokens=5))[0] is True + + def test_legacy_unit_without_cost_keys_reads_as_unknown(self): + assert unit_cost({"model": MINI, "input_tokens": 10, "output_tokens": 2}) == (True, None, "") + + def test_none_tokens_from_an_old_stub_do_not_crash(self): + assert unit_cost({"model": "", "input_tokens": None, "output_tokens": None}) == (False, None, "") + + def test_an_invalid_cost_reads_as_none_and_drops_the_source(self): + assert unit_cost(_unit(cost_usd=-1.0, cost_source="usage.cost")) == (True, None, "") + + +class TestRunCost: + def test_passthrough_all_units_reported_sums_and_is_not_estimated(self): + units = [_unit(cost_usd=0.0004, cost_source="usage.cost"), _unit(cost_usd=0.0003, cost_source="usage.cost")] + assert run_cost(units, None) == (pytest.approx(0.0007), False, []) + + def test_provider_cost_wins_over_a_table_entry(self): + usd, estimated, unpriced = run_cost([_unit(cost_usd=0.5, cost_source="litellm")], TABLE) + assert (usd, estimated, unpriced) == (0.5, False, []) + + def test_estimate_fills_a_unit_without_a_reported_cost_and_flags_the_run(self): + usd, estimated, unpriced = run_cost([_unit(input_tokens=9, output_tokens=2, cost_usd=None)], TABLE) + assert usd == pytest.approx(2.55e-06) + assert estimated is True + assert unpriced == [] + + def test_mixed_reported_and_estimated_is_estimated(self): + units = [_unit(cost_usd=0.001, cost_source="usage.cost"), _unit(input_tokens=1000, output_tokens=0)] + usd, estimated, _ = run_cost(units, TABLE) + assert usd == pytest.approx(0.001 + 1000 * 0.15 / 1e6) + assert estimated is True + + def test_unknown_is_null_never_zero(self): + usd, estimated, unpriced = run_cost([_unit(model="vendor/unpriced", cost_usd=None)], TABLE) + assert usd is None + assert usd != 0 + assert estimated is False + assert unpriced == ["vendor/unpriced"] + + def test_a_partial_sum_is_never_reported(self): + units = [_unit(cost_usd=0.001, cost_source="usage.cost") for _ in range(3)] + [_unit(model="x/unpriced")] + assert run_cost(units, TABLE) == (None, False, ["x/unpriced"]) + + def test_an_estimated_unit_next_to_an_unknown_one_is_still_unknown(self): + assert run_cost([_unit(), _unit(model="x/unpriced")], TABLE) == (None, False, ["x/unpriced"]) + + def test_a_unit_without_usage_is_not_estimated_to_zero(self): + assert run_cost([_unit(input_tokens=0, output_tokens=0)], TABLE) == (None, False, [MINI]) + + def test_units_that_raised_are_skipped(self): + units = [_raised_unit(), _unit(cost_usd=0.002, cost_source="usage.cost")] + assert run_cost(units, None) == (0.002, False, []) + + def test_no_completion_received_is_null(self): + assert run_cost([_raised_unit(), _raised_unit()], TABLE) == (None, False, []) + assert run_cost([], TABLE) == (None, False, []) + + def test_reported_zero_is_a_real_zero(self): + usd, estimated, unpriced = run_cost([_unit(cost_usd=0.0, cost_source="usage.cost")], None) + assert usd == 0.0 + assert usd is not None + assert (estimated, unpriced) == (False, []) + + def test_legacy_unit_without_cost_keys_counts_as_unknown(self): + legacy = {"findings": [], "error": None, "model": MINI, "input_tokens": 10, "output_tokens": 2} + assert run_cost([legacy], None) == (None, False, [MINI]) + + def test_a_received_unit_without_a_model_is_named_as_unknown_model(self): + assert run_cost([_unit(model="", input_tokens=5)], TABLE) == (None, False, [""]) + + def test_unpriced_models_are_sorted_and_unique(self): + units = [_unit(model="b/m"), _unit(model="a/m"), _unit(model="b/m")] + assert run_cost(units, {})[2] == ["a/m", "b/m"] + + def test_the_total_is_rounded_to_remove_float_noise(self): + assert 9 * 0.15 / 1e6 + 2 * 0.60 / 1e6 != 2.55e-06 + usd, _, _ = run_cost([_unit(input_tokens=9, output_tokens=2)], TABLE) + assert usd == 2.55e-06 + + def test_accepts_a_generator_of_units(self): + usd, _, _ = run_cost((u for u in [_unit(cost_usd=0.25, cost_source="litellm")]), None) + assert usd == 0.25 + + +class TestFormatUsd: + @pytest.mark.parametrize( + ("value", "text"), + [ + (0, "$0.00"), + (0.0, "$0.00"), + (0.00000001, "<$0.0001"), + (0.0000999, "<$0.0001"), + (0.0001, "$0.0001"), + (0.00070635, "$0.0007"), + (0.0123, "$0.0123"), + (0.99994, "$0.9999"), + (0.99996, "$1.00"), + (1, "$1.00"), + (1.234, "$1.23"), + (12.5, "$12.50"), + ], + ) + def test_format_usd_cases(self, value, text): + assert format_usd(value) == text + + @pytest.mark.parametrize("value", [1e-12, 1e-9, 4e-5, 5e-5, 9.9999e-5, 0.00004999]) + def test_nonzero_cost_never_renders_as_zero(self, value): + assert format_usd(value) != "$0.00" + assert format_usd(value) != "$0.0000" + + @pytest.mark.parametrize("value", [-0.01, float("nan"), float("inf"), None, "0.1", True, 10**400]) + def test_format_usd_rejects_what_is_not_a_dollar_amount(self, value): + with pytest.raises(ValueError, match="not a dollar amount"): + format_usd(value) + + +class TestCostLabel: + def test_unknown(self): + assert cost_label(None, False) == "cost unknown" + assert cost_label(None, True) == "cost unknown" + + def test_reported(self): + assert cost_label(0.0007, False) == "$0.0007" + + def test_estimated(self): + assert cost_label(0.0007, True) == "~$0.0007 (est.)" + + def test_a_known_zero_is_zero_not_unknown(self): + assert cost_label(0.0, False) == "$0.00" + + @pytest.mark.parametrize("value", [float("nan"), -1.0, True]) + def test_an_unusable_figure_is_unknown_not_a_crash(self, value): + assert cost_label(value, False) == "cost unknown" + + +class TestModuleIsALeaf: + def test_imports_only_the_standard_library_and_the_config_error(self): + tree = ast.parse(Path(costs.__file__).read_text(encoding="utf-8")) + stdlib: set[str] = set() + package: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + stdlib.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + package.add(f"{node.module}:{','.join(a.name for a in node.names)}") + else: + stdlib.add((node.module or "").split(".")[0]) + assert stdlib <= set(sys.stdlib_module_names) | {"__future__"}, stdlib + assert package == {"llm:ConfigError"} diff --git a/tests/test_markers.py b/tests/test_markers.py new file mode 100644 index 00000000..c69dbfc0 --- /dev/null +++ b/tests/test_markers.py @@ -0,0 +1,135 @@ +"""Tests for prxref.markers (the one glyph table) and the ticket-scope +vocabulary in prxref.triage that the out-of-ticket marker keys on. +""" +from __future__ import annotations + +import ast +import sys +from pathlib import Path + +import pytest + +from prxref import markers +from prxref.markers import ( + FALLBACK_MARKER, + OUT_OF_TICKET_MARKER, + SCOPE_LABELS, + SEVERITY_MARKERS, + marker_for, + severity_marker, +) +from prxref.quality import SEVERITIES +from prxref.triage import SCOPE_IN, SCOPE_OUT, SCOPE_UNKNOWN, SCOPES, normalize_scope + + +class TestSeverityTable: + def test_the_table_is_owner_decision_2(self): + assert list(SEVERITY_MARKERS.items()) == [ + ("error", "πŸŸ₯"), + ("warning", "🟧"), + ("spec", "πŸ”"), + ("outofscope", "⬜"), + ] + + def test_every_severity_the_gate_accepts_has_a_glyph(self): + assert set(SEVERITY_MARKERS) == set(SEVERITIES) + + def test_glyphs_are_distinct(self): + assert len(set(SEVERITY_MARKERS.values())) == len(SEVERITY_MARKERS) + + def test_the_out_of_ticket_marker_is_never_a_severity_glyph(self): + assert OUT_OF_TICKET_MARKER == "🟦" + assert OUT_OF_TICKET_MARKER not in SEVERITY_MARKERS.values() + + def test_an_unknown_severity_renders_as_the_minor_class(self): + assert FALLBACK_MARKER == "⬜" + assert FALLBACK_MARKER == SEVERITY_MARKERS["outofscope"] + + def test_the_tables_are_read_only(self): + with pytest.raises(TypeError): + SEVERITY_MARKERS["error"] = "X" # type: ignore[index] + with pytest.raises(TypeError): + SCOPE_LABELS[SCOPE_OUT] = "X" # type: ignore[index] + + def test_scope_labels(self): + assert dict(SCOPE_LABELS) == {"out": "OUTSIDE TICKET"} + + +class TestSeverityMarker: + @pytest.mark.parametrize(("severity", "glyph"), list(SEVERITY_MARKERS.items())) + def test_known_severities(self, severity, glyph): + assert severity_marker(severity) == glyph + + @pytest.mark.parametrize("severity", ["blocker", "", "ERROR", "minor"]) + def test_anything_else_falls_back(self, severity): + assert severity_marker(severity) == FALLBACK_MARKER + + +_D64_TABLE = [ + ("error", "πŸŸ₯"), + ("warning", "🟧"), + ("spec", "πŸ”"), + ("outofscope", "⬜"), + ("blocker", "⬜"), +] + + +class TestMarkerFor: + """Design #64 Β§10.1: the scope prefix goes in front of the severity glyph.""" + + @pytest.mark.parametrize(("severity", "glyph"), _D64_TABLE) + def test_in_and_unknown_scope_add_nothing(self, severity, glyph): + assert marker_for(severity, SCOPE_IN) == glyph + assert marker_for(severity, SCOPE_UNKNOWN) == glyph + + @pytest.mark.parametrize(("severity", "glyph"), _D64_TABLE) + def test_out_of_ticket_prefixes_the_severity_glyph(self, severity, glyph): + assert marker_for(severity, SCOPE_OUT) == f"🟦 {glyph}" + + def test_scope_is_matched_exactly(self): + assert marker_for("error", "OUT") == "πŸŸ₯" + + +class TestScopeVocabulary: + def test_constants(self): + assert (SCOPE_IN, SCOPE_OUT, SCOPE_UNKNOWN) == ("in", "out", "unknown") + assert SCOPES == ("in", "out", "unknown") + assert isinstance(SCOPES, tuple) + + @pytest.mark.parametrize( + ("raw", "expected"), + [ + ("in", "in"), + ("out", "out"), + ("unknown", "unknown"), + (" OUT \n", "out"), + ("In", "in"), + ("UNKNOWN", "unknown"), + ], + ) + def test_the_three_words_survive_case_and_whitespace(self, raw, expected): + assert normalize_scope(raw) == expected + + @pytest.mark.parametrize( + "raw", + ["In scope", "yes", "no", "outside", "in-scope", "", " ", None, True, False, 1, 0, ["out"], {"scope": "out"}], + ) + def test_everything_else_is_unknown(self, raw): + assert normalize_scope(raw) == SCOPE_UNKNOWN + + +class TestModuleIsALeaf: + def test_markers_imports_only_the_standard_library_and_triage(self): + tree = ast.parse(Path(markers.__file__).read_text(encoding="utf-8")) + stdlib: set[str] = set() + package: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + stdlib.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + if node.level: + package.add(node.module or "") + else: + stdlib.add((node.module or "").split(".")[0]) + assert stdlib <= set(sys.stdlib_module_names) | {"__future__"}, stdlib + assert package == {"triage"} diff --git a/tests/test_text_inputs.py b/tests/test_text_inputs.py new file mode 100644 index 00000000..662b2a69 --- /dev/null +++ b/tests/test_text_inputs.py @@ -0,0 +1,400 @@ +"""Tests for prxref.text_inputs: the bounded, fingerprinted loader shared by +review rules, ticket context and local spec sources. + +The confinement cases build real symlinks under pytest's tmp dirs, with the +escape target in a sibling directory that is never under the working +directory, so a pass means the loader refused a real escape. +""" +from __future__ import annotations + +import ast +import dataclasses +import errno +import hashlib +import json +import os +import sys +import threading +import tracemalloc +from pathlib import Path + +import pytest + +from prxref import text_inputs +from prxref.text_inputs import ( + CappedText, + cap_text, + check_readable_path, + confine_to_cwd, + decode_text, + read_capped_file, +) + +SECRET = "SECRET-TOKEN-CONTENT" + + +@pytest.fixture +def workdir(tmp_path, monkeypatch): + """A fresh working directory, made the process cwd for the test.""" + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + return work + + +@pytest.fixture +def outside(tmp_path_factory): + """A directory that is never under the working directory, holding a secret.""" + out = tmp_path_factory.mktemp("outside") + (out / "secret.txt").write_text(SECRET, encoding="utf-8") + return out + + +def _write(path: Path, data: bytes) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path + + +class TestFingerprint: + def test_sha256_matches_the_raw_file_bytes(self, workdir): + data = b"\xef\xbb\xbfrule one\r\nrule two\rrule three\n" + path = _write(workdir / "rules.md", data) + capped = read_capped_file(path, 5) + assert capped.sha256 == hashlib.sha256(data).hexdigest() + + def test_hash_does_not_change_with_the_cap(self, workdir): + path = _write(workdir / "rules.md", b"abcdefghij" * 100) + assert read_capped_file(path, 5).sha256 == read_capped_file(path, 5000).sha256 + + def test_one_changed_byte_changes_the_hash(self, workdir): + first = read_capped_file(_write(workdir / "a.md", b"abcdef"), 3) + second = read_capped_file(_write(workdir / "b.md", b"abcdeF"), 3) + assert first.text == second.text + assert first.sha256 != second.sha256 + + def test_empty_file_hashes_the_empty_string(self, workdir): + capped = read_capped_file(_write(workdir / "empty.md", b""), 10) + assert capped == CappedText("", hashlib.sha256(b"").hexdigest(), 0, False, 10) + + +class TestCap: + def test_len_equal_to_cap_is_not_truncated(self, workdir): + capped = read_capped_file(_write(workdir / "f.md", b"abcde"), 5) + assert (capped.text, capped.chars, capped.truncated) == ("abcde", 5, False) + + def test_one_over_the_cap_is_truncated(self, workdir): + capped = read_capped_file(_write(workdir / "f.md", b"abcdef"), 5) + assert (capped.text, capped.chars, capped.truncated) == ("abcde", 6, True) + + def test_kept_text_is_exactly_the_prefix_and_carries_no_marker(self, workdir): + body = "line one\nline two\nline three\n" + capped = read_capped_file(_write(workdir / "f.md", body.encode()), 12) + assert capped.text == body[:12] + assert "truncated" not in capped.text + + def test_chars_counts_the_full_text_not_the_kept_text(self, workdir): + capped = read_capped_file(_write(workdir / "f.md", "Γ©".encode() * 1000), 10) + assert capped.chars == 1000 + assert len(capped.text) == 10 + + @pytest.mark.parametrize("cap", [0, -1]) + def test_cap_below_one_raises_value_error(self, workdir, cap): + with pytest.raises(ValueError, match="max_chars"): + cap_text("text", cap) + with pytest.raises(ValueError, match="max_chars"): + read_capped_file(_write(workdir / "f.md", b"text"), cap) + + def test_cap_is_checked_before_the_filesystem_is_touched(self, workdir): + with pytest.raises(ValueError, match="max_chars"): + read_capped_file(workdir / "missing.md", 0) + + +class TestDecoding: + def test_bom_stripped_and_crlf_and_cr_normalised(self, workdir): + capped = read_capped_file(_write(workdir / "f.md", b"\xef\xbb\xbfa\r\nb\rc\n"), 100) + assert capped.text == "a\nb\nc\n" + assert capped.chars == 6 + + def test_only_a_leading_bom_is_dropped(self): + assert decode_text(b"\xef\xbb\xbfa\xef\xbb\xbfb") == "aο»Ώb" + + def test_cr_before_crlf_is_two_newlines(self): + assert decode_text(b"a\r\r\nb") == "a\n\nb" + + def test_invalid_utf8_raises_unicode_decode_error(self, workdir): + with pytest.raises(UnicodeDecodeError): + read_capped_file(_write(workdir / "f.md", b"ok\xff\xfe"), 100) + with pytest.raises(UnicodeDecodeError): + decode_text(b"ok\xff\xfe") + + def test_invalid_utf8_after_the_cap_still_raises(self, workdir): + with pytest.raises(UnicodeDecodeError): + read_capped_file(_write(workdir / "f.md", b"a" * 200_000 + b"\xff"), 10) + + def test_a_truncated_multibyte_sequence_at_eof_raises(self, workdir): + with pytest.raises(UnicodeDecodeError): + read_capped_file(_write(workdir / "f.md", b"abc" + "Γ©".encode()[:1]), 100) + + @pytest.mark.parametrize("split", [65535, 65536, 65537]) + def test_crlf_split_across_a_read_chunk_counts_once(self, workdir, split): + data = b"x" * (split - 1) + b"\r\n" + b"y" * 10 + capped = read_capped_file(_write(workdir / "f.md", data), 10**7) + assert capped.chars == (split - 1) + 1 + 10 + assert capped.text.count("\n") == 1 + + @pytest.mark.parametrize("split", [65535, 65536]) + def test_multibyte_char_split_across_a_read_chunk(self, workdir, split): + data = b"x" * (split - 1) + "Γ©".encode() + "🟦".encode() + b"z" + capped = read_capped_file(_write(workdir / "f.md", data), 10**7) + assert capped.chars == (split - 1) + 3 + assert capped.text.endswith("é🟦z") + + +_PARITY_SAMPLES = [ + b"", + b"plain ascii text", + b"\xef\xbb\xbfbom then text", + b"\xef\xbb\xbf", + b"a\r\nb\r\nc", + b"a\rb\rc\r", + b"\r\n\r\n\r", + b"a\r\r\nb\n\rc", + "cafΓ© β€” \U0001f7e6 done\r\n".encode(), + "\U0001f7e5\r\U0001f7e7\r\nΓ©".encode(), +] + + +class TestStreamingMatchesWholeFileDecoding: + """The streaming reader and ``cap_text(decode_text(...))`` must agree on + every cut point, so rules (whole read) and ticket/spec (streamed) record + identical fingerprints for identical files. + """ + + @pytest.mark.parametrize("data", _PARITY_SAMPLES, ids=range(len(_PARITY_SAMPLES))) + def test_every_cap_agrees(self, workdir, data): + path = _write(workdir / "f.md", data) + digest = hashlib.sha256(data).hexdigest() + full = decode_text(data) + for cap in range(1, len(full) + 3): + assert read_capped_file(path, cap) == cap_text(full, cap, sha256=digest), cap + + @pytest.mark.parametrize("cap", [65534, 65535, 65536, 65537]) + def test_caps_at_the_chunk_boundary_agree(self, workdir, cap): + data = b"x" * 65535 + b"\r\n" + "Γ©\r".encode() * 20 + path = _write(workdir / "f.md", data) + expected = cap_text(decode_text(data), cap, sha256=hashlib.sha256(data).hexdigest()) + assert read_capped_file(path, cap) == expected + + +class TestCapText: + def test_default_sha256_covers_the_whole_text_before_the_cap(self): + capped = cap_text("abcdef", 3) + assert capped.sha256 == hashlib.sha256(b"abcdef").hexdigest() + assert (capped.text, capped.chars, capped.truncated, capped.max_chars) == ("abc", 6, True, 3) + + def test_an_explicit_sha256_is_recorded_as_given(self): + assert cap_text("abc", 10, sha256="f" * 64).sha256 == "f" * 64 + + def test_a_lone_surrogate_does_not_crash_the_default_hash(self): + assert len(cap_text("a\ud800b", 10).sha256) == 64 + + +class TestRecord: + def test_record_is_json_serialisable_with_fixed_keys(self, workdir): + capped = read_capped_file(_write(workdir / "f.md", b"abcdef"), 3) + record = capped.record() + assert list(record) == ["sha256", "chars", "max_chars", "truncated"] + assert json.loads(json.dumps(record)) == { + "sha256": hashlib.sha256(b"abcdef").hexdigest(), + "chars": 6, + "max_chars": 3, + "truncated": True, + } + + def test_the_record_never_carries_the_text(self): + assert "SECRET" not in json.dumps(cap_text(SECRET, 100).record()) + + def test_capped_text_is_frozen(self): + with pytest.raises(dataclasses.FrozenInstanceError): + cap_text("abc", 10).text = "changed" + + +class TestNonRegularFiles: + def test_missing_file_raises_file_not_found_naming_the_path_as_given(self, workdir): + with pytest.raises(FileNotFoundError) as info: + read_capped_file("missing.md", 10) + assert info.value.filename == "missing.md" + + def test_directory_is_refused(self, workdir): + (workdir / "sub").mkdir() + with pytest.raises(IsADirectoryError) as info: + read_capped_file("sub", 10) + assert info.value.errno == errno.EISDIR + assert info.value.filename == "sub" + with pytest.raises(IsADirectoryError): + check_readable_path("sub") + + @pytest.mark.skipif(not hasattr(os, "mkfifo"), reason="needs POSIX FIFOs") + def test_fifo_is_refused_without_blocking(self, workdir): + os.mkfifo("pipe.md") + outcome: list[BaseException | None] = [] + + def attempt() -> None: + try: + read_capped_file("pipe.md", 10) + outcome.append(None) + except BaseException as e: + outcome.append(e) + + worker = threading.Thread(target=attempt, daemon=True) + worker.start() + worker.join(5) + assert not worker.is_alive(), "reading a FIFO blocked" + assert type(outcome[0]) is OSError + assert outcome[0].strerror == "not a regular file" + assert outcome[0].filename == "pipe.md" + + @pytest.mark.skipif(not os.path.exists("/dev/zero"), reason="needs /dev/zero") + def test_device_is_refused(self): + with pytest.raises(OSError) as info: + read_capped_file("/dev/zero", 10, confine=False) + assert info.value.strerror == "not a regular file" + + +class TestBoundedMemory: + def test_a_large_file_is_scanned_not_held(self, workdir): + line = b"line of text\n" + data = line * (4 * 1024 * 1024 // len(line)) + path = _write(workdir / "big.md", data) + tracemalloc.start() + try: + capped = read_capped_file(path, 1000) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + assert capped.truncated + assert capped.chars == len(data) + assert capped.sha256 == hashlib.sha256(data).hexdigest() + assert peak < 1024 * 1024, f"peak {peak} bytes for a {len(data)} byte file" + + +class TestConfinement: + """SEC-3 rule (a): a path under the working directory must not resolve out of it.""" + + def test_relative_file_symlink_escaping_the_cwd_is_refused(self, workdir, outside): + (workdir / "docs").mkdir() + os.symlink(outside / "secret.txt", workdir / "docs" / "SPEC.md") + with pytest.raises(PermissionError) as info: + read_capped_file("docs/SPEC.md", 100) + assert info.value.errno == errno.EACCES + assert info.value.strerror == "resolves outside the working directory" + assert info.value.filename == "docs/SPEC.md" + + def test_the_refusal_never_names_the_symlink_target(self, workdir, outside): + os.symlink(outside / "secret.txt", workdir / "SPEC.md") + with pytest.raises(PermissionError) as info: + read_capped_file("SPEC.md", 100) + assert str(outside) not in str(info.value) + assert "secret" not in str(info.value) + + def test_symlinked_directory_root_is_refused(self, workdir, outside): + (workdir / "docs").mkdir() + os.symlink(outside, workdir / "docs" / "specs") + with pytest.raises(PermissionError): + confine_to_cwd("docs/specs") + with pytest.raises(PermissionError): + read_capped_file("docs/specs/secret.txt", 100) + + def test_symlinked_parent_component_is_refused(self, workdir, outside): + (workdir / "p").mkdir() + os.symlink(outside, workdir / "p" / "docs") + with pytest.raises(PermissionError): + read_capped_file(os.path.join("p", "docs", "secret.txt"), 100) + + def test_dotdot_after_a_symlinked_directory_is_refused(self, workdir, outside): + deep = outside / "deep" / "dir" + deep.mkdir(parents=True) + _write(outside / "deep" / "x.md", b"escaped") + os.symlink(deep, workdir / "sub") + with pytest.raises(PermissionError): + read_capped_file("sub/../x.md", 100) + + def test_absolute_spelling_of_the_cwd_is_confined_too(self, workdir, outside): + os.symlink(outside / "secret.txt", workdir / "SPEC.md") + with pytest.raises(PermissionError): + read_capped_file(str(workdir / "SPEC.md"), 100) + + def test_the_cwd_reached_through_a_symlinked_alias_is_confined(self, workdir, outside, tmp_path): + (workdir / "docs").mkdir() + os.symlink(outside / "secret.txt", workdir / "docs" / "SPEC.md") + _write(workdir / "docs" / "ok.md", b"fine") + alias = tmp_path / "alias" + os.symlink(workdir, alias) + with pytest.raises(PermissionError): + read_capped_file(str(alias / "docs" / "SPEC.md"), 100) + assert read_capped_file(str(alias / "docs" / "ok.md"), 100).text == "fine" + + def test_a_symlinked_alias_into_a_subdirectory_is_confined(self, workdir, outside, tmp_path): + (workdir / "docs").mkdir() + os.symlink(outside / "secret.txt", workdir / "docs" / "SPEC.md") + alias = tmp_path / "docs-alias" + os.symlink(workdir / "docs", alias) + with pytest.raises(PermissionError): + read_capped_file(str(alias / "SPEC.md"), 100) + + def test_dangling_escape_is_refused_before_existence_is_checked(self, workdir, outside): + os.symlink(outside / "no-such-file", workdir / "SPEC.md") + with pytest.raises(PermissionError): + read_capped_file("SPEC.md", 100) + + def test_dangling_symlink_inside_the_cwd_is_just_missing(self, workdir): + os.symlink(workdir / "no-such-file", workdir / "SPEC.md") + with pytest.raises(FileNotFoundError): + read_capped_file("SPEC.md", 100) + + def test_symlink_that_stays_inside_the_cwd_is_allowed(self, workdir): + _write(workdir / "real.md", b"inside") + os.symlink("real.md", workdir / "link.md") + assert read_capped_file("link.md", 100).text == "inside" + assert confine_to_cwd("link.md") == os.path.realpath(workdir / "real.md") + + def test_absolute_path_outside_cwd_allowed(self, workdir, outside): + capped = read_capped_file(str(outside / "secret.txt"), 100) + assert capped.text == SECRET + + def test_relative_dotdot_path_outside_cwd_is_the_operators_choice(self, workdir, outside, monkeypatch): + monkeypatch.chdir(outside) + (outside / "inner").mkdir() + monkeypatch.chdir(outside / "inner") + assert read_capped_file("../secret.txt", 100).text == SECRET + + def test_confine_false_follows_an_escaping_link(self, workdir, outside): + os.symlink(outside / "secret.txt", workdir / "link.md") + assert read_capped_file("link.md", 100, confine=False).text == SECRET + + def test_confine_to_cwd_returns_the_realpath_and_needs_no_file(self, workdir): + assert confine_to_cwd("not-yet/there.md") == os.path.realpath(workdir / "not-yet" / "there.md") + assert confine_to_cwd(".") == os.path.realpath(workdir) + + def test_a_directory_inside_the_cwd_is_allowed(self, workdir): + (workdir / "docs" / "specs").mkdir(parents=True) + assert confine_to_cwd(Path("docs/specs")) == os.path.realpath(workdir / "docs" / "specs") + + def test_check_readable_path_returns_the_resolved_regular_file(self, workdir): + _write(workdir / "real.md", b"x") + os.symlink("real.md", workdir / "link.md") + assert check_readable_path("link.md") == os.path.realpath(workdir / "real.md") + + +class TestModuleIsALeaf: + def test_imports_only_the_standard_library(self): + tree = ast.parse(Path(text_inputs.__file__).read_text(encoding="utf-8")) + imported: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + assert node.level == 0, "text_inputs must not import from the package" + imported.add((node.module or "").split(".")[0]) + assert imported <= set(sys.stdlib_module_names) | {"__future__"}, imported From ea69c052c0d3329970f14ee890ebdc3e30216740 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 13:39:26 -0700 Subject: [PATCH 14/61] fix: spec units group wrapped blocks, sentence-split, standalone pins, normative tokens unscored MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec constraint extraction matched one physical line at a time, so a hard-wrapped MUST lost its tail, a paragraph holding several keywords became one mislabelled unit, a `# comment` in a code fence was read as a heading, and any line near a date became a MUST (COR-1 / LIVE-3, backlog 8). _spec_units now groups lines into blocks first: a paragraph or list item joins its wrapped and indented continuations; blank lines, headings, setext underlines, fences, table rows and new list items end a block. Table rows and fenced lines stay single-line units and are never headings. Blocks over 400 chars or with two or more matching sentences are split by an abbreviation-aware sentence splitter (e.g., i.e., initialisms, code spans) into one capped unit per matching sentence with its own strength; otherwise the block is kept whole. A prose `can`/`discouraged` or bare pin keeps just its sentence. A unit ending its block with `:` carries the following list items up to the cap. Units anchor L{n} and doc_idx on the block's first original line. _is_version_pin_line now uses _STANDALONE_PIN_RE after peeling list markers and decoration, so only a pin on its own line becomes a MUST. COR-5: _rank_units excludes the new _NORMATIVE_TOKENS from relevance, so a `must` comment or `required=True` in the diff no longer reorders the digest. quality._STOPWORDS is untouched. New tests/test_specs_units.py covers the eval corpus (case-001/002/003), block boundaries, sentence units, colon lead-ins, pins and COR-5. πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/specs.py | 242 ++++++++++++++++++++---- tests/test_specs_units.py | 376 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 582 insertions(+), 36 deletions(-) create mode 100644 tests/test_specs_units.py diff --git a/src/prxref/specs.py b/src/prxref/specs.py index 1e630b15..4d11dcc8 100644 --- a/src/prxref/specs.py +++ b/src/prxref/specs.py @@ -432,62 +432,232 @@ def _strength(line: str) -> int | None: return None +_FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") + +_LIST_ITEM_RE = re.compile(r"^(\s*)(?:[-*+]|\d+[.)])\s") + +_TABLE_ROW_RE = re.compile(r"^\s*\|") + +_QUOTE_RE = re.compile(r"^\s*(?:>\s?)+") + +_CODE_SPAN_RE = re.compile(r"`[^`]*`") + +_SENTENCE_BREAK_RE = re.compile(r"[.;!?][\"'”’)\]]*\s+") + +_INITIALISM_RE = re.compile(r"(?:[a-z]\.)+") + +_ABBREVIATIONS = frozenset({"etc.", "vs.", "cf.", "incl.", "approx.", "fig.", "sec.", "resp.", "viz."}) + +_PIN_DECORATION_RE = re.compile(r"[`\"'β€œβ€β€˜β€™*_()\[\]]") + +_NORMATIVE_TOKENS = frozenset( + {"must", "shall", "required", "recommended", "forbidden", "discouraged", "should", "never", "optional"} +) + + def _is_version_pin_line(line: str) -> bool: - if _VERSION_DATE_RE.search(line): - return True - return bool(_VERSION_NUM_RE.search(line) and _VERSION_CONTEXT_RE.search(line)) + """True when ``line`` is nothing but a version pin. + A list marker and markdown decoration (backticks, quotes, emphasis, + brackets) are peeled first, so ``- `"2026-07-28"`.`` pins. A date or + version number inside prose does not: it rides along verbatim when its + sentence is a kept constraint, and is dropped otherwise. + """ + bare = _PIN_DECORATION_RE.sub("", _LIST_ITEM_RE.sub("", line, count=1)).strip() + return _STANDALONE_PIN_RE.match(bare) is not None + + +def _classify(sentence: str) -> tuple[int, bool] | None: + if _NAMING_RE.search(sentence): + return 3, True + strength = _strength(sentence) + if strength is not None: + return strength, strength > 1 or _STRENGTH1_RE.search(sentence) is not None + if _is_version_pin_line(sentence): + return 3, False + return None -def _spec_units(src: SpecSource, source_idx: int) -> list[_Unit]: - short = _origin_short(src.origin) - lines = src.text.splitlines() - units: list[_Unit] = [] - heading: tuple[str, str] | None = None + +def _split_sentences(text: str) -> list[str]: + masked = _CODE_SPAN_RE.sub(lambda m: "x" * len(m.group()), text) + sentences: list[str] = [] + start = 0 + for m in _SENTENCE_BREAK_RE.finditer(masked): + if masked[m.start()] == ".": + word_start = max(start, masked.rfind(" ", start, m.start()) + 1) + word = masked[word_start : m.start() + 1].lstrip("(\"'β€œβ€˜[").lower() + if word in _ABBREVIATIONS or _INITIALISM_RE.fullmatch(word): + continue + sentences.append(text[start : m.end()].strip()) + start = m.end() + tail = text[start:].strip() + if tail: + sentences.append(tail) + return sentences + + +@dataclass +class _Block: + """One structural element of a spec text, anchored on its first line.""" + + line: int + kind: str + indent: int + parts: list[str] + + def text(self) -> str: + return " ".join(" ".join(self.parts).split()) + + +def _blocks(text: str) -> list[_Block]: + lines = text.splitlines() + blocks: list[_Block] = [] + current: _Block | None = None + current_quoted = False + fence: str | None = None i = 0 while i < len(lines): line = lines[i] + stripped = line.strip() + i += 1 + if fence is not None: + if stripped and set(stripped) == {fence[0]} and len(stripped) >= len(fence): + fence = None + blocks.append(_Block(i, "break", 0, [])) + elif stripped: + blocks.append(_Block(i, "code", 0, [stripped])) + continue + opener = _FENCE_RE.match(line) + if opener: + fence = opener.group(1) + current = None + blocks.append(_Block(i, "break", 0, [])) + continue + if not stripped: + current = None + continue md = _MD_HEADING_RE.match(line) setext = ( md is None and _SETEXT_TEXT_RE.match(line) is not None - and i + 1 < len(lines) - and _SETEXT_UNDER_RE.match(lines[i + 1]) is not None + and i < len(lines) + and _SETEXT_UNDER_RE.match(lines[i]) is not None and _strength(line) is None ) if md or setext: - text = (md.group(1) if md else line).strip() + current = None + blocks.append(_Block(i, "heading", 0, [(md.group(1) if md else line).strip()])) + i += 1 if setext else 0 + continue + if _SETEXT_UNDER_RE.match(line): + current = None + blocks.append(_Block(i, "break", 0, [])) + continue + if _TABLE_ROW_RE.match(line): + current = None + blocks.append(_Block(i, "row", 0, [stripped])) + continue + quote = _QUOTE_RE.match(line) + content = line[quote.end() :] if quote else line + if not content.strip(): + current = None + continue + if current is not None and current_quoted != bool(quote): + current = None + current_quoted = bool(quote) + item = _LIST_ITEM_RE.match(content) + if item: + current = _Block(i, "item", len(item.group(1).expandtabs(4)), [content.strip()]) + blocks.append(current) + elif current is not None: + current.parts.append(content.strip()) + else: + current = _Block(i, "text", 0, [content.strip()]) + blocks.append(current) + return blocks + + +def _block_statements(block: _Block) -> list[tuple[int, str]]: + text = block.text() + if block.kind in ("row", "code"): + hit = _classify(text) + return [(hit[0], _one_line(text, _STATEMENT_MAX_CHARS))] if hit else [] + hits = [(s, c) for s in _split_sentences(text) if (c := _classify(s)) is not None] + if not hits: + return [] + if len(text) > _STATEMENT_MAX_CHARS or len(hits) >= 2: + return [(strength, _one_line(s, _STATEMENT_MAX_CHARS)) for s, (strength, _) in hits] + sentence, (strength, strong) = hits[0] + return [(strength, text if strong else sentence)] + + +def _attach_list(statement: str, blocks: list[_Block], pos: int) -> str: + lead = blocks[pos] + parts = [statement] + size = len(statement) + for block in blocks[pos + 1 :]: + if block.kind != "item" or (lead.kind == "item" and block.indent <= lead.indent): + break + item = block.text() + if size + 1 + len(item) > _STATEMENT_MAX_CHARS: + break + parts.append(item) + size += 1 + len(item) + return " ".join(parts) + + +def _spec_units(src: SpecSource, source_idx: int) -> list[_Unit]: + """Extract the constraint units of one non-ticket source, in document order. + + Lines are grouped into blocks before anything is matched: a paragraph or + list item joins its wrapped and indented continuation lines, while a + blank line, heading, setext underline, code fence, table row or new list + item ends a block. Table rows and fenced lines are single-line units and + never headings. Each block is split into sentences (``e.g.``/``i.e.`` + and code spans never end one) and every sentence is matched on its own + for RFC-2119 keywords, naming rules and standalone version pins. A + block over :data:`_STATEMENT_MAX_CHARS` or holding two or more matching + sentences yields one capped unit per matching sentence, each with its + own strength; otherwise the block is kept whole, unless its only match + is a prose ``can``/``discouraged`` or a bare pin, which keeps just that + sentence. A unit that ends its block with ``:`` carries the following + list items (nested ones only, under a list item) while they fit the + cap; those items are still matched on their own. Every unit is anchored + ``L{n}`` on its block's first original line. + """ + short = _origin_short(src.origin) + blocks = _blocks(src.text) + units: list[_Unit] = [] + heading: tuple[str, str] | None = None + for pos, block in enumerate(blocks): + if block.kind == "heading": + text = block.parts[0] slug = _heading_slug(text) heading = ( f"[spec:{short}#{slug}] (heading) {_one_line(text, _STATEMENT_MAX_CHARS)}", slug, ) - i += 2 if setext else 1 - continue - i += 1 - stripped = line.strip() - if not stripped: continue - strength = 3 if _NAMING_RE.search(stripped) else _strength(stripped) - if strength is None and _is_version_pin_line(stripped): - strength = 3 - if strength is None: + if block.kind == "break": continue - anchor = f"L{i}" - statement = _one_line(stripped, _STATEMENT_MAX_CHARS) - label = {3: "MUST", 2: "SHOULD", 1: "MAY"}[strength] - units.append( - _Unit( - origin=src.origin, - source_idx=source_idx, - doc_idx=i, - strength=strength, - is_ticket=False, - render=f"[spec:{short}#{anchor}] ({label}) {statement}", - heading_render=heading[0] if heading else None, - heading_key=heading[1] if heading else None, - tokens=frozenset(_evidence_tokens(statement)), + ends_with_colon = block.kind in ("text", "item") and block.text().endswith(":") + for strength, statement in _block_statements(block): + if ends_with_colon and statement.endswith(":"): + statement = _attach_list(statement, blocks, pos) + label = {3: "MUST", 2: "SHOULD", 1: "MAY"}[strength] + units.append( + _Unit( + origin=src.origin, + source_idx=source_idx, + doc_idx=block.line, + strength=strength, + is_ticket=False, + render=f"[spec:{short}#L{block.line}] ({label}) {statement}", + heading_render=heading[0] if heading else None, + heading_key=heading[1] if heading else None, + tokens=frozenset(_evidence_tokens(statement)), + ) ) - ) return units @@ -540,7 +710,7 @@ def _rank_units(sources: list[SpecSource], diff_toks: frozenset[str]) -> list[_U for u in units: if u.is_ticket: continue - u.score = len(u.tokens & diff_toks) + u.score = len((u.tokens & diff_toks) - _NORMATIVE_TOKENS) rest.append(u) relevant = sorted( (u for u in rest if u.score >= 1), diff --git a/tests/test_specs_units.py b/tests/test_specs_units.py new file mode 100644 index 00000000..f11f8273 --- /dev/null +++ b/tests/test_specs_units.py @@ -0,0 +1,376 @@ +"""Spec unit extraction: block grouping, sentence units, pins, relevance scoring. + +Covers COR-1/LIVE-3 (hard-wrapped normative statements), COR-5 (normative +keywords never count as relevance overlap) and COR backlog 8 (only a +standalone version pin becomes its own constraint). +""" +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +import prxref.specs as specs +from prxref.specs import SpecSource, build_spec_digest +from prxref.triage import parse_unified_diff + +EVALS = Path(__file__).parent / "evals" + +_RENDER_RE = re.compile(r"^\[spec:[^\]#]+#L(\d+)\] \((MUST|SHOULD|MAY)\) (.+)$") + + +def _src(text: str, origin: str = "/docs/spec.md") -> SpecSource: + return SpecSource(origin=origin, kind="file", text=text, error="") + + +def _units(text: str) -> list[specs._Unit]: + return specs._spec_units(_src(text), 0) + + +def _parsed(text: str) -> list[tuple[int, str, str]]: + out = [] + for unit in _units(text): + m = _RENDER_RE.match(unit.render) + assert m, unit.render + out.append((int(m.group(1)), m.group(2), m.group(3))) + return out + + +def _statements(text: str) -> list[str]: + return [statement for _, _, statement in _parsed(text)] + + +def _added_file(path: str, lines: list[str]) -> str: + body = "".join(f"+{text}\n" for text in lines) + return ( + f"diff --git a/{path} b/{path}\n" + "new file mode 100644\n" + "--- /dev/null\n" + f"+++ b/{path}\n" + f"@@ -0,0 +1,{len(lines)} @@\n" + f"{body}" + ) + + +def _eval_sources(case: str) -> list[SpecSource]: + root = EVALS / case + paths = [root / "ticket.md", *sorted((root / "docs").iterdir())] + return [_src(p.read_text(encoding="utf-8"), origin=str(p)) for p in paths] + + +def _eval_digest(case: str, *, with_diff: bool) -> str: + files = parse_unified_diff((EVALS / case / "diff.patch").read_text()) if with_diff else [] + return build_spec_digest(_eval_sources(case), files, token_budget=3000) + + +def _eval_doc(case: str) -> str: + (doc,) = sorted((EVALS / case / "docs").iterdir()) + return doc.read_text(encoding="utf-8") + + +_FILLER = ( + "The ingest service accepts webhook deliveries from partner systems. " + "Each delivery carries a JSON body and a small set of headers. " + "Deliveries arrive in bursts during the nightly settlement window. " + "The queue absorbs those bursts before the workers pick them up. " + "Operators watch the queue depth on the shared ingest dashboard. " + "Partners are onboarded through the integrations team." +) + + +class TestWrappedStatements: + @pytest.mark.parametrize("with_diff", [False, True]) + def test_case_002_digest_keeps_the_wrapped_subjects(self, with_diff): + digest = _eval_digest("case-002-session-token-logging", with_diff=with_diff) + for needle in ("APP_", "VITE_", "Authorization", "Tokens, session tokens"): + assert needle in digest, needle + + def test_rfc_wrapped_paragraph_keeps_its_middle_line(self): + text = ( + " A client MUST NOT send a request body larger than the limit\n" + " advertised by the server in the max-body-size field, and it\n" + " MUST close the connection if the server responds with 413.\n" + ) + assert _statements(text) == [ + "A client MUST NOT send a request body larger than the limit advertised by the " + "server in the max-body-size field, and it MUST close the connection if the " + "server responds with 413." + ] + digest = build_spec_digest([_src(text, origin="rfc.txt")], [], token_budget=3000) + assert "max-body-size" in digest + + def test_a_must_wrapped_over_two_lines_keeps_both_halves(self): + text = "Session cookies MUST set the\n`Secure` and `HttpOnly` attributes.\n" + assert _statements(text) == ["Session cookies MUST set the `Secure` and `HttpOnly` attributes."] + + def test_an_indented_continuation_joins_its_list_item(self): + text = "- Clients MUST send the\n `X-Trace` header on every retry.\n- Unrelated item.\n" + assert _statements(text) == ["- Clients MUST send the `X-Trace` header on every retry."] + + def test_blockquote_markers_are_stripped_and_quoted_lines_join(self): + text = "> Relays MUST forward the\n> `traceparent` header unchanged.\n" + assert _statements(text) == ["Relays MUST forward the `traceparent` header unchanged."] + + def test_a_keyword_free_block_keeps_nothing(self): + assert _units(_FILLER.replace(". ", ".\n")) == [] + + +class TestSentenceUnits: + def test_a_late_must_in_a_paragraph_over_400_chars_keeps_its_clause(self): + text = ( + _FILLER.replace(". ", ".\n") + + " Webhook signatures MUST be verified with the\nshared HMAC key.\n" + ) + assert len(" ".join(text.split())) > specs._STATEMENT_MAX_CHARS + assert _parsed(text) == [(1, "MUST", "Webhook signatures MUST be verified with the shared HMAC key.")] + + def test_consecutive_keyword_lines_keep_their_own_labels(self): + text = ( + "Clients MUST send the header.\n" + "Servers SHOULD log the request.\n" + "Proxies MAY cache the response.\n" + ) + assert [(label, s) for _, label, s in _parsed(text)] == [ + ("MUST", "Clients MUST send the header."), + ("SHOULD", "Servers SHOULD log the request."), + ("MAY", "Proxies MAY cache the response."), + ] + + def test_semicolon_clauses_with_two_keywords_split_with_their_labels(self): + text = ( + "An initialize request missing any required field MUST be treated as a spec\n" + "violation; conforming servers SHOULD reject it.\n" + ) + assert [(label, s) for _, label, s in _parsed(text)] == [ + ("MUST", "An initialize request missing any required field MUST be treated as a spec violation;"), + ("SHOULD", "conforming servers SHOULD reject it."), + ] + + def test_eg_and_backticked_dots_do_not_split_a_sentence(self): + text = ( + _FILLER + + " Server-side secrets MUST be read from variables carrying the\n" + "`APP_` prefix only (e.g. `APP_SESSION_SECRET` or `app.secret. key`), i.e. never\n" + "from a `VITE_` variable.\n" + ) + assert _statements(text) == [ + "Server-side secrets MUST be read from variables carrying the `APP_` prefix only " + "(e.g. `APP_SESSION_SECRET` or `app.secret. key`), i.e. never from a `VITE_` variable." + ] + + def test_a_short_block_with_one_keyword_sentence_stays_whole(self): + text = ( + "Tokens and `Authorization` header values MUST NEVER be written\n" + "to logs. Log the opaque session ID instead; the session ID is not a secret.\n" + ) + assert _statements(text) == [ + "Tokens and `Authorization` header values MUST NEVER be written to logs. " + "Log the opaque session ID instead; the session ID is not a secret." + ] + + def test_can_runs_per_sentence_not_per_block(self): + text = ( + "Support tickets spike after every deploy: the admin console keeps sessions in\n" + "a process-local dict, so any restart invalidates them. Introduce a durable\n" + "session store and wire it into the app. Add an issuance log line so support\n" + "can correlate a session with a user when triaging incidents.\n" + ) + assert _parsed(text) == [ + (1, "MAY", "Add an issuance log line so support can correlate a session with a user " + "when triaging incidents."), + ] + + def test_every_unit_is_one_capped_line(self): + text = "Clients MUST " + "retry " * 120 + "forever.\n" + (statement,) = _statements(text) + assert len(statement) == specs._STATEMENT_MAX_CHARS + assert "\n" not in statement + + +class TestBlockBoundaries: + def test_table_rows_are_their_own_units(self): + text = ( + "| Field | Rule |\n" + "|---|---|\n" + "| `id` | MUST be a UUID |\n" + "| `name` | SHOULD be short |\n" + ) + assert _parsed(text) == [ + (3, "MUST", "| `id` | MUST be a UUID |"), + (4, "SHOULD", "| `name` | SHOULD be short |"), + ] + + def test_fenced_lines_are_never_joined_or_read_as_headings(self): + text = ( + "The client MUST send pings.\n" + "```python\n" + "# The client MUST retry on 503\n" + "x = 1\n" + "```\n" + "Servers SHOULD answer pings.\n" + ) + units = _units(text) + assert _parsed(text) == [ + (1, "MUST", "The client MUST send pings."), + (3, "MUST", "# The client MUST retry on 503"), + (6, "SHOULD", "Servers SHOULD answer pings."), + ] + assert all(u.heading_render is None for u in units) + + def test_a_setext_heading_scopes_and_an_underline_never_joins(self): + text = ( + "Transport Rules\n" + "---------------\n" + "Clients MUST use TLS.\n" + "===\n" + "Servers SHOULD pin certificates.\n" + ) + units = _units(text) + assert _statements(text) == ["Clients MUST use TLS.", "Servers SHOULD pin certificates."] + assert [u.heading_key for u in units] == ["transport-rules", "transport-rules"] + + def test_anchor_and_doc_idx_come_from_the_block_first_original_line(self): + text = "Intro prose.\n\nMore prose.\n\n- The retry header is\n REQUIRED on every retry.\n" + (unit,) = _units(text) + assert unit.render.startswith("[spec:spec.md#L5] (MUST) - The retry header is REQUIRED") + assert unit.doc_idx == 5 + + def test_sentence_units_share_the_block_anchor(self): + text = "Intro prose.\n\nClients MUST send the header.\nServers SHOULD log it.\n" + assert [line for line, _, _ in _parsed(text)] == [3, 3] + + +class TestColonLeadIn: + def test_case_003_naming_rule_carries_its_list(self): + statements = _statements(_eval_doc("case-003-config-schema-pin")) + (naming,) = [s for s in statements if "MUST be named" in s] + assert "EXACTLY two underscores" in naming + assert "lowercase words separated by single underscores" in naming + + def test_case_001_required_fields_lead_in_carries_its_list(self): + statements = _statements(_eval_doc("case-001-mcp-protocol-upgrade")) + (lead,) = [s for s in statements if "MUST include ALL of:" in s] + for needle in ("`protocolVersion`", "`capabilities`", "`clientInfo`", "`name`", "`version`"): + assert needle in lead, needle + assert any(s.startswith("- `clientInfo`: object with REQUIRED") for s in statements) + + def test_a_list_item_lead_in_attaches_only_its_children(self): + text = ( + "- Tokens MUST include:\n" + " - an issuer\n" + " - an expiry\n" + "- Refresh tokens MUST rotate on use.\n" + ) + assert _statements(text) == [ + "- Tokens MUST include: - an issuer - an expiry", + "- Refresh tokens MUST rotate on use.", + ] + + def test_the_attached_list_stops_at_the_cap_on_an_item_boundary(self): + items = [f"- `feature_{i:02d}`: a descriptive capability entry" for i in range(20)] + text = "Clients MUST support:\n\n" + "\n".join(items) + "\n- `final`: MUST be negotiated last\n" + statements = _statements(text) + lead = statements[0] + assert lead.startswith("Clients MUST support: - `feature_00`") + assert len(lead) <= specs._STATEMENT_MAX_CHARS + assert lead.endswith("a descriptive capability entry") + assert statements[1:] == ["- `final`: MUST be negotiated last"] + + def test_a_paragraph_after_the_list_is_not_attached(self): + text = "Clients MUST send:\n\n- a name\n\nThe rest is prose.\n" + assert _statements(text) == ["Clients MUST send: - a name"] + + +class TestVersionPins: + @pytest.mark.parametrize( + ("line", "expected"), + [ + ("2026-07-28", True), + ('`"2026-07-28"`.', True), + ("- 2026-07-28", True), + ("v1.2", True), + ("3.0.1:", True), + ("Last updated 2025-03-01 by the docs team.", False), + ("Version 2025-06-18", False), + ('"protocolVersion": "2025-06-18",', False), + ("Protocol version 3.2 is the floor.", False), + ], + ) + def test_only_a_standalone_pin_is_a_pin(self, line, expected): + assert specs._is_version_pin_line(line) is expected + + def test_dated_prose_is_not_a_must(self): + assert _units("Last updated 2025-03-01 by the docs team.\n\nVersion 2025-06-18\n") == [] + + def test_a_standalone_pin_sentence_is_its_own_must(self): + text = "Protocol version 3.2 is REQUIRED for all clients.\n2026-07-28\n" + assert [(label, s) for _, label, s in _parsed(text)] == [ + ("MUST", "Protocol version 3.2 is REQUIRED for all clients."), + ("MUST", "2026-07-28"), + ] + + def test_case_001_pin_stays_inside_its_must_sentence(self): + statements = _statements(_eval_doc("case-001-mcp-protocol-upgrade")) + assert "- The initialize request MUST carry `protocolVersion` exactly `\"2026-07-28\"`." in statements + assert all(s.strip("`\".") != "2026-07-28" for s in statements) + assert any("request header on every subsequent request" in s for s in statements) + + +class TestEvalCorpusUnits: + @pytest.mark.parametrize("case", sorted(p.name for p in EVALS.glob("case-*"))) + def test_units_are_verbatim_single_capped_lines(self, case): + for src in _eval_sources(case): + flat = " ".join(src.text.split()) + for unit in specs._spec_units(src, 0): + m = _RENDER_RE.match(unit.render) + assert m, unit.render + statement = m.group(3) + assert statement in flat, statement + assert len(statement) <= specs._STATEMENT_MAX_CHARS + + @pytest.mark.parametrize("case", sorted(p.name for p in EVALS.glob("case-*"))) + def test_digest_is_deterministic(self, case): + assert _eval_digest(case, with_diff=True) == _eval_digest(case, with_diff=True) + + +class TestNormativeTokensNeverScore: + _SHOULD = "Exports SHOULD include the invoice currency." + + def _digest(self, spec_text: str, diff_lines: list[str], budget: int) -> str: + diff = parse_unified_diff(_added_file("src/export.py", diff_lines)) + return build_spec_digest([_src(spec_text)], diff, token_budget=budget) + + def test_the_set_covers_every_strength_word(self): + assert { + "must", "shall", "required", "recommended", "forbidden", + "discouraged", "should", "never", "optional", + } <= specs._NORMATIVE_TOKENS + + def test_a_must_in_the_diff_does_not_reorder_the_digest(self): + spec_text = ( + "The audit ledger MUST be append-only.\n\n" + "Webhooks MUST be signed.\n\n" + "Backups MUST be encrypted at rest.\n\n" + f"{self._SHOULD}\n" + ) + control = self._digest(spec_text, ["def export(currency):"], budget=70) + keyword = self._digest(spec_text, ["def export(currency):", " # currency must be ISO-4217"], budget=70) + assert "invoice currency" in control + assert keyword == control + + def test_required_true_in_the_diff_does_not_reorder_the_digest(self): + spec_text = ( + "A signing key is REQUIRED for uploads.\n\n" + "An audit trail is REQUIRED for deletes.\n\n" + f"{self._SHOULD}\n" + ) + control = self._digest(spec_text, ['parser.add_argument("--currency")'], budget=70) + keyword = self._digest(spec_text, ['parser.add_argument("--currency", required=True)'], budget=70) + assert "invoice currency" in control + assert keyword == control + + def test_content_overlap_still_ranks(self): + spec_text = "Webhooks MUST be signed.\n\nThe invoice currency MUST be ISO-4217.\n" + digest = self._digest(spec_text, ["# currency must be set"], budget=3000) + assert digest.index("invoice currency MUST") < digest.index("Webhooks MUST be signed") From f3203972de5cca4b06da1b97e7cd2d5dea364237 Mon Sep 17 00:00:00 2001 From: Stephen Blatt <5125883+sblattj@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:46:12 -0700 Subject: [PATCH 15/61] feat: PromptContext plumbing, single-pass template fill, Finding.scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundation stage C, seat F2-DATA (#63, #64, #67 plumbing). reviewer.py: new public fill_template(template, values) fills every named placeholder in ONE re.sub pass, so a PR description, ticket or rules text that quotes {diff} (or any other placeholder) renders literally instead of receiving that value; unknown braces stay literal. Both renderers use it. New frozen PromptContext (rules_worker, rules_sweep, ticket_scope, ticket_context, spec_digest; scope_active = bool(ticket_scope)) and NO_PROMPT_CONTEXT replace the spec branch's spec_digest= keyword on every hop. Assembly order per contract section 5.1: SYSTEM = template head, then rules (worker or sweep framing), then ticket scope, joined by blank lines; USER = Review Context, then {ticket_context} (always ending in one blank line when set), then Spec constraints, then diff/digest. With nothing set the system prompt is the template head and the user prompt is unchanged. _finding_from/_invoke_and_parse gain accept_scope (scope read through normalize_scope only when the prompt asked for it). The reviewer meta and .meta.json reserve cost_usd=None and cost_source="" (8 base keys) for W67A to fill. triage.py: Finding.scope = "unknown", appended as the last field. orchestrator.py: _run_workers, _run_worker (both _invoke_chunk calls, timeout retry included), _invoke_chunk and _run_sweep thread prompt_context; _coerce_finding(accept_scope) gates the dict-path scope; orchestrate_review builds the one PromptContext from the spec digest. prompts: worker.md and systemic.md gain the {ticket_context} slot between Repo and Spec constraints. tests: the 7 strict doubles in test_orchestrator.py take prompt_context; test_reviewer.py's three spec_digest= call sites pass a PromptContext; new tests/test_prompt_context.py (66 tests). πŸ€– Authored with Claude Code β€” Claude Opus 5.5 via Claude Code Claude-Session-Id: f915b0ff-b27c-46f3-bc7c-430dca61d459 --- src/prxref/orchestrator.py | 49 +-- src/prxref/prompts/systemic.md | 2 +- src/prxref/prompts/worker.md | 2 +- src/prxref/reviewer.py | 198 ++++++++---- src/prxref/triage.py | 4 + tests/test_orchestrator.py | 17 +- tests/test_prompt_context.py | 532 +++++++++++++++++++++++++++++++++ tests/test_reviewer.py | 11 +- 8 files changed, 731 insertions(+), 84 deletions(-) create mode 100644 tests/test_prompt_context.py diff --git a/src/prxref/orchestrator.py b/src/prxref/orchestrator.py index 018aef00..6e4f6c1b 100644 --- a/src/prxref/orchestrator.py +++ b/src/prxref/orchestrator.py @@ -147,14 +147,17 @@ finding_rank_key, finding_sort_key, ) +from .reviewer import NO_PROMPT_CONTEXT, PromptContext from .trace import Tracer, get_tracer from .triage import ( DEFAULT_CONTEXT_LINES, DEFAULT_MAX_FILES_PER_CHUNK, DEFAULT_TOKEN_BUDGET, + SCOPE_UNKNOWN, Finding, added_lines_by_file, build_chunks, + normalize_scope, parse_unified_diff, ) @@ -540,12 +543,13 @@ def orchestrate_review( logger.error("spec grounding failed (best-effort): %s", e) tracer.event("specs", "fail", error=e.__class__.__name__) + prompt_context = PromptContext(spec_digest=spec_digest) reader = _make_file_reader(forge, ref, pr) results = _run_workers( llm, chunks, pr, max_tokens=max_tokens, max_workers=max_workers, context_lines=context_lines, tracer=tracer, reader=reader, all_files=files, trace_dir=trace_dir, - spec_digest=spec_digest, + prompt_context=prompt_context, ) # One more worker-style unit, not inside the pool: the sweep digests the @@ -558,7 +562,7 @@ def orchestrate_review( llm, files, pr, max_tokens=max_tokens, token_budget=token_budget, tracer=tracer, threads=threads, trace_dir=trace_dir, - spec_digest=spec_digest, + prompt_context=prompt_context, ) ) @@ -940,7 +944,8 @@ def _run_workers( llm: LLMClient, chunks, pr: PRData, *, max_tokens: int | None = None, max_workers: int = MAX_WORKERS, context_lines: int | None = None, tracer: Tracer | None = None, reader=None, all_files=None, - trace_dir: str | None = None, spec_digest: str = "", + trace_dir: str | None = None, + prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> list[dict]: # Never below 1: ThreadPoolExecutor rejects a zero-width pool, and a # library caller is not gated by config's range check. @@ -976,7 +981,7 @@ def _heartbeat() -> None: _run_worker, i + 1, len(chunks), llm, chunk, pr, max_tokens, context_lines, tracer, reader, all_files, trace_label=f"chunk{i}", trace_dir=trace_dir, - spec_digest=spec_digest, + prompt_context=prompt_context, ) for i, chunk in enumerate(chunks) ] @@ -1023,7 +1028,7 @@ def _invoke_chunk( max_tokens: int | None, context_lines: int | None, reader=None, *, include_definitions: bool = True, all_files=None, trace_label: str = "", trace_dir: str | None = None, - spec_digest: str = "", + prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> dict: """One normalized :func:`reviewer.review_chunk` call; never raises. @@ -1039,7 +1044,10 @@ def _invoke_chunk( retry, whose whole purpose is a smaller prompt. ``all_files`` is the PR's full parsed file list; the reviewer reduces it to the bounded sibling summary, which survives the retry because refuting evidence is not - bulk context. + bulk context. ``prompt_context`` (rules, ticket, spec digest) is passed + unchanged on both attempts: it is intent, not bulk context, and a + dict-shaped finding keeps its ``scope`` only when + :attr:`reviewer.PromptContext.scope_active`. """ blocks = _context_blocks(chunk, reader, include_definitions=include_definitions) try: @@ -1048,7 +1056,7 @@ def _invoke_chunk( max_tokens=max_tokens, context_lines=context_lines, context_blocks=blocks, sibling_files=all_files or (), trace_label=trace_label, trace_dir=trace_dir or "", - spec_digest=spec_digest, + prompt_context=prompt_context, ) except Exception as e: # noqa: BLE001 return { @@ -1071,7 +1079,7 @@ def _invoke_chunk( findings = [] for item in res.get("findings") or []: - finding = _coerce_finding(item) + finding = _coerce_finding(item, accept_scope=prompt_context.scope_active) if finding is not None: findings.append(finding) @@ -1090,7 +1098,7 @@ def _run_worker( max_tokens: int | None = None, context_lines: int | None = None, tracer: Tracer | None = None, reader=None, all_files=None, trace_label: str = "", trace_dir: str | None = None, - *, spec_digest: str = "", + *, prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> dict: tracer = tracer if tracer is not None else get_tracer() t0 = time.perf_counter() @@ -1107,7 +1115,7 @@ def _run_worker( ) res = _invoke_chunk( llm, chunk, pr, max_tokens, context_lines, reader, all_files=all_files, - trace_label=trace_label, trace_dir=trace_dir, spec_digest=spec_digest, + trace_label=trace_label, trace_dir=trace_dir, prompt_context=prompt_context, ) if ( res["error"] @@ -1133,7 +1141,7 @@ def _run_worker( llm, chunk, pr, max_tokens, _TIMEOUT_RETRY_CONTEXT_LINES, reader, include_definitions=False, all_files=all_files, trace_label=trace_label, trace_dir=trace_dir, - spec_digest=spec_digest, + prompt_context=prompt_context, ) error = res["error"] @@ -1172,7 +1180,7 @@ def _run_sweep( tracer: Tracer | None = None, threads: Sequence[Thread] = (), trace_dir: str | None = None, - spec_digest: str = "", + prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> dict: """Run the whole-PR systemic sweep as one worker-style review unit. @@ -1180,8 +1188,11 @@ def _run_sweep( ``token_budget``), makes ONE single-shot call through :func:`reviewer.review_systemic` β€” so ``PRXREF_LLM_MAX_TOKENS``, the timeout, and the model fallback chain all apply as to any chunk β€” and - returns the same result shape a chunk worker does. ``spec_digest`` rides - along into the sweep prompt's Spec constraints block. A failure is that + returns the same result shape a chunk worker does. ``prompt_context`` + rides along into the sweep prompt (sweep rules and ticket scope in the + system half, ticket context and the spec digest in the user half), and a + dict-shaped finding keeps its ``scope`` only when + :attr:`reviewer.PromptContext.scope_active`. A failure is that shape with ``error`` set prefixed ``systemic sweep:``, so the partial-review banner names the unit that failed; it counts as one failed chunk in the caller's coverage accounting. @@ -1204,7 +1215,7 @@ def _run_sweep( llm, digest, pr_title=pr.title, pr_description=pr.description, max_tokens=max_tokens, threads=discussion, trace_label="sweep", trace_dir=trace_dir or "", - spec_digest=spec_digest, + prompt_context=prompt_context, ) except Exception as e: # noqa: BLE001 logger.error("[sweep] raised: %s", e) @@ -1220,7 +1231,7 @@ def _run_sweep( findings = [] for item in findings_raw: - finding = _coerce_finding(item) + finding = _coerce_finding(item, accept_scope=prompt_context.scope_active) if finding is not None: findings.append(finding) @@ -1251,7 +1262,7 @@ def _run_sweep( } -def _coerce_finding(item) -> Finding | None: +def _coerce_finding(item, *, accept_scope: bool = False) -> Finding | None: if isinstance(item, Finding): return item if isinstance(item, dict): @@ -1263,6 +1274,10 @@ def _coerce_finding(item) -> Finding | None: confidence=float(item.get("confidence") or 0.0), title=str(item.get("title") or ""), body=str(item.get("body") or ""), + scope=( + normalize_scope(item.get("scope")) if accept_scope + else SCOPE_UNKNOWN + ), ) except (KeyError, TypeError, ValueError) as e: logger.warning("dropping malformed finding %r: %s", item, e) diff --git a/src/prxref/prompts/systemic.md b/src/prxref/prompts/systemic.md index cc675d2d..b5c5a544 100644 --- a/src/prxref/prompts/systemic.md +++ b/src/prxref/prompts/systemic.md @@ -45,7 +45,7 @@ PR description: Repo: {repo_hint} -### Spec constraints +{ticket_context}### Spec constraints {spec_digest} diff --git a/src/prxref/prompts/worker.md b/src/prxref/prompts/worker.md index 2bb52177..1622cdf5 100644 --- a/src/prxref/prompts/worker.md +++ b/src/prxref/prompts/worker.md @@ -42,7 +42,7 @@ PR description: Repo: {repo_hint} -### Spec constraints +{ticket_context}### Spec constraints {spec_digest} diff --git a/src/prxref/reviewer.py b/src/prxref/reviewer.py index db780d8d..43274dc9 100644 --- a/src/prxref/reviewer.py +++ b/src/prxref/reviewer.py @@ -22,6 +22,16 @@ :func:`_render_discussion_block` under the ``DISCUSSION_MAX_*`` caps) so the sweep stops re-raising subjects the team already argued out. +Inputs an operator or a ticket supplies for the whole run ride one frozen +:class:`PromptContext` through every hop, in one fixed order: team rules and +the ticket-scope instructions are appended to the SYSTEM half (policy), while +the ticket context and the spec digest are filled into the USER half ahead of +the diff (per-PR data). Each template is filled in one pass by +:func:`fill_template`, so a value that contains another placeholder (a PR +description quoting ``{diff}``) renders literally. With +:data:`NO_PROMPT_CONTEXT` the system prompt is the template head unchanged and +the user prompt gains nothing. + Both ``prompts/worker.md`` and ``prompts/systemic.md`` require a throw/panic/crash/unhandled-rejection finding to name its containment boundary; :func:`prxref.quality.apply_containment_note` @@ -38,8 +48,10 @@ import json import logging import os +import re import time -from collections.abc import Sequence +from collections.abc import Mapping, Sequence +from dataclasses import dataclass from importlib import resources from typing import Any @@ -47,7 +59,7 @@ from .forges.base import Thread from .llm import LLMClient from .parser import loads_lenient -from .triage import FileDiff, Finding, trim_hunk_context +from .triage import SCOPE_UNKNOWN, FileDiff, Finding, normalize_scope, trim_hunk_context logger = logging.getLogger("prxref") @@ -112,6 +124,65 @@ def load_prompt(name: str) -> str: return resources.files("prxref").joinpath("prompts").joinpath(fname).read_text(encoding="utf-8") +def fill_template(template: str, values: Mapping[str, str]) -> str: + """Replace each ``{name}`` in ``template`` whose name is a key of ``values``. + + One :func:`re.sub` pass over the template: substituted text is never + scanned again, so a value that itself contains ``{diff}`` or any other + placeholder renders literally instead of receiving that placeholder's + value. Braces whose name is not a key (the JSON example in + ``## Output Format``, a literal ``{foo}``) stay as written. Values must + be strings and are inserted verbatim; backslashes are not interpreted. + An empty ``values`` returns the template unchanged. + """ + if not values: + return template + pattern = re.compile(r"\{(" + "|".join(re.escape(k) for k in values) + r")\}") + return pattern.sub(lambda m: values[m.group(1)], template) + + +@dataclass(frozen=True) +class PromptContext: + """Run-wide inputs injected into every review unit's prompt, in one fixed order. + + SYSTEM half, appended to the template head in this order: + ``rules_worker`` for chunk units or ``rules_sweep`` for the sweep (the + team review rules), then ``ticket_scope`` (the instructions that ask the + model for a per-finding ``scope``). USER half, after the Review Context + lines: ``ticket_context`` (the fenced ticket text), then ``spec_digest`` + (the Spec constraints block), then the diff or digest. + + Every field defaults to ``""``, which injects nothing; ``spec_digest`` + empty renders ``(no specs provided for this review)`` as before. + :attr:`scope_active` is true only when the scope instructions are in the + prompt, and it alone decides whether a model-supplied ``scope`` is read. + """ + + rules_worker: str = "" + rules_sweep: str = "" + ticket_scope: str = "" + ticket_context: str = "" + spec_digest: str = "" + + @property + def scope_active(self) -> bool: + """True when the prompt asks for ``scope``, so the answer may be kept.""" + return bool(self.ticket_scope) + + +NO_PROMPT_CONTEXT = PromptContext() + + +def _append_block(system: str, block: str) -> str: + block = block.strip() + return f"{system}\n\n{block}" if block else system + + +def _ticket_context_value(prompt_context: PromptContext) -> str: + block = prompt_context.ticket_context.strip() + return f"{block}\n\n" if block else "" + + def _render_file(f: FileDiff, context_lines: int | None = None) -> str: old = f.old_path or f.new_path or f.path new = f.new_path or f.old_path or f.path @@ -157,7 +228,8 @@ def _render_prompt( context_lines: int | None = None, context_blocks: str = "", sibling_files: Sequence[FileDiff] = (), - spec_digest: str = "", + *, + prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> tuple[str, str]: template = load_prompt("worker.md") head, marker, tail = template.partition(_CONTEXT_MARKER) @@ -165,22 +237,18 @@ def _render_prompt( raise ValueError(f"worker.md is missing the {_CONTEXT_MARKER!r} split marker") sibling_block = sibling_summary_block(chunk, sibling_files) blocks = "\n\n".join(b for b in (sibling_block, context_blocks.strip()) if b) - user = ( - marker + tail - ).replace( - "{pr_title}", pr_title.strip() or "(untitled)" - ).replace( - "{pr_description}", pr_description.strip() or "(none)" - ).replace( - "{repo_hint}", repo_hint.strip() or "(unspecified)" - ).replace( - "{context_blocks}", blocks - ).replace( - "{spec_digest}", spec_digest.strip() or _NO_SPECS_TEXT - ).replace( - "{diff}", render_chunk(chunk, context_lines) or "(empty chunk)" - ) - return head.strip(), user.strip() + user = fill_template(marker + tail, { + "pr_title": pr_title.strip() or "(untitled)", + "pr_description": pr_description.strip() or "(none)", + "repo_hint": repo_hint.strip() or "(unspecified)", + "ticket_context": _ticket_context_value(prompt_context), + "spec_digest": prompt_context.spec_digest.strip() or _NO_SPECS_TEXT, + "context_blocks": blocks, + "diff": render_chunk(chunk, context_lines) or "(empty chunk)", + }) + system = _append_block(head.strip(), prompt_context.rules_worker) + system = _append_block(system, prompt_context.ticket_scope) + return system, user.strip() def _render_discussion_block(threads: Sequence[Thread]) -> str: @@ -222,30 +290,28 @@ def _render_systemic_prompt( pr_description: str, repo_hint: str, threads: Sequence[Thread] = (), - spec_digest: str = "", + *, + prompt_context: PromptContext = NO_PROMPT_CONTEXT, ) -> tuple[str, str]: template = load_prompt("systemic.md") head, marker, tail = template.partition(_CONTEXT_MARKER) if not marker: raise ValueError(f"systemic.md is missing the {_CONTEXT_MARKER!r} split marker") - user = ( - marker + tail - ).replace( - "{pr_title}", pr_title.strip() or "(untitled)" - ).replace( - "{pr_description}", pr_description.strip() or "(none)" - ).replace( - "{repo_hint}", repo_hint.strip() or "(unspecified)" - ).replace( - "{spec_digest}", spec_digest.strip() or _NO_SPECS_TEXT - ).replace( - "{digest}", digest.strip() or "(empty digest)" - ) + user = fill_template(marker + tail, { + "pr_title": pr_title.strip() or "(untitled)", + "pr_description": pr_description.strip() or "(none)", + "repo_hint": repo_hint.strip() or "(unspecified)", + "ticket_context": _ticket_context_value(prompt_context), + "spec_digest": prompt_context.spec_digest.strip() or _NO_SPECS_TEXT, + "digest": digest.strip() or "(empty digest)", + }) discussion = _render_discussion_block(threads) user = user.strip() if discussion: user = f"{user}\n\n{discussion}" - return head.strip(), user + system = _append_block(head.strip(), prompt_context.rules_sweep) + system = _append_block(system, prompt_context.ticket_scope) + return system, user def _as_int(value: Any, default: int = 0) -> int: @@ -255,7 +321,7 @@ def _as_int(value: Any, default: int = 0) -> int: return default -def _finding_from(raw: Any) -> Finding | None: +def _finding_from(raw: Any, *, accept_scope: bool = False) -> Finding | None: if not isinstance(raw, dict): return None file = str(raw.get("file") or raw.get("path") or "").strip() @@ -272,6 +338,7 @@ def _finding_from(raw: Any) -> Finding | None: confidence=confidence, title=str(raw.get("title") or "").strip(), body=str(raw.get("body") or "").strip(), + scope=normalize_scope(raw.get("scope")) if accept_scope else SCOPE_UNKNOWN, ) @@ -290,7 +357,9 @@ def _write_trace_files( prompt halves, ``