Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions bin/findings.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@
"excessive-post-ship-iteration",
# v1.1 Fix 3 — behavioral claim with structural-only verification
"verification-too-shallow-for-claim",
# v1.2 Fix C — Tier-3 thin negative-path coverage alongside-finding
"tier3-negative-paths-thin-coverage",
}

# Severity mapping for Tier 3 contradiction tuple kinds (v0.5.2).
Expand All @@ -118,6 +120,8 @@
"adversarial-pathway": "block",
# v0.8 — contract-filter audit sentinel
"tier3-filter-applied": "info",
# v1.2 Fix C — thin negative-path coverage alongside demotion
"tier3-negative-paths-thin-coverage": "warn",
}

SEVERITIES = {"block", "warn", "info"}
Expand Down
54 changes: 50 additions & 4 deletions bin/llm_judge.py
Original file line number Diff line number Diff line change
Expand Up @@ -1005,11 +1005,29 @@ def _faithfulness_malformed_finding() -> Finding:
)


def _thin_coverage_finding(original_finding: Finding) -> Finding:
"""Return a tier3-negative-paths-thin-coverage warn alongside a demoted negative-path-omission."""
step_n = original_finding.location.step
msg = (
f"Step {step_n} has fewer than 3 negative-paths entries (thin coverage); "
"consider adding more failure branches."
)[:findings.MAX_MESSAGE_LEN]
return Finding(
tier=3,
kind="tier3-negative-paths-thin-coverage",
severity="warn",
location=original_finding.location,
message=msg,
dismissable=True,
)


def _verify_block_tuples_with_citations(
tuple_findings: list[Finding],
step_table: dict,
*,
config: JudgeConfig,
step_objects: list | None = None,
) -> list[Finding]:
"""Run a second batched API call to verify block-severity contradiction tuples.

Expand All @@ -1032,15 +1050,41 @@ def _verify_block_tuples_with_citations(
Returns a new finding list with the same non-block findings plus either
verified-or-demoted block findings.
"""
# Build negative-paths count lookup from step_objects (Fix C).
# Done first so thin-coverage logic can run even when no block findings exist.
_neg_paths_count: dict[int, int] = {}
if step_objects:
for obj in step_objects:
if isinstance(obj, dict):
sn = obj.get("step")
np = obj.get("negative_paths") or obj.get("negative-paths") or []
else:
sn = getattr(obj, "step", None)
np = getattr(obj, "negative_paths", None) or []
if sn is not None:
_neg_paths_count[int(sn)] = len(np) if np else 0

# Separate block (verifiable) from non-block (pass through).
block_indices: list[int] = []
for idx, f in enumerate(tuple_findings):
if f.kind in _BLOCK_CONTRADICTION_KINDS and f.severity == "block":
block_indices.append(idx)

# Short-circuit: nothing to verify.
# Fix C: emit thin-coverage alongside-finding for any negative-path-omission
# finding whose step has fewer than 3 negative-paths entries.
# This runs regardless of whether block findings exist.
thin_coverage_additions: list[Finding] = []
for f in tuple_findings:
if f.kind == "negative-path-omission":
step_n = f.location.step
np_count = _neg_paths_count.get(step_n, 0) if step_n is not None else 0
if np_count < 3:
thin_coverage_additions.append(_thin_coverage_finding(f))

# Short-circuit: nothing to verify via faithfulness check.
if not block_indices:
return list(tuple_findings)
result = list(tuple_findings) + thin_coverage_additions
return result

# Build a minimal representation of block findings to send to DeepSeek.
block_summaries = []
Expand Down Expand Up @@ -1149,7 +1193,7 @@ def _verify_block_tuples_with_citations(
if should_demote:
result[orig_idx] = _faithfulness_demote_finding(original_finding)

return result
return result + thin_coverage_additions


# ── Deterministic contract-resolution post-filter ────────────────────────────
Expand Down Expand Up @@ -1404,4 +1448,6 @@ def evaluate(

# Second pass: cite-and-verify for block-severity tuples (v0.6 faithfulness check).
# Zero extra cost when no block tuples; one batched call otherwise.
return _verify_block_tuples_with_citations(primary_findings, step_table, config=config)
return _verify_block_tuples_with_citations(
primary_findings, step_table, config=config, step_objects=step_objects
)
6 changes: 3 additions & 3 deletions bin/spec_ast.py
Original file line number Diff line number Diff line change
Expand Up @@ -1641,7 +1641,7 @@ def _check_self_cycle_produces(steps: list[dict]) -> list[_findings.Finding]:
continue
# Self-cycle confirmed
display = tok if len(tok) <= 60 else "..." + tok[-57:]
msg = f"Step {step_n} action consumes {display!r} which it also produces (self-cycle)."
msg = f"Step {step_n} action references {display!r} which it also declares in produces (possible self-cycle)."
if len(msg) > 140:
msg = msg[:137] + "..."
results.append(_findings.Finding(
Expand All @@ -1653,8 +1653,8 @@ def _check_self_cycle_produces(steps: list[dict]) -> list[_findings.Finding]:
),
message=msg,
suggested_fix=(
"Move the file to a prior step's produces:, or remove it from "
"this step's produces: if it is an input, not an output."
"If action only names the path, remove from produces:. "
"If action reads X, change verification to assert idempotency."
)[:140],
))

Expand Down
140 changes: 140 additions & 0 deletions bin/walker.py
Original file line number Diff line number Diff line change
Expand Up @@ -315,6 +315,12 @@ def record_answer(state: WalkState, *, concern_id: str, answer: str) -> WalkStat
state.answered[concern_id] = answer
del state.pending[i]
state.round_count += 1
# Fix D: per-round visibility for operators.
try:
from bin import _status as _st
_st.emit("info", "walker.round", round=state.round_count, pending=len(state.pending))
except Exception: # noqa: BLE001
pass
# Flip seed-family flags
if concern_id == "seed-lifecycle":
state.lifecycle_asked = True
Expand Down Expand Up @@ -1392,6 +1398,140 @@ def generate_negative_path_concerns(
return concerns


# ── Step-action precision concerns (Fix B) ───────────────────────────────────

# pip install without pinned version: no `==`, no `@`, no `-r` constraint file, no `--constraint`.
_PRECISION_PIP_UNVERSIONED_RE = re.compile(
r"\bpip\s+install\b(?!.*(?:==|@\s*\S|(?:-r|-c|--constraint|--requirement)\s+\S))",
)
# python -m <pkg> with no subcommand/argument after the package name.
_PRECISION_PYTHON_M_BARE_RE = re.compile(
r"\bpython3?\s+-m\s+([\w.]+)\s*$"
)
# Bare URL with no version token nearby (http(s):// not followed by a version token on the same line).
_PRECISION_BARE_URL_RE = re.compile(
r"https?://\S+"
)
_PRECISION_VERSION_TOKEN_RE = re.compile(
r"(?:==|@\s*v?[\d.]+|/v[\d.]+/|/[\d.]+/)"
)
# LLM vendor SDK + bare model identifier patterns.
_PRECISION_LLM_VENDOR_RE = re.compile(
r"\b(anthropic|openai|deepseek|ollama|client\.messages\.create|client\.chat\.completions|client\.responses\.create)\b",
re.IGNORECASE,
)
_PRECISION_BARE_MODEL_ID_RE = re.compile(
r"\b(gpt-\d+[a-z0-9-]*|claude-[a-z0-9-]+|deepseek-[a-z0-9-]+|llama-[a-z0-9-]+|mistral-[a-z0-9-]+|gemma-[a-z0-9-]+|command-[a-z0-9-]+)\b",
re.IGNORECASE,
)


def _check_step_precision(step_n: int, action: str) -> list[tuple[str, str]]:
"""Return list of (concern_id, summary) for vague shapes in *action*.

Checks (structural patterns):
1. pip install without version pin.
2. python -m <pkg> with no subcommand args.
3. Bare URL with no version-pin verification nearby.
4. Bare model ID alongside LLM vendor SDK call.
"""
concerns: list[tuple[str, str]] = []
a = action.strip()

# 1. pip install without version pin
if _PRECISION_PIP_UNVERSIONED_RE.search(a):
concerns.append((
f"precision-pip-{step_n}",
(
f"Step {step_n} action `{a[:60]}` has `pip install` without a pinned version — "
"add `==X.Y.Z` or a constraint file (e.g. `-c constraints.txt`) to make the "
"build reproducible."
)[:280],
))

# 2. python -m <pkg> with no subcommand args
m = _PRECISION_PYTHON_M_BARE_RE.search(a)
if m:
pkg = m.group(1)
concerns.append((
f"precision-python-m-{step_n}",
(
f"Step {step_n} action `{a[:60]}` invokes `python -m {pkg}` with no subcommand "
"or arguments — specify the subcommand (e.g. `python -m myapp serve`) so the "
"intent is unambiguous."
)[:280],
))

# 3. Bare URL with no version-pin token in the same action
url_m = _PRECISION_BARE_URL_RE.search(a)
if url_m:
url = url_m.group(0)[:60]
if not _PRECISION_VERSION_TOKEN_RE.search(a):
concerns.append((
f"precision-url-{step_n}",
(
f"Step {step_n} action contains a bare URL (`{url}`) with no version pin — "
"pin to a specific version tag or commit so the download is reproducible."
)[:280],
))

# 4. Bare model ID alongside LLM vendor SDK
if _PRECISION_LLM_VENDOR_RE.search(a):
model_m = _PRECISION_BARE_MODEL_ID_RE.search(a)
if model_m:
model_id = model_m.group(0)
concerns.append((
f"precision-model-{step_n}",
(
f"Step {step_n} action uses bare model ID `{model_id}` alongside a vendor SDK — "
"document the model selection logic (env var, config key, or version-locked constant) "
"so the choice is explicit and auditable."
)[:280],
))

return concerns


def generate_step_precision_concerns(
state: WalkState,
steps: list[dict],
) -> list[Concern]:
"""Emit edge-case concerns for vague action shapes in each step.

Checks four structural patterns per step action (Fix B):
1. pip install without version pin.
2. python -m <pkg> with no subcommand.
3. Bare URL with no version-pin verification.
4. Bare model ID alongside LLM vendor SDK.

Idempotent: never emits a concern whose id already exists in state.
"""
existing_ids: set[str] = (
{c.id for c in state.asked}
| {c.id for c in state.pending}
| set(state.answered)
)
concerns: list[Concern] = []
for step in steps:
step_n = step.get("step")
if step_n is None:
continue
action: str = step.get("action", "") or ""
if not action:
continue
for concern_id, summary in _check_step_precision(step_n, action):
if concern_id in existing_ids:
continue
concerns.append(Concern(
id=concern_id,
kind="edge-case",
receivers=["human"],
depends_on=[],
summary=summary,
))
return concerns


# ── Scaffold-precondition concern ─────────────────────────────────────────────

# Stdlib top-level modules for `python -m <pkg>` heuristic — these do NOT need
Expand Down
18 changes: 18 additions & 0 deletions docs/glossary.md
Original file line number Diff line number Diff line change
Expand Up @@ -1189,3 +1189,21 @@ Status codes: dotted identifiers like `walker.init`. Terms: `term:<noun>` prefix
- user_action: Run `spectre catalog upgrade-taxonomy --spec <slug> --to <version>` when you want to consider the newer axes, or ignore.
- related: term:taxonomy-version
- since: v1.0

## tier3-negative-paths-thin-coverage
- kind: finding
- dev: Emitted alongside any `negative-path-omission` finding whose step has fewer than 3 `negative-paths:` entries. No demotion is involved — `negative-path-omission` is info-severity and never enters the faithfulness demotion path. The pairing signals "the LLM judge flagged a missing failure branch AND the step's structural coverage is thin." Tier-3 warn, dismissable.
- pm: A step's failure-branch coverage is thin (fewer than 3 entries) and the automated review flagged a missing failure scenario. Consider adding more failure scenarios to the step's negative-paths section.
- triggered_by: Co-occurrence of a `negative-path-omission` finding (LLM-judge output) and `< 3` negative-paths entries on the affected step.
- user_action: Add more negative-paths entries to the flagged step (at least 3 entries covering different failure modes), or dismiss if the step genuinely has only one or two realistic failure branches.
- related: negative-path-omission
- since: v1.2

## walker.round
- kind: status
- dev: Emitted after each concern answer in the walker interview loop. Fields: round=N (1-based count of answered concerns), pending=K (remaining non-stale concerns). Provides per-round visibility into walk progress without exposing convergence decisions.
- pm: The walker just finished interview round N. There are K questions still to answer.
- triggered_by: walker.record_answer increments round_count.
- user_action: No action required. Monitor round and pending counts to gauge walk progress. Operator interpretation only — walker.round does not imply any threshold or convergence signal.
- related: walker.yield, walker.coverage
- since: v1.2
Loading
Loading