From c46f4d2debea29ce0483e1fdd6ff3e0a684e9121 Mon Sep 17 00:00:00 2001 From: Copilot Date: Fri, 15 May 2026 23:49:50 +0200 Subject: [PATCH] feat(layer5): v1.3 reasoning-trace prototype at three choice points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three commit sites instrument machine-parseable reasoning traces: 1. walker-concern — `bin/walker.py::record_answer` emits a trace before `state.answered[concern_id] = answer` when the concern has >=2 prefab options. Exemplar-binding concerns (6 known IDs) emit choice_point= "exemplar-binding" with validation_anchor="tier2-cross-view-gate" instead. 2. substitution — `bin/eval_metadata.py::write_sidecar` accepts a new `layer5_trace: list[dict] | None` kwarg and writes it under the top-level `layer5_trace` key. `bin/_layer5.project_substitution_trace` projects an extended substitution dict (carrying optional_considered + rationale) into a trace record. One event source, two payload keys. 3. exemplar-binding — handled through the same `record_answer` path as walker-concern (the concern carries catalog slugs as prefab_options). Schema: six fields + timestamp — choice_point, step_or_concern_id, options_considered, selected, rationale, validation_anchor, source_anchor. Lives in `.eval.json` sidecar under `layer5_trace: [...]`. No new file type. New `bin/_layer5.py` module owns `build_trace_record()` and `project_substitution_trace()`. `SPECTRE_LAYER5=off` disables emission entirely for token-cost A/B tests. Skips when options_considered <= 1. WalkState gains `layer5_trace: list[dict]` field, persisted + loaded (backwards-compat: defaults to [] on old state files). Both JSON-mode CLI payloads (init-or-resume, get-state) include the field. Glossary: two new term entries — `term:layer5-trace` and `term:layer5-choice-point`. Tests: 20 new tests across three files, all green. Full suite 1987 passed. Co-Authored-By: Claude Opus 4.7 --- bin/_layer5.py | 113 ++++++++++++++++++ bin/eval_metadata.py | 15 +++ bin/walker.py | 49 ++++++++ docs/glossary.md | 14 +++ tests/test_layer5_exemplar_trace.py | 150 ++++++++++++++++++++++++ tests/test_layer5_substitution_trace.py | 149 +++++++++++++++++++++++ tests/test_layer5_walker_trace.py | 125 ++++++++++++++++++++ 7 files changed, 615 insertions(+) create mode 100644 bin/_layer5.py create mode 100644 tests/test_layer5_exemplar_trace.py create mode 100644 tests/test_layer5_substitution_trace.py create mode 100644 tests/test_layer5_walker_trace.py diff --git a/bin/_layer5.py b/bin/_layer5.py new file mode 100644 index 0000000..2d35aec --- /dev/null +++ b/bin/_layer5.py @@ -0,0 +1,113 @@ +"""bin/_layer5.py — Layer 5 self-interrogation trace helper (v1.3 prototype). + +Provides build_trace_record() used by walker and eval_metadata at the three +named choice points: + - walker-concern : concern-resolution in walker.record_answer + - substitution : substitution logged in eval_metadata.write_sidecar + - exemplar-binding : exemplar binding committed in walker + +Schema (six fields + timestamp, machine-parseable JSON): + { + "choice_point": "walker-concern" | "substitution" | "exemplar-binding", + "step_or_concern_id": "", + "options_considered": ["", "", ""], + "selected": "", + "rationale": "", + "validation_anchor": "", + "source_anchor": "", + "timestamp": "" + } + +Emission is skipped when: + - SPECTRE_LAYER5=off environment variable is set + - len(options_considered) <= 1 (no real choice) + +Stdlib only. No third-party dependencies. +""" +from __future__ import annotations + +import os +from datetime import datetime, timezone +from typing import Any + +# Valid choice-point names. +CHOICE_POINTS: frozenset[str] = frozenset({ + "walker-concern", + "substitution", + "exemplar-binding", +}) + +_LAYER5_DISABLED_SENTINEL = "off" + + +def _layer5_enabled() -> bool: + """Return False when SPECTRE_LAYER5=off disables trace emission.""" + return os.environ.get("SPECTRE_LAYER5", "").lower() != _LAYER5_DISABLED_SENTINEL + + +def build_trace_record( + *, + choice_point: str, + step_or_concern_id: str, + options_considered: list[str], + selected: str, + rationale: str, + validation_anchor: str | None, + source_anchor: str | None, +) -> dict[str, Any] | None: + """Build a Layer 5 trace record dict and return it, or None if emission is skipped. + + Returns None (skips emission) when: + - SPECTRE_LAYER5=off + - len(options_considered) <= 1 (no real choice to trace) + + The caller appends the returned dict to state.layer5_trace (walker) or + includes it in the sidecar payload projection (eval_metadata). + """ + if not _layer5_enabled(): + return None + if len(options_considered) <= 1: + return None + if choice_point not in CHOICE_POINTS: + raise ValueError( + f"unknown choice_point {choice_point!r}; expected one of {sorted(CHOICE_POINTS)}" + ) + return { + "choice_point": choice_point, + "step_or_concern_id": step_or_concern_id, + "options_considered": list(options_considered), + "selected": selected, + "rationale": rationale, + "validation_anchor": validation_anchor, + "source_anchor": source_anchor, + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + } + + +def project_substitution_trace(substitution: dict[str, Any]) -> dict[str, Any] | None: + """Project a substitution dict into a layer5_trace record. + + The substitution dict (v1.2.1 #6 schema extended by v1.3 #10) may carry + optional layer5 fields: + options_considered, selected, rationale, validation_anchor, source_anchor + + Returns a trace record if enough fields are present and emission is not + disabled, else None. + """ + options = substitution.get("options_considered") + if not isinstance(options, list): + return None + selected = substitution.get("selected", substitution.get("to", "")) + rationale = substitution.get("rationale", substitution.get("reason", "")) + step_id = substitution.get("step_id", "") + validation_anchor = substitution.get("validation_anchor") + source_anchor = substitution.get("source_anchor") + return build_trace_record( + choice_point="substitution", + step_or_concern_id=step_id, + options_considered=options, + selected=selected, + rationale=rationale, + validation_anchor=validation_anchor, + source_anchor=source_anchor, + ) diff --git a/bin/eval_metadata.py b/bin/eval_metadata.py index d1d8824..422c73c 100644 --- a/bin/eval_metadata.py +++ b/bin/eval_metadata.py @@ -241,6 +241,7 @@ def write_sidecar( substrate_resolution: dict | None = None, findings_inline: list[dict] | None = None, substitutions: list[dict] | None = None, + layer5_trace: list[dict] | None = None, ) -> pathlib.Path: """Atomic write of .eval.json next to the spec file. @@ -275,6 +276,12 @@ def write_sidecar( "tier1_check_name": , "step_id": }`` recording an agent's rewrite of action content or verification commands to satisfy a Tier-1 check. Not a finding — contemporaneous evidence the operator can audit. + + *layer5_trace* — v1.3 #10 reasoning-trace records. When non-None, written + verbatim under the ``layer5_trace`` key. Each entry is a six-field dict + (choice_point, step_or_concern_id, options_considered, selected, rationale, + validation_anchor, source_anchor, timestamp). Projected from walker state + or from substitution dicts that carry the extended schema. """ spec_path = pathlib.Path(spec_path) sidecar_path = sidecar_path_for(spec_path) @@ -323,6 +330,13 @@ def write_sidecar( if substitutions is not None: payload["substitutions"] = substitutions + # v1.3 #10: Layer 5 reasoning-trace records. Each entry is a six-field + # dict (choice_point, step_or_concern_id, options_considered, selected, + # rationale, validation_anchor, source_anchor, timestamp). Projected + # from walker state or from substitution dicts with extended schema. + if layer5_trace is not None: + payload["layer5_trace"] = layer5_trace + # Atomic write: mkstemp + os.replace fd, tmp = tempfile.mkstemp( dir=sidecar_path.parent, prefix=sidecar_path.name, suffix=".tmp" @@ -506,6 +520,7 @@ def write_envelope_alongside_sidecar( substrate_resolution=payload.get("substrate_resolution"), findings_inline=payload.get("findings_inline"), substitutions=payload.get("substitutions"), + layer5_trace=payload.get("layer5_trace"), ) except KeyError as exc: _status.emit("error", "eval_metadata.sidecar_missing_field", dest="stderr", diff --git a/bin/walker.py b/bin/walker.py index c19c5a9..463e42c 100644 --- a/bin/walker.py +++ b/bin/walker.py @@ -26,6 +26,7 @@ sys.path.insert(0, str(_ROOT)) from bin import _catalog # noqa: E402 +from bin import _layer5 # noqa: E402 WALKER_VERSION = "1.0.0" @@ -94,6 +95,8 @@ class WalkState: human_user_asked: bool = False integrator_asked: bool = False operator_asked: bool = False + # v1.3 #10 — Layer 5 reasoning-trace records accumulated during this walk. + layer5_trace: list[dict] = field(default_factory=list) _OQ_INLINE_RE = re.compile( @@ -312,6 +315,32 @@ def record_answer(state: WalkState, *, concern_id: str, answer: str) -> WalkStat for i, c in enumerate(state.pending): if c.id == concern_id: state.asked.append(c) + # v1.3 #10 — emit layer5 trace when a real choice exists. + # Skip when prefab_options has 0 or 1 entry (no meaningful choice). + if len(c.prefab_options) > 1: + _is_exemplar = concern_id in _EXEMPLAR_BINDING_CONCERN_IDS + _choice_point = "exemplar-binding" if _is_exemplar else "walker-concern" + _rationale = ( + f"Operator bound exemplar '{answer}' for view concern '{concern_id}' " + f"from {len(c.prefab_options)} catalog options." + if _is_exemplar else + f"Operator selected '{answer}' for concern '{concern_id}' " + f"({c.summary[:80]}…) after considering " + f"{len(c.prefab_options)} prefab options." + ) + _trace = _layer5.build_trace_record( + choice_point=_choice_point, + step_or_concern_id=concern_id, + options_considered=list(c.prefab_options), + selected=answer, + rationale=_rationale, + validation_anchor=( + "tier2-cross-view-gate" if _is_exemplar else "tier1-structural" + ), + source_anchor=None, + ) + if _trace is not None: + state.layer5_trace.append(_trace) state.answered[concern_id] = answer del state.pending[i] state.round_count += 1 @@ -507,6 +536,8 @@ def persist(state: WalkState, path: pathlib.Path) -> None: "human_user_asked": state.human_user_asked, "integrator_asked": state.integrator_asked, "operator_asked": state.operator_asked, + # v1.3 #10 — layer5 reasoning-trace records + "layer5_trace": list(state.layer5_trace), } path.parent.mkdir(parents=True, exist_ok=True) fd, tmp = tempfile.mkstemp(dir=str(path.parent), prefix=path.name, suffix=".tmp") @@ -569,6 +600,8 @@ def load(path: pathlib.Path) -> WalkState | None: human_user_asked=data.get("human_user_asked", False), integrator_asked=data.get("integrator_asked", False), operator_asked=data.get("operator_asked", False), + # v1.3 #10 — layer5 trace (defaults to empty list for backwards compat) + layer5_trace=list(data.get("layer5_trace", [])), ) @@ -918,6 +951,18 @@ def generate_semantic_criteria_concern( # _VIEW_SCOPE_CONCERN_IDS, kept alongside for co-location). _SCOPE_CONCERN_TO_VIEW: dict[str, str] = {v: k for k, v in _VIEW_SCOPE_CONCERN_IDS.items()} +# v1.3 #10 — Concern IDs that represent exemplar-binding choices (catalog slug +# selection). When one of these is answered via record_answer, a layer5 trace +# with choice_point="exemplar-binding" is emitted instead of "walker-concern". +_EXEMPLAR_BINDING_CONCERN_IDS: frozenset[str] = frozenset({ + "input-exemplar-pi", + "help-text-style-hu", + "error-text-style-hu", + "api-exemplar-int", + "log-format-style-op", + "observability-style-op", +}) + def _exemplar_options_for( fingerprint: str | None, @@ -2416,6 +2461,8 @@ def _drive_to_completeness_satisfied( "human_user_asked": state.human_user_asked, "integrator_asked": state.integrator_asked, "operator_asked": state.operator_asked, + # v1.3 #10 — layer5 reasoning-trace records + "layer5_trace": list(state.layer5_trace), } print(json.dumps(payload, indent=2, sort_keys=True)) else: @@ -2591,6 +2638,8 @@ def _drive_to_completeness_satisfied( "human_user_asked": state.human_user_asked, "integrator_asked": state.integrator_asked, "operator_asked": state.operator_asked, + # v1.3 #10 — layer5 reasoning-trace records + "layer5_trace": list(state.layer5_trace), } print(json.dumps(payload, indent=2, sort_keys=True)) else: diff --git a/docs/glossary.md b/docs/glossary.md index 51f00f2..91fb2a6 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1234,3 +1234,17 @@ Status codes: dotted identifiers like `walker.init`. Terms: `term:` prefix - user_action: No action required. Monitor round and pending counts to gauge walk progress. Operator interpretation only — walker.round does not imply any threshold or convergence signal. - related: walker.yield, walker.coverage - since: v1.2 + +## term:layer5-trace +- kind: term +- dev: A Layer 5 reasoning-trace record emitted at a named choice point (walker-concern, substitution, or exemplar-binding). Six fields: choice_point, step_or_concern_id, options_considered, selected, rationale, validation_anchor, source_anchor, plus timestamp. Written to the `.eval.json` sidecar under `layer5_trace: [...]`. Skipped when SPECTRE_LAYER5=off or when options_considered has <=1 entry (no real choice). +- pm: A logged record of why the system made a specific choice at a key decision point. This audit trail lets reviewers verify the reasoning behind each significant design or binding selection. +- related: term:layer5-choice-point, eval.substitutions +- since: v1.3 + +## term:layer5-choice-point +- kind: term +- dev: One of three named decision sites where Layer 5 reasoning-trace records are emitted. (1) walker-concern — operator selects from prefab options when answering a walker concern. (2) substitution — agent rewrites action or verification content to satisfy a Tier-1 check. (3) exemplar-binding — operator selects a catalog exemplar for a view. Traces are emitted only when a real choice exists (options_considered > 1). +- pm: The three types of decisions the system records reasoning traces for: answering a walker question, rewriting content to pass a check, or picking a catalog example to follow. +- related: term:layer5-trace +- since: v1.3 diff --git a/tests/test_layer5_exemplar_trace.py b/tests/test_layer5_exemplar_trace.py new file mode 100644 index 0000000..3c80612 --- /dev/null +++ b/tests/test_layer5_exemplar_trace.py @@ -0,0 +1,150 @@ +"""Layer 5 exemplar-binding trace (v1.3 #10). + +Assert that answering an exemplar-binding concern via walker.record_answer +emits a trace record with choice_point="exemplar-binding", not "walker-concern". +Also verifies that the options_considered contain the catalog slugs that were +presented, and that the record is correctly identified for known exemplar IDs. +""" +import pathlib + +import pytest + +from bin import walker + + +_EXEMPLAR_CONCERN_IDS = [ + "input-exemplar-pi", + "help-text-style-hu", + "error-text-style-hu", + "api-exemplar-int", + "log-format-style-op", + "observability-style-op", +] + + +def _make_state(tmp_path: pathlib.Path) -> walker.WalkState: + draft = tmp_path / "draft.spec.md" + draft.write_text("# test spec\n", encoding="utf-8") + return walker.WalkState( + spec_intent="test intent", + spec_draft_path=draft, + ) + + +def _make_exemplar_concern(concern_id: str, slugs: list[str]) -> walker.Concern: + return walker.Concern( + id=concern_id, + kind="receiver-clarification", + receivers=["human"], + depends_on=[], + summary=f"Exemplar binding for {concern_id}", + prefab_options=slugs, + ) + + +def test_exemplar_binding_trace_emitted(tmp_path: pathlib.Path): + """Answering an exemplar-binding concern emits choice_point=exemplar-binding.""" + state = _make_state(tmp_path) + concern = _make_exemplar_concern( + "help-text-style-hu", + ["spectre-cli-help", "cargo-help-text", "kubectl-help-text"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="help-text-style-hu", answer="spectre-cli-help") + + assert len(state.layer5_trace) == 1 + rec = state.layer5_trace[0] + assert rec["choice_point"] == "exemplar-binding" + assert rec["step_or_concern_id"] == "help-text-style-hu" + assert rec["selected"] == "spectre-cli-help" + assert "spectre-cli-help" in rec["options_considered"] + assert len(rec["options_considered"]) == 3 + + +def test_exemplar_binding_trace_six_fields(tmp_path: pathlib.Path): + """Exemplar-binding trace carries all six required fields plus timestamp.""" + state = _make_state(tmp_path) + concern = _make_exemplar_concern( + "api-exemplar-int", + ["rest-json-api", "graphql-api", "grpc-api"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="api-exemplar-int", answer="rest-json-api") + + assert len(state.layer5_trace) == 1 + rec = state.layer5_trace[0] + required = {"choice_point", "step_or_concern_id", "options_considered", + "selected", "rationale", "validation_anchor", "source_anchor", "timestamp"} + assert required <= set(rec.keys()) + + +def test_exemplar_binding_validation_anchor_is_cross_view(tmp_path: pathlib.Path): + """Exemplar-binding traces carry tier2-cross-view-gate as validation_anchor.""" + state = _make_state(tmp_path) + concern = _make_exemplar_concern( + "log-format-style-op", + ["json-lines-logfmt", "plain-logfmt"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="log-format-style-op", answer="json-lines-logfmt") + + assert state.layer5_trace[0]["validation_anchor"] == "tier2-cross-view-gate" + + +def test_exemplar_binding_not_emitted_for_non_exemplar_concern(tmp_path: pathlib.Path): + """Non-exemplar concerns with multiple options emit choice_point=walker-concern.""" + state = _make_state(tmp_path) + concern = _make_exemplar_concern( + "scope-product-input", + ["human-typed", "programmatic-trusted", "not-applicable"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="scope-product-input", answer="human-typed") + + assert state.layer5_trace[0]["choice_point"] == "walker-concern" + + +def test_input_exemplar_pi_emits_exemplar_binding(tmp_path: pathlib.Path): + """input-exemplar-pi concern emits choice_point=exemplar-binding.""" + state = _make_state(tmp_path) + state.pending.append(_make_exemplar_concern("input-exemplar-pi", ["slug-a", "slug-b", "slug-c"])) + walker.record_answer(state, concern_id="input-exemplar-pi", answer="slug-a") + assert len(state.layer5_trace) == 1 + assert state.layer5_trace[0]["choice_point"] == "exemplar-binding" + + +def test_error_text_style_hu_emits_exemplar_binding(tmp_path: pathlib.Path): + """error-text-style-hu concern emits choice_point=exemplar-binding.""" + state = _make_state(tmp_path) + state.pending.append(_make_exemplar_concern("error-text-style-hu", ["slug-a", "slug-b", "slug-c"])) + walker.record_answer(state, concern_id="error-text-style-hu", answer="slug-b") + assert len(state.layer5_trace) == 1 + assert state.layer5_trace[0]["choice_point"] == "exemplar-binding" + + +def test_observability_style_op_emits_exemplar_binding(tmp_path: pathlib.Path): + """observability-style-op concern emits choice_point=exemplar-binding.""" + state = _make_state(tmp_path) + state.pending.append(_make_exemplar_concern("observability-style-op", ["slug-a", "slug-b", "slug-c"])) + walker.record_answer(state, concern_id="observability-style-op", answer="slug-c") + assert len(state.layer5_trace) == 1 + assert state.layer5_trace[0]["choice_point"] == "exemplar-binding" + + +def test_exemplar_trace_disabled_by_env(tmp_path: pathlib.Path, monkeypatch): + """SPECTRE_LAYER5=off disables exemplar-binding trace emission.""" + monkeypatch.setenv("SPECTRE_LAYER5", "off") + state = _make_state(tmp_path) + concern = _make_exemplar_concern( + "help-text-style-hu", + ["slug-a", "slug-b"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="help-text-style-hu", answer="slug-a") + + assert state.layer5_trace == [] diff --git a/tests/test_layer5_substitution_trace.py b/tests/test_layer5_substitution_trace.py new file mode 100644 index 0000000..4b32008 --- /dev/null +++ b/tests/test_layer5_substitution_trace.py @@ -0,0 +1,149 @@ +"""Layer 5 substitution trace (v1.3 #10). + +Assert that write_sidecar with a layer5_trace kwarg writes the layer5_trace +key to the sidecar payload. Also tests _layer5.project_substitution_trace +which projects an extended substitution dict into a trace record. +""" +import json +import pathlib + +from bin import eval_metadata +from bin import _layer5 + + +def test_layer5_trace_omitted_when_none(tmp_path: pathlib.Path): + """layer5_trace not written when kwarg is None.""" + spec = tmp_path / "x.spec.md" + spec.write_text("# x\n", encoding="utf-8") + sidecar = eval_metadata.write_sidecar( + spec, + evaluator_version="1.0.0", + tiers_run=[1], + findings=[], + dismissals=[], + config_path=None, + config_hash=None, + deepseek_model_version=None, + policy_hash="abc", + layer5_trace=None, + ) + payload = json.loads(sidecar.read_text(encoding="utf-8")) + assert "layer5_trace" not in payload + + +def test_layer5_trace_written_as_empty_list(tmp_path: pathlib.Path): + """layer5_trace=[] writes an empty array to the sidecar.""" + spec = tmp_path / "x.spec.md" + spec.write_text("# x\n", encoding="utf-8") + sidecar = eval_metadata.write_sidecar( + spec, + evaluator_version="1.0.0", + tiers_run=[1], + findings=[], + dismissals=[], + config_path=None, + config_hash=None, + deepseek_model_version=None, + policy_hash="abc", + layer5_trace=[], + ) + payload = json.loads(sidecar.read_text(encoding="utf-8")) + assert payload["layer5_trace"] == [] + + +def test_layer5_trace_round_trips_through_sidecar(tmp_path: pathlib.Path): + """A populated layer5_trace round-trips through write_sidecar unchanged.""" + spec = tmp_path / "x.spec.md" + spec.write_text("# x\n", encoding="utf-8") + trace_record = { + "choice_point": "walker-concern", + "step_or_concern_id": "scope-product-input", + "options_considered": ["human-typed", "programmatic-trusted"], + "selected": "human-typed", + "rationale": "Operator chose human-typed for interactive CLI.", + "validation_anchor": "tier1-structural", + "source_anchor": None, + "timestamp": "2026-05-15T12:00:00Z", + } + sidecar = eval_metadata.write_sidecar( + spec, + evaluator_version="1.0.0", + tiers_run=[1], + findings=[], + dismissals=[], + config_path=None, + config_hash=None, + deepseek_model_version=None, + policy_hash="abc", + layer5_trace=[trace_record], + ) + payload = json.loads(sidecar.read_text(encoding="utf-8")) + assert payload["layer5_trace"] == [trace_record] + + +def test_project_substitution_trace_with_options(tmp_path: pathlib.Path): + """project_substitution_trace returns a valid trace for an extended substitution.""" + sub = { + "from": "python3 -c 'import x'", + "to": "pip show x >/dev/null 2>&1", + "reason": "shell-eval false-positive", + "tier1_check_name": "untrusted-flow-unguarded", + "step_id": "step-3", + "options_considered": ["pip show x >/dev/null 2>&1", "python3 -m x --check"], + "selected": "pip show x >/dev/null 2>&1", + "rationale": "pip show is idempotent and doesn't exec user data.", + "validation_anchor": "tier1-structural", + "source_anchor": None, + } + record = _layer5.project_substitution_trace(sub) + assert record is not None + assert record["choice_point"] == "substitution" + assert record["step_or_concern_id"] == "step-3" + assert record["options_considered"] == ["pip show x >/dev/null 2>&1", "python3 -m x --check"] + assert record["selected"] == "pip show x >/dev/null 2>&1" + assert record["rationale"] == "pip show is idempotent and doesn't exec user data." + assert record["validation_anchor"] == "tier1-structural" + assert "timestamp" in record + + +def test_project_substitution_trace_none_when_no_options(): + """project_substitution_trace returns None when options_considered is absent.""" + sub = { + "from": "old", "to": "new", + "reason": "fix", "tier1_check_name": "soft-verification", "step_id": "step-1", + } + result = _layer5.project_substitution_trace(sub) + assert result is None + + +def test_layer5_and_substitutions_coexist(tmp_path: pathlib.Path): + """layer5_trace and substitutions can both appear in the same sidecar.""" + spec = tmp_path / "x.spec.md" + spec.write_text("# x\n", encoding="utf-8") + sub_entry = {"from": "a", "to": "b", "reason": "r", "tier1_check_name": "t", "step_id": "s-1"} + trace_record = { + "choice_point": "substitution", + "step_or_concern_id": "s-1", + "options_considered": ["b", "c"], + "selected": "b", + "rationale": "b is safer", + "validation_anchor": "tier1-structural", + "source_anchor": None, + "timestamp": "2026-05-15T12:00:00Z", + } + sidecar = eval_metadata.write_sidecar( + spec, + evaluator_version="1.0.0", + tiers_run=[1], + findings=[], + dismissals=[], + config_path=None, + config_hash=None, + deepseek_model_version=None, + policy_hash="abc", + substitutions=[sub_entry], + layer5_trace=[trace_record], + ) + payload = json.loads(sidecar.read_text(encoding="utf-8")) + assert payload["substitutions"] == [sub_entry] + assert payload["layer5_trace"] == [trace_record] diff --git a/tests/test_layer5_walker_trace.py b/tests/test_layer5_walker_trace.py new file mode 100644 index 0000000..4d8172a --- /dev/null +++ b/tests/test_layer5_walker_trace.py @@ -0,0 +1,125 @@ +"""Layer 5 walker concern-resolution trace (v1.3 #10). + +Assert that calling walker.record_answer with a concern whose prefab_options +has >=2 entries appends a trace record to state.layer5_trace, and that the +trace has all six required fields populated. Also confirms that concerns with +<=1 prefab option produce no trace. +""" +import os +import pathlib + +import pytest + +from bin import walker + + +def _make_state(tmp_path: pathlib.Path) -> walker.WalkState: + draft = tmp_path / "draft.spec.md" + draft.write_text("# test spec\n", encoding="utf-8") + return walker.WalkState( + spec_intent="test intent", + spec_draft_path=draft, + ) + + +def _make_concern( + concern_id: str, + prefab_options: list[str], +) -> walker.Concern: + return walker.Concern( + id=concern_id, + kind="receiver-clarification", + receivers=["human"], + depends_on=[], + summary=f"Test concern for {concern_id}", + prefab_options=prefab_options, + ) + + +def test_walker_concern_trace_appended_on_multichoice(tmp_path: pathlib.Path): + """record_answer with >=2 prefab options appends a trace record.""" + state = _make_state(tmp_path) + concern = _make_concern( + "scope-product-input", + ["human-typed", "programmatic-trusted", "programmatic-untrusted", "not-applicable"], + ) + state.pending.append(concern) + + walker.record_answer(state, concern_id="scope-product-input", answer="human-typed") + + assert len(state.layer5_trace) == 1 + record = state.layer5_trace[0] + assert record["choice_point"] == "walker-concern" + assert record["step_or_concern_id"] == "scope-product-input" + assert record["options_considered"] == [ + "human-typed", "programmatic-trusted", "programmatic-untrusted", "not-applicable" + ] + assert record["selected"] == "human-typed" + assert isinstance(record["rationale"], str) and len(record["rationale"]) > 0 + assert "validation_anchor" in record + assert "source_anchor" in record + assert "timestamp" in record + + +def test_walker_concern_trace_six_fields_all_present(tmp_path: pathlib.Path): + """Each trace record carries all six required fields plus timestamp.""" + state = _make_state(tmp_path) + concern = _make_concern("scope-human-user", ["cli-user", "web-user", "not-applicable"]) + state.pending.append(concern) + + walker.record_answer(state, concern_id="scope-human-user", answer="cli-user") + + assert len(state.layer5_trace) == 1 + rec = state.layer5_trace[0] + required = {"choice_point", "step_or_concern_id", "options_considered", + "selected", "rationale", "validation_anchor", "source_anchor", "timestamp"} + assert required <= set(rec.keys()), f"Missing fields: {required - set(rec.keys())}" + + +def test_walker_concern_no_trace_when_zero_prefab(tmp_path: pathlib.Path): + """Concerns with no prefab options produce no trace (open-ended answer).""" + state = _make_state(tmp_path) + concern = _make_concern("seed-lifecycle", []) + state.pending.append(concern) + + walker.record_answer(state, concern_id="seed-lifecycle", answer="systemd unit") + + assert state.layer5_trace == [] + + +def test_walker_concern_no_trace_when_single_prefab(tmp_path: pathlib.Path): + """Concerns with exactly one prefab option produce no trace (no real choice).""" + state = _make_state(tmp_path) + concern = _make_concern("only-option-concern", ["the-only-choice"]) + state.pending.append(concern) + + walker.record_answer(state, concern_id="only-option-concern", answer="the-only-choice") + + assert state.layer5_trace == [] + + +def test_walker_concern_trace_accumulates_multiple(tmp_path: pathlib.Path): + """Multiple answered concerns with prefab options accumulate multiple trace records.""" + state = _make_state(tmp_path) + c1 = _make_concern("scope-product-input", ["human-typed", "programmatic-trusted", "not-applicable"]) + c2 = _make_concern("scope-product-output", ["human-reader", "programmatic-consumer", "not-applicable"]) + state.pending.extend([c1, c2]) + + walker.record_answer(state, concern_id="scope-product-input", answer="human-typed") + walker.record_answer(state, concern_id="scope-product-output", answer="programmatic-consumer") + + assert len(state.layer5_trace) == 2 + assert state.layer5_trace[0]["step_or_concern_id"] == "scope-product-input" + assert state.layer5_trace[1]["step_or_concern_id"] == "scope-product-output" + + +def test_walker_concern_trace_disabled_by_env(tmp_path: pathlib.Path, monkeypatch): + """SPECTRE_LAYER5=off disables trace emission entirely.""" + monkeypatch.setenv("SPECTRE_LAYER5", "off") + state = _make_state(tmp_path) + concern = _make_concern("scope-product-input", ["human-typed", "programmatic-trusted"]) + state.pending.append(concern) + + walker.record_answer(state, concern_id="scope-product-input", answer="human-typed") + + assert state.layer5_trace == []