fix(sdlc): activate inert INV-1..5 trace-checker + unify auto-mint onto the live escape-grant contract

scripts/hapax-sdlc-invariants

@@ -9,12 +9,44 @@
 #
 # ADVISORY-WITH-LEDGER ONLY. A violation is ledgered to
 # ~/.cache/hapax/sdlc-invariant-findings.jsonl; INV-3/4/5 violations additionally
-# AUTO-MINT a signed escape grant into ~/.cache/hapax/escape-grants/ so a stuck
-# lane is unblocked rather than frozen. This evaluator NEVER blocks and always
-# exits 0 — a self-blocking proof gate would rebuild the freeze-blocks-thaw
-# catch-22 in the verification layer.
+# AUTO-MINT a signed escape grant into the canonical coord grants dir
+# (~/.cache/hapax/coord/grants, signed with ~/.cache/hapax/coord/grant-key) that
+# the live escape-grant.sh shim reads, so a stuck lane is unblocked rather than
+# frozen. This evaluator NEVER blocks and always exits 0 — a self-blocking proof
+# gate would rebuild the freeze-blocks-thaw catch-22 in the verification layer.
 set -euo pipefail
 
-REPO="${HAPAX_COUNCIL_REPO:-$HOME/projects/hapax-council}"
+# Resolve the repo from THIS script's own location — the stable deploy worktree
+# the unit's ExecStart points at — NOT a hardcoded ~/projects/hapax-council. A
+# lane may park the primary worktree on a feature branch where this script +
+# shared/sdlc_invariants.py do not exist, which crash-looped the unit 203/EXEC
+# (#3820 → reform-inv-trace-checker-activate). HAPAX_COUNCIL_REPO still wins when
+# the unit sets it explicitly.
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO="${HAPAX_COUNCIL_REPO:-$(cd "$SCRIPT_DIR/.." && pwd)}"
+
+# Resolve uv even under a minimal systemd-user PATH (the unit also sets PATH).
+UV="$(command -v uv 2>/dev/null || true)"
+[[ -n "$UV" ]] || UV="$HOME/.local/bin/uv"
+
+# Deploy-time / start-time --verify: assert the ExecStart target resolves to a
+# usable evaluator (the resolved repo carries the module AND uv is runnable) so a
+# misdeploy fails LOUDLY here — caught by the unit's ExecStartPre → OnFailure ntfy,
+# or skipped by its ConditionPathExists — instead of silently crash-looping
+# 203/EXEC. The deploy step (scripts/rebuild-service.sh) may also call this.
+if [[ "${1:-}" == "--verify" ]]; then
+  rc=0
+  if [[ ! -f "$REPO/shared/sdlc_invariants.py" ]]; then
+    printf 'hapax-sdlc-invariants: verify FAIL — missing %s/shared/sdlc_invariants.py\n' "$REPO" >&2
+    rc=1
+  fi
+  if [[ ! -x "$UV" ]] && ! command -v uv >/dev/null 2>&1; then
+    printf 'hapax-sdlc-invariants: verify FAIL — uv not executable (%s)\n' "$UV" >&2
+    rc=1
+  fi
+  [[ "$rc" -eq 0 ]] && printf 'hapax-sdlc-invariants: verify ok (repo=%s uv=%s)\n' "$REPO" "$UV"
+  exit "$rc"
+fi
+
 cd "$REPO"
-exec uv run python -m shared.sdlc_invariants --mint-escapes "$@"
+exec "$UV" run python -m shared.sdlc_invariants --mint-escapes "$@"

shared/sdlc_invariants.py

@@ -32,6 +32,7 @@
 from datetime import UTC, datetime
 from pathlib import Path
 
+from shared.coord_event_log import default_grant_dir, default_grant_key
 from shared.governance.coord_capabilities import (
     EscapeGrant,
     mint_escape_grant,
@@ -46,20 +47,17 @@
 DEFAULT_AUTHORITY_LEDGER = Path(os.path.expanduser("~/.cache/hapax/authority-case-ledger.jsonl"))
 #: Advisory findings ledger this monitor writes (violations only).
 DEFAULT_INVARIANT_LEDGER = Path(os.path.expanduser("~/.cache/hapax/sdlc-invariant-findings.jsonl"))
-#: HMAC key the auto-mint signs escape grants with. The monitor is the first
-#: runtime minter, so it CREATES this key (0600) if absent; the daemon-independent
-#: shim (which honors grants kernel-down) reads the SAME path. Override via
-#: ``--key-file`` / ``HAPAX_COORD_GRANT_KEY_FILE``.
-DEFAULT_GRANT_KEY_FILE = Path(
-    os.environ.get(
-        "HAPAX_COORD_GRANT_KEY_FILE", os.path.expanduser("~/.config/hapax/coord-capability.key")
-    )
-)
-#: Directory the auto-minted escape grant FILES land in. The shim reads grants
-#: directly from disk here — no RPC, so a grant is honored with the kernel dead.
-DEFAULT_ESCAPE_GRANT_DIR = Path(
-    os.environ.get("HAPAX_ESCAPE_GRANT_DIR", os.path.expanduser("~/.cache/hapax/escape-grants"))
-)
+#: The auto-mint writes signed grant FILES the daemon-independent shim reads
+#: directly off disk (a pure file read, no RPC). Their directory and signing key
+#: MUST resolve identically to ``hooks/scripts/escape-grant.sh`` and
+#: ``scripts/coord-grant-mint`` — otherwise a minted grant is invisible (wrong
+#: dir/extension) or unverifiable (wrong key) to the live shim and the escape is
+#: inert. So BOTH are resolved at CALL TIME through the single canonical SSOT,
+#: ``shared.coord_event_log`` (``default_grant_dir`` → ``<coord>/grants``,
+#: ``default_grant_key`` → ``<coord>/grant-key``; ``HAPAX_COORD_DIR`` redirects the
+#: whole tree for tests). There is deliberately NO module-level path constant: an
+#: import-time snapshot of a divergent path is exactly the regression this monitor
+#: exists to prevent (reform-inv-trace-checker-activate).
 #: Only the escape-class invariants auto-mint a grant on violation (§4.5). INV-1/2
 #: are statechart/liveness properties — they ledger an advisory alert, never mint.
 AUTO_MINT_INVARIANTS = frozenset({"INV-3", "INV-4", "INV-5"})
@@ -327,15 +325,18 @@ def record_invariant_findings(
 # unblocks a lane regardless of daemon liveness (INV-4).
 
 
-def load_or_create_grant_key(path: str | os.PathLike[str] = DEFAULT_GRANT_KEY_FILE) -> bytes:
+def load_or_create_grant_key(path: str | os.PathLike[str] | None = None) -> bytes:
     """Load the coord-capability HMAC key, creating a fresh 32-byte key (mode 0600) if absent.
 
-    The monitor is the first runtime minter, so it establishes the key the
-    daemon-independent shim later reads from the SAME path. Returns ``b""`` only if
+    ``path`` defaults to the canonical coord signing key
+    (``shared.coord_event_log.default_grant_key`` — the SAME key
+    ``hooks/scripts/escape-grant.sh`` and ``scripts/coord-grant-mint`` use), so an
+    auto-minted grant verifies against the live shim. The monitor may be the first
+    runtime minter, so it establishes that key if absent. Returns ``b""`` only if
     the key can neither be read nor persisted — the caller then degrades to
     ledger-only (no verifiable grant can be signed). Never raises.
     """
-    target = Path(path)
+    target = Path(path) if path is not None else default_grant_key()
     try:
         return target.read_bytes()
     except OSError:
@@ -388,17 +389,19 @@ def mint_escape_for_violation(
     result: InvariantResult,
     *,
     key: bytes,
-    grant_dir: str | os.PathLike[str] = DEFAULT_ESCAPE_GRANT_DIR,
+    grant_dir: str | os.PathLike[str] | None = None,
     now: float,
     ttl_s: float = ESCAPE_GRANT_TTL_S,
 ) -> EscapeGrant | None:
     """Auto-mint a signed universal ("*") escape grant for an INV-3/4/5 violation.
 
     INV-3/4/5 violations all mean the escape machinery itself may be compromised, so
     the never-freeze response is a broad daemon-independent escape (the triggering
-    invariant is stamped in the grant ``reason`` for legibility). Returns the minted
-    grant written to ``grant_dir``, or ``None`` when the invariant holds, is not an
-    auto-mint class, or no signing key is available. Never raises — advisory.
+    invariant is stamped in the grant ``reason`` for legibility). The grant is
+    written as ``<grant_id>.grant`` — the extension the shim globs — into
+    ``grant_dir`` (default: the canonical ``default_grant_dir()``, the SAME dir the
+    shim reads). Returns the minted grant, or ``None`` when the invariant holds, is
+    not an auto-mint class, or no signing key is available. Never raises — advisory.
     """
     if result.holds or result.invariant not in AUTO_MINT_INVARIANTS or not key:
         return None
@@ -414,8 +417,8 @@ def mint_escape_for_violation(
             key=key,
             now=now,
         )
-        slug = result.invariant.lower().replace("-", "")
-        write_grant_file(grant, Path(grant_dir) / f"{slug}-{grant.grant_id}.json")
+        target_dir = Path(grant_dir) if grant_dir is not None else default_grant_dir()
+        write_grant_file(grant, target_dir / f"{grant.grant_id}.grant")
         return grant
     except Exception:  # noqa: BLE001 — auto-mint is advisory; a mint/IO failure must not block.
         return None
@@ -457,7 +460,7 @@ def run_evaluator(
     stale_after_s: float = 86400.0,
     ladder: Ladder = SDLC_LADDER,
     findings_path: str | os.PathLike[str] = DEFAULT_INVARIANT_LEDGER,
-    grant_dir: str | os.PathLike[str] = DEFAULT_ESCAPE_GRANT_DIR,
+    grant_dir: str | os.PathLike[str] | None = None,
     key: bytes = b"",
     alert: bool = True,
 ) -> EvaluationReport:
@@ -541,8 +544,17 @@ def main(argv: list[str] | None = None) -> int:
         action="store_true",
         help="auto-mint the relevant escape grant on each INV-3/4/5 violation (§4.5)",
     )
-    parser.add_argument("--grant-dir", default=str(DEFAULT_ESCAPE_GRANT_DIR))
-    parser.add_argument("--key-file", dest="key_file", default=str(DEFAULT_GRANT_KEY_FILE))
+    parser.add_argument(
+        "--grant-dir",
+        default=None,
+        help="override the escape-grant directory (default: the canonical coord grants dir)",
+    )
+    parser.add_argument(
+        "--key-file",
+        dest="key_file",
+        default=None,
+        help="override the signing key path (default: the canonical coord grant key)",
+    )
     parser.add_argument(
         "--no-alert", dest="alert", action="store_false", help="suppress the operator notification"
     )

systemd/units/hapax-sdlc-invariants.service

@@ -1,14 +1,28 @@
 [Unit]
 Description=Hapax SDLC never-stuck invariant evaluator (INV-1..5, advisory-with-ledger)
-Documentation=file://%h/projects/hapax-council/docs/formal/sdlc-ladder.tla
+Documentation=file://%h/.cache/hapax/rebuild/worktree/docs/formal/sdlc-ladder.tla
 After=network-online.target
+OnFailure=notify-failure@%n.service
+# Run from the STABLE deploy worktree kept fresh on origin/main by the
+# source-activation / rebuild chain (reform-deploy-chain-repair) — NEVER the
+# primary worktree %h/projects/hapax-council, which a lane may park on a feature
+# branch where scripts/hapax-sdlc-invariants + shared/sdlc_invariants.py do not
+# exist (the #3820 203/EXEC crash-loop root cause). ConditionPathExists turns an
+# absent / not-yet-deployed target into a CLEAN SKIP instead of a crash-loop —
+# the same deploy-worktree pattern the hapax-audio-health-* units use.
+ConditionPathExists=%h/.cache/hapax/rebuild/worktree/scripts/hapax-sdlc-invariants
 
 [Service]
 Type=oneshot
 # Advisory-with-ledger: evaluates INV-1..5 against the live authority-case ledger,
 # ledgers every violation, and AUTO-MINTS the relevant escape on INV-3/4/5 so a
 # stuck lane is unblocked, never frozen. The evaluator always exits 0 (never a gate).
-ExecStart=%h/projects/hapax-council/scripts/hapax-sdlc-invariants
+# ExecStartPre --verify catches a misdeployed target LOUDLY (→ OnFailure ntfy)
+# rather than via a silent 203/EXEC.
+ExecStartPre=%h/.cache/hapax/rebuild/worktree/scripts/hapax-sdlc-invariants --verify
+ExecStart=%h/.cache/hapax/rebuild/worktree/scripts/hapax-sdlc-invariants
+Environment=HAPAX_COUNCIL_REPO=%h/.cache/hapax/rebuild/worktree
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
 Environment=NTFY_URL=https://ntfy.sh
 Environment=NTFY_TOPIC=hapax-ops
 NoNewPrivileges=true

tests/test_sdlc_invariants_chaos.py

@@ -123,10 +123,11 @@ def test_inv3_4_5_violation_mints_universal_grant(self, tmp_path):
             assert grant.scope == "*"
             assert grant.grantor == ESCAPE_GRANTOR
             assert inv in grant.reason
-            # the grant was written to disk and round-trips
-            files = list(tmp_path.glob(f"{inv.lower().replace('-', '')}-*.json"))
-            assert len(files) == 1
-            assert read_grant_file(files[0]) == grant
+            # the grant was written to disk as <grant_id>.grant — the extension the
+            # live shim globs, NOT the old <slug>-<id>.json — and round-trips
+            grant_file = tmp_path / f"{grant.grant_id}.grant"
+            assert grant_file.exists()
+            assert read_grant_file(grant_file) == grant
 
     def test_minted_grant_actually_unblocks_a_lane(self, tmp_path):
         grant = mint_escape_for_violation(
@@ -223,7 +224,7 @@ def test_inv3_violation_auto_mints_escape(self, tmp_path):
         # exactly one grant minted — for INV-3 (the auto-mint class), never for INV-1
         assert len(report.minted) == 1
         assert "INV-3" in report.minted[0].reason
-        assert list(grant_dir.glob("inv3-*.json"))
+        assert (grant_dir / f"{report.minted[0].grant_id}.grant").exists()
 
     def test_no_key_ledgers_but_cannot_mint(self, tmp_path):
         report = run_evaluator(
@@ -318,7 +319,7 @@ def test_handwritten_grant_unblocks_lane_with_daemon_dead(self, tmp_path):
                 key=_KEY,
                 now=_NOW,
             )
-            grant_file = tmp_path / "escape.json"
+            grant_file = tmp_path / "escape.grant"
             write_grant_file(grant, grant_file)
 
             # 5. the lane reads the grant directly from disk — the daemon is STILL dead —

tests/test_sdlc_invariants_escape_grant_integration.py

@@ -0,0 +1,148 @@
+"""Cross-boundary integration: the SDLC auto-mint must produce a grant the REAL
+``escape-grant.sh`` shim accepts.
+
+reform-inv-trace-checker-activate (CASE-SDLC-REFORM-001). The in-process chaos
+test (``test_sdlc_invariants_chaos.py``) mints and verifies with a *shared
+tmp_path key* — which MASKS the production disconnect this task fixes: the
+auto-mint historically wrote ``<slug>-<id>.json`` into ``~/.cache/hapax/escape-grants``
+signed with ``~/.config/hapax/coord-capability.key`` — none of which the live shim
+reads (it globs ``<coord>/grants/*.grant`` verified with ``<coord>/grant-key``).
+
+This test drives BOTH the Python minter and the bash shim through the SAME
+canonical coord resolvers (``shared.coord_event_log`` redirected to a tmp tree via
+``HAPAX_COORD_DIR``) and asserts the shim's real glob + signature-verify path
+ACCEPTS the auto-minted grant. It FAILS if the directory, file extension, or
+signing key diverge across the Python→bash boundary — exactly the regression this
+task closes. It never touches the operator's real ``~/.cache``.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import time
+from pathlib import Path
+
+import pytest
+
+from shared.coord_event_log import default_grant_dir, default_grant_key
+from shared.governance.coord_capabilities import load_or_create_key
+from shared.sdlc_invariants import (
+    InvariantResult,
+    Ladder,
+    mint_escape_for_violation,
+    run_evaluator,
+)
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+SHIM = REPO_ROOT / "hooks" / "scripts" / "escape-grant.sh"
+
+#: A ladder whose BLOCKED state has no escape edge → a genuine INV-3 violation
+#: that ``run_evaluator`` auto-mints for (mirrors the chaos test's no-escape
+#: ladder, but here the minted grant must survive the REAL shim, not an
+#: in-process key).
+_NO_ESCAPE_LADDER = Ladder(
+    stages=("S0", "S11", "BLOCKED"),
+    transitions={"S0": frozenset({"S11"}), "S11": frozenset(), "BLOCKED": frozenset()},
+    terminal=frozenset({"S11"}),
+    blocked=frozenset({"BLOCKED"}),
+)
+
+
+def _shim_allows(gate: str, env: dict[str, str]) -> tuple[bool, str]:
+    """Invoke the REAL ``escape-grant.sh`` exactly as an irreversible-harm shim
+    does: set ``SCRIPT_DIR``, ``source`` the shim, then call ``escape_grant_allows``
+    from inside an ``if`` (the shim's documented ``set -e``-safe contract)."""
+    script = (
+        "set -uo pipefail; "
+        f"export SCRIPT_DIR={SHIM.parent!s}; "
+        f"source {SHIM!s}; "
+        f"if escape_grant_allows {gate}; then echo __ALLOW__; else echo __DENY__; fi"
+    )
+    proc = subprocess.run(
+        ["bash", "-c", script],
+        capture_output=True,
+        text=True,
+        env=env,
+        cwd=str(REPO_ROOT),
+    )
+    return ("__ALLOW__" in proc.stdout), (proc.stdout + proc.stderr)
+
+
+@pytest.fixture
+def coord_env(tmp_path, monkeypatch):
+    """Redirect the canonical coord tree (BOTH minter and shim) to a hermetic tmp
+    dir, and clear any explicit per-surface overrides so resolution flows through
+    ``HAPAX_COORD_DIR`` identically on each side."""
+    coord = tmp_path / "coord"
+    for var in (
+        "HAPAX_COORD_GRANT_DIR",
+        "HAPAX_COORD_GRANT_KEY",
+        "HAPAX_ESCAPE_GRANT_DIR",  # the OLD divergent dir override
+        "HAPAX_COORD_GRANT_KEY_FILE",  # the OLD divergent key override
+    ):
+        monkeypatch.delenv(var, raising=False)
+    monkeypatch.setenv("HAPAX_COORD_DIR", str(coord))
+    # keep the shim's "grant honored" ledger out of the operator's real cache
+    monkeypatch.setenv("HAPAX_METHODOLOGY_LEDGER", str(tmp_path / "methodology.jsonl"))
+    env = dict(os.environ)
+    return coord, env
+
+
+@pytest.mark.skipif(not SHIM.exists(), reason="escape-grant.sh shim absent")
+@pytest.mark.skipif(
+    shutil.which("bash") is None or shutil.which("python3") is None,
+    reason="bash + python3 required for the real shim round-trip",
+)
+class TestAutoMintShimRoundTrip:
+    def test_run_evaluator_inv3_mint_is_accepted_by_real_shim(self, coord_env):
+        coord, env = coord_env
+
+        # Control: with no grant minted yet, the REAL shim MUST deny — proves this
+        # test is capable of failing (not trivially green).
+        allowed_before, out = _shim_allows("floor:merge", env)
+        assert not allowed_before, f"shim allowed with an empty grant dir:\n{out}"
+
+        grant_dir = default_grant_dir()
+        key = load_or_create_key(default_grant_key())
+
+        report = run_evaluator(
+            (),
+            now=time.time(),
+            ladder=_NO_ESCAPE_LADDER,
+            findings_path=coord / "findings.jsonl",
+            key=key,
+            alert=False,
+        )
+        assert "INV-3" in report.violations
+        inv3 = [g for g in report.minted if "INV-3" in g.reason]
+        assert inv3, f"INV-3 did not auto-mint; minted reasons={[g.reason for g in report.minted]}"
+
+        # Canonical contract: <coord>/grants/<grant_id>.grant (NOT <slug>-<id>.json).
+        minted = grant_dir / f"{inv3[0].grant_id}.grant"
+        found = sorted(p.name for p in grant_dir.glob("*")) if grant_dir.exists() else "NO DIR"
+        assert minted.exists(), (
+            f"auto-mint did not write {minted} — dir/extension divergence ({found})"
+        )
+
+        # THE cross-boundary assertion: the REAL shim's glob + HMAC verify accepts it.
+        allowed, out = _shim_allows("floor:merge", env)
+        assert allowed, f"real escape-grant.sh REJECTED the auto-minted grant:\n{out}"
+
+    def test_mint_escape_for_violation_is_accepted_by_real_shim(self, coord_env):
+        coord, env = coord_env
+        key = load_or_create_key(default_grant_key())
+
+        grant = mint_escape_for_violation(
+            InvariantResult("INV-4", "authority", holds=False, violations=("boom",), advisory="x"),
+            key=key,
+            now=time.time(),
+        )
+        assert grant is not None
+        minted = default_grant_dir() / f"{grant.grant_id}.grant"
+        assert minted.exists(), f"auto-mint did not write {minted}"
+
+        # universal "*" scope covers any gate the shim asks about
+        allowed, out = _shim_allows("cc-task-gate", env)
+        assert allowed, f"real escape-grant.sh REJECTED the auto-minted grant:\n{out}"