Fix review findings: console 403, governance safety, consumer supervision, learning-loop plumbing

common/config.py

@@ -20,7 +20,14 @@ class Settings(BaseSettings):
     hitl_poll_interval_seconds: float = 0.5
     training_store_path: str = "data/training.jsonl"
     reliability_suppress_threshold: float = 0.8
+    # Labelled outcomes a signature needs before it can be suppressed at all.
+    reliability_suppress_min_samples: int = 3
+    # How often RCA re-reads the training store for its runbook track records.
+    reliability_refresh_seconds: float = 60.0
     graduation_min_successes: int = 3
+    # Whether a dry-run SUCCESS counts toward graduating a playbook to AUTO. Off:
+    # a simulation always "succeeds", so it is no evidence a fix is safe unattended.
+    graduation_count_simulated: bool = False
 
     # --- live-stack settings (test-safe defaults) ---
     telemetry_mode: str = "file"  # "file" | "prometheus"
@@ -51,6 +58,9 @@ class Settings(BaseSettings):
     correlation_seasonal_buckets: int = 24
     correlation_robust_window: int = 128
     correlation_robust_warmup: int = 30
+    # "metric" (default, historical) pools every service's samples of a metric into
+    # one baseline; "series" keeps one per (metric, service). See RobustCorrelator.
+    correlation_robust_key_by: str = "metric"  # "metric" | "series"
     governance_mode: str = "in_process"  # "in_process" | "http"
     governance_url: str = "http://localhost:8005"
     # read-service asks rca for the authoritative LLM config rather than

common/contracts.py

@@ -258,6 +258,11 @@ class TrainingRecord(BaseModel):
     result: RemediationResult
     worked: bool
     ts: datetime
+    # How the remediation ran ("dry_run" | "k8s" | ...), copied from the outcome.
+    # Additive and optional: records written before it existed load as None.
+    # Graduation needs it - a simulated success is not evidence that a playbook
+    # is safe to run unattended on a real cluster.
+    mode: str | None = None
 
 
 class TraceStepKind(str, Enum):

common/supervise.py

@@ -0,0 +1,65 @@
+"""Keep a background worker alive across unexpected exceptions.
+
+Every service runs its bus consumer on a daemon thread. Before this, any
+exception a handler didn't catch - a governance restart surfacing as a
+ConnectError from the HTTP gate, a Postgres blip in an audit write - ended the
+thread for good. Nothing noticed: /health is static, so Kubernetes never
+restarted the pod, and the service sat "ready" while processing nothing.
+
+`start_supervised` runs the target again after a crash, with capped exponential
+backoff, until the stop event is set. A target that RETURNS is done (the stream
+ended, or it saw the stop event) and is not restarted.
+
+Restarting is safe under both delivery modes: at-least-once re-serves the
+un-acked entry on the next consume() (and the DLQ parks it after
+bus_max_delivery_attempts, so a poison event can't loop forever); at-most-once
+has already acked it, which is exactly the loss that mode accepts.
+"""
+
+from __future__ import annotations
+
+import logging
+import threading
+from collections.abc import Callable
+
+logger = logging.getLogger("intelliops.supervise")
+
+
+def supervise(
+    name: str,
+    target: Callable[..., object],
+    stop_event: threading.Event,
+    args: tuple = (),
+    kwargs: dict | None = None,
+    backoff_seconds: float = 1.0,
+    max_backoff_seconds: float = 30.0,
+) -> None:
+    kwargs = kwargs or {}
+    delay = backoff_seconds
+    while not stop_event.is_set():
+        try:
+            target(*args, **kwargs)
+            return
+        except Exception:
+            logger.exception("%s crashed; restarting in %.1fs", name, delay)
+            if stop_event.wait(delay):
+                return
+            delay = min(delay * 2, max_backoff_seconds)
+
+
+def start_supervised(
+    name: str,
+    target: Callable[..., object],
+    stop_event: threading.Event,
+    args: tuple = (),
+    kwargs: dict | None = None,
+) -> threading.Thread:
+    """Start `target(*args, **kwargs)` on a supervised daemon thread."""
+    thread = threading.Thread(
+        target=supervise,
+        name=name,
+        args=(name, target, stop_event, args, kwargs),
+        daemon=True,
+    )
+    thread.start()
+    return thread

deploy/Dockerfile

@@ -1,7 +1,8 @@
 # syntax=docker/dockerfile:1
 FROM python:3.11-slim AS builder-base
 
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+# Pinned: :latest made every image build depend on whatever uv shipped that day.
+COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv
 
 WORKDIR /app
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy

deploy/Dockerfile.frontend

@@ -12,6 +12,10 @@ ENV VITE_READ_URL=/api/read
 ENV VITE_GOV_URL=/api/gov
 ENV VITE_CORR_URL=/api/corr
 ENV VITE_RCA_URL=/api/rca
+# The human identity the console acts as. It must name an approver in
+# policies/rbac_policy.yaml or every decision is rejected with 403.
+ARG VITE_OPERATOR_NAME=demo-operator
+ENV VITE_OPERATOR_NAME=${VITE_OPERATOR_NAME}
 RUN npm run build
 
 FROM nginx:1.27-alpine

deploy/Dockerfile.meridian

@@ -13,7 +13,8 @@ RUN npm run build
 
 FROM python:3.11-slim
 
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
+# Pinned: :latest made every image build depend on whatever uv shipped that day.
+COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv
 
 WORKDIR /app
 ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy

deploy/k8s/platform/templates/configmap.yaml

@@ -41,6 +41,7 @@ data:
   # this correlator's name is meant to buy.
   INTELLIOPS_CORRELATION_ROBUST_WINDOW: {{ .Values.env.CORRELATION_ROBUST_WINDOW | default "128" | quote }}
   INTELLIOPS_CORRELATION_ROBUST_WARMUP: {{ .Values.env.CORRELATION_ROBUST_WARMUP | default "30" | quote }}
+  INTELLIOPS_CORRELATION_ROBUST_KEY_BY: {{ .Values.env.CORRELATION_ROBUST_KEY_BY | default "metric" | quote }}
   INTELLIOPS_DETECTION_POLICY: {{ .Values.env.DETECTION_POLICY | quote }}
   INTELLIOPS_RUNBOOK_SELECTOR_MODE: {{ .Values.env.RUNBOOK_SELECTOR_MODE | quote }}
   INTELLIOPS_REMEDIATOR_MODE: {{ .Values.env.REMEDIATOR_MODE | quote }}

deploy/k8s/platform/templates/service-deployment.yaml

@@ -28,6 +28,13 @@ metadata:
     {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 4 }}
 spec:
   replicas: 1
+  # Recreate, not the default RollingUpdate. Every consumer uses one stable
+  # consumer name per group (ADR-033), and a rolling update briefly runs the old
+  # and new pod side by side - two consumers sharing one name and one pending
+  # list, both able to act on the same entry. replicas: 1 only holds if the
+  # rollout never surges.
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 6 }}

deploy/k8s/platform/values.yaml

@@ -59,7 +59,7 @@ env:
   DETECTION_POLICY: "off"           # on in values-live (Phase 2)
   RUNBOOK_SELECTOR_MODE: "off"      # embedding in values-live (Phase 3)
   REMEDIATOR_MODE: "dry_run"        # k8s in values-live (real remediation)
-  SANDBOX_MODE: "off"               # k8s in values-live (pre-flight rehearsal)
+  SANDBOX_MODE: "off"               # off in values-live too (ADR-029); k8s via docker-compose.k8s.yml
   HEALTH_CHECK_MODE: "always"       # k8s in values-live (Phase 4 per-metric verify)
   # Must stay comfortably above the remediated workload's termination grace
   # period; see the configmap comment. 30 (the old hardcoded value) equalled
@@ -75,6 +75,10 @@ env:
   CORRELATION_MIN_EVENTS: "1"   # values-live raises this; 1 = historical behaviour
   CORRELATION_ROBUST_WINDOW: "128"
   CORRELATION_ROBUST_WARMUP: "30"   # values-live requires a FULL window instead
+  # "series" gives each (metric, service) its own baseline instead of pooling
+  # every service into one per metric. Re-tune warm-up before flipping it: a
+  # per-series window fills ~5x slower (one sample per poll, not five).
+  CORRELATION_ROBUST_KEY_BY: "metric"
   GOVERNANCE_MODE: "in_process"     # "http" in values-live — action + governance are
   GOVERNANCE_URL: "http://governance:8000"  # separate pods, so approvals must go over HTTP
   HITL_POLL_TIMEOUT_SECONDS: "30"   # values-live raises this so a human has time to click Approve

docs/OPERATIONS.md

@@ -29,7 +29,8 @@ need a token).
 
 In `token` mode **all** endpoints are gated — including the internal
 service-to-service paths on governance (`POST /audit`, `POST /rbac/check`,
-`POST /approvals`, `GET /approvals/{id}`, `POST /playbooks/{id}/graduate`).
+`POST /approvals`, `GET /approvals/{id}`, `POST /approvals/{id}/expire`,
+`POST /playbooks/{id}/graduate`, `POST /playbooks/{id}/demote`).
 Internal callers (action's `HttpGovernanceGate`, feedback's graduator)
 attach the shared `Bearer` token to their requests automatically.
 
@@ -65,7 +66,7 @@ that talk to governance over REST:
 | --- | --- |
 | `governance` | Validates incoming tokens on all endpoints. |
 | `action` | `HttpGovernanceGate` calls `POST /rbac/check`, `POST /audit`, `POST /approvals`, `GET /approvals/{id}`. |
-| `feedback` | `_make_graduator` calls `POST /playbooks/{id}/graduate`. |
+| `feedback` | `_make_graduator` / `_make_demoter` call `POST /playbooks/{id}/graduate` and `/demote`. |
 | `rca` | Uses the shared factory; gated if exposed. |
 
 Example compose environment block (add to each service above):

flow.md

@@ -243,10 +243,12 @@ the unmodified z-score (default `off`, byte-identical). See
 |----------|--------------|-----|-----------|
 | `check_rbac(actor, action, resource) → allow \| deny` | Decides whether an actor may perform an action. | Every automated action passes here — the enforceable RBAC guarantee. | RBAC policy |
 | `create_approval_request() → ApprovalRequest` | Opens a pending approval for a `hitl` playbook. | Materializes the human decision point. | `contracts.ApprovalRequest` |
-| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. | The approval interface the React console's Approve/Reject buttons drive. | — |
+| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. Only `approved`/`rejected` are accepted (422 otherwise), the decider needs the matching `approve`/`reject` permission, a request that is no longer pending is a 409, and the decision is written to the audit log under the human's name. | The approval interface the React console's Approve/Reject buttons drive. | RBAC, `AuditSink` |
+| `expire(id)` | `POST /approvals/{id}/expire` — the requester (action-service) marks a request it stopped waiting for as `expired` at its HITL timeout. | Without it a timed-out request stayed `pending` forever and the console offered an Approve that could no longer do anything. | `AuditSink` |
 | `get_approval(id)` / `list_approvals()` | REST reads (`GET /approvals/{id}`, `GET /approvals`) of the pending queue. | Lets the HTTP gate poll for a decision across containers, and the dashboard show what's pending. | — |
 | `write_audit(record)` | Appends an immutable `AuditRecord`. | The compliance backbone (NIST AI RMF / DORA / EU AI Act). | `AuditSink` |
-| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. | Standardization — playbooks are shared, not reinvented per team. | playbook store |
+| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. `POST /playbooks?registered_by=` needs the `register` permission (coe-admin), refuses `hitl_mode=auto` unless the actor may also `graduate`, and is audited. | Standardization — playbooks are shared, not reinvented per team. A direct write bypasses the AI-proposal review, so it is an admin action, not an open one. | playbook store, RBAC |
+| `graduate(id)` / `demote(id)` | `POST /playbooks/{id}/graduate` (hitl → auto) and `POST /playbooks/{id}/demote` (auto → hitl), both behind the `graduate` permission and audited. feedback-service calls graduate on clean **real** evidence (dry-run successes don't count unless `GRADUATION_COUNT_SIMULATED`) and demote when an auto playbook fails or rolls back. | Automation expands on evidence and contracts on failure. | RBAC, playbook store, `AuditSink` |
 | `propose_playbook(situation) → ProposedPlaybook` | `POST /playbooks/proposed`: calls a `RunbookAuthor` (LLM) to draft a typed runbook for a gap, forces `hitl_mode=HITL` + a server-assigned id, stores it as a **proposal** (not the live registry); 422 if the author declines. | The AI **proposes** a runbook for a gap; `model_validate` rejects any unsafe draft ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | `RunbookAuthor`, proposed store |
 | `approve_proposed(id)` / `reject_proposed(id)` | `POST /playbooks/proposed/{id}/approve\|reject` — RBAC-gated (reuses `approve`/`reject`), audited. **Approve registers the inner playbook into the live registry**; reject does not. | The human **disposes** — the only path from an AI draft to the live registry ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | RBAC, playbook store, `AuditSink` |