diff --git a/common/config.py b/common/config.py index 72c0d17..9329287 100644 --- a/common/config.py +++ b/common/config.py @@ -20,7 +20,14 @@ class Settings(BaseSettings): hitl_poll_interval_seconds: float = 0.5 training_store_path: str = "data/training.jsonl" reliability_suppress_threshold: float = 0.8 + # Labelled outcomes a signature needs before it can be suppressed at all. + reliability_suppress_min_samples: int = 3 + # How often RCA re-reads the training store for its runbook track records. + reliability_refresh_seconds: float = 60.0 graduation_min_successes: int = 3 + # Whether a dry-run SUCCESS counts toward graduating a playbook to AUTO. Off: + # a simulation always "succeeds", so it is no evidence a fix is safe unattended. + graduation_count_simulated: bool = False # --- live-stack settings (test-safe defaults) --- telemetry_mode: str = "file" # "file" | "prometheus" @@ -51,6 +58,9 @@ class Settings(BaseSettings): correlation_seasonal_buckets: int = 24 correlation_robust_window: int = 128 correlation_robust_warmup: int = 30 + # "metric" (default, historical) pools every service's samples of a metric into + # one baseline; "series" keeps one per (metric, service). See RobustCorrelator. + correlation_robust_key_by: str = "metric" # "metric" | "series" governance_mode: str = "in_process" # "in_process" | "http" governance_url: str = "http://localhost:8005" # read-service asks rca for the authoritative LLM config rather than diff --git a/common/contracts.py b/common/contracts.py index 3a711dd..c0f0bee 100644 --- a/common/contracts.py +++ b/common/contracts.py @@ -258,6 +258,11 @@ class TrainingRecord(BaseModel): result: RemediationResult worked: bool ts: datetime + # How the remediation ran ("dry_run" | "k8s" | ...), copied from the outcome. + # Additive and optional: records written before it existed load as None. + # Graduation needs it - a simulated success is not evidence that a playbook + # is safe to run unattended on a real cluster. + mode: str | None = None class TraceStepKind(str, Enum): diff --git a/common/supervise.py b/common/supervise.py new file mode 100644 index 0000000..0cf9e92 --- /dev/null +++ b/common/supervise.py @@ -0,0 +1,65 @@ +"""Keep a background worker alive across unexpected exceptions. + +Every service runs its bus consumer on a daemon thread. Before this, any +exception a handler didn't catch - a governance restart surfacing as a +ConnectError from the HTTP gate, a Postgres blip in an audit write - ended the +thread for good. Nothing noticed: /health is static, so Kubernetes never +restarted the pod, and the service sat "ready" while processing nothing. + +`start_supervised` runs the target again after a crash, with capped exponential +backoff, until the stop event is set. A target that RETURNS is done (the stream +ended, or it saw the stop event) and is not restarted. + +Restarting is safe under both delivery modes: at-least-once re-serves the +un-acked entry on the next consume() (and the DLQ parks it after +bus_max_delivery_attempts, so a poison event can't loop forever); at-most-once +has already acked it, which is exactly the loss that mode accepts. +""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Callable + +logger = logging.getLogger("intelliops.supervise") + + +def supervise( + name: str, + target: Callable[..., object], + stop_event: threading.Event, + args: tuple = (), + kwargs: dict | None = None, + backoff_seconds: float = 1.0, + max_backoff_seconds: float = 30.0, +) -> None: + kwargs = kwargs or {} + delay = backoff_seconds + while not stop_event.is_set(): + try: + target(*args, **kwargs) + return + except Exception: + logger.exception("%s crashed; restarting in %.1fs", name, delay) + if stop_event.wait(delay): + return + delay = min(delay * 2, max_backoff_seconds) + + +def start_supervised( + name: str, + target: Callable[..., object], + stop_event: threading.Event, + args: tuple = (), + kwargs: dict | None = None, +) -> threading.Thread: + """Start `target(*args, **kwargs)` on a supervised daemon thread.""" + thread = threading.Thread( + target=supervise, + name=name, + args=(name, target, stop_event, args, kwargs), + daemon=True, + ) + thread.start() + return thread diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 07f35b9..c1cf955 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:1 FROM python:3.11-slim AS builder-base -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +# Pinned: :latest made every image build depend on whatever uv shipped that day. +COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv WORKDIR /app ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy diff --git a/deploy/Dockerfile.frontend b/deploy/Dockerfile.frontend index 9db8b0c..b08b6e7 100644 --- a/deploy/Dockerfile.frontend +++ b/deploy/Dockerfile.frontend @@ -12,6 +12,10 @@ ENV VITE_READ_URL=/api/read ENV VITE_GOV_URL=/api/gov ENV VITE_CORR_URL=/api/corr ENV VITE_RCA_URL=/api/rca +# The human identity the console acts as. It must name an approver in +# policies/rbac_policy.yaml or every decision is rejected with 403. +ARG VITE_OPERATOR_NAME=demo-operator +ENV VITE_OPERATOR_NAME=${VITE_OPERATOR_NAME} RUN npm run build FROM nginx:1.27-alpine diff --git a/deploy/Dockerfile.meridian b/deploy/Dockerfile.meridian index 3fa13be..b40ace6 100644 --- a/deploy/Dockerfile.meridian +++ b/deploy/Dockerfile.meridian @@ -13,7 +13,8 @@ RUN npm run build FROM python:3.11-slim -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +# Pinned: :latest made every image build depend on whatever uv shipped that day. +COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv WORKDIR /app ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy diff --git a/deploy/k8s/platform/templates/configmap.yaml b/deploy/k8s/platform/templates/configmap.yaml index 62b807d..42b5e4c 100644 --- a/deploy/k8s/platform/templates/configmap.yaml +++ b/deploy/k8s/platform/templates/configmap.yaml @@ -41,6 +41,7 @@ data: # this correlator's name is meant to buy. INTELLIOPS_CORRELATION_ROBUST_WINDOW: {{ .Values.env.CORRELATION_ROBUST_WINDOW | default "128" | quote }} INTELLIOPS_CORRELATION_ROBUST_WARMUP: {{ .Values.env.CORRELATION_ROBUST_WARMUP | default "30" | quote }} + INTELLIOPS_CORRELATION_ROBUST_KEY_BY: {{ .Values.env.CORRELATION_ROBUST_KEY_BY | default "metric" | quote }} INTELLIOPS_DETECTION_POLICY: {{ .Values.env.DETECTION_POLICY | quote }} INTELLIOPS_RUNBOOK_SELECTOR_MODE: {{ .Values.env.RUNBOOK_SELECTOR_MODE | quote }} INTELLIOPS_REMEDIATOR_MODE: {{ .Values.env.REMEDIATOR_MODE | quote }} diff --git a/deploy/k8s/platform/templates/service-deployment.yaml b/deploy/k8s/platform/templates/service-deployment.yaml index 785f160..458ee45 100644 --- a/deploy/k8s/platform/templates/service-deployment.yaml +++ b/deploy/k8s/platform/templates/service-deployment.yaml @@ -28,6 +28,13 @@ metadata: {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 4 }} spec: replicas: 1 + # Recreate, not the default RollingUpdate. Every consumer uses one stable + # consumer name per group (ADR-033), and a rolling update briefly runs the old + # and new pod side by side - two consumers sharing one name and one pending + # list, both able to act on the same entry. replicas: 1 only holds if the + # rollout never surges. + strategy: + type: Recreate selector: matchLabels: {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 6 }} diff --git a/deploy/k8s/platform/values.yaml b/deploy/k8s/platform/values.yaml index 698aa1a..0b73b45 100644 --- a/deploy/k8s/platform/values.yaml +++ b/deploy/k8s/platform/values.yaml @@ -59,7 +59,7 @@ env: DETECTION_POLICY: "off" # on in values-live (Phase 2) RUNBOOK_SELECTOR_MODE: "off" # embedding in values-live (Phase 3) REMEDIATOR_MODE: "dry_run" # k8s in values-live (real remediation) - SANDBOX_MODE: "off" # k8s in values-live (pre-flight rehearsal) + SANDBOX_MODE: "off" # off in values-live too (ADR-029); k8s via docker-compose.k8s.yml HEALTH_CHECK_MODE: "always" # k8s in values-live (Phase 4 per-metric verify) # Must stay comfortably above the remediated workload's termination grace # period; see the configmap comment. 30 (the old hardcoded value) equalled @@ -75,6 +75,10 @@ env: CORRELATION_MIN_EVENTS: "1" # values-live raises this; 1 = historical behaviour CORRELATION_ROBUST_WINDOW: "128" CORRELATION_ROBUST_WARMUP: "30" # values-live requires a FULL window instead + # "series" gives each (metric, service) its own baseline instead of pooling + # every service into one per metric. Re-tune warm-up before flipping it: a + # per-series window fills ~5x slower (one sample per poll, not five). + CORRELATION_ROBUST_KEY_BY: "metric" GOVERNANCE_MODE: "in_process" # "http" in values-live — action + governance are GOVERNANCE_URL: "http://governance:8000" # separate pods, so approvals must go over HTTP HITL_POLL_TIMEOUT_SECONDS: "30" # values-live raises this so a human has time to click Approve diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 7eec784..87a770e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -29,7 +29,8 @@ need a token). In `token` mode **all** endpoints are gated — including the internal service-to-service paths on governance (`POST /audit`, `POST /rbac/check`, -`POST /approvals`, `GET /approvals/{id}`, `POST /playbooks/{id}/graduate`). +`POST /approvals`, `GET /approvals/{id}`, `POST /approvals/{id}/expire`, +`POST /playbooks/{id}/graduate`, `POST /playbooks/{id}/demote`). Internal callers (action's `HttpGovernanceGate`, feedback's graduator) attach the shared `Bearer` token to their requests automatically. @@ -65,7 +66,7 @@ that talk to governance over REST: | --- | --- | | `governance` | Validates incoming tokens on all endpoints. | | `action` | `HttpGovernanceGate` calls `POST /rbac/check`, `POST /audit`, `POST /approvals`, `GET /approvals/{id}`. | -| `feedback` | `_make_graduator` calls `POST /playbooks/{id}/graduate`. | +| `feedback` | `_make_graduator` / `_make_demoter` call `POST /playbooks/{id}/graduate` and `/demote`. | | `rca` | Uses the shared factory; gated if exposed. | Example compose environment block (add to each service above): diff --git a/flow.md b/flow.md index 6cbaafb..17303ee 100644 --- a/flow.md +++ b/flow.md @@ -243,10 +243,12 @@ the unmodified z-score (default `off`, byte-identical). See |----------|--------------|-----|-----------| | `check_rbac(actor, action, resource) → allow \| deny` | Decides whether an actor may perform an action. | Every automated action passes here — the enforceable RBAC guarantee. | RBAC policy | | `create_approval_request() → ApprovalRequest` | Opens a pending approval for a `hitl` playbook. | Materializes the human decision point. | `contracts.ApprovalRequest` | -| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. | The approval interface the React console's Approve/Reject buttons drive. | — | +| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. Only `approved`/`rejected` are accepted (422 otherwise), the decider needs the matching `approve`/`reject` permission, a request that is no longer pending is a 409, and the decision is written to the audit log under the human's name. | The approval interface the React console's Approve/Reject buttons drive. | RBAC, `AuditSink` | +| `expire(id)` | `POST /approvals/{id}/expire` — the requester (action-service) marks a request it stopped waiting for as `expired` at its HITL timeout. | Without it a timed-out request stayed `pending` forever and the console offered an Approve that could no longer do anything. | `AuditSink` | | `get_approval(id)` / `list_approvals()` | REST reads (`GET /approvals/{id}`, `GET /approvals`) of the pending queue. | Lets the HTTP gate poll for a decision across containers, and the dashboard show what's pending. | — | | `write_audit(record)` | Appends an immutable `AuditRecord`. | The compliance backbone (NIST AI RMF / DORA / EU AI Act). | `AuditSink` | -| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. | Standardization — playbooks are shared, not reinvented per team. | playbook store | +| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. `POST /playbooks?registered_by=` needs the `register` permission (coe-admin), refuses `hitl_mode=auto` unless the actor may also `graduate`, and is audited. | Standardization — playbooks are shared, not reinvented per team. A direct write bypasses the AI-proposal review, so it is an admin action, not an open one. | playbook store, RBAC | +| `graduate(id)` / `demote(id)` | `POST /playbooks/{id}/graduate` (hitl → auto) and `POST /playbooks/{id}/demote` (auto → hitl), both behind the `graduate` permission and audited. feedback-service calls graduate on clean **real** evidence (dry-run successes don't count unless `GRADUATION_COUNT_SIMULATED`) and demote when an auto playbook fails or rolls back. | Automation expands on evidence and contracts on failure. | RBAC, playbook store, `AuditSink` | | `propose_playbook(situation) → ProposedPlaybook` | `POST /playbooks/proposed`: calls a `RunbookAuthor` (LLM) to draft a typed runbook for a gap, forces `hitl_mode=HITL` + a server-assigned id, stores it as a **proposal** (not the live registry); 422 if the author declines. | The AI **proposes** a runbook for a gap; `model_validate` rejects any unsafe draft ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | `RunbookAuthor`, proposed store | | `approve_proposed(id)` / `reject_proposed(id)` | `POST /playbooks/proposed/{id}/approve\|reject` — RBAC-gated (reuses `approve`/`reject`), audited. **Approve registers the inner playbook into the live registry**; reject does not. | The human **disposes** — the only path from an AI draft to the live registry ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | RBAC, playbook store, `AuditSink` | diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ec992f4..bd6a070 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,7 +10,6 @@ "dependencies": { "@phosphor-icons/react": "^2.1.7", "framer-motion": "^11.11.17", - "geist": "^1.3.1", "react": "^18.3.1", "react-dom": "^18.3.1" }, @@ -38,593 +37,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", - "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", - "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", - "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "peer": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", - "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", - "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", - "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", - "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", - "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", - "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", - "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", - "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", - "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", - "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", - "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", - "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", - "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", - "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", - "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", - "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", - "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", - "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", - "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/runtime": "^1.11.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", - "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", - "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", - "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": "^20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", - "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -664,161 +76,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@next/env": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.4.tgz", - "integrity": "sha512-cjWZnUUa6jZq2kFaNe/ZyJdZonOZ/QoN0Zka2nz/FLOrfx14pQuM9c5RaSVkWMqgdt4ksgPAMWPyHSs/CyV48Q==", - "license": "MIT", - "peer": true - }, - "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.4.tgz", - "integrity": "sha512-iBr3I5LZNk5/bgl5//iTgD2tcym14MX0Xo7fD//u9dYAEgGzza1y9oywluPtf74YnOswVdH1908aK9xVz7zQTw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.4.tgz", - "integrity": "sha512-2dpiSyl2Jw/NrBPaU2MAKGSa+2MR82pJIn4Sm5Rjr+gxAeuh0z158Su3Z2O8zn7UNNq+ej4bToed6RcRN/Lydg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.4.tgz", - "integrity": "sha512-+t+U8HZT+fApePCS5h89CSH3datz29MkzyfCn+6fpsZBG/oiEOhINcb9rtkv6sdpToLGFn2e6146NzaKCXkqrA==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.4.tgz", - "integrity": "sha512-mx03GNs1ocQA5JQ4FxDMmIsNkdrZh8cuezKCrId28e5/gIPU/l7Kcy2+vmCCzdjnnmXJy+iOAu+7K0QppO6Urg==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.4.tgz", - "integrity": "sha512-YIhGY6fSMfha52bnVxnzc9zaVBzJg+cqQTOD8tXIBSx4fuv0pVMxQTE0PaS59YhnMOiYiG09IMwxJAf/CFm/Dw==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.4.tgz", - "integrity": "sha512-+eaaX6axpDb0yF1GCpiERe6njplvdC+nks/fKfcHu3XPGRrald8P3/X7yv7QLdjA51knnxwl9pxdIJsg+w1L+Q==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.4.tgz", - "integrity": "sha512-0jcXW7Xs/uzICrmgV3MhDYDeRy++1CqnpDIerlPIqYO4bhzB4WNbX/aRnQclustsAyTkFKB0z6rbcjmNg5tR8A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.4.tgz", - "integrity": "sha512-vvBzwu1pYQCp92maZCFCIw/XgOTMR5tur9GjakwIo2cmwRTMKajRZZDS9+e4KsUZWKu1E007WUeAFXRRjZeuzw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1160,16 +417,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@swc/helpers": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", - "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/@types/prop-types": { "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", @@ -1297,6 +544,7 @@ "version": "2.11.14", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.14.tgz", "integrity": "sha512-JyJ954WzuIR8/FFzX0o5krdSTrBAkcCSRfWSleRsIHSWV+cZe2FI1PKggVkFke1hBldRs+LRxUczzE9iPmgZww==", + "dev": true, "license": "Apache-2.0", "bin": { "baseline-browser-mapping": "dist/cli.cjs" @@ -1379,6 +627,7 @@ "version": "1.0.30001809", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz", "integrity": "sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==", + "dev": true, "funding": [ { "type": "opencollective", @@ -1433,13 +682,6 @@ "node": ">= 6" } }, - "node_modules/client-only": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", - "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", - "license": "MIT", - "peer": true - }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -1474,7 +716,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -1640,15 +882,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/geist": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/geist/-/geist-1.7.2.tgz", - "integrity": "sha512-Gu5lDFa3pLRyoBlBPf0QIFHVdWAnpco7fS1bJm41jyLPFoguBgiubseUN2oLXMgqZ7uxAxDoXcHMhCY/fOTTgg==", - "license": "SIL OPEN FONT LICENSE", - "peerDependencies": { - "next": ">=13.2.0" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2113,6 +1346,7 @@ "version": "3.3.18", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, "funding": [ { "type": "github", @@ -2127,89 +1361,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/next": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.4.tgz", - "integrity": "sha512-/Ztf6CeRH+ejEXUrYtqI4gkS66eFIHuSwqi60RgcpWKodxFZx2/dqVCMKBwILfAHXQ+F1b1vAudgj3mnxqtoIA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@next/env": "16.3.4", - "@swc/helpers": "0.5.23", - "baseline-browser-mapping": "^2.9.19", - "caniuse-lite": "^1.0.30001579", - "postcss": "8.5.23", - "styled-jsx": "5.1.6" - }, - "bin": { - "next": "dist/bin/next" - }, - "engines": { - "node": ">=20.9.0" - }, - "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.4", - "@next/swc-darwin-x64": "16.3.4", - "@next/swc-linux-arm64-gnu": "16.3.4", - "@next/swc-linux-arm64-musl": "16.3.4", - "@next/swc-linux-x64-gnu": "16.3.4", - "@next/swc-linux-x64-musl": "16.3.4", - "@next/swc-win32-arm64-msvc": "16.3.4", - "@next/swc-win32-x64-msvc": "16.3.4", - "sharp": "^0.35.4" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0", - "@playwright/test": "^1.51.1", - "babel-plugin-react-compiler": "*", - "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", - "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", - "sass": "^1.3.0" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@playwright/test": { - "optional": true - }, - "babel-plugin-react-compiler": { - "optional": true - }, - "sass": { - "optional": true - } - } - }, - "node_modules/next/node_modules/postcss": { - "version": "8.5.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", - "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "nanoid": "^3.3.16", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, "node_modules/node-releases": { "version": "2.0.53", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.53.tgz", @@ -2261,6 +1412,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -2628,104 +1780,16 @@ "loose-envify": "^1.1.0" } }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", - "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "@img/colour": "^1.1.0", - "detect-libc": "^2.1.2", - "semver": "^7.8.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.4", - "@img/sharp-darwin-x64": "0.35.4", - "@img/sharp-freebsd-wasm32": "0.35.4", - "@img/sharp-libvips-darwin-arm64": "1.3.3", - "@img/sharp-libvips-darwin-x64": "1.3.3", - "@img/sharp-libvips-linux-arm": "1.3.3", - "@img/sharp-libvips-linux-arm64": "1.3.3", - "@img/sharp-libvips-linux-ppc64": "1.3.3", - "@img/sharp-libvips-linux-riscv64": "1.3.3", - "@img/sharp-libvips-linux-s390x": "1.3.3", - "@img/sharp-libvips-linux-x64": "1.3.3", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", - "@img/sharp-libvips-linuxmusl-x64": "1.3.3", - "@img/sharp-linux-arm": "0.35.4", - "@img/sharp-linux-arm64": "0.35.4", - "@img/sharp-linux-ppc64": "0.35.4", - "@img/sharp-linux-riscv64": "0.35.4", - "@img/sharp-linux-s390x": "0.35.4", - "@img/sharp-linux-x64": "0.35.4", - "@img/sharp-linuxmusl-arm64": "0.35.4", - "@img/sharp-linuxmusl-x64": "0.35.4", - "@img/sharp-webcontainers-wasm32": "0.35.4", - "@img/sharp-win32-arm64": "0.35.4", - "@img/sharp-win32-ia32": "0.35.4", - "@img/sharp-win32-x64": "0.35.4" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } }, - "node_modules/styled-jsx": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", - "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==", - "license": "MIT", - "peer": true, - "dependencies": { - "client-only": "0.0.1" - }, - "engines": { - "node": ">= 12.0.0" - }, - "peerDependencies": { - "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "babel-plugin-macros": { - "optional": true - } - } - }, "node_modules/sucrase": { "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 9e50660..0bc2b87 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.0.0", "type": "module", - "description": "IntelliOps CoE \u2014 internal operator console (Overview \u00b7 Incidents \u00b7 Governance)", + "description": "IntelliOps CoE — internal operator console (Overview · Incidents · Governance)", "scripts": { "dev": "vite", "build": "node scripts/check-contrast.mjs && tsc -b && vite build", @@ -13,7 +13,6 @@ "dependencies": { "@phosphor-icons/react": "^2.1.7", "framer-motion": "^11.11.17", - "geist": "^1.3.1", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/frontend/src/data/api.ts b/frontend/src/data/api.ts index 4832e8a..2db9afc 100644 --- a/frontend/src/data/api.ts +++ b/frontend/src/data/api.ts @@ -71,9 +71,6 @@ export async function decideApproval( export const loadProposals = () => getJSON(`${GOV}/playbooks/proposed`); -export const proposePlaybook = (situation: Situation, requestedBy: string) => - postJSON(`${GOV}/playbooks/proposed`, { situation, requested_by: requestedBy }); - export const approveProposal = (id: string, decidedBy: string) => postJSON(`${GOV}/playbooks/proposed/${id}/approve`, { decided_by: decidedBy }); diff --git a/frontend/src/data/source.ts b/frontend/src/data/source.ts index ffab9c0..83d04f8 100644 --- a/frontend/src/data/source.ts +++ b/frontend/src/data/source.ts @@ -61,33 +61,6 @@ export const testLlmConfig = LIVE export const loadProposals = LIVE ? api.loadProposals : async () => _mockProposals; -export const proposePlaybook = LIVE - ? api.proposePlaybook - : async (situation: Situation, _requestedBy: string): Promise => { - // mock mode: fabricate a proposal the same shape the server would return - // (server-assigned id, forced hitl) — honest about being a stub draft. - const proposal: ProposedPlaybook = { - id: `prop-mock-${Date.now().toString(36)}`, - playbook: { - id: `ai-${situation.signature}-mock`, - name: `Drafted fix · ${situation.service}`, - match_rule: situation.id, - steps: [{ action: "restart", note: "mock draft — no LLM configured" }], - hitl_mode: "hitl", - reversible: true, - rollback_steps: [], - }, - status: "proposed", - proposed_by: "runbook-author", - rationale: "mock mode: no LLM configured — this is a stub draft, not a real AI proposal.", - source_situation_id: situation.id, - decided_by: null, - ts: Date.now(), - }; - _mockProposals.push(proposal); - return proposal; - }; - export const approveProposal = LIVE ? api.approveProposal : async (id: string, decidedBy: string): Promise => { diff --git a/frontend/src/data/types.ts b/frontend/src/data/types.ts index eb95035..90549de 100644 --- a/frontend/src/data/types.ts +++ b/frontend/src/data/types.ts @@ -22,7 +22,15 @@ export type RemediationResult = "success" | "failure" | "rolled_back" | "escalat export type OutcomeReason = | "healthy" | "unhealthy:rolled-back" + | "unhealthy:no-rollback" + | "unhealthy:rollback-failed" | "execute-failed" + | "preflight-failed" + | "interrupted:unknown" + | "denied:unsafe-scale" + | "denied:unsafe-limits" + | "denied:unsafe-probe" + | "denied:unsafe-revision" | "denied:rbac" | "refused:not-reversible" | "aborted:rejected" diff --git a/policies/rbac_policy.yaml b/policies/rbac_policy.yaml index b1ee857..62ea493 100644 --- a/policies/rbac_policy.yaml +++ b/policies/rbac_policy.yaml @@ -10,8 +10,15 @@ roles: - {action: reject, resource: "playbook:*"} coe-admin: - {action: graduate, resource: "playbook:*"} + # Direct writes to the live playbook registry (POST /playbooks). Admin-only: + # it bypasses the AI-proposal review, so it is not an approver permission. + - {action: register, resource: "playbook:*"} actors: rca-service: [operator] action-service: [operator, executor] oncall-alice: [operator, approver] + # The console's default identity (VITE_OPERATOR_NAME, frontend/src/data/api.ts). + # Without a role here every Approve / Reject / Draft click from a stock build + # is a 403. Rebuild the console with a different VITE_OPERATOR_NAME to change it. + demo-operator: [operator, approver] feedback-service: [coe-admin] diff --git a/services/action/adapters/governance_gate.py b/services/action/adapters/governance_gate.py index 77b351d..a578c76 100644 --- a/services/action/adapters/governance_gate.py +++ b/services/action/adapters/governance_gate.py @@ -48,6 +48,13 @@ def await_decision(self, approval_id: str, timeout_seconds: float) -> ApprovalRe def write_audit(self, record: AuditRecord) -> None: self._audit_sink.write(record) + def expire(self, approval_id: str, actor: str) -> None: + req = self._approvals.get(approval_id) + if req is not None and req.status == "pending" and req.requested_by == actor: + self._approvals[approval_id] = req.model_copy( + update={"status": "expired", "decided_by": actor} + ) + class HttpGovernanceGate: """The cross-process gate: action talks to governance over REST. @@ -119,3 +126,6 @@ def await_decision(self, approval_id: str, timeout_seconds: float) -> ApprovalRe def write_audit(self, record: AuditRecord) -> None: self._client.post(f"{self._base}/audit", json=record.model_dump(mode="json")) + + def expire(self, approval_id: str, actor: str) -> None: + self._client.post(f"{self._base}/approvals/{approval_id}/expire", json={"actor": actor}) diff --git a/services/action/adapters/k8s_health.py b/services/action/adapters/k8s_health.py index 6b5bd0f..abf9337 100644 --- a/services/action/adapters/k8s_health.py +++ b/services/action/adapters/k8s_health.py @@ -97,15 +97,22 @@ def _pod_ready(self, target: RemediationTarget) -> bool: # error would escape the try (Python does not consult sibling excepts for # an error raised while matching an except type). So: one catch, no gaps. try: - st = ( - self._api() - .read_namespaced_deployment_status(target.deployment, target.namespace) - .status - ) + dep = self._api().read_namespaced_deployment_status(target.deployment, target.namespace) + st = dep.status except Exception: # noqa: BLE001 — any client/config/connection error → not-yet-ready return False + # Right after the patch the controller may not have seen the new spec + # yet, and the status still describes the OLD ReplicaSet - fully ready. + # Checking ready == replicas alone could pass on the pre-restart pods. + generation = getattr(getattr(dep, "metadata", None), "generation", None) + observed = getattr(st, "observed_generation", None) + if generation is not None and observed is not None and observed < generation: + return False ready = st.ready_replicas or 0 desired = st.replicas or 0 + updated = getattr(st, "updated_replicas", None) + if updated is not None and updated < desired: + return False return desired > 0 and ready == desired def _safe_metric(self, predicate) -> bool: diff --git a/services/action/adapters/k8s_remediator.py b/services/action/adapters/k8s_remediator.py index d83de15..fca8aa4 100644 --- a/services/action/adapters/k8s_remediator.py +++ b/services/action/adapters/k8s_remediator.py @@ -37,6 +37,11 @@ def __init__(self, namespace_default: str, apps_v1=None, exc_type=None) -> None: self._ns_default = namespace_default self._apps_v1 = apps_v1 # injected in tests; loaded lazily if None self._exc_type = exc_type # the exception class to treat as a safe failure + # Replica count each deployment had before execute() first scaled it, so + # rollback() can put back exactly that. Undoing a scale with the inverse + # DELTA was wrong whenever the forward step was clamped: 9 replicas + 2 + # clamps to 10, and 10 - 2 = 8, one below where we started. + self._pre_scale: dict[tuple[str, str], int] = {} def _api(self): if self._apps_v1 is None: @@ -49,18 +54,25 @@ def _exc(self): return self._exc_type def execute(self, plan: RemediationPlan) -> bool: - return self._run(plan.target, plan.steps) + ns = plan.target.namespace or self._ns_default + self._pre_scale.pop((ns, plan.target.deployment), None) + return self._run(plan.target, plan.steps, rolling_back=False) def rollback(self, plan: RemediationPlan) -> bool: - return self._run(plan.target, plan.rollback_steps) - - def _run(self, target: RemediationTarget, steps: list[RemediationStep]) -> bool: + ok = self._run(plan.target, plan.rollback_steps, rolling_back=True) + ns = plan.target.namespace or self._ns_default + self._pre_scale.pop((ns, plan.target.deployment), None) + return ok + + def _run( + self, target: RemediationTarget, steps: list[RemediationStep], rolling_back: bool = False + ) -> bool: ns = target.namespace or self._ns_default try: exc_type = self._exc() api = self._api() for step in steps: - self._dispatch(api, ns, target.deployment, step) + self._dispatch(api, ns, target.deployment, step, rolling_back) except exc_type as exc: # any K8s API error → safe failure logger.warning("k8s remediation failed on %s/%s: %s", ns, target.deployment, exc) return False @@ -69,7 +81,9 @@ def _run(self, target: RemediationTarget, steps: list[RemediationStep]) -> bool: return False return True - def _dispatch(self, api, ns: str, deployment: str, step: RemediationStep) -> None: + def _dispatch( + self, api, ns: str, deployment: str, step: RemediationStep, rolling_back: bool = False + ) -> None: if step.action == "wait": return # readiness is the health checker's job if step.action == "restart": @@ -84,7 +98,17 @@ def _dispatch(self, api, ns: str, deployment: str, step: RemediationStep) -> Non api.patch_namespaced_deployment(deployment, ns, body) return if step.action == "scale": + key = (ns, deployment) + if rolling_back and key in self._pre_scale: + # Restore the recorded original; the step's delta is only a hint. + desired = self._pre_scale[key] + api.patch_namespaced_deployment_scale( + deployment, ns, {"spec": {"replicas": desired}} + ) + return current = api.read_namespaced_deployment(deployment, ns).spec.replicas or 1 + if not rolling_back: + self._pre_scale.setdefault(key, current) desired = max(_MIN_REPLICAS, min(_MAX_REPLICAS, current + (step.replicas or 0))) api.patch_namespaced_deployment_scale(deployment, ns, {"spec": {"replicas": desired}}) return diff --git a/services/action/app.py b/services/action/app.py index 14b272e..0ecb48a 100644 --- a/services/action/app.py +++ b/services/action/app.py @@ -10,6 +10,7 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.action.adapters.governance_gate import ( HttpGovernanceGate, InProcessGovernanceGate, @@ -121,9 +122,11 @@ async def lifespan(app: FastAPI): app.state.db_engine = stores.engine store = stores.playbook_store gate = _make_gate(settings, stores.audit_sink) - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "action-consumer", + run_consumer, + stop_event, + ( app.state.bus, store, gate, @@ -135,9 +138,7 @@ async def lifespan(app: FastAPI): stop_event, make_guard(settings, app.state.bus), ), - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/action/remediate.py b/services/action/remediate.py index c56c9c3..2f39797 100644 --- a/services/action/remediate.py +++ b/services/action/remediate.py @@ -168,6 +168,20 @@ def _audit(gate, situation: Situation, playbook: Playbook, decision: str) -> Non ) +def _expire_best_effort(gate, approval_id: str) -> None: + """We stopped waiting; say so, or the request sits "pending" forever and the + console keeps offering an Approve that can no longer do anything. Best effort: + the outcome below is already fail-closed, and a gate without expire() (older + fakes) simply leaves the request as it was.""" + expire = getattr(gate, "expire", None) + if expire is None: + return + try: + expire(approval_id, _ACTOR) + except Exception: # noqa: BLE001, S110 - never let bookkeeping change the outcome + pass + + def execute_remediation( situation: Situation, playbook: Playbook, @@ -241,6 +255,8 @@ def execute_remediation( decided = gate.await_decision(request.id, timeout_seconds) if decided.status != "approved": reason = "aborted:rejected" if decided.status == "rejected" else "aborted:timeout" + if decided.status == "pending": + _expire_best_effort(gate, request.id) _audit(gate, situation, playbook, "abort") return _outcome( situation, playbook, RemediationResult.FAILURE, reason, preflight=preflight @@ -272,7 +288,32 @@ def execute_remediation( preflight=preflight, ) - remediator.rollback(plan) + # Unhealthy. Only report ROLLED_BACK when something was actually undone: a + # playbook with no rollback steps (restart-pod ships with none) used to be + # reported as rolled back when nothing ran, and a rollback that FAILED was + # reported the same way as one that worked. + if not plan.rollback_steps: + _audit(gate, situation, playbook, "unhealthy-no-rollback") + return _outcome( + situation, + playbook, + RemediationResult.FAILURE, + "unhealthy:no-rollback", + steps=steps, + mode=mode, + preflight=preflight, + ) + if not remediator.rollback(plan): + _audit(gate, situation, playbook, "rollback-failed") + return _outcome( + situation, + playbook, + RemediationResult.FAILURE, + "unhealthy:rollback-failed", + steps=steps, + mode=mode, + preflight=preflight, + ) _audit(gate, situation, playbook, "rolled-back") return _outcome( situation, diff --git a/services/action/tests/test_k8s_health.py b/services/action/tests/test_k8s_health.py index 0b208a7..14ec679 100644 --- a/services/action/tests/test_k8s_health.py +++ b/services/action/tests/test_k8s_health.py @@ -141,3 +141,46 @@ def test_back_compat_injected_metric_healthy_still_used(): exc_type=FakeExc, ) assert checker.check(_sit(), _tgt()) is False # injected predicate honored + + +class _RolloutApps: + """Status whose controller has not caught up with the latest spec yet.""" + + def __init__(self, generation, observed, updated, ready=1, desired=1): + self.generation, self.observed, self.updated = generation, observed, updated + self.ready, self.desired = ready, desired + + def read_namespaced_deployment_status(self, name, namespace): + outer = self + + class _Meta: + generation = outer.generation + + class _St: + observed_generation = outer.observed + updated_replicas = outer.updated + ready_replicas = outer.ready + replicas = outer.desired + + class _Dep: + metadata = _Meta() + status = _St() + + return _Dep() + + +def test_stale_status_from_before_the_patch_is_not_ready(): + # The old ReplicaSet is fully ready, but the controller has not observed + # the restart yet - that is not a recovered deployment. + apps = _RolloutApps(generation=5, observed=4, updated=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is False + + +def test_old_pods_still_serving_is_not_ready(): + apps = _RolloutApps(generation=5, observed=5, updated=0, ready=1, desired=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is False + + +def test_converged_rollout_is_ready(): + apps = _RolloutApps(generation=5, observed=5, updated=1, ready=1, desired=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is True diff --git a/services/action/tests/test_k8s_remediator.py b/services/action/tests/test_k8s_remediator.py index a86aab9..97a78ac 100644 --- a/services/action/tests/test_k8s_remediator.py +++ b/services/action/tests/test_k8s_remediator.py @@ -219,3 +219,24 @@ def list_namespaced_replica_set(self, namespace, **kwargs): step = RemediationStep(action="rollback_to_revision", revision=3) # Should fail-safe (revision not found) rather than patching the wrong template assert r.execute(_plan(step)) is False + + +class _StatefulApps(FakeAppsV1): + """Remembers the replica count it was scaled to.""" + + def patch_namespaced_deployment_scale(self, name, namespace, body): + super().patch_namespaced_deployment_scale(name, namespace, body) + self._replicas = body["spec"]["replicas"] + + +def test_rollback_restores_the_original_count_even_after_a_clamp(): + # 9 + 2 clamps to 10; undoing with the inverse delta gave 8. + apps = _StatefulApps(replicas=9) + r = KubernetesRemediator("ns", apps_v1=apps, exc_type=FakeApiException) + plan = RemediationPlan( + target=RemediationTarget(namespace="ns", deployment="demo-app"), + steps=[RemediationStep(action="scale", replicas=2)], + rollback_steps=[RemediationStep(action="scale", replicas=-2)], + ) + assert r.execute(plan) and apps._replicas == 10 + assert r.rollback(plan) and apps._replicas == 9 diff --git a/services/action/tests/test_remediate.py b/services/action/tests/test_remediate.py index 58fbc8d..3adbcc4 100644 --- a/services/action/tests/test_remediate.py +++ b/services/action/tests/test_remediate.py @@ -211,3 +211,58 @@ def test_successful_outcome_records_steps_and_mode(): assert out.steps # non-empty, human-readable assert any("scale" in s for s in out.steps) assert out.mode in ("dry_run", "k8s") + + +# --- A timed-out approval is expired, not left pending forever --- + + +class _ExpiringGate(FakeGate): + def __init__(self, status, raises=False): + super().__init__(decision_status=status) + self.expired = [] + self._raises = raises + + def expire(self, approval_id, actor): + if self._raises: + raise RuntimeError("governance down") + self.expired.append((approval_id, actor)) + + +def test_timeout_expires_the_approval_request(): + gate = _ExpiringGate("pending") + r = RecordingRemediator() + out = _run(_playbook(hitl=HitlMode.HITL), gate, r, FixedHealthChecker(True)) + assert out.health_after == "aborted:timeout" + assert gate.expired == [("appr-s1", "action-service")] + assert r.executed_plan is None + + +def test_rejection_is_not_expired(): + gate = _ExpiringGate("rejected") + _run(_playbook(hitl=HitlMode.HITL), gate, RecordingRemediator(), FixedHealthChecker(True)) + assert gate.expired == [] + + +def test_expire_failure_does_not_change_the_outcome(): + gate = _ExpiringGate("pending", raises=True) + out = _run(_playbook(hitl=HitlMode.HITL), gate, RecordingRemediator(), FixedHealthChecker(True)) + assert out.health_after == "aborted:timeout" + + +# --- An unhealthy result is only "rolled back" when something was undone --- + + +def test_no_rollback_steps_is_a_failure_not_a_rollback(): + pb = _playbook().model_copy(update={"rollback_steps": []}) + r = RecordingRemediator() + out = _run(pb, FakeGate(), r, FixedHealthChecker(False)) + assert out.result == RemediationResult.FAILURE + assert out.health_after == "unhealthy:no-rollback" + assert r.rolled_back_plan is None + + +def test_a_failed_rollback_is_reported_as_such(): + r = RecordingRemediator(rollback_result=False) + out = _run(_playbook(), FakeGate(), r, FixedHealthChecker(False)) + assert out.result == RemediationResult.FAILURE + assert out.health_after == "unhealthy:rollback-failed" diff --git a/services/correlation/adapters/__init__.py b/services/correlation/adapters/__init__.py index 68cf9f9..f006138 100644 --- a/services/correlation/adapters/__init__.py +++ b/services/correlation/adapters/__init__.py @@ -56,6 +56,7 @@ def make_correlator(settings: Settings) -> BaseCorrelator: seasonal_buckets=settings.correlation_seasonal_buckets, window_size=settings.correlation_robust_window, detection_policy=policy, + key_by=settings.correlation_robust_key_by, ) if kind == "trained": from services.correlation.adapters.trained_correlator import TrainedCorrelator diff --git a/services/correlation/adapters/base_correlator.py b/services/correlation/adapters/base_correlator.py index fa24f97..870d85a 100644 --- a/services/correlation/adapters/base_correlator.py +++ b/services/correlation/adapters/base_correlator.py @@ -30,6 +30,8 @@ def __init__( self._z_threshold = z_threshold self._warmup_samples = warmup_samples self._reliability: dict[str, float] = {} + # How many labelled outcomes each reliability figure rests on. + self._samples: dict[str, int] = {} self._policy = ( detection_policy if detection_policy is not None else DetectionPolicy(enabled=False) ) @@ -46,6 +48,23 @@ def snapshot(self) -> list[dict]: ... @abstractmethod def load(self, rows: list[dict]) -> None: ... + def _clone_kwargs(self) -> dict: + """Constructor kwargs that reproduce this correlator's configuration. + Subclasses with extra settings extend this.""" + return { + "z_threshold": self._z_threshold, + "warmup_samples": self._warmup_samples, + "detection_policy": self._policy, + } + + def clone_empty(self) -> BaseCorrelator: + """A fresh correlator with the same configuration and no learned state. + + CorrelationEngine.reset() used to rebuild via type(c)(z_threshold=, + warmup_samples=, detection_policy=), which quietly reset every other + setting - window size, seasonal buckets - to its default.""" + return type(self)(**self._clone_kwargs()) + def is_anomaly(self, event: TelemetryEvent) -> bool: return self.is_anomaly_scored(event, self.detect(event)) @@ -62,11 +81,16 @@ def retrain(self, training_data: list[dict]) -> None: if record.get("worked"): worked[sig] = worked.get(sig, 0) + 1 self._reliability = {sig: worked.get(sig, 0) / n for sig, n in total.items()} + self._samples = dict(total) def reliability(self, signature: str) -> float: return self._reliability.get(signature, 0.0) - def should_suppress(self, signature: str, threshold: float) -> bool: + def should_suppress(self, signature: str, threshold: float, min_samples: int = 1) -> bool: + # min_samples: a single success is reliability 1/1 = 1.0, which cleared any + # threshold. One data point is not a track record. + if self._samples.get(signature, 0) < min_samples: + return False return self.reliability(signature) >= threshold def _severity_band(self, score: float) -> str: @@ -78,5 +102,13 @@ def _severity_band(self, score: float) -> str: @staticmethod def _signature(events: list[TelemetryEvent]) -> str: - joined = "|".join(sorted(e.fingerprint for e in events)) + # The SET of series involved, not the multiset of samples. A fingerprint + # identifies a series (source + name + labels), so every sample of one + # series shares it; hashing the raw list made the signature depend on how + # many anomalous samples happened to land in the window - six vs seven + # samples of the same fault on the same service gave two unrelated + # signatures, and everything keyed on the signature (suppression, + # graduation evidence, RCA reliability, the author's past decisions) + # only matched by luck. + joined = "|".join(sorted({e.fingerprint for e in events})) return hashlib.sha1(joined.encode()).hexdigest()[:16] diff --git a/services/correlation/adapters/robust_correlator.py b/services/correlation/adapters/robust_correlator.py index aedce54..68e2905 100644 --- a/services/correlation/adapters/robust_correlator.py +++ b/services/correlation/adapters/robust_correlator.py @@ -53,25 +53,72 @@ def __init__( seasonal_buckets: int = 24, window_size: int = 128, detection_policy: DetectionPolicy | None = None, + key_by: str = "metric", ) -> None: # sets _z_threshold/_warmup_samples/_reliability/_policy super().__init__(z_threshold, warmup_samples, detection_policy=detection_policy) self._n_buckets = seasonal_buckets self._window_size = window_size - self._windows: dict[tuple[str, int], collections.deque] = {} + # "metric" (default, historical): one window per metric NAME, pooling every + # service's samples. A fault on a quiet service is then judged against a + # busier one's normal - a 10 -> 45 CPU jump scores ~0 beside a service + # idling at 80. "series": one window per (metric, service label). + # Per-series windows fill more slowly (one sample per poll instead of one + # per service per poll), so pair it with a smaller warm-up. + self._key_by = key_by + self._windows: dict[tuple, collections.deque] = {} + + def _clone_kwargs(self) -> dict: + return { + **super()._clone_kwargs(), + "seasonal_buckets": self._n_buckets, + "window_size": self._window_size, + "key_by": self._key_by, + } def _bucket(self, event: TelemetryEvent) -> int: return event.ts.hour % self._n_buckets + def _series(self, event: TelemetryEvent) -> str: + if self._key_by != "series": + return "" + return event.labels.get("service") or event.labels.get("job") or "" + + def _key(self, name: str, series: str, bucket: int) -> tuple: + # The default mode keeps the historical 2-tuple so snapshots and any + # external reader of _windows see exactly the old shape. + return (name, series, bucket) if self._key_by == "series" else (name, bucket) + + def _scoring_window(self, name: str, series: str, win) -> collections.deque: + """The window to score against: this hour's, or - while it is still + warming - the fullest warm window of the same series from another hour. + + Without the fallback every top-of-hour opened a fresh, empty bucket and the + detector scored 0 for everything until it re-warmed (~2 minutes under the + live posture) - a blackout on the hour, every hour.""" + if len(win) >= self._warmup_samples: + return win + best = win + for key, other in list(self._windows.items()): + if other is win or len(other) < self._warmup_samples: + continue + if key[0] != name or (self._key_by == "series" and key[1] != series): + continue + if len(other) > len(best): + best = other + return best + def detect(self, event: TelemetryEvent) -> float: if event.value is None: return 0.0 - key = (event.name, self._bucket(event)) + series = self._series(event) + key = self._key(event.name, series, self._bucket(event)) win = self._windows.setdefault(key, collections.deque(maxlen=self._window_size)) - if len(win) < self._warmup_samples: + ref = self._scoring_window(event.name, series, win) + if len(ref) < self._warmup_samples: score = 0.0 else: - arr = np.fromiter(win, dtype=float, count=len(win)) + arr = np.fromiter(ref, dtype=float, count=len(ref)) med = np.median(arr) mad = np.median(np.abs(arr - med)) deviation = abs(event.value - med) @@ -130,9 +177,9 @@ def baseline_snapshot(self) -> dict: produced the situation. """ pooled: dict[str, list[float]] = {} - for (name, _bucket), win in list(self._windows.items()): # list() = live-resize guard + for key, win in list(self._windows.items()): # list() = live-resize guard if win: - pooled.setdefault(name, []).extend(win) + pooled.setdefault(key[0], []).extend(win) out: dict = {} for name, samples in pooled.items(): @@ -147,13 +194,16 @@ def baseline_snapshot(self) -> dict: def snapshot(self) -> list[dict]: out: list[dict] = [] - for (name, bucket), win in list(self._windows.items()): # list() = live-resize guard - out.append({"metric_name": name, "bucket": bucket, "n": len(win), "window": list(win)}) + for key, win in list(self._windows.items()): # list() = live-resize guard + row = {"metric_name": key[0], "bucket": key[-1], "n": len(win), "window": list(win)} + if self._key_by == "series": + row["series"] = key[1] + out.append(row) return out def load(self, rows: list[dict]) -> None: for r in rows: - key = (r["metric_name"], int(r["bucket"])) + key = self._key(r["metric_name"], r.get("series", ""), int(r["bucket"])) self._windows[key] = collections.deque( (float(x) for x in r["window"]), maxlen=self._window_size ) diff --git a/services/correlation/adapters/trained_correlator.py b/services/correlation/adapters/trained_correlator.py index 71867e8..d36abfb 100644 --- a/services/correlation/adapters/trained_correlator.py +++ b/services/correlation/adapters/trained_correlator.py @@ -92,6 +92,8 @@ def __init__( super().__init__(z_threshold, warmup_samples, detection_policy=detection_policy) self._min_fit_samples = min_fit_samples self._contamination = contamination + self._seasonal_buckets = seasonal_buckets + self._window_size = window_size # The online path: a composed RobustCorrelator built with the same params. # Deliberately NOT given detection_policy: the engine decides anomaly-ness # using the OUTER (trained) correlator's _policy; the inner robust is used @@ -107,6 +109,15 @@ def __init__( self._features: collections.deque[list[float]] = collections.deque(maxlen=4096) self._model = None # sklearn IsolationForest once fitted, else None + def _clone_kwargs(self) -> dict: + return { + **super()._clone_kwargs(), + "seasonal_buckets": self._seasonal_buckets, + "window_size": self._window_size, + "min_fit_samples": self._min_fit_samples, + "contamination": self._contamination, + } + # --- featurization -------------------------------------------------------- @staticmethod diff --git a/services/correlation/app.py b/services/correlation/app.py index 759a69e..654eed4 100644 --- a/services/correlation/app.py +++ b/services/correlation/app.py @@ -14,6 +14,7 @@ from common.envelope import publish_model from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.correlation.adapters import make_correlator from services.correlation.consumer import ( @@ -105,6 +106,10 @@ async def lifespan(app: FastAPI): window_seconds=settings.correlation_window_seconds, group_by=settings.correlation_group_by, min_events=settings.correlation_min_events, + # Both were settings the service never passed: the threshold was dead + # config and the engine always used its 0.8 default. + suppress_threshold=settings.reliability_suppress_threshold, + suppress_min_samples=settings.reliability_suppress_min_samples, ) app.state.engine = engine # Reload-on-boot: restore the durable baseline + reliability BEFORE the @@ -139,15 +144,17 @@ async def lifespan(app: FastAPI): _reload_model(engine, model_store) app.state.baseline_store = baseline_store app.state.model_store = model_store - thread = threading.Thread( - target=run_consumer, - args=(app.state.bus, engine, stop_event, make_guard(settings, app.state.bus)), - daemon=True, + thread = start_supervised( + "correlation-consumer", + run_consumer, + stop_event, + (app.state.bus, engine, stop_event, make_guard(settings, app.state.bus)), ) - thread.start() - flusher = threading.Thread( - target=run_flusher, - args=( + flusher = start_supervised( + "correlation-flusher", + run_flusher, + stop_event, + ( app.state.bus, engine, settings.correlation_window_seconds, @@ -155,9 +162,7 @@ async def lifespan(app: FastAPI): baseline_store, settings.baseline_snapshot_seconds, ), - daemon=True, ) - flusher.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread app.state.flusher_thread = flusher diff --git a/services/correlation/consumer.py b/services/correlation/consumer.py index d837737..1799699 100644 --- a/services/correlation/consumer.py +++ b/services/correlation/consumer.py @@ -29,8 +29,7 @@ def _snapshot_baseline_once(engine, baseline_store) -> None: def _drain_suppressed(bus, engine: CorrelationEngine) -> None: - s = engine.pop_suppressed() - if s is not None: + while (s := engine.pop_suppressed()) is not None: publish_model(bus, "situations.suppressed", s) diff --git a/services/correlation/engine.py b/services/correlation/engine.py index 53596ec..56fdf75 100644 --- a/services/correlation/engine.py +++ b/services/correlation/engine.py @@ -38,15 +38,21 @@ def __init__( suppress_threshold: float = 0.8, group_by: str = "window", min_events: int = 1, + suppress_min_samples: int = 1, ) -> None: self._correlator = correlator - self._correlator_factory = lambda: type(correlator)( - z_threshold=correlator._z_threshold, - warmup_samples=correlator._warmup_samples, - detection_policy=correlator._policy, + self._correlator_factory = ( + correlator.clone_empty + if hasattr(correlator, "clone_empty") + else lambda: type(correlator)( + z_threshold=correlator._z_threshold, + warmup_samples=correlator._warmup_samples, + detection_policy=correlator._policy, + ) ) self._window = window_seconds self._suppress_threshold = suppress_threshold + self._suppress_min_samples = max(1, int(suppress_min_samples)) self._group_by = group_by # How many anomalous events a window must hold before it is an incident. # @@ -64,7 +70,9 @@ def __init__( # this is the old single-buffer behaviour with one dict lookup. self._buffers: dict[str, list[TelemetryEvent]] = {} self._max_scores: dict[str, float] = {} - self._suppressed: Situation | None = None + # A queue, not a slot: one flush_all() can suppress several buckets, and a + # single slot kept only the last, silently dropping the rest. + self._suppressed: list[Situation] = [] # Guards _buffer/_max_score so a background time-flush (see the service # lifespan) can run concurrently with add() on the consumer thread. # Single-threaded callers (tests) are unaffected — the lock is uncontended. @@ -155,8 +163,10 @@ def _correlate_buffer(self, key: str = _ALL) -> Situation | None: self._buffers.pop(key, None) self._max_scores.pop(key, None) # Closed loop: suppress a Situation whose signature reliably self-heals. - if self._correlator.should_suppress(sit.signature, self._suppress_threshold): - self._suppressed = sit + if self._correlator.should_suppress( + sit.signature, self._suppress_threshold, self._suppress_min_samples + ): + self._suppressed.append(sit) return None return sit @@ -169,14 +179,13 @@ def load(self, rows: list[dict]) -> None: self._correlator.load(rows) def pop_suppressed(self) -> Situation | None: + """Oldest suppressed Situation not yet published, or None.""" with self._lock: - s = self._suppressed - self._suppressed = None - return s + return self._suppressed.pop(0) if self._suppressed else None def reset(self) -> None: with self._lock: self._correlator = self._correlator_factory() self._buffers = {} self._max_scores = {} - self._suppressed = None + self._suppressed = [] diff --git a/services/correlation/tests/test_engine_suppress.py b/services/correlation/tests/test_engine_suppress.py index e59e23c..eddcab5 100644 --- a/services/correlation/tests/test_engine_suppress.py +++ b/services/correlation/tests/test_engine_suppress.py @@ -70,3 +70,27 @@ def test_engine_still_emits_unreliable_signature(): engine.add(_event(120.0, "b", 4)) still_emitted = engine.flush() assert still_emitted is not None # unreliable signature is NOT suppressed + + +def test_every_bucket_suppressed_in_one_flush_is_reported(): + # With group_by="service", one flush_all() can suppress several buckets. The + # engine used to keep suppressed situations in a single slot, so only the + # last one ever reached situations.suppressed. + correlator = RiverCorrelator(z_threshold=3.0) + engine = CorrelationEngine(correlator, window_seconds=30, group_by="service") + ev_a = _event(1.0, "fa").model_copy(update={"labels": {"service": "a"}}) + ev_b = _event(1.0, "fb").model_copy(update={"labels": {"service": "b"}}) + engine._buffers = {"a": [ev_a], "b": [ev_b]} + engine._max_scores = {"a": 9.0, "b": 9.0} + correlator.retrain( + [ + {"signature": correlator._signature([ev_a]), "worked": True}, + {"signature": correlator._signature([ev_b]), "worked": True}, + ] + ) + + assert engine.flush_all() == [] + suppressed = [] + while (s := engine.pop_suppressed()) is not None: + suppressed.append(s) + assert len(suppressed) == 2 diff --git a/services/correlation/tests/test_retrain.py b/services/correlation/tests/test_retrain.py index 99e5794..c0cff70 100644 --- a/services/correlation/tests/test_retrain.py +++ b/services/correlation/tests/test_retrain.py @@ -43,3 +43,14 @@ def test_retrain_replaces_prior(): assert c.reliability("a") == 0.0 c.retrain([{"signature": "a", "worked": True}, {"signature": "a", "worked": True}]) assert c.reliability("a") == 1.0 + + +def test_one_success_is_not_a_track_record(): + # reliability 1/1 = 1.0 used to clear any threshold, so a single fixed + # incident was enough to silence that signature. + c = RiverCorrelator() + c.retrain([{"signature": "s", "worked": True}]) + assert c.should_suppress("s", 0.8) is True # historical default: min_samples=1 + assert c.should_suppress("s", 0.8, min_samples=3) is False + c.retrain([{"signature": "s", "worked": True}] * 3) + assert c.should_suppress("s", 0.8, min_samples=3) is True diff --git a/services/correlation/tests/test_robust_correlator.py b/services/correlation/tests/test_robust_correlator.py index 837e3a3..725b342 100644 --- a/services/correlation/tests/test_robust_correlator.py +++ b/services/correlation/tests/test_robust_correlator.py @@ -231,3 +231,70 @@ def test_baseline_snapshot_pools_across_seasonal_buckets(): def test_baseline_snapshot_is_empty_before_any_observation(): assert RobustCorrelator(window_size=50, warmup_samples=3).baseline_snapshot() == {} + + +# --- per-series baselines, the top-of-hour blackout, config-preserving reset --- + + +def _svc_event(svc, value, ts): + return TelemetryEvent( + source="prom", + kind=TelemetryKind.METRIC, + name="cpu_usage", + value=value, + labels={"service": svc}, + ts=ts, + fingerprint=f"fp-{svc}", + ) + + +def _two_services(c): + ts0 = datetime(2026, 8, 13, 0, 0, 0, tzinfo=UTC) + for i in range(40): # a idles near 80% cpu, b near 10% + c.detect(_svc_event("a", 80.0 + (i % 3), ts0 + timedelta(seconds=i))) + c.detect(_svc_event("b", 10.0 + (i % 3), ts0 + timedelta(seconds=i))) + return ts0 + timedelta(seconds=41) + + +def test_metric_keying_masks_a_fault_on_a_quiet_service(): + # The historical default, pinned so the difference below is visible. + c = RobustCorrelator(warmup_samples=40) + t = _two_services(c) + assert c.detect(_svc_event("b", 45.0, t)) < 3.0 + + +def test_series_keying_catches_it(): + c = RobustCorrelator(warmup_samples=40, key_by="series") + t = _two_services(c) + assert c.detect(_svc_event("b", 45.0, t)) > 3.0 + assert c.detect(_svc_event("a", 81.0, t)) < 3.0 # a's normal is still normal + + +def test_series_snapshot_round_trips(): + c = RobustCorrelator(warmup_samples=40, key_by="series") + t = _two_services(c) + rows = c.snapshot() + assert {r["series"] for r in rows} == {"a", "b"} + c2 = RobustCorrelator(warmup_samples=40, key_by="series") + c2.load(rows) + assert c2.detect(_svc_event("b", 45.0, t)) == c.detect(_svc_event("b", 45.0, t)) + + +def test_a_new_hour_scores_against_the_last_warm_hour(): + c = RobustCorrelator(z_threshold=3.0, warmup_samples=30) + _feed_flat(c, name="cpu", value=10.0, n=40, hour=0) + for i in range(40): # give the flat window a little spread + c.detect(_event(value=10.0 + (i % 3) * 0.1, ts=datetime(2026, 8, 13, 0, 1, i, tzinfo=UTC))) + one_am = datetime(2026, 8, 13, 1, 0, 0, tzinfo=UTC) + # the hour-1 bucket is empty; the spike used to score 0.0 here + assert c.detect(_event(value=500.0, ts=one_am)) > 3.0 + + +def test_reset_keeps_the_configuration(): + c = RobustCorrelator( + z_threshold=4.0, warmup_samples=7, seasonal_buckets=6, window_size=33, key_by="series" + ) + fresh = c.clone_empty() + assert (fresh._z_threshold, fresh._warmup_samples) == (4.0, 7) + assert (fresh._n_buckets, fresh._window_size, fresh._key_by) == (6, 33, "series") + assert fresh._windows == {} diff --git a/services/correlation/tests/test_suppressed_signal.py b/services/correlation/tests/test_suppressed_signal.py index 450347e..4f61d72 100644 --- a/services/correlation/tests/test_suppressed_signal.py +++ b/services/correlation/tests/test_suppressed_signal.py @@ -21,7 +21,7 @@ def _ev(v, i, t0): def test_pop_suppressed_returns_suppressed_situation(): # Force suppression: a correlator whose should_suppress is always True. class AlwaysSuppress(RiverCorrelator): - def should_suppress(self, signature, threshold): + def should_suppress(self, signature, threshold, min_samples=1): return True c = AlwaysSuppress(z_threshold=0.0, warmup_samples=0) diff --git a/services/feedback/app.py b/services/feedback/app.py index d1b8d6a..8ebcc09 100644 --- a/services/feedback/app.py +++ b/services/feedback/app.py @@ -11,18 +11,21 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.feedback.consumer import run_consumer from services.feedback.metrics import compute_metrics -def _make_graduator(rbac_actor: str = "feedback-service"): - # In the running service, graduation calls governance's REST endpoint. - # Inside the docker-compose network, governance listens on its internal - # PORT=8000 (8005 is only the host-side port mapping). Best-effort, fire-and- - # forget: a failed graduation is logged by governance's own audit, and the - # next matching outcome will retry on a fresh process. Kept simple here. - def graduate(playbook_id: str) -> None: +def _make_playbook_call(verb: str, rbac_actor: str = "feedback-service"): + """POST governance's /playbooks/{id}/ (graduate or demote). + + Inside the docker-compose network, governance listens on its internal + PORT=8000 (8005 is only the host-side port mapping). Best-effort, fire-and- + forget: governance's own audit is the record, and the next matching outcome + retries on a fresh process.""" + + def call(playbook_id: str) -> None: try: settings = get_settings() headers = ( @@ -31,7 +34,7 @@ def graduate(playbook_id: str) -> None: else {} ) httpx.post( - f"http://governance:8000/playbooks/{playbook_id}/graduate", + f"http://governance:8000/playbooks/{playbook_id}/{verb}", json={"decided_by": rbac_actor}, headers=headers, timeout=5.0, @@ -39,7 +42,15 @@ def graduate(playbook_id: str) -> None: except Exception: # noqa: BLE001, S110 — best-effort; governance's own audit is the record pass - return graduate + return call + + +def _make_graduator(rbac_actor: str = "feedback-service"): + return _make_playbook_call("graduate", rbac_actor) + + +def _make_demoter(rbac_actor: str = "feedback-service"): + return _make_playbook_call("demote", rbac_actor) @asynccontextmanager @@ -50,9 +61,11 @@ async def lifespan(app: FastAPI): app.state.db_engine = stores.engine store = stores.training_store app.state.training_store = store - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "feedback-consumer", + run_consumer, + stop_event, + ( app.state.bus, store, _make_graduator(), @@ -60,9 +73,11 @@ async def lifespan(app: FastAPI): stop_event, make_guard(settings, app.state.bus), ), - daemon=True, + { + "demoter": _make_demoter(), + "count_simulated": settings.graduation_count_simulated, + }, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/feedback/consumer.py b/services/feedback/consumer.py index 0574ffc..e7c720b 100644 --- a/services/feedback/consumer.py +++ b/services/feedback/consumer.py @@ -12,7 +12,7 @@ import threading from collections.abc import Callable -from common.contracts import RemediationOutcome, RemediationResult +from common.contracts import HitlMode, RemediationOutcome, RemediationResult from common.envelope import iter_models from common.idempotency import NullGuard from services.feedback.graduate import playbook_stats, should_graduate @@ -26,6 +26,8 @@ def run_consumer( min_successes: int, stop_event: threading.Event, guard=None, + demoter: Callable[[str], None] | None = None, + count_simulated: bool = False, ) -> None: guard = guard if guard is not None else NullGuard() graduated: set[str] = set() @@ -42,8 +44,20 @@ def run_consumer( continue store.append(label_outcome(outcome)) pid = outcome.playbook_id + if ( + pid + and demoter is not None + and outcome.hitl_mode == HitlMode.AUTO + and outcome.result in (RemediationResult.FAILURE, RemediationResult.ROLLED_BACK) + ): + # An auto playbook just failed unattended. Graduation only ever + # expanded automation; this is the matching contraction, so the next + # run goes back behind a human. + demoter(pid) + graduated.discard(pid) + continue if pid and pid not in graduated: - stats = playbook_stats(store.read_all(), pid) + stats = playbook_stats(store.read_all(), pid, count_simulated=count_simulated) if should_graduate(stats, min_successes): graduator(pid) graduated.add(pid) diff --git a/services/feedback/graduate.py b/services/feedback/graduate.py index 22953ab..a9eec4b 100644 --- a/services/feedback/graduate.py +++ b/services/feedback/graduate.py @@ -9,13 +9,26 @@ from common.contracts import RemediationResult, TrainingRecord +# Modes in which nothing real was touched. DryRunRemediator + AlwaysHealthyChecker +# report SUCCESS unconditionally, so three approved dry runs used to graduate a +# playbook to AUTO - and flipping REMEDIATOR_MODE=k8s afterwards meant real +# changes with no human in the loop, on evidence from a simulation. +_SIMULATED_MODES = frozenset({"dry_run"}) -def playbook_stats(records: list[TrainingRecord], playbook_id: str) -> dict: + +def playbook_stats( + records: list[TrainingRecord], playbook_id: str, count_simulated: bool = False +) -> dict: + """Per-playbook evidence. A simulated success counts toward nothing unless + `count_simulated` - it proves the gates ran, not that the fix works. Records + from before `mode` existed (None) keep counting as they always did.""" successes = failures = rollbacks = 0 for r in records: if r.playbook_id != playbook_id: continue if r.result == RemediationResult.SUCCESS: + if r.mode in _SIMULATED_MODES and not count_simulated: + continue successes += 1 elif r.result == RemediationResult.FAILURE: failures += 1 diff --git a/services/feedback/label.py b/services/feedback/label.py index 3142544..9f53a08 100644 --- a/services/feedback/label.py +++ b/services/feedback/label.py @@ -21,4 +21,5 @@ def label_outcome(outcome: RemediationOutcome) -> TrainingRecord: result=outcome.result, worked=outcome.result == RemediationResult.SUCCESS, ts=outcome.ts, + mode=outcome.mode, ) diff --git a/services/feedback/tests/test_consumer.py b/services/feedback/tests/test_consumer.py index f518f08..93fa5e7 100644 --- a/services/feedback/tests/test_consumer.py +++ b/services/feedback/tests/test_consumer.py @@ -10,7 +10,14 @@ def _raw_outcome(result, playbook="restart-pod"): o = RemediationOutcome( - situation_id="sit-abc", playbook_id=playbook, result=result, health_after="healthy", ts=NOW + situation_id="sit-abc", + playbook_id=playbook, + result=result, + health_after="healthy", + ts=NOW, + # Real runs: a dry-run success is not graduation evidence + # (see test_graduation_safety.py). + mode="k8s", ) return {"data": o.model_dump_json()} diff --git a/services/feedback/tests/test_graduation_safety.py b/services/feedback/tests/test_graduation_safety.py new file mode 100644 index 0000000..5f529ad --- /dev/null +++ b/services/feedback/tests/test_graduation_safety.py @@ -0,0 +1,118 @@ +"""Graduation must rest on real evidence, and must be reversible. + +DryRunRemediator + AlwaysHealthyChecker report SUCCESS for every run, so three +approved dry runs used to graduate a playbook to AUTO. And nothing ever demoted +an auto playbook after it failed unattended. +""" + +from __future__ import annotations + +import threading +from datetime import UTC, datetime + +from common.contracts import HitlMode, RemediationOutcome, RemediationResult, TrainingRecord +from common.envelope import publish_model +from services.feedback.adapters.training_store import InMemoryTrainingStore +from services.feedback.consumer import run_consumer +from services.feedback.graduate import playbook_stats, should_graduate +from services.feedback.label import label_outcome + +NOW = datetime(2026, 9, 23, tzinfo=UTC) + + +def _rec(mode, result=RemediationResult.SUCCESS): + return TrainingRecord( + situation_id="sit-x", + signature="x", + playbook_id="pb", + result=result, + worked=result == RemediationResult.SUCCESS, + ts=NOW, + mode=mode, + ) + + +def _outcome(result, mode="k8s", hitl=HitlMode.HITL, pid="pb"): + return RemediationOutcome( + situation_id="sit-x", + playbook_id=pid, + result=result, + health_after="x", + ts=NOW, + hitl_mode=hitl, + mode=mode, + ) + + +def test_label_carries_the_mode(): + assert label_outcome(_outcome(RemediationResult.SUCCESS, mode="dry_run")).mode == "dry_run" + + +def test_dry_run_successes_do_not_graduate_by_default(): + recs = [_rec("dry_run") for _ in range(5)] + stats = playbook_stats(recs, "pb") + assert stats["successes"] == 0 + assert should_graduate(stats, 3) is False + + +def test_dry_run_successes_count_when_explicitly_allowed(): + recs = [_rec("dry_run") for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb", count_simulated=True), 3) is True + + +def test_real_successes_still_graduate(): + recs = [_rec("k8s") for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb"), 3) is True + + +def test_legacy_records_without_mode_keep_counting(): + recs = [_rec(None) for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb"), 3) is True + + +class _ListBus: + """Minimal bus: publish appends; consume yields everything published so far.""" + + def __init__(self): + self.entries: list[dict] = [] + + def publish(self, topic, message): + self.entries.append(message) + + def consume(self, topic, group): + yield from list(self.entries) + + def send_to_dlq(self, *a, **k): + return False + + +def _drive(outcomes, **kwargs): + bus = _ListBus() + for o in outcomes: + publish_model(bus, "remediation.outcomes", o) + graduated, demoted = [], [] + run_consumer( + bus, + InMemoryTrainingStore(), + graduated.append, + 3, + threading.Event(), + demoter=demoted.append, + **kwargs, + ) + return graduated, demoted + + +def test_consumer_does_not_graduate_on_dry_runs(): + graduated, _ = _drive([_outcome(RemediationResult.SUCCESS, mode="dry_run")] * 4) + assert graduated == [] + + +def test_consumer_demotes_an_auto_playbook_that_fails(): + _, demoted = _drive([_outcome(RemediationResult.ROLLED_BACK, hitl=HitlMode.AUTO)]) + assert demoted == ["pb"] + + +def test_consumer_leaves_hitl_failures_alone(): + _, demoted = _drive([_outcome(RemediationResult.FAILURE, hitl=HitlMode.HITL)]) + assert demoted == [] diff --git a/services/governance/app.py b/services/governance/app.py index 81d1e08..5aa0d76 100644 --- a/services/governance/app.py +++ b/services/governance/app.py @@ -30,6 +30,7 @@ ) from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.governance.adapters.author_tools import AuthorToolbox from services.governance.adapters.runbook_author import ( @@ -120,17 +121,17 @@ async def lifespan(app: FastAPI): # see services/governance/consumer.py) and disposes the engine on shutdown, # matching feedback's lifespan. stop_event = threading.Event() - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "governance-consumer", + run_consumer, + stop_event, + ( app.state.bus, app.state.author_decision_store, stop_event, make_guard(get_settings(), app.state.bus), ), - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread # The draft-async worker threads publish trace steps via AgentRunHub from @@ -207,9 +208,40 @@ def query_audit(correlation_id: str | None = None) -> list[AuditRecord]: return app.state.audit_sink.records(correlation_id) +def _audit(actor: str, action: str, resource: str, correlation_id: str, decision="allow") -> None: + app.state.audit_sink.write( + AuditRecord( + actor=actor, + action=action, + resource=resource, + decision=decision, + ts=datetime.now(UTC), + correlation_id=correlation_id, + ) + ) + + @app.post("/playbooks") -def register_playbook(playbook: Playbook) -> dict[str, str]: +def register_playbook(playbook: Playbook, registered_by: str) -> dict[str, str]: + """Write a playbook straight into the live registry. + + This used to take no identity at all, which made it a back door around every + other control here: anyone who could reach governance could overwrite a seed + playbook with hitl_mode=auto and skip both the HITL gate and the + AI-proposal review. It is now an audited admin action, and AUTO is only + reachable by an actor who could have graduated the playbook anyway. + """ + resource = f"playbook:{playbook.id}" + if not app.state.rbac.check(registered_by, "register", resource): + raise HTTPException(status_code=403, detail="actor lacks register permission") + if playbook.hitl_mode == HitlMode.AUTO and not app.state.rbac.check( + registered_by, "graduate", resource + ): + raise HTTPException( + status_code=403, detail="only an actor with graduate permission may register auto" + ) app.state.playbook_store.register(playbook) + _audit(registered_by, "register", resource, resource) return {"status": "ok"} @@ -258,16 +290,62 @@ def get_approval(approval_id: str) -> ApprovalRequest: return req +# What a human may set an approval to. action-service reads `status != "approved"` +# as a refusal, so any other string used to be accepted, stored, and then reported +# downstream as a timeout. +_HUMAN_DECISIONS = {"approved": "approve", "rejected": "reject"} + + @app.post("/approvals/{approval_id}/decide") def decide_approval(approval_id: str, decision: Decision) -> ApprovalRequest: req = app.state.approval_store.get(approval_id) if req is None: raise HTTPException(status_code=404, detail="approval not found") - if not app.state.rbac.check(decision.decided_by, "approve", f"playbook:{req.playbook_id}"): - raise HTTPException(status_code=403, detail="decider lacks approve permission") + permission = _HUMAN_DECISIONS.get(decision.decision) + if permission is None: + raise HTTPException(status_code=422, detail="decision must be 'approved' or 'rejected'") + if not app.state.rbac.check(decision.decided_by, permission, f"playbook:{req.playbook_id}"): + raise HTTPException(status_code=403, detail=f"decider lacks {permission} permission") + if req.status != "pending": + # Deciding twice would rewrite history (approved -> rejected after the fix + # already ran), and deciding an expired request does nothing but look like + # it did. + raise HTTPException(status_code=409, detail=f"approval is already {req.status}") updated = app.state.approval_store.decide( approval_id, status=decision.decision, decided_by=decision.decided_by ) + # The human decision is the governance event this service exists to record, + # and until now it only lived inside the approvals row. + _audit( + decision.decided_by, + permission, + f"playbook:{req.playbook_id}", + req.situation_id, + ) + return updated + + +class Expire(BaseModel): + actor: str + + +@app.post("/approvals/{approval_id}/expire") +def expire_approval(approval_id: str, body: Expire) -> ApprovalRequest: + """Mark a request nobody decided in time as expired. + + action-service stops waiting at its HITL timeout, but the request used to stay + "pending" forever: the console kept offering an Approve button that could no + longer do anything. Only the requester may expire its own request. + """ + req = app.state.approval_store.get(approval_id) + if req is None: + raise HTTPException(status_code=404, detail="approval not found") + if body.actor != req.requested_by: + raise HTTPException(status_code=403, detail="only the requester may expire a request") + if req.status != "pending": + return req + updated = app.state.approval_store.decide(approval_id, status="expired", decided_by=body.actor) + _audit(body.actor, "expire", f"playbook:{req.playbook_id}", req.situation_id) return updated @@ -292,16 +370,33 @@ def graduate_playbook(playbook_id: str, body: Graduate) -> Playbook: raise HTTPException(status_code=403, detail="actor lacks graduate permission") updated = pb.model_copy(update={"hitl_mode": HitlMode.AUTO}) app.state.playbook_store.register(updated) - app.state.audit_sink.write( - AuditRecord( - actor=body.decided_by, - action="graduate", - resource=f"playbook:{playbook_id}", - decision="allow", - ts=datetime.now(UTC), - correlation_id=f"playbook:{playbook_id}", - ) - ) + _audit(body.decided_by, "graduate", f"playbook:{playbook_id}", f"playbook:{playbook_id}") + return updated + + +class Demote(BaseModel): + decided_by: str + reason: str = "" + + +@app.post("/playbooks/{playbook_id}/demote") +def demote_playbook(playbook_id: str, body: Demote) -> Playbook: + """Put an auto playbook back behind a human (auto -> hitl). + + Graduation expanded automation on evidence but nothing ever contracted it: a + playbook that failed or rolled back after going auto stayed auto. Same + permission as graduation, since it is the same decision in reverse. + """ + pb = app.state.playbook_store.get(playbook_id) + if pb is None: + raise HTTPException(status_code=404, detail="playbook not found") + if not app.state.rbac.check(body.decided_by, "graduate", f"playbook:{playbook_id}"): + raise HTTPException(status_code=403, detail="actor lacks graduate permission") + if pb.hitl_mode != HitlMode.AUTO: + return pb + updated = pb.model_copy(update={"hitl_mode": HitlMode.HITL}) + app.state.playbook_store.register(updated) + _audit(body.decided_by, "demote", f"playbook:{playbook_id}", f"playbook:{playbook_id}") return updated @@ -579,6 +674,10 @@ def approve_proposed(proposal_id: str, body: ProposalDecision) -> ProposedPlaybo raise HTTPException(status_code=404, detail="proposal not found") if not app.state.rbac.check(body.decided_by, "approve", f"playbook:{p.playbook.id}"): raise HTTPException(status_code=403, detail="decider lacks approve permission") + if p.status != ProposedPlaybookStatus.PROPOSED: + # A rejected proposal must not be approvable later (that would put a + # runbook a human turned down into the live registry), and vice versa. + raise HTTPException(status_code=409, detail=f"proposal is already {p.status.value}") updated = app.state.proposed_store.set_status( proposal_id, ProposedPlaybookStatus.APPROVED, body.decided_by ) @@ -611,6 +710,10 @@ def reject_proposed(proposal_id: str, body: ProposalDecision) -> ProposedPlayboo raise HTTPException(status_code=404, detail="proposal not found") if not app.state.rbac.check(body.decided_by, "reject", f"playbook:{p.playbook.id}"): raise HTTPException(status_code=403, detail="decider lacks reject permission") + if p.status != ProposedPlaybookStatus.PROPOSED: + # A rejected proposal must not be approvable later (that would put a + # runbook a human turned down into the live registry), and vice versa. + raise HTTPException(status_code=409, detail=f"proposal is already {p.status.value}") updated = app.state.proposed_store.set_status( proposal_id, ProposedPlaybookStatus.REJECTED, body.decided_by ) diff --git a/services/governance/tests/test_governance_api.py b/services/governance/tests/test_governance_api.py index 4d5eb20..51bd14b 100644 --- a/services/governance/tests/test_governance_api.py +++ b/services/governance/tests/test_governance_api.py @@ -20,8 +20,9 @@ def _client(): roles={ "operator": [{"action": "diagnose", "resource": "situation:*"}], "approver": [{"action": "approve", "resource": "playbook:*"}], + "admin": [{"action": "register", "resource": "playbook:*"}], }, - actors={"rca-service": ["operator"], "oncall-alice": ["approver"]}, + actors={"rca-service": ["operator"], "oncall-alice": ["approver"], "cara": ["admin"]}, ) app.state.approval_store = InMemoryApprovalStore() return TestClient(app) @@ -59,7 +60,7 @@ def test_playbook_register_and_list(): steps=[RemediationStep(action="restart")], hitl_mode=HitlMode.HITL, ).model_dump(mode="json") - assert c.post("/playbooks", json=pb).status_code == 200 + assert c.post("/playbooks", params={"registered_by": "cara"}, json=pb).status_code == 200 assert c.get("/playbooks/restart-pod").json()["name"] == "Restart Pod" assert [p["id"] for p in c.get("/playbooks").json()] == ["restart-pod"] assert c.get("/playbooks/missing").status_code == 404 diff --git a/services/governance/tests/test_governance_safety.py b/services/governance/tests/test_governance_safety.py new file mode 100644 index 0000000..df09a20 --- /dev/null +++ b/services/governance/tests/test_governance_safety.py @@ -0,0 +1,168 @@ +"""Governance controls that used to be missing or bypassable. + +- POST /playbooks took no identity, so anyone could overwrite a seed playbook + with hitl_mode=auto and skip the HITL gate and the AI-proposal review. +- A human's approve/reject was never written to the audit log. +- /decide accepted any status string and could re-decide a closed request. +- A request action-service stopped waiting for stayed "pending" forever. +- A graduated (auto) playbook could never be put back behind a human. +""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from common.contracts import HitlMode, Playbook, RemediationStep +from services.governance.adapters.approval_store import InMemoryApprovalStore +from services.governance.adapters.audit_sink import InMemoryAuditSink +from services.governance.adapters.playbook_store import InMemoryPlaybookStore +from services.governance.rbac import RbacPolicy + + +def _client(): + from services.governance.app import app + + app.state.audit_sink = InMemoryAuditSink() + app.state.playbook_store = InMemoryPlaybookStore() + app.state.rbac = RbacPolicy.from_file("policies/rbac_policy.yaml") + app.state.approval_store = InMemoryApprovalStore() + return TestClient(app), app + + +def _pb(mode=HitlMode.HITL) -> dict: + return Playbook( + id="restart-pod", + name="Restart Pod", + match_rule="x", + steps=[RemediationStep(action="restart")], + hitl_mode=mode, + ).model_dump(mode="json") + + +def _approval(c, aid="a1"): + c.post( + "/approvals", + json={ + "id": aid, + "situation_id": "sit-1", + "playbook_id": "restart-pod", + "requested_by": "action-service", + }, + ) + + +# --- POST /playbooks ------------------------------------------------------- + + +def test_register_requires_an_identity(): + c, _ = _client() + assert c.post("/playbooks", json=_pb()).status_code == 422 + + +def test_register_refused_for_an_approver_or_stranger(): + c, app = _client() + for who in ("oncall-alice", "nobody"): + r = c.post("/playbooks", params={"registered_by": who}, json=_pb()) + assert r.status_code == 403, who + assert app.state.playbook_store.get("restart-pod") is None + + +def test_register_allowed_for_coe_admin_and_audited(): + c, app = _client() + r = c.post("/playbooks", params={"registered_by": "feedback-service"}, json=_pb()) + assert r.status_code == 200 + recs = app.state.audit_sink.records("playbook:restart-pod") + assert [(x.actor, x.action) for x in recs] == [("feedback-service", "register")] + + +def test_register_auto_needs_graduate_permission(): + c, app = _client() + app.state.rbac = RbacPolicy( + roles={"reg": [{"action": "register", "resource": "playbook:*"}]}, + actors={"registrar": ["reg"]}, + ) + r = c.post("/playbooks", params={"registered_by": "registrar"}, json=_pb(HitlMode.AUTO)) + assert r.status_code == 403 + r = c.post("/playbooks", params={"registered_by": "registrar"}, json=_pb(HitlMode.HITL)) + assert r.status_code == 200 + + +# --- /approvals/{id}/decide -------------------------------------------------- + + +def test_decide_writes_an_audit_record_naming_the_human(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + assert r.status_code == 200 + recs = app.state.audit_sink.records("sit-1") + assert [(x.actor, x.action, x.resource) for x in recs] == [ + ("oncall-alice", "approve", "playbook:restart-pod") + ] + + +def test_decide_rejects_an_unknown_decision_string(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/decide", json={"decision": "APPROVED", "decided_by": "oncall-alice"}) + assert r.status_code == 422 + assert app.state.approval_store.get("a1").status == "pending" + + +def test_decide_cannot_rewrite_a_closed_request(): + c, app = _client() + _approval(c) + c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + r = c.post("/approvals/a1/decide", json={"decision": "rejected", "decided_by": "oncall-alice"}) + assert r.status_code == 409 + assert app.state.approval_store.get("a1").status == "approved" + + +# --- /approvals/{id}/expire -------------------------------------------------- + + +def test_requester_can_expire_a_pending_request(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/expire", json={"actor": "action-service"}) + assert r.status_code == 200 and r.json()["status"] == "expired" + assert app.state.approval_store.list_pending() == [] + # an expired request can no longer be approved + r = c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + assert r.status_code == 409 + + +def test_only_the_requester_can_expire(): + c, app = _client() + _approval(c) + assert c.post("/approvals/a1/expire", json={"actor": "oncall-alice"}).status_code == 403 + assert app.state.approval_store.get("a1").status == "pending" + + +def test_expiring_a_decided_request_leaves_it_alone(): + c, _app = _client() + _approval(c) + c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + r = c.post("/approvals/a1/expire", json={"actor": "action-service"}) + assert r.status_code == 200 and r.json()["status"] == "approved" + + +# --- /playbooks/{id}/demote -------------------------------------------------- + + +def test_demote_puts_an_auto_playbook_back_behind_a_human(): + c, app = _client() + app.state.playbook_store.register(Playbook.model_validate(_pb(HitlMode.AUTO))) + r = c.post("/playbooks/restart-pod/demote", json={"decided_by": "feedback-service"}) + assert r.status_code == 200 and r.json()["hitl_mode"] == "hitl" + assert app.state.playbook_store.get("restart-pod").hitl_mode == HitlMode.HITL + actions = [x.action for x in app.state.audit_sink.records("playbook:restart-pod")] + assert actions == ["demote"] + + +def test_demote_requires_graduate_permission(): + c, app = _client() + app.state.playbook_store.register(Playbook.model_validate(_pb(HitlMode.AUTO))) + r = c.post("/playbooks/restart-pod/demote", json={"decided_by": "oncall-alice"}) + assert r.status_code == 403 + assert app.state.playbook_store.get("restart-pod").hitl_mode == HitlMode.AUTO diff --git a/services/rca/app.py b/services/rca/app.py index d8b35fc..6cba465 100644 --- a/services/rca/app.py +++ b/services/rca/app.py @@ -4,6 +4,7 @@ import logging import threading +import time from contextlib import asynccontextmanager from fastapi import FastAPI @@ -12,6 +13,7 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.rca.adapters.context_provider import FileContextProvider from services.rca.adapters.explanation_provider import ( @@ -43,29 +45,59 @@ def _make_runbook_selector(settings): return NullRunbookSelector() -def _build_reliability_provider(training_store): - """Best-effort: per-signature worked/total from training records, same - math as RiverCorrelator/BaseCorrelator.retrain. Returns None if the read - fails, so RCA ranking degrades gracefully to rule-only behavior.""" - try: - records = training_store.read_all() - except Exception: - logger.exception("failed to read training store; ranking without reliability boost") - return None - - worked: dict[str, int] = {} - total: dict[str, int] = {} - for record in records: - sig = record.signature - total[sig] = total.get(sig, 0) + 1 - if record.worked: - worked[sig] = worked.get(sig, 0) + 1 - reliability = {sig: worked.get(sig, 0) / n for sig, n in total.items()} - - def _reliability_provider(signature: str) -> float: - return reliability.get(signature, 0.0) - - return _reliability_provider +class ReliabilityProvider: + """(signature, runbook_id) -> worked/total, re-read from the training store. + + Two things were wrong with the boot-time closure this replaces: + + - It was built once in the lifespan, so every outcome recorded after RCA + started was invisible to ranking until the pod restarted. + - It was keyed on signature alone, and rank_hypotheses added that one number + to every runbook-bearing hypothesis alike - which cannot change their order. + + Refreshes lazily, at most every `refresh_seconds`, on the consumer thread + that calls it. Best-effort: a failed read keeps the last good table (or + none), so ranking degrades to rule-only instead of raising. + """ + + def __init__(self, training_store, refresh_seconds: float = 60.0, clock=time.monotonic): + self._store = training_store + self._refresh = refresh_seconds + self._clock = clock + self._loaded_at: float | None = None + self._by_runbook: dict[tuple[str, str], float] = {} + self._by_signature: dict[str, float] = {} + + def _maybe_reload(self) -> None: + now = self._clock() + if self._loaded_at is not None and now - self._loaded_at < self._refresh: + return + self._loaded_at = now + try: + records = self._store.read_all() + except Exception: + logger.exception("failed to read training store; keeping the last reliability table") + return + worked: dict = {} + total: dict = {} + for r in records: + for key in ((r.signature, r.playbook_id), r.signature): + total[key] = total.get(key, 0) + 1 + if r.worked: + worked[key] = worked.get(key, 0) + 1 + ratio = {k: worked.get(k, 0) / n for k, n in total.items()} + self._by_runbook = {k: v for k, v in ratio.items() if isinstance(k, tuple)} + self._by_signature = {k: v for k, v in ratio.items() if isinstance(k, str)} + + def __call__(self, signature: str, runbook_id: str | None = None) -> float: + self._maybe_reload() + if runbook_id is None: + return self._by_signature.get(signature, 0.0) + return self._by_runbook.get((signature, runbook_id), 0.0) + + +def _build_reliability_provider(training_store, refresh_seconds: float = 60.0): + return ReliabilityProvider(training_store, refresh_seconds=refresh_seconds) @asynccontextmanager @@ -79,19 +111,21 @@ async def lifespan(app: FastAPI): audit_sink = stores.audit_sink holder = ProviderHolder(make_explanation_provider(settings)) app.state.provider_holder = holder - reliability_provider = _build_reliability_provider(stores.training_store) + reliability_provider = _build_reliability_provider( + stores.training_store, settings.reliability_refresh_seconds + ) selector = _make_runbook_selector(settings) - thread = threading.Thread( - target=run_consumer, - args=(app.state.bus, provider, store, audit_sink, holder.get, stop_event), - kwargs={ + thread = start_supervised( + "rca-consumer", + run_consumer, + stop_event, + (app.state.bus, provider, store, audit_sink, holder.get, stop_event), + { "reliability_provider": reliability_provider, "selector": selector, "guard": make_guard(settings, app.state.bus), }, - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/rca/rank.py b/services/rca/rank.py index ec51766..09ddcd5 100644 --- a/services/rca/rank.py +++ b/services/rca/rank.py @@ -4,10 +4,10 @@ sorted best-first. A low-confidence fallback guarantees a non-empty result so downstream always has something to act on (see flow.md 5.3). -An optional `reliability_provider` (situation.signature -> float in [0, 1], -e.g. the correlator's learned worked/total track record) can boost a -hypothesis whose suggested runbook has proven reliable for this signature. -Passing None preserves the original rule-only ranking exactly.""" +An optional `reliability_provider` ((signature, runbook_id) -> float in [0, 1], +the worked/total track record of THAT runbook on THAT signature) can boost a +hypothesis whose suggested runbook has proven reliable here. Passing None +preserves the original rule-only ranking exactly.""" from __future__ import annotations @@ -42,7 +42,7 @@ def _service_labels(situation: Situation) -> set[str]: def rank_hypotheses( situation: Situation, context: EnrichmentContext, - reliability_provider: Callable[[str], float] | None = None, + reliability_provider: Callable[[str, str | None], float] | None = None, store=None, selector=None, ) -> list[RootCauseHypothesis]: @@ -208,12 +208,21 @@ def rank_hypotheses( # runbook — the fallback (runbook_id=None) is never boosted, so the top # suggestion after ranking still resolves to a real playbook id whenever # any rule-based hypothesis fired. - reliability = reliability_provider(situation.signature) if situation.signature else 0.0 - reliability = max(0.0, min(1.0, reliability)) + # + # The track record is per (signature, runbook). It used to be one number per + # signature, added equally to every runbook-bearing hypothesis - a uniform + # offset that could never change their order, so the boost did nothing. + def _reliability(h: RootCauseHypothesis) -> float: + if h.suggested_runbook_id is None or not situation.signature: + return 0.0 + value = reliability_provider(situation.signature, h.suggested_runbook_id) + return max(0.0, min(1.0, value)) - def _score(h: RootCauseHypothesis) -> float: - boost = _RELIABILITY_WEIGHT * reliability if h.suggested_runbook_id is not None else 0.0 - return min(1.0, h.confidence + boost) + def _score(h: RootCauseHypothesis) -> tuple[float, float]: + boosted = min(1.0, h.confidence + _RELIABILITY_WEIGHT * _reliability(h)) + # Confidence breaks ties, so two hypotheses both clamped at 1.0 keep + # their confidence order instead of falling back to rule order. + return (boosted, h.confidence) hypotheses.sort(key=_score, reverse=True) return hypotheses diff --git a/services/rca/tests/test_rank.py b/services/rca/tests/test_rank.py index f9d66fb..4f2eaa5 100644 --- a/services/rca/tests/test_rank.py +++ b/services/rca/tests/test_rank.py @@ -184,7 +184,7 @@ def test_reliability_provider_boosts_proven_runbook(): ctx = EnrichmentContext() # no deploys -> only saturation rule fires situation = _situation(name="cpu_usage", labels={"service": "web"}) - def reliability(signature: str) -> float: + def reliability(signature: str, runbook_id: str | None = None) -> float: assert signature == situation.signature return 1.0 @@ -207,7 +207,7 @@ def test_reliability_provider_never_boosts_fallback_hypothesis(): recent_deploys=[{"service": "web", "version": "v2", "ts": NOW.isoformat()}] ) situation = _situation(name="cpu", labels={"service": "web"}) - hyps = rank_hypotheses(situation, ctx, lambda sig: 1.0) + hyps = rank_hypotheses(situation, ctx, lambda sig, rb=None: 1.0) assert hyps[0].suggested_runbook_id is not None @@ -471,3 +471,19 @@ def test_otel_duration_metrics_map_to_the_latency_rule(): for metric in ("otel_http_server_request_duration_ms", "otel_rpc_server_duration_ms"): hyps = rank_hypotheses(_situation(name=metric), ctx) assert hyps[0].suggested_runbook_id == "scale-service", metric + + +def test_reliability_is_per_runbook_and_can_reorder(): + # memory (restart-pod, 0.65) and cpu saturation (scale-service, 0.6) both + # fire. A proven track record for scale-service on THIS signature - and a + # poor one for restart-pod - must be able to put scale-service first. A + # per-signature figure applied to both equally never could. + situation = _situation_with_metric("memory_usage_mb cpu_usage", value=900.0) + ctx = EnrichmentContext() + record = {"scale-service": 1.0, "restart-pod": 0.0} + + unboosted = rank_hypotheses(situation, ctx, None) + boosted = rank_hypotheses(situation, ctx, lambda sig, rb=None: record.get(rb, 0.0)) + + assert unboosted[0].suggested_runbook_id == "restart-pod" + assert boosted[0].suggested_runbook_id == "scale-service" diff --git a/services/rca/tests/test_reliability_provider.py b/services/rca/tests/test_reliability_provider.py new file mode 100644 index 0000000..1e8309a --- /dev/null +++ b/services/rca/tests/test_reliability_provider.py @@ -0,0 +1,78 @@ +from datetime import UTC, datetime + +from common.contracts import RemediationResult, TrainingRecord +from services.rca.app import ReliabilityProvider + +NOW = datetime(2026, 9, 23, tzinfo=UTC) + + +def _rec(sig, pid, worked): + return TrainingRecord( + situation_id=f"sit-{sig}", + signature=sig, + playbook_id=pid, + result=RemediationResult.SUCCESS if worked else RemediationResult.ROLLED_BACK, + worked=worked, + ts=NOW, + ) + + +class _Store: + def __init__(self, records): + self.records = records + self.reads = 0 + + def read_all(self): + self.reads += 1 + return list(self.records) + + +class _Clock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +def test_track_record_is_per_runbook(): + store = _Store( + [ + _rec("s", "scale-service", True), + _rec("s", "restart-pod", False), + _rec("s", "restart-pod", True), + ] + ) + rel = ReliabilityProvider(store) + assert rel("s", "scale-service") == 1.0 + assert rel("s", "restart-pod") == 0.5 + assert rel("s") == 2 / 3 + assert rel("other", "scale-service") == 0.0 + + +def test_new_outcomes_are_seen_after_the_refresh_period(): + store = _Store([]) + clock = _Clock() + rel = ReliabilityProvider(store, refresh_seconds=60, clock=clock) + assert rel("s", "scale-service") == 0.0 + + store.records.append(_rec("s", "scale-service", True)) + clock.t = 30 + assert rel("s", "scale-service") == 0.0 # still cached + clock.t = 61 + assert rel("s", "scale-service") == 1.0 # the boot-time closure never got here + assert store.reads == 2 + + +def test_a_failed_read_keeps_the_last_table(): + store = _Store([_rec("s", "scale-service", True)]) + clock = _Clock() + rel = ReliabilityProvider(store, refresh_seconds=1, clock=clock) + assert rel("s", "scale-service") == 1.0 + + def boom(): + raise RuntimeError("db down") + + store.read_all = boom + clock.t = 5 + assert rel("s", "scale-service") == 1.0 diff --git a/services/read/consumer.py b/services/read/consumer.py index 196d9a6..4260893 100644 --- a/services/read/consumer.py +++ b/services/read/consumer.py @@ -18,6 +18,7 @@ from common.contracts import DiagnosedSituation, RemediationOutcome, Situation from common.envelope import iter_models +from common.supervise import start_supervised from services.read.projection import ReadModel _GROUP = "read-model" @@ -53,11 +54,11 @@ def _run_topic( def run_consumer(bus, model: ReadModel, stop_event: threading.Event) -> list[threading.Thread]: threads = [] for topic, model_type, method in _TOPICS: - t = threading.Thread( - target=_run_topic, - args=(bus, model, topic, model_type, method, stop_event, bus), - daemon=True, + t = start_supervised( + f"read-{topic}", + _run_topic, + stop_event, + (bus, model, topic, model_type, method, stop_event, bus), ) - t.start() threads.append(t) return threads diff --git a/services/read/projection.py b/services/read/projection.py index 7f769f5..b3648dd 100644 --- a/services/read/projection.py +++ b/services/read/projection.py @@ -69,6 +69,11 @@ def __init__( self._subscribers: set[asyncio.Queue] = set() self._subs_lock = threading.Lock() self._loop: asyncio.AbstractEventLoop | None = None + # Guards _sits/_outcomes/_suppressed_count. Four consumer threads (one per + # topic) write them while request threads read and prune them; two + # concurrent /situations calls could both try to delete the same aged-out + # entry, and iterating a dict another thread is resizing raises. + self._lock = threading.RLock() def bind_loop(self, loop: asyncio.AbstractEventLoop) -> None: """Called once from the async lifespan so consumer threads can hand off.""" @@ -110,14 +115,47 @@ def _deliver(self, q: asyncio.Queue, event: dict) -> None: pass def apply_detected(self, s: Situation) -> None: + with self._lock: + applied = self._apply_detected(s) + if applied: + self.publish({"type": "changed"}) + + def _apply_detected(self, s: Situation) -> bool: + """Fold a detection in. False when the event is stale and was ignored. + + A Situation's id is its signature, so the same incident recurring later + arrives under the same id. first_seen tells the occurrences apart: + + - older than what we hold: a late event from an earlier occurrence (the + topics are consumed on separate threads, and a rebuild replays them + topic by topic). Applying it would roll the card back in time. + - newer, and the held one is finished: a new occurrence. Start clean, + or the new card inherits the previous run's outcome and timeline. + - equal: the same occurrence seen again (e.g. via its diagnosis). Never + move its status backwards - a detected event that lost the race to its + own outcome must not reopen a resolved card. + """ + first_ms = _epoch_ms(s.first_seen) existing = self._sits.get(s.id, {}) + if existing: + held_first = existing.get("first_seen", first_ms) + if first_ms < held_first: + return False + if first_ms > held_first and existing.get("status") in self._EVICTABLE: + existing = {} + status = s.status.value if isinstance(s.status, SituationStatus) else str(s.status) + if existing.get("first_seen") == first_ms and existing.get("status") not in ( + None, + "detected", + ): + status = existing["status"] self._sits[s.id] = { **existing, "id": s.id, "signature": s.signature, "service": self._service_of(s), "title": s.signature, - "status": s.status.value if isinstance(s.status, SituationStatus) else str(s.status), + "status": status, "severity": _SEVERITY_MAP.get(s.severity, "medium"), "memberCount": len(s.member_events), "member_events": _project_events(s), @@ -140,14 +178,22 @@ def apply_detected(self, s: Situation) -> None: stages = self._sits[s.id].get("stages", {}) stages.setdefault("detected", _epoch_ms(s.first_seen)) self._sits[s.id]["stages"] = stages - self.publish({"type": "changed"}) + return True def apply_suppressed(self, s: Situation) -> None: - self._suppressed_count += 1 + with self._lock: + self._suppressed_count += 1 self.publish({"type": "changed"}) def apply_diagnosed(self, d: DiagnosedSituation) -> None: - self.apply_detected(d.situation) + with self._lock: + applied = self._apply_diagnosed(d) + if applied: + self.publish({"type": "changed"}) + + def _apply_diagnosed(self, d: DiagnosedSituation) -> bool: + if not self._apply_detected(d.situation): + return False hyps = [ { "description": h.description, @@ -162,20 +208,29 @@ def apply_diagnosed(self, d: DiagnosedSituation) -> None: ] service = self._sits[d.situation.id].get("service", "unknown") title = f"{hyps[0]['description']} · {service}" if hyps else d.situation.signature - stages = self._sits[d.situation.id].get("stages", {}) + sit = self._sits[d.situation.id] + stages = sit.get("stages", {}) stages["diagnosed"] = _epoch_ms(d.situation.last_seen) - self._sits[d.situation.id].update( + sit.update( { - "status": "diagnosed", "hypotheses": hyps, "suggested_runbook_id": d.suggested_runbook_id, "title": title, "stages": stages, } ) - self.publish({"type": "changed"}) + # Same no-going-backwards rule as _apply_detected: a diagnosis that + # arrives after its outcome must not reopen the card. + if sit.get("status") in (None, "detected"): + sit["status"] = "diagnosed" + return True def apply_outcome(self, o: RemediationOutcome) -> None: + with self._lock: + self._apply_outcome(o) + self.publish({"type": "changed"}) + + def _apply_outcome(self, o: RemediationOutcome) -> None: if o.situation_id in self._sits: self._sits[o.situation_id]["status"] = _RESULT_STATUS.get(o.result, "failed") self._sits[o.situation_id]["last_activity"] = _epoch_ms(o.ts) @@ -229,7 +284,6 @@ def apply_outcome(self, o: RemediationOutcome) -> None: }, ) del self._outcomes[self._max :] - self.publish({"type": "changed"}) _TERMINAL: ClassVar[set[str]] = {"resolved", "failed"} @@ -265,29 +319,37 @@ def _prune(self, now_ms: int) -> None: self._enforce_cap() def situations(self, now_ms: int | None = None) -> list[dict]: - if now_ms is not None: - self._prune(now_ms) - else: - self._enforce_cap() - return list(self._sits.values()) + with self._lock: + if now_ms is not None: + self._prune(now_ms) + else: + self._enforce_cap() + return list(self._sits.values()) def outcomes(self) -> list[dict]: - return list(self._outcomes) + with self._lock: + return list(self._outcomes) def situation(self, sid: str) -> dict | None: - s = self._sits.get(sid) - return dict(s) if s is not None else None + with self._lock: + s = self._sits.get(sid) + return dict(s) if s is not None else None def reset(self) -> None: - self._sits.clear() - self._outcomes.clear() - self._suppressed_count = 0 + with self._lock: + self._sits.clear() + self._outcomes.clear() + self._suppressed_count = 0 # needs_attention is outstanding work, so it counts as open even though it is # not in-flight. approvalsPending narrows to diagnosed/acting on its own. _OPEN: ClassVar[set[str]] = {"detected", "diagnosed", "acting", "needs_attention"} def metrics(self) -> dict: + with self._lock: + return self._metrics() + + def _metrics(self) -> dict: # Enforce the same cap situations() does. Without this the two endpoints # count different sets, and the console renders them side by side: the # noise-reduction tile said "N alerts -> 3 open" while the incident list diff --git a/services/read/tests/test_occurrences.py b/services/read/tests/test_occurrences.py new file mode 100644 index 0000000..6ad6f6e --- /dev/null +++ b/services/read/tests/test_occurrences.py @@ -0,0 +1,102 @@ +"""The same incident recurring under the same id, and events arriving out of order. + +A Situation's id is its signature, so a recurrence reuses the id; and the read +model consumes each topic on its own thread, so a detection can land after its +own diagnosis or outcome. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from common.contracts import ( + DiagnosedSituation, + RemediationOutcome, + RemediationResult, + RootCauseHypothesis, + Situation, + SituationStatus, +) +from services.read.projection import ReadModel + +T1 = datetime(2026, 9, 23, 10, tzinfo=UTC) +T2 = T1 + timedelta(minutes=20) + + +def _sit(first, status=SituationStatus.DETECTED): + return Situation( + id="sit-a", + status=status, + member_events=[], + severity="high", + first_seen=first, + last_seen=first, + signature="a", + ) + + +def _diag(first): + return DiagnosedSituation( + situation=_sit(first, SituationStatus.DIAGNOSED), + hypotheses=[ + RootCauseHypothesis( + situation_id="sit-a", + description="saturation", + confidence=0.6, + suggested_runbook_id="scale-service", + ) + ], + suggested_runbook_id="scale-service", + ) + + +def _outcome(ts, result=RemediationResult.SUCCESS): + return RemediationOutcome( + situation_id="sit-a", + playbook_id="scale-service", + result=result, + health_after="healthy", + ts=ts, + ) + + +def test_a_recurrence_starts_a_clean_card(): + rm = ReadModel() + rm.apply_detected(_sit(T1)) + rm.apply_diagnosed(_diag(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + assert rm.situation("sit-a")["status"] == "resolved" + + rm.apply_detected(_sit(T2)) + card = rm.situation("sit-a") + assert card["status"] == "detected" + assert "outcome" not in card # not the previous run's result + assert card["hypotheses"] == [] + assert set(card["stages"]) == {"detected"} + + +def test_a_late_detection_does_not_reopen_a_resolved_card(): + rm = ReadModel() + rm.apply_diagnosed(_diag(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + rm.apply_detected(_sit(T1)) # lost the race to its own outcome + assert rm.situation("sit-a")["status"] == "resolved" + + +def test_a_late_diagnosis_does_not_reopen_a_resolved_card(): + rm = ReadModel() + rm.apply_detected(_sit(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + rm.apply_diagnosed(_diag(T1)) + card = rm.situation("sit-a") + assert card["status"] == "resolved" + assert card["suggested_runbook_id"] == "scale-service" # the data still lands + + +def test_an_event_from_an_earlier_occurrence_is_ignored(): + rm = ReadModel() + rm.apply_detected(_sit(T2)) + rm.apply_diagnosed(_diag(T1)) # replayed out of order during a rebuild + card = rm.situation("sit-a") + assert card["first_seen"] == int(T2.timestamp() * 1000) + assert card["hypotheses"] == [] diff --git a/tests/test_console_operator_rbac.py b/tests/test_console_operator_rbac.py new file mode 100644 index 0000000..3fc054f --- /dev/null +++ b/tests/test_console_operator_rbac.py @@ -0,0 +1,38 @@ +"""The console's built-in identity must be allowed to do what the console asks of it. + +The console sends its operator name as `decided_by` / `requested_by` on every +approval, rejection and AI-draft request. When that default named nobody in the +RBAC policy, every one of those clicks came back 403 from a stock build. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from services.governance.rbac import RbacPolicy + + +def _console_default_operator() -> str: + src = Path("frontend/src/data/api.ts").read_text(encoding="utf-8") + m = re.search(r'VITE_OPERATOR_NAME\s*\|\|\s*"([^"]+)"', src) + assert m, "could not find the console's default OPERATOR_NAME in api.ts" + return m.group(1) + + +def _dockerfile_default_operator() -> str: + src = Path("deploy/Dockerfile.frontend").read_text(encoding="utf-8") + m = re.search(r"ARG VITE_OPERATOR_NAME=(\S+)", src) + assert m, "deploy/Dockerfile.frontend must pin VITE_OPERATOR_NAME" + return m.group(1) + + +def test_console_default_operator_can_decide_and_draft(): + policy = RbacPolicy.from_file("policies/rbac_policy.yaml") + for who in {_console_default_operator(), _dockerfile_default_operator()}: + # decide_approval / approve_proposed: approve on the specific playbook + assert policy.check(who, "approve", "playbook:restart-pod"), who + # reject_proposed + assert policy.check(who, "reject", "playbook:restart-pod"), who + # propose / draft-async gate on the wildcard resource + assert policy.check(who, "approve", "playbook:*"), who diff --git a/tests/test_slice4_acceptance.py b/tests/test_slice4_acceptance.py index 875bed2..0d65022 100644 --- a/tests/test_slice4_acceptance.py +++ b/tests/test_slice4_acceptance.py @@ -146,6 +146,9 @@ def graduator(pid: str) -> None: result=RemediationResult.SUCCESS, health_after="healthy", ts=NOW, + # Graduation counts real runs only; dry-run successes are no + # evidence a fix is safe unattended. + mode="k8s", ), ) run_consumer(bus, tstore, graduator, min_successes=3, stop_event=threading.Event()) diff --git a/tests/test_supervise.py b/tests/test_supervise.py new file mode 100644 index 0000000..f8682da --- /dev/null +++ b/tests/test_supervise.py @@ -0,0 +1,43 @@ +import threading + +from common.supervise import start_supervised, supervise + + +def test_restarts_after_a_crash_then_finishes(): + calls = [] + + def flaky(): + calls.append(1) + if len(calls) < 3: + raise RuntimeError("boom") + + supervise("t", flaky, threading.Event(), backoff_seconds=0.0) + assert len(calls) == 3 + + +def test_a_normal_return_is_not_restarted(): + calls = [] + supervise("t", lambda: calls.append(1), threading.Event(), backoff_seconds=0.0) + assert calls == [1] + + +def test_stop_event_ends_the_retry_loop(): + stop = threading.Event() + calls = [] + + def always_fails(): + calls.append(1) + stop.set() + raise RuntimeError("boom") + + supervise("t", always_fails, stop, backoff_seconds=0.0) + assert calls == [1] + + +def test_passes_args_and_kwargs_on_a_real_thread(): + got = [] + t = start_supervised( + "t", lambda a, b=None: got.append((a, b)), threading.Event(), (1,), {"b": 2} + ) + t.join(timeout=5) + assert got == [(1, 2)]