From baf9b3130827227c116325f78459fdc790ddbce4 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 00:25:38 +0530 Subject: [PATCH 1/7] fix(console): give the console's default operator an approver role PR #72 made the console act as VITE_OPERATOR_NAME, defaulting to "demo-operator", but that name was in no role in rbac_policy.yaml and the console image never set the variable. Every Approve, Reject and "Draft a runbook with AI" click from a stock build was a 403. Grant demo-operator [operator, approver], pin it as a build ARG in Dockerfile.frontend so the identity is explicit and overridable, and add a test that reads the default from api.ts and the Dockerfile and checks it against the policy. Co-Authored-By: Claude Opus 5.5 --- deploy/Dockerfile.frontend | 4 +++ policies/rbac_policy.yaml | 4 +++ tests/test_console_operator_rbac.py | 38 +++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 tests/test_console_operator_rbac.py diff --git a/deploy/Dockerfile.frontend b/deploy/Dockerfile.frontend index 9db8b0c..b08b6e7 100644 --- a/deploy/Dockerfile.frontend +++ b/deploy/Dockerfile.frontend @@ -12,6 +12,10 @@ ENV VITE_READ_URL=/api/read ENV VITE_GOV_URL=/api/gov ENV VITE_CORR_URL=/api/corr ENV VITE_RCA_URL=/api/rca +# The human identity the console acts as. It must name an approver in +# policies/rbac_policy.yaml or every decision is rejected with 403. +ARG VITE_OPERATOR_NAME=demo-operator +ENV VITE_OPERATOR_NAME=${VITE_OPERATOR_NAME} RUN npm run build FROM nginx:1.27-alpine diff --git a/policies/rbac_policy.yaml b/policies/rbac_policy.yaml index b1ee857..b4c1ded 100644 --- a/policies/rbac_policy.yaml +++ b/policies/rbac_policy.yaml @@ -14,4 +14,8 @@ actors: rca-service: [operator] action-service: [operator, executor] oncall-alice: [operator, approver] + # The console's default identity (VITE_OPERATOR_NAME, frontend/src/data/api.ts). + # Without a role here every Approve / Reject / Draft click from a stock build + # is a 403. Rebuild the console with a different VITE_OPERATOR_NAME to change it. + demo-operator: [operator, approver] feedback-service: [coe-admin] diff --git a/tests/test_console_operator_rbac.py b/tests/test_console_operator_rbac.py new file mode 100644 index 0000000..3fc054f --- /dev/null +++ b/tests/test_console_operator_rbac.py @@ -0,0 +1,38 @@ +"""The console's built-in identity must be allowed to do what the console asks of it. + +The console sends its operator name as `decided_by` / `requested_by` on every +approval, rejection and AI-draft request. When that default named nobody in the +RBAC policy, every one of those clicks came back 403 from a stock build. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from services.governance.rbac import RbacPolicy + + +def _console_default_operator() -> str: + src = Path("frontend/src/data/api.ts").read_text(encoding="utf-8") + m = re.search(r'VITE_OPERATOR_NAME\s*\|\|\s*"([^"]+)"', src) + assert m, "could not find the console's default OPERATOR_NAME in api.ts" + return m.group(1) + + +def _dockerfile_default_operator() -> str: + src = Path("deploy/Dockerfile.frontend").read_text(encoding="utf-8") + m = re.search(r"ARG VITE_OPERATOR_NAME=(\S+)", src) + assert m, "deploy/Dockerfile.frontend must pin VITE_OPERATOR_NAME" + return m.group(1) + + +def test_console_default_operator_can_decide_and_draft(): + policy = RbacPolicy.from_file("policies/rbac_policy.yaml") + for who in {_console_default_operator(), _dockerfile_default_operator()}: + # decide_approval / approve_proposed: approve on the specific playbook + assert policy.check(who, "approve", "playbook:restart-pod"), who + # reject_proposed + assert policy.check(who, "reject", "playbook:restart-pod"), who + # propose / draft-async gate on the wildcard resource + assert policy.check(who, "approve", "playbook:*"), who From 42e32000271fcbe40d1d2ce498f21d142d8c2f47 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 00:38:42 +0530 Subject: [PATCH 2/7] fix(governance): close the playbook back door, audit human decisions, make graduation reversible - POST /playbooks took no identity: anyone reaching governance could overwrite a seed playbook with hitl_mode=auto and skip both the HITL gate and the AI-proposal review. It now requires ?registered_by= with the new `register` permission (coe-admin), refuses AUTO unless the actor may also graduate, and writes an audit record. - A human's approve/reject was never written to the audit log. It is now. - /decide accepted any string (a typo was stored and later reported as a timeout) and could re-decide a closed request. It now takes approved|rejected only (422), checks the matching approve/reject permission, and returns 409 unless the request is still pending. Proposal approve/reject get the same pending check, so a rejected AI runbook can't be approved into the registry. - A request action-service stopped waiting for stayed "pending" forever and the console kept offering a dead Approve button. Action now expires it on timeout (POST /approvals/{id}/expire, requester only, best effort). - Graduation counted dry-run successes (DryRunRemediator + AlwaysHealthyChecker always succeed), so three approved simulations could make a playbook AUTO for a later real cluster. TrainingRecord now carries the outcome's mode (additive, payload-only) and simulated successes don't count unless GRADUATION_COUNT_SIMULATED is set. - Nothing ever demoted an auto playbook. Feedback now calls the new POST /playbooks/{id}/demote when an auto playbook fails or rolls back. Co-Authored-By: Claude Opus 5.5 --- common/config.py | 3 + common/contracts.py | 5 + policies/rbac_policy.yaml | 3 + services/action/adapters/governance_gate.py | 10 ++ services/action/remediate.py | 16 ++ services/action/tests/test_remediate.py | 36 ++++ services/feedback/app.py | 32 +++- services/feedback/consumer.py | 18 +- services/feedback/graduate.py | 15 +- services/feedback/label.py | 1 + services/feedback/tests/test_consumer.py | 9 +- .../feedback/tests/test_graduation_safety.py | 118 ++++++++++++ services/governance/app.py | 128 +++++++++++-- .../governance/tests/test_governance_api.py | 5 +- .../tests/test_governance_safety.py | 168 ++++++++++++++++++ tests/test_slice4_acceptance.py | 3 + 16 files changed, 542 insertions(+), 28 deletions(-) create mode 100644 services/feedback/tests/test_graduation_safety.py create mode 100644 services/governance/tests/test_governance_safety.py diff --git a/common/config.py b/common/config.py index 72c0d17..45ac1b4 100644 --- a/common/config.py +++ b/common/config.py @@ -21,6 +21,9 @@ class Settings(BaseSettings): training_store_path: str = "data/training.jsonl" reliability_suppress_threshold: float = 0.8 graduation_min_successes: int = 3 + # Whether a dry-run SUCCESS counts toward graduating a playbook to AUTO. Off: + # a simulation always "succeeds", so it is no evidence a fix is safe unattended. + graduation_count_simulated: bool = False # --- live-stack settings (test-safe defaults) --- telemetry_mode: str = "file" # "file" | "prometheus" diff --git a/common/contracts.py b/common/contracts.py index 3a711dd..c0f0bee 100644 --- a/common/contracts.py +++ b/common/contracts.py @@ -258,6 +258,11 @@ class TrainingRecord(BaseModel): result: RemediationResult worked: bool ts: datetime + # How the remediation ran ("dry_run" | "k8s" | ...), copied from the outcome. + # Additive and optional: records written before it existed load as None. + # Graduation needs it - a simulated success is not evidence that a playbook + # is safe to run unattended on a real cluster. + mode: str | None = None class TraceStepKind(str, Enum): diff --git a/policies/rbac_policy.yaml b/policies/rbac_policy.yaml index b4c1ded..62ea493 100644 --- a/policies/rbac_policy.yaml +++ b/policies/rbac_policy.yaml @@ -10,6 +10,9 @@ roles: - {action: reject, resource: "playbook:*"} coe-admin: - {action: graduate, resource: "playbook:*"} + # Direct writes to the live playbook registry (POST /playbooks). Admin-only: + # it bypasses the AI-proposal review, so it is not an approver permission. + - {action: register, resource: "playbook:*"} actors: rca-service: [operator] action-service: [operator, executor] diff --git a/services/action/adapters/governance_gate.py b/services/action/adapters/governance_gate.py index 77b351d..a578c76 100644 --- a/services/action/adapters/governance_gate.py +++ b/services/action/adapters/governance_gate.py @@ -48,6 +48,13 @@ def await_decision(self, approval_id: str, timeout_seconds: float) -> ApprovalRe def write_audit(self, record: AuditRecord) -> None: self._audit_sink.write(record) + def expire(self, approval_id: str, actor: str) -> None: + req = self._approvals.get(approval_id) + if req is not None and req.status == "pending" and req.requested_by == actor: + self._approvals[approval_id] = req.model_copy( + update={"status": "expired", "decided_by": actor} + ) + class HttpGovernanceGate: """The cross-process gate: action talks to governance over REST. @@ -119,3 +126,6 @@ def await_decision(self, approval_id: str, timeout_seconds: float) -> ApprovalRe def write_audit(self, record: AuditRecord) -> None: self._client.post(f"{self._base}/audit", json=record.model_dump(mode="json")) + + def expire(self, approval_id: str, actor: str) -> None: + self._client.post(f"{self._base}/approvals/{approval_id}/expire", json={"actor": actor}) diff --git a/services/action/remediate.py b/services/action/remediate.py index c56c9c3..885965a 100644 --- a/services/action/remediate.py +++ b/services/action/remediate.py @@ -168,6 +168,20 @@ def _audit(gate, situation: Situation, playbook: Playbook, decision: str) -> Non ) +def _expire_best_effort(gate, approval_id: str) -> None: + """We stopped waiting; say so, or the request sits "pending" forever and the + console keeps offering an Approve that can no longer do anything. Best effort: + the outcome below is already fail-closed, and a gate without expire() (older + fakes) simply leaves the request as it was.""" + expire = getattr(gate, "expire", None) + if expire is None: + return + try: + expire(approval_id, _ACTOR) + except Exception: # noqa: BLE001, S110 - never let bookkeeping change the outcome + pass + + def execute_remediation( situation: Situation, playbook: Playbook, @@ -241,6 +255,8 @@ def execute_remediation( decided = gate.await_decision(request.id, timeout_seconds) if decided.status != "approved": reason = "aborted:rejected" if decided.status == "rejected" else "aborted:timeout" + if decided.status == "pending": + _expire_best_effort(gate, request.id) _audit(gate, situation, playbook, "abort") return _outcome( situation, playbook, RemediationResult.FAILURE, reason, preflight=preflight diff --git a/services/action/tests/test_remediate.py b/services/action/tests/test_remediate.py index 58fbc8d..b4450df 100644 --- a/services/action/tests/test_remediate.py +++ b/services/action/tests/test_remediate.py @@ -211,3 +211,39 @@ def test_successful_outcome_records_steps_and_mode(): assert out.steps # non-empty, human-readable assert any("scale" in s for s in out.steps) assert out.mode in ("dry_run", "k8s") + + +# --- A timed-out approval is expired, not left pending forever --- + + +class _ExpiringGate(FakeGate): + def __init__(self, status, raises=False): + super().__init__(decision_status=status) + self.expired = [] + self._raises = raises + + def expire(self, approval_id, actor): + if self._raises: + raise RuntimeError("governance down") + self.expired.append((approval_id, actor)) + + +def test_timeout_expires_the_approval_request(): + gate = _ExpiringGate("pending") + r = RecordingRemediator() + out = _run(_playbook(hitl=HitlMode.HITL), gate, r, FixedHealthChecker(True)) + assert out.health_after == "aborted:timeout" + assert gate.expired == [("appr-s1", "action-service")] + assert r.executed_plan is None + + +def test_rejection_is_not_expired(): + gate = _ExpiringGate("rejected") + _run(_playbook(hitl=HitlMode.HITL), gate, RecordingRemediator(), FixedHealthChecker(True)) + assert gate.expired == [] + + +def test_expire_failure_does_not_change_the_outcome(): + gate = _ExpiringGate("pending", raises=True) + out = _run(_playbook(hitl=HitlMode.HITL), gate, RecordingRemediator(), FixedHealthChecker(True)) + assert out.health_after == "aborted:timeout" diff --git a/services/feedback/app.py b/services/feedback/app.py index d1b8d6a..b316727 100644 --- a/services/feedback/app.py +++ b/services/feedback/app.py @@ -16,13 +16,15 @@ from services.feedback.metrics import compute_metrics -def _make_graduator(rbac_actor: str = "feedback-service"): - # In the running service, graduation calls governance's REST endpoint. - # Inside the docker-compose network, governance listens on its internal - # PORT=8000 (8005 is only the host-side port mapping). Best-effort, fire-and- - # forget: a failed graduation is logged by governance's own audit, and the - # next matching outcome will retry on a fresh process. Kept simple here. - def graduate(playbook_id: str) -> None: +def _make_playbook_call(verb: str, rbac_actor: str = "feedback-service"): + """POST governance's /playbooks/{id}/ (graduate or demote). + + Inside the docker-compose network, governance listens on its internal + PORT=8000 (8005 is only the host-side port mapping). Best-effort, fire-and- + forget: governance's own audit is the record, and the next matching outcome + retries on a fresh process.""" + + def call(playbook_id: str) -> None: try: settings = get_settings() headers = ( @@ -31,7 +33,7 @@ def graduate(playbook_id: str) -> None: else {} ) httpx.post( - f"http://governance:8000/playbooks/{playbook_id}/graduate", + f"http://governance:8000/playbooks/{playbook_id}/{verb}", json={"decided_by": rbac_actor}, headers=headers, timeout=5.0, @@ -39,7 +41,15 @@ def graduate(playbook_id: str) -> None: except Exception: # noqa: BLE001, S110 — best-effort; governance's own audit is the record pass - return graduate + return call + + +def _make_graduator(rbac_actor: str = "feedback-service"): + return _make_playbook_call("graduate", rbac_actor) + + +def _make_demoter(rbac_actor: str = "feedback-service"): + return _make_playbook_call("demote", rbac_actor) @asynccontextmanager @@ -60,6 +70,10 @@ async def lifespan(app: FastAPI): stop_event, make_guard(settings, app.state.bus), ), + kwargs={ + "demoter": _make_demoter(), + "count_simulated": settings.graduation_count_simulated, + }, daemon=True, ) thread.start() diff --git a/services/feedback/consumer.py b/services/feedback/consumer.py index 0574ffc..e7c720b 100644 --- a/services/feedback/consumer.py +++ b/services/feedback/consumer.py @@ -12,7 +12,7 @@ import threading from collections.abc import Callable -from common.contracts import RemediationOutcome, RemediationResult +from common.contracts import HitlMode, RemediationOutcome, RemediationResult from common.envelope import iter_models from common.idempotency import NullGuard from services.feedback.graduate import playbook_stats, should_graduate @@ -26,6 +26,8 @@ def run_consumer( min_successes: int, stop_event: threading.Event, guard=None, + demoter: Callable[[str], None] | None = None, + count_simulated: bool = False, ) -> None: guard = guard if guard is not None else NullGuard() graduated: set[str] = set() @@ -42,8 +44,20 @@ def run_consumer( continue store.append(label_outcome(outcome)) pid = outcome.playbook_id + if ( + pid + and demoter is not None + and outcome.hitl_mode == HitlMode.AUTO + and outcome.result in (RemediationResult.FAILURE, RemediationResult.ROLLED_BACK) + ): + # An auto playbook just failed unattended. Graduation only ever + # expanded automation; this is the matching contraction, so the next + # run goes back behind a human. + demoter(pid) + graduated.discard(pid) + continue if pid and pid not in graduated: - stats = playbook_stats(store.read_all(), pid) + stats = playbook_stats(store.read_all(), pid, count_simulated=count_simulated) if should_graduate(stats, min_successes): graduator(pid) graduated.add(pid) diff --git a/services/feedback/graduate.py b/services/feedback/graduate.py index 22953ab..a9eec4b 100644 --- a/services/feedback/graduate.py +++ b/services/feedback/graduate.py @@ -9,13 +9,26 @@ from common.contracts import RemediationResult, TrainingRecord +# Modes in which nothing real was touched. DryRunRemediator + AlwaysHealthyChecker +# report SUCCESS unconditionally, so three approved dry runs used to graduate a +# playbook to AUTO - and flipping REMEDIATOR_MODE=k8s afterwards meant real +# changes with no human in the loop, on evidence from a simulation. +_SIMULATED_MODES = frozenset({"dry_run"}) -def playbook_stats(records: list[TrainingRecord], playbook_id: str) -> dict: + +def playbook_stats( + records: list[TrainingRecord], playbook_id: str, count_simulated: bool = False +) -> dict: + """Per-playbook evidence. A simulated success counts toward nothing unless + `count_simulated` - it proves the gates ran, not that the fix works. Records + from before `mode` existed (None) keep counting as they always did.""" successes = failures = rollbacks = 0 for r in records: if r.playbook_id != playbook_id: continue if r.result == RemediationResult.SUCCESS: + if r.mode in _SIMULATED_MODES and not count_simulated: + continue successes += 1 elif r.result == RemediationResult.FAILURE: failures += 1 diff --git a/services/feedback/label.py b/services/feedback/label.py index 3142544..9f53a08 100644 --- a/services/feedback/label.py +++ b/services/feedback/label.py @@ -21,4 +21,5 @@ def label_outcome(outcome: RemediationOutcome) -> TrainingRecord: result=outcome.result, worked=outcome.result == RemediationResult.SUCCESS, ts=outcome.ts, + mode=outcome.mode, ) diff --git a/services/feedback/tests/test_consumer.py b/services/feedback/tests/test_consumer.py index f518f08..93fa5e7 100644 --- a/services/feedback/tests/test_consumer.py +++ b/services/feedback/tests/test_consumer.py @@ -10,7 +10,14 @@ def _raw_outcome(result, playbook="restart-pod"): o = RemediationOutcome( - situation_id="sit-abc", playbook_id=playbook, result=result, health_after="healthy", ts=NOW + situation_id="sit-abc", + playbook_id=playbook, + result=result, + health_after="healthy", + ts=NOW, + # Real runs: a dry-run success is not graduation evidence + # (see test_graduation_safety.py). + mode="k8s", ) return {"data": o.model_dump_json()} diff --git a/services/feedback/tests/test_graduation_safety.py b/services/feedback/tests/test_graduation_safety.py new file mode 100644 index 0000000..5f529ad --- /dev/null +++ b/services/feedback/tests/test_graduation_safety.py @@ -0,0 +1,118 @@ +"""Graduation must rest on real evidence, and must be reversible. + +DryRunRemediator + AlwaysHealthyChecker report SUCCESS for every run, so three +approved dry runs used to graduate a playbook to AUTO. And nothing ever demoted +an auto playbook after it failed unattended. +""" + +from __future__ import annotations + +import threading +from datetime import UTC, datetime + +from common.contracts import HitlMode, RemediationOutcome, RemediationResult, TrainingRecord +from common.envelope import publish_model +from services.feedback.adapters.training_store import InMemoryTrainingStore +from services.feedback.consumer import run_consumer +from services.feedback.graduate import playbook_stats, should_graduate +from services.feedback.label import label_outcome + +NOW = datetime(2026, 9, 23, tzinfo=UTC) + + +def _rec(mode, result=RemediationResult.SUCCESS): + return TrainingRecord( + situation_id="sit-x", + signature="x", + playbook_id="pb", + result=result, + worked=result == RemediationResult.SUCCESS, + ts=NOW, + mode=mode, + ) + + +def _outcome(result, mode="k8s", hitl=HitlMode.HITL, pid="pb"): + return RemediationOutcome( + situation_id="sit-x", + playbook_id=pid, + result=result, + health_after="x", + ts=NOW, + hitl_mode=hitl, + mode=mode, + ) + + +def test_label_carries_the_mode(): + assert label_outcome(_outcome(RemediationResult.SUCCESS, mode="dry_run")).mode == "dry_run" + + +def test_dry_run_successes_do_not_graduate_by_default(): + recs = [_rec("dry_run") for _ in range(5)] + stats = playbook_stats(recs, "pb") + assert stats["successes"] == 0 + assert should_graduate(stats, 3) is False + + +def test_dry_run_successes_count_when_explicitly_allowed(): + recs = [_rec("dry_run") for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb", count_simulated=True), 3) is True + + +def test_real_successes_still_graduate(): + recs = [_rec("k8s") for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb"), 3) is True + + +def test_legacy_records_without_mode_keep_counting(): + recs = [_rec(None) for _ in range(3)] + assert should_graduate(playbook_stats(recs, "pb"), 3) is True + + +class _ListBus: + """Minimal bus: publish appends; consume yields everything published so far.""" + + def __init__(self): + self.entries: list[dict] = [] + + def publish(self, topic, message): + self.entries.append(message) + + def consume(self, topic, group): + yield from list(self.entries) + + def send_to_dlq(self, *a, **k): + return False + + +def _drive(outcomes, **kwargs): + bus = _ListBus() + for o in outcomes: + publish_model(bus, "remediation.outcomes", o) + graduated, demoted = [], [] + run_consumer( + bus, + InMemoryTrainingStore(), + graduated.append, + 3, + threading.Event(), + demoter=demoted.append, + **kwargs, + ) + return graduated, demoted + + +def test_consumer_does_not_graduate_on_dry_runs(): + graduated, _ = _drive([_outcome(RemediationResult.SUCCESS, mode="dry_run")] * 4) + assert graduated == [] + + +def test_consumer_demotes_an_auto_playbook_that_fails(): + _, demoted = _drive([_outcome(RemediationResult.ROLLED_BACK, hitl=HitlMode.AUTO)]) + assert demoted == ["pb"] + + +def test_consumer_leaves_hitl_failures_alone(): + _, demoted = _drive([_outcome(RemediationResult.FAILURE, hitl=HitlMode.HITL)]) + assert demoted == [] diff --git a/services/governance/app.py b/services/governance/app.py index 81d1e08..682aa58 100644 --- a/services/governance/app.py +++ b/services/governance/app.py @@ -207,9 +207,40 @@ def query_audit(correlation_id: str | None = None) -> list[AuditRecord]: return app.state.audit_sink.records(correlation_id) +def _audit(actor: str, action: str, resource: str, correlation_id: str, decision="allow") -> None: + app.state.audit_sink.write( + AuditRecord( + actor=actor, + action=action, + resource=resource, + decision=decision, + ts=datetime.now(UTC), + correlation_id=correlation_id, + ) + ) + + @app.post("/playbooks") -def register_playbook(playbook: Playbook) -> dict[str, str]: +def register_playbook(playbook: Playbook, registered_by: str) -> dict[str, str]: + """Write a playbook straight into the live registry. + + This used to take no identity at all, which made it a back door around every + other control here: anyone who could reach governance could overwrite a seed + playbook with hitl_mode=auto and skip both the HITL gate and the + AI-proposal review. It is now an audited admin action, and AUTO is only + reachable by an actor who could have graduated the playbook anyway. + """ + resource = f"playbook:{playbook.id}" + if not app.state.rbac.check(registered_by, "register", resource): + raise HTTPException(status_code=403, detail="actor lacks register permission") + if playbook.hitl_mode == HitlMode.AUTO and not app.state.rbac.check( + registered_by, "graduate", resource + ): + raise HTTPException( + status_code=403, detail="only an actor with graduate permission may register auto" + ) app.state.playbook_store.register(playbook) + _audit(registered_by, "register", resource, resource) return {"status": "ok"} @@ -258,16 +289,62 @@ def get_approval(approval_id: str) -> ApprovalRequest: return req +# What a human may set an approval to. action-service reads `status != "approved"` +# as a refusal, so any other string used to be accepted, stored, and then reported +# downstream as a timeout. +_HUMAN_DECISIONS = {"approved": "approve", "rejected": "reject"} + + @app.post("/approvals/{approval_id}/decide") def decide_approval(approval_id: str, decision: Decision) -> ApprovalRequest: req = app.state.approval_store.get(approval_id) if req is None: raise HTTPException(status_code=404, detail="approval not found") - if not app.state.rbac.check(decision.decided_by, "approve", f"playbook:{req.playbook_id}"): - raise HTTPException(status_code=403, detail="decider lacks approve permission") + permission = _HUMAN_DECISIONS.get(decision.decision) + if permission is None: + raise HTTPException(status_code=422, detail="decision must be 'approved' or 'rejected'") + if not app.state.rbac.check(decision.decided_by, permission, f"playbook:{req.playbook_id}"): + raise HTTPException(status_code=403, detail=f"decider lacks {permission} permission") + if req.status != "pending": + # Deciding twice would rewrite history (approved -> rejected after the fix + # already ran), and deciding an expired request does nothing but look like + # it did. + raise HTTPException(status_code=409, detail=f"approval is already {req.status}") updated = app.state.approval_store.decide( approval_id, status=decision.decision, decided_by=decision.decided_by ) + # The human decision is the governance event this service exists to record, + # and until now it only lived inside the approvals row. + _audit( + decision.decided_by, + permission, + f"playbook:{req.playbook_id}", + req.situation_id, + ) + return updated + + +class Expire(BaseModel): + actor: str + + +@app.post("/approvals/{approval_id}/expire") +def expire_approval(approval_id: str, body: Expire) -> ApprovalRequest: + """Mark a request nobody decided in time as expired. + + action-service stops waiting at its HITL timeout, but the request used to stay + "pending" forever: the console kept offering an Approve button that could no + longer do anything. Only the requester may expire its own request. + """ + req = app.state.approval_store.get(approval_id) + if req is None: + raise HTTPException(status_code=404, detail="approval not found") + if body.actor != req.requested_by: + raise HTTPException(status_code=403, detail="only the requester may expire a request") + if req.status != "pending": + return req + updated = app.state.approval_store.decide(approval_id, status="expired", decided_by=body.actor) + _audit(body.actor, "expire", f"playbook:{req.playbook_id}", req.situation_id) return updated @@ -292,16 +369,33 @@ def graduate_playbook(playbook_id: str, body: Graduate) -> Playbook: raise HTTPException(status_code=403, detail="actor lacks graduate permission") updated = pb.model_copy(update={"hitl_mode": HitlMode.AUTO}) app.state.playbook_store.register(updated) - app.state.audit_sink.write( - AuditRecord( - actor=body.decided_by, - action="graduate", - resource=f"playbook:{playbook_id}", - decision="allow", - ts=datetime.now(UTC), - correlation_id=f"playbook:{playbook_id}", - ) - ) + _audit(body.decided_by, "graduate", f"playbook:{playbook_id}", f"playbook:{playbook_id}") + return updated + + +class Demote(BaseModel): + decided_by: str + reason: str = "" + + +@app.post("/playbooks/{playbook_id}/demote") +def demote_playbook(playbook_id: str, body: Demote) -> Playbook: + """Put an auto playbook back behind a human (auto -> hitl). + + Graduation expanded automation on evidence but nothing ever contracted it: a + playbook that failed or rolled back after going auto stayed auto. Same + permission as graduation, since it is the same decision in reverse. + """ + pb = app.state.playbook_store.get(playbook_id) + if pb is None: + raise HTTPException(status_code=404, detail="playbook not found") + if not app.state.rbac.check(body.decided_by, "graduate", f"playbook:{playbook_id}"): + raise HTTPException(status_code=403, detail="actor lacks graduate permission") + if pb.hitl_mode != HitlMode.AUTO: + return pb + updated = pb.model_copy(update={"hitl_mode": HitlMode.HITL}) + app.state.playbook_store.register(updated) + _audit(body.decided_by, "demote", f"playbook:{playbook_id}", f"playbook:{playbook_id}") return updated @@ -579,6 +673,10 @@ def approve_proposed(proposal_id: str, body: ProposalDecision) -> ProposedPlaybo raise HTTPException(status_code=404, detail="proposal not found") if not app.state.rbac.check(body.decided_by, "approve", f"playbook:{p.playbook.id}"): raise HTTPException(status_code=403, detail="decider lacks approve permission") + if p.status != ProposedPlaybookStatus.PROPOSED: + # A rejected proposal must not be approvable later (that would put a + # runbook a human turned down into the live registry), and vice versa. + raise HTTPException(status_code=409, detail=f"proposal is already {p.status.value}") updated = app.state.proposed_store.set_status( proposal_id, ProposedPlaybookStatus.APPROVED, body.decided_by ) @@ -611,6 +709,10 @@ def reject_proposed(proposal_id: str, body: ProposalDecision) -> ProposedPlayboo raise HTTPException(status_code=404, detail="proposal not found") if not app.state.rbac.check(body.decided_by, "reject", f"playbook:{p.playbook.id}"): raise HTTPException(status_code=403, detail="decider lacks reject permission") + if p.status != ProposedPlaybookStatus.PROPOSED: + # A rejected proposal must not be approvable later (that would put a + # runbook a human turned down into the live registry), and vice versa. + raise HTTPException(status_code=409, detail=f"proposal is already {p.status.value}") updated = app.state.proposed_store.set_status( proposal_id, ProposedPlaybookStatus.REJECTED, body.decided_by ) diff --git a/services/governance/tests/test_governance_api.py b/services/governance/tests/test_governance_api.py index 4d5eb20..51bd14b 100644 --- a/services/governance/tests/test_governance_api.py +++ b/services/governance/tests/test_governance_api.py @@ -20,8 +20,9 @@ def _client(): roles={ "operator": [{"action": "diagnose", "resource": "situation:*"}], "approver": [{"action": "approve", "resource": "playbook:*"}], + "admin": [{"action": "register", "resource": "playbook:*"}], }, - actors={"rca-service": ["operator"], "oncall-alice": ["approver"]}, + actors={"rca-service": ["operator"], "oncall-alice": ["approver"], "cara": ["admin"]}, ) app.state.approval_store = InMemoryApprovalStore() return TestClient(app) @@ -59,7 +60,7 @@ def test_playbook_register_and_list(): steps=[RemediationStep(action="restart")], hitl_mode=HitlMode.HITL, ).model_dump(mode="json") - assert c.post("/playbooks", json=pb).status_code == 200 + assert c.post("/playbooks", params={"registered_by": "cara"}, json=pb).status_code == 200 assert c.get("/playbooks/restart-pod").json()["name"] == "Restart Pod" assert [p["id"] for p in c.get("/playbooks").json()] == ["restart-pod"] assert c.get("/playbooks/missing").status_code == 404 diff --git a/services/governance/tests/test_governance_safety.py b/services/governance/tests/test_governance_safety.py new file mode 100644 index 0000000..df09a20 --- /dev/null +++ b/services/governance/tests/test_governance_safety.py @@ -0,0 +1,168 @@ +"""Governance controls that used to be missing or bypassable. + +- POST /playbooks took no identity, so anyone could overwrite a seed playbook + with hitl_mode=auto and skip the HITL gate and the AI-proposal review. +- A human's approve/reject was never written to the audit log. +- /decide accepted any status string and could re-decide a closed request. +- A request action-service stopped waiting for stayed "pending" forever. +- A graduated (auto) playbook could never be put back behind a human. +""" + +from __future__ import annotations + +from fastapi.testclient import TestClient + +from common.contracts import HitlMode, Playbook, RemediationStep +from services.governance.adapters.approval_store import InMemoryApprovalStore +from services.governance.adapters.audit_sink import InMemoryAuditSink +from services.governance.adapters.playbook_store import InMemoryPlaybookStore +from services.governance.rbac import RbacPolicy + + +def _client(): + from services.governance.app import app + + app.state.audit_sink = InMemoryAuditSink() + app.state.playbook_store = InMemoryPlaybookStore() + app.state.rbac = RbacPolicy.from_file("policies/rbac_policy.yaml") + app.state.approval_store = InMemoryApprovalStore() + return TestClient(app), app + + +def _pb(mode=HitlMode.HITL) -> dict: + return Playbook( + id="restart-pod", + name="Restart Pod", + match_rule="x", + steps=[RemediationStep(action="restart")], + hitl_mode=mode, + ).model_dump(mode="json") + + +def _approval(c, aid="a1"): + c.post( + "/approvals", + json={ + "id": aid, + "situation_id": "sit-1", + "playbook_id": "restart-pod", + "requested_by": "action-service", + }, + ) + + +# --- POST /playbooks ------------------------------------------------------- + + +def test_register_requires_an_identity(): + c, _ = _client() + assert c.post("/playbooks", json=_pb()).status_code == 422 + + +def test_register_refused_for_an_approver_or_stranger(): + c, app = _client() + for who in ("oncall-alice", "nobody"): + r = c.post("/playbooks", params={"registered_by": who}, json=_pb()) + assert r.status_code == 403, who + assert app.state.playbook_store.get("restart-pod") is None + + +def test_register_allowed_for_coe_admin_and_audited(): + c, app = _client() + r = c.post("/playbooks", params={"registered_by": "feedback-service"}, json=_pb()) + assert r.status_code == 200 + recs = app.state.audit_sink.records("playbook:restart-pod") + assert [(x.actor, x.action) for x in recs] == [("feedback-service", "register")] + + +def test_register_auto_needs_graduate_permission(): + c, app = _client() + app.state.rbac = RbacPolicy( + roles={"reg": [{"action": "register", "resource": "playbook:*"}]}, + actors={"registrar": ["reg"]}, + ) + r = c.post("/playbooks", params={"registered_by": "registrar"}, json=_pb(HitlMode.AUTO)) + assert r.status_code == 403 + r = c.post("/playbooks", params={"registered_by": "registrar"}, json=_pb(HitlMode.HITL)) + assert r.status_code == 200 + + +# --- /approvals/{id}/decide -------------------------------------------------- + + +def test_decide_writes_an_audit_record_naming_the_human(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + assert r.status_code == 200 + recs = app.state.audit_sink.records("sit-1") + assert [(x.actor, x.action, x.resource) for x in recs] == [ + ("oncall-alice", "approve", "playbook:restart-pod") + ] + + +def test_decide_rejects_an_unknown_decision_string(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/decide", json={"decision": "APPROVED", "decided_by": "oncall-alice"}) + assert r.status_code == 422 + assert app.state.approval_store.get("a1").status == "pending" + + +def test_decide_cannot_rewrite_a_closed_request(): + c, app = _client() + _approval(c) + c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + r = c.post("/approvals/a1/decide", json={"decision": "rejected", "decided_by": "oncall-alice"}) + assert r.status_code == 409 + assert app.state.approval_store.get("a1").status == "approved" + + +# --- /approvals/{id}/expire -------------------------------------------------- + + +def test_requester_can_expire_a_pending_request(): + c, app = _client() + _approval(c) + r = c.post("/approvals/a1/expire", json={"actor": "action-service"}) + assert r.status_code == 200 and r.json()["status"] == "expired" + assert app.state.approval_store.list_pending() == [] + # an expired request can no longer be approved + r = c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + assert r.status_code == 409 + + +def test_only_the_requester_can_expire(): + c, app = _client() + _approval(c) + assert c.post("/approvals/a1/expire", json={"actor": "oncall-alice"}).status_code == 403 + assert app.state.approval_store.get("a1").status == "pending" + + +def test_expiring_a_decided_request_leaves_it_alone(): + c, _app = _client() + _approval(c) + c.post("/approvals/a1/decide", json={"decision": "approved", "decided_by": "oncall-alice"}) + r = c.post("/approvals/a1/expire", json={"actor": "action-service"}) + assert r.status_code == 200 and r.json()["status"] == "approved" + + +# --- /playbooks/{id}/demote -------------------------------------------------- + + +def test_demote_puts_an_auto_playbook_back_behind_a_human(): + c, app = _client() + app.state.playbook_store.register(Playbook.model_validate(_pb(HitlMode.AUTO))) + r = c.post("/playbooks/restart-pod/demote", json={"decided_by": "feedback-service"}) + assert r.status_code == 200 and r.json()["hitl_mode"] == "hitl" + assert app.state.playbook_store.get("restart-pod").hitl_mode == HitlMode.HITL + actions = [x.action for x in app.state.audit_sink.records("playbook:restart-pod")] + assert actions == ["demote"] + + +def test_demote_requires_graduate_permission(): + c, app = _client() + app.state.playbook_store.register(Playbook.model_validate(_pb(HitlMode.AUTO))) + r = c.post("/playbooks/restart-pod/demote", json={"decided_by": "oncall-alice"}) + assert r.status_code == 403 + assert app.state.playbook_store.get("restart-pod").hitl_mode == HitlMode.AUTO diff --git a/tests/test_slice4_acceptance.py b/tests/test_slice4_acceptance.py index 875bed2..0d65022 100644 --- a/tests/test_slice4_acceptance.py +++ b/tests/test_slice4_acceptance.py @@ -146,6 +146,9 @@ def graduator(pid: str) -> None: result=RemediationResult.SUCCESS, health_after="healthy", ts=NOW, + # Graduation counts real runs only; dry-run successes are no + # evidence a fix is safe unattended. + mode="k8s", ), ) run_consumer(bus, tstore, graduator, min_successes=3, stop_event=threading.Event()) From 463b298d2c7bf7b490a67c13ce9a7b0a95a101fb Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 00:50:30 +0530 Subject: [PATCH 3/7] fix(reliability): supervise consumer threads, recreate on rollout, stop losing suppressions - Consumer threads died for good on any unhandled exception (a governance restart surfacing as a ConnectError from the HTTP gate, a Postgres blip in an audit write). /health is static, so Kubernetes never restarted the pod and the service sat "ready" processing nothing. common/supervise.py restarts a crashed worker with capped backoff; every service's consumer (and the correlation flusher, and all four read-model topic threads) now runs under it. At-least-once re-serves the in-flight entry and the DLQ bounds a poison one. - The Helm Deployments used the default RollingUpdate, so each upgrade briefly ran two pods consuming under the same consumer name - the exact overlap ADR-033's replicas:1 assumption rules out. They are now strategy: Recreate. - CorrelationEngine kept suppressed situations in a single slot, so a flush_all() that suppressed two buckets reported one. It is a queue now and the drain loops until empty. - The read model had no lock while four consumer threads and the request threads mutated and pruned it. It now takes an RLock around every mutation and read. It is also occurrence-aware: a recurrence of a finished incident (same id, newer first_seen) starts a clean card instead of inheriting the last run's outcome and timeline; a late detection/diagnosis never moves a card's status backwards; and an event from an earlier occurrence is ignored. - Fix the stale values.yaml comment claiming values-live turns the sandbox on. Co-Authored-By: Claude Opus 5.5 --- common/supervise.py | 65 +++++++++++ .../templates/service-deployment.yaml | 7 ++ deploy/k8s/platform/values.yaml | 2 +- services/action/app.py | 11 +- services/correlation/app.py | 21 ++-- services/correlation/consumer.py | 3 +- services/correlation/engine.py | 13 +-- .../correlation/tests/test_engine_suppress.py | 24 +++++ services/feedback/app.py | 13 +-- services/governance/app.py | 11 +- services/rca/app.py | 13 +-- services/read/consumer.py | 11 +- services/read/projection.py | 102 ++++++++++++++---- services/read/tests/test_occurrences.py | 102 ++++++++++++++++++ tests/test_supervise.py | 43 ++++++++ 15 files changed, 375 insertions(+), 66 deletions(-) create mode 100644 common/supervise.py create mode 100644 services/read/tests/test_occurrences.py create mode 100644 tests/test_supervise.py diff --git a/common/supervise.py b/common/supervise.py new file mode 100644 index 0000000..0cf9e92 --- /dev/null +++ b/common/supervise.py @@ -0,0 +1,65 @@ +"""Keep a background worker alive across unexpected exceptions. + +Every service runs its bus consumer on a daemon thread. Before this, any +exception a handler didn't catch - a governance restart surfacing as a +ConnectError from the HTTP gate, a Postgres blip in an audit write - ended the +thread for good. Nothing noticed: /health is static, so Kubernetes never +restarted the pod, and the service sat "ready" while processing nothing. + +`start_supervised` runs the target again after a crash, with capped exponential +backoff, until the stop event is set. A target that RETURNS is done (the stream +ended, or it saw the stop event) and is not restarted. + +Restarting is safe under both delivery modes: at-least-once re-serves the +un-acked entry on the next consume() (and the DLQ parks it after +bus_max_delivery_attempts, so a poison event can't loop forever); at-most-once +has already acked it, which is exactly the loss that mode accepts. +""" + +from __future__ import annotations + +import logging +import threading +from collections.abc import Callable + +logger = logging.getLogger("intelliops.supervise") + + +def supervise( + name: str, + target: Callable[..., object], + stop_event: threading.Event, + args: tuple = (), + kwargs: dict | None = None, + backoff_seconds: float = 1.0, + max_backoff_seconds: float = 30.0, +) -> None: + kwargs = kwargs or {} + delay = backoff_seconds + while not stop_event.is_set(): + try: + target(*args, **kwargs) + return + except Exception: + logger.exception("%s crashed; restarting in %.1fs", name, delay) + if stop_event.wait(delay): + return + delay = min(delay * 2, max_backoff_seconds) + + +def start_supervised( + name: str, + target: Callable[..., object], + stop_event: threading.Event, + args: tuple = (), + kwargs: dict | None = None, +) -> threading.Thread: + """Start `target(*args, **kwargs)` on a supervised daemon thread.""" + thread = threading.Thread( + target=supervise, + name=name, + args=(name, target, stop_event, args, kwargs), + daemon=True, + ) + thread.start() + return thread diff --git a/deploy/k8s/platform/templates/service-deployment.yaml b/deploy/k8s/platform/templates/service-deployment.yaml index 785f160..458ee45 100644 --- a/deploy/k8s/platform/templates/service-deployment.yaml +++ b/deploy/k8s/platform/templates/service-deployment.yaml @@ -28,6 +28,13 @@ metadata: {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 4 }} spec: replicas: 1 + # Recreate, not the default RollingUpdate. Every consumer uses one stable + # consumer name per group (ADR-033), and a rolling update briefly runs the old + # and new pod side by side - two consumers sharing one name and one pending + # list, both able to act on the same entry. replicas: 1 only holds if the + # rollout never surges. + strategy: + type: Recreate selector: matchLabels: {{- include "intelliops.selectorLabels" (dict "root" $ "name" .name) | nindent 6 }} diff --git a/deploy/k8s/platform/values.yaml b/deploy/k8s/platform/values.yaml index 698aa1a..a2722b7 100644 --- a/deploy/k8s/platform/values.yaml +++ b/deploy/k8s/platform/values.yaml @@ -59,7 +59,7 @@ env: DETECTION_POLICY: "off" # on in values-live (Phase 2) RUNBOOK_SELECTOR_MODE: "off" # embedding in values-live (Phase 3) REMEDIATOR_MODE: "dry_run" # k8s in values-live (real remediation) - SANDBOX_MODE: "off" # k8s in values-live (pre-flight rehearsal) + SANDBOX_MODE: "off" # off in values-live too (ADR-029); k8s via docker-compose.k8s.yml HEALTH_CHECK_MODE: "always" # k8s in values-live (Phase 4 per-metric verify) # Must stay comfortably above the remediated workload's termination grace # period; see the configmap comment. 30 (the old hardcoded value) equalled diff --git a/services/action/app.py b/services/action/app.py index 14b272e..0ecb48a 100644 --- a/services/action/app.py +++ b/services/action/app.py @@ -10,6 +10,7 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.action.adapters.governance_gate import ( HttpGovernanceGate, InProcessGovernanceGate, @@ -121,9 +122,11 @@ async def lifespan(app: FastAPI): app.state.db_engine = stores.engine store = stores.playbook_store gate = _make_gate(settings, stores.audit_sink) - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "action-consumer", + run_consumer, + stop_event, + ( app.state.bus, store, gate, @@ -135,9 +138,7 @@ async def lifespan(app: FastAPI): stop_event, make_guard(settings, app.state.bus), ), - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/correlation/app.py b/services/correlation/app.py index 759a69e..d418797 100644 --- a/services/correlation/app.py +++ b/services/correlation/app.py @@ -14,6 +14,7 @@ from common.envelope import publish_model from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.correlation.adapters import make_correlator from services.correlation.consumer import ( @@ -139,15 +140,17 @@ async def lifespan(app: FastAPI): _reload_model(engine, model_store) app.state.baseline_store = baseline_store app.state.model_store = model_store - thread = threading.Thread( - target=run_consumer, - args=(app.state.bus, engine, stop_event, make_guard(settings, app.state.bus)), - daemon=True, + thread = start_supervised( + "correlation-consumer", + run_consumer, + stop_event, + (app.state.bus, engine, stop_event, make_guard(settings, app.state.bus)), ) - thread.start() - flusher = threading.Thread( - target=run_flusher, - args=( + flusher = start_supervised( + "correlation-flusher", + run_flusher, + stop_event, + ( app.state.bus, engine, settings.correlation_window_seconds, @@ -155,9 +158,7 @@ async def lifespan(app: FastAPI): baseline_store, settings.baseline_snapshot_seconds, ), - daemon=True, ) - flusher.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread app.state.flusher_thread = flusher diff --git a/services/correlation/consumer.py b/services/correlation/consumer.py index d837737..1799699 100644 --- a/services/correlation/consumer.py +++ b/services/correlation/consumer.py @@ -29,8 +29,7 @@ def _snapshot_baseline_once(engine, baseline_store) -> None: def _drain_suppressed(bus, engine: CorrelationEngine) -> None: - s = engine.pop_suppressed() - if s is not None: + while (s := engine.pop_suppressed()) is not None: publish_model(bus, "situations.suppressed", s) diff --git a/services/correlation/engine.py b/services/correlation/engine.py index 53596ec..b2865cc 100644 --- a/services/correlation/engine.py +++ b/services/correlation/engine.py @@ -64,7 +64,9 @@ def __init__( # this is the old single-buffer behaviour with one dict lookup. self._buffers: dict[str, list[TelemetryEvent]] = {} self._max_scores: dict[str, float] = {} - self._suppressed: Situation | None = None + # A queue, not a slot: one flush_all() can suppress several buckets, and a + # single slot kept only the last, silently dropping the rest. + self._suppressed: list[Situation] = [] # Guards _buffer/_max_score so a background time-flush (see the service # lifespan) can run concurrently with add() on the consumer thread. # Single-threaded callers (tests) are unaffected — the lock is uncontended. @@ -156,7 +158,7 @@ def _correlate_buffer(self, key: str = _ALL) -> Situation | None: self._max_scores.pop(key, None) # Closed loop: suppress a Situation whose signature reliably self-heals. if self._correlator.should_suppress(sit.signature, self._suppress_threshold): - self._suppressed = sit + self._suppressed.append(sit) return None return sit @@ -169,14 +171,13 @@ def load(self, rows: list[dict]) -> None: self._correlator.load(rows) def pop_suppressed(self) -> Situation | None: + """Oldest suppressed Situation not yet published, or None.""" with self._lock: - s = self._suppressed - self._suppressed = None - return s + return self._suppressed.pop(0) if self._suppressed else None def reset(self) -> None: with self._lock: self._correlator = self._correlator_factory() self._buffers = {} self._max_scores = {} - self._suppressed = None + self._suppressed = [] diff --git a/services/correlation/tests/test_engine_suppress.py b/services/correlation/tests/test_engine_suppress.py index e59e23c..eddcab5 100644 --- a/services/correlation/tests/test_engine_suppress.py +++ b/services/correlation/tests/test_engine_suppress.py @@ -70,3 +70,27 @@ def test_engine_still_emits_unreliable_signature(): engine.add(_event(120.0, "b", 4)) still_emitted = engine.flush() assert still_emitted is not None # unreliable signature is NOT suppressed + + +def test_every_bucket_suppressed_in_one_flush_is_reported(): + # With group_by="service", one flush_all() can suppress several buckets. The + # engine used to keep suppressed situations in a single slot, so only the + # last one ever reached situations.suppressed. + correlator = RiverCorrelator(z_threshold=3.0) + engine = CorrelationEngine(correlator, window_seconds=30, group_by="service") + ev_a = _event(1.0, "fa").model_copy(update={"labels": {"service": "a"}}) + ev_b = _event(1.0, "fb").model_copy(update={"labels": {"service": "b"}}) + engine._buffers = {"a": [ev_a], "b": [ev_b]} + engine._max_scores = {"a": 9.0, "b": 9.0} + correlator.retrain( + [ + {"signature": correlator._signature([ev_a]), "worked": True}, + {"signature": correlator._signature([ev_b]), "worked": True}, + ] + ) + + assert engine.flush_all() == [] + suppressed = [] + while (s := engine.pop_suppressed()) is not None: + suppressed.append(s) + assert len(suppressed) == 2 diff --git a/services/feedback/app.py b/services/feedback/app.py index b316727..8ebcc09 100644 --- a/services/feedback/app.py +++ b/services/feedback/app.py @@ -11,6 +11,7 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.feedback.consumer import run_consumer from services.feedback.metrics import compute_metrics @@ -60,9 +61,11 @@ async def lifespan(app: FastAPI): app.state.db_engine = stores.engine store = stores.training_store app.state.training_store = store - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "feedback-consumer", + run_consumer, + stop_event, + ( app.state.bus, store, _make_graduator(), @@ -70,13 +73,11 @@ async def lifespan(app: FastAPI): stop_event, make_guard(settings, app.state.bus), ), - kwargs={ + { "demoter": _make_demoter(), "count_simulated": settings.graduation_count_simulated, }, - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/governance/app.py b/services/governance/app.py index 682aa58..5aa0d76 100644 --- a/services/governance/app.py +++ b/services/governance/app.py @@ -30,6 +30,7 @@ ) from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.governance.adapters.author_tools import AuthorToolbox from services.governance.adapters.runbook_author import ( @@ -120,17 +121,17 @@ async def lifespan(app: FastAPI): # see services/governance/consumer.py) and disposes the engine on shutdown, # matching feedback's lifespan. stop_event = threading.Event() - thread = threading.Thread( - target=run_consumer, - args=( + thread = start_supervised( + "governance-consumer", + run_consumer, + stop_event, + ( app.state.bus, app.state.author_decision_store, stop_event, make_guard(get_settings(), app.state.bus), ), - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread # The draft-async worker threads publish trace steps via AgentRunHub from diff --git a/services/rca/app.py b/services/rca/app.py index d8b35fc..151b691 100644 --- a/services/rca/app.py +++ b/services/rca/app.py @@ -12,6 +12,7 @@ from common.config import get_settings from common.idempotency import make_guard from common.stores import make_stores +from common.supervise import start_supervised from services.base import create_app, db_ready from services.rca.adapters.context_provider import FileContextProvider from services.rca.adapters.explanation_provider import ( @@ -81,17 +82,17 @@ async def lifespan(app: FastAPI): app.state.provider_holder = holder reliability_provider = _build_reliability_provider(stores.training_store) selector = _make_runbook_selector(settings) - thread = threading.Thread( - target=run_consumer, - args=(app.state.bus, provider, store, audit_sink, holder.get, stop_event), - kwargs={ + thread = start_supervised( + "rca-consumer", + run_consumer, + stop_event, + (app.state.bus, provider, store, audit_sink, holder.get, stop_event), + { "reliability_provider": reliability_provider, "selector": selector, "guard": make_guard(settings, app.state.bus), }, - daemon=True, ) - thread.start() app.state.consumer_stop = stop_event app.state.consumer_thread = thread try: diff --git a/services/read/consumer.py b/services/read/consumer.py index 196d9a6..4260893 100644 --- a/services/read/consumer.py +++ b/services/read/consumer.py @@ -18,6 +18,7 @@ from common.contracts import DiagnosedSituation, RemediationOutcome, Situation from common.envelope import iter_models +from common.supervise import start_supervised from services.read.projection import ReadModel _GROUP = "read-model" @@ -53,11 +54,11 @@ def _run_topic( def run_consumer(bus, model: ReadModel, stop_event: threading.Event) -> list[threading.Thread]: threads = [] for topic, model_type, method in _TOPICS: - t = threading.Thread( - target=_run_topic, - args=(bus, model, topic, model_type, method, stop_event, bus), - daemon=True, + t = start_supervised( + f"read-{topic}", + _run_topic, + stop_event, + (bus, model, topic, model_type, method, stop_event, bus), ) - t.start() threads.append(t) return threads diff --git a/services/read/projection.py b/services/read/projection.py index 7f769f5..b3648dd 100644 --- a/services/read/projection.py +++ b/services/read/projection.py @@ -69,6 +69,11 @@ def __init__( self._subscribers: set[asyncio.Queue] = set() self._subs_lock = threading.Lock() self._loop: asyncio.AbstractEventLoop | None = None + # Guards _sits/_outcomes/_suppressed_count. Four consumer threads (one per + # topic) write them while request threads read and prune them; two + # concurrent /situations calls could both try to delete the same aged-out + # entry, and iterating a dict another thread is resizing raises. + self._lock = threading.RLock() def bind_loop(self, loop: asyncio.AbstractEventLoop) -> None: """Called once from the async lifespan so consumer threads can hand off.""" @@ -110,14 +115,47 @@ def _deliver(self, q: asyncio.Queue, event: dict) -> None: pass def apply_detected(self, s: Situation) -> None: + with self._lock: + applied = self._apply_detected(s) + if applied: + self.publish({"type": "changed"}) + + def _apply_detected(self, s: Situation) -> bool: + """Fold a detection in. False when the event is stale and was ignored. + + A Situation's id is its signature, so the same incident recurring later + arrives under the same id. first_seen tells the occurrences apart: + + - older than what we hold: a late event from an earlier occurrence (the + topics are consumed on separate threads, and a rebuild replays them + topic by topic). Applying it would roll the card back in time. + - newer, and the held one is finished: a new occurrence. Start clean, + or the new card inherits the previous run's outcome and timeline. + - equal: the same occurrence seen again (e.g. via its diagnosis). Never + move its status backwards - a detected event that lost the race to its + own outcome must not reopen a resolved card. + """ + first_ms = _epoch_ms(s.first_seen) existing = self._sits.get(s.id, {}) + if existing: + held_first = existing.get("first_seen", first_ms) + if first_ms < held_first: + return False + if first_ms > held_first and existing.get("status") in self._EVICTABLE: + existing = {} + status = s.status.value if isinstance(s.status, SituationStatus) else str(s.status) + if existing.get("first_seen") == first_ms and existing.get("status") not in ( + None, + "detected", + ): + status = existing["status"] self._sits[s.id] = { **existing, "id": s.id, "signature": s.signature, "service": self._service_of(s), "title": s.signature, - "status": s.status.value if isinstance(s.status, SituationStatus) else str(s.status), + "status": status, "severity": _SEVERITY_MAP.get(s.severity, "medium"), "memberCount": len(s.member_events), "member_events": _project_events(s), @@ -140,14 +178,22 @@ def apply_detected(self, s: Situation) -> None: stages = self._sits[s.id].get("stages", {}) stages.setdefault("detected", _epoch_ms(s.first_seen)) self._sits[s.id]["stages"] = stages - self.publish({"type": "changed"}) + return True def apply_suppressed(self, s: Situation) -> None: - self._suppressed_count += 1 + with self._lock: + self._suppressed_count += 1 self.publish({"type": "changed"}) def apply_diagnosed(self, d: DiagnosedSituation) -> None: - self.apply_detected(d.situation) + with self._lock: + applied = self._apply_diagnosed(d) + if applied: + self.publish({"type": "changed"}) + + def _apply_diagnosed(self, d: DiagnosedSituation) -> bool: + if not self._apply_detected(d.situation): + return False hyps = [ { "description": h.description, @@ -162,20 +208,29 @@ def apply_diagnosed(self, d: DiagnosedSituation) -> None: ] service = self._sits[d.situation.id].get("service", "unknown") title = f"{hyps[0]['description']} · {service}" if hyps else d.situation.signature - stages = self._sits[d.situation.id].get("stages", {}) + sit = self._sits[d.situation.id] + stages = sit.get("stages", {}) stages["diagnosed"] = _epoch_ms(d.situation.last_seen) - self._sits[d.situation.id].update( + sit.update( { - "status": "diagnosed", "hypotheses": hyps, "suggested_runbook_id": d.suggested_runbook_id, "title": title, "stages": stages, } ) - self.publish({"type": "changed"}) + # Same no-going-backwards rule as _apply_detected: a diagnosis that + # arrives after its outcome must not reopen the card. + if sit.get("status") in (None, "detected"): + sit["status"] = "diagnosed" + return True def apply_outcome(self, o: RemediationOutcome) -> None: + with self._lock: + self._apply_outcome(o) + self.publish({"type": "changed"}) + + def _apply_outcome(self, o: RemediationOutcome) -> None: if o.situation_id in self._sits: self._sits[o.situation_id]["status"] = _RESULT_STATUS.get(o.result, "failed") self._sits[o.situation_id]["last_activity"] = _epoch_ms(o.ts) @@ -229,7 +284,6 @@ def apply_outcome(self, o: RemediationOutcome) -> None: }, ) del self._outcomes[self._max :] - self.publish({"type": "changed"}) _TERMINAL: ClassVar[set[str]] = {"resolved", "failed"} @@ -265,29 +319,37 @@ def _prune(self, now_ms: int) -> None: self._enforce_cap() def situations(self, now_ms: int | None = None) -> list[dict]: - if now_ms is not None: - self._prune(now_ms) - else: - self._enforce_cap() - return list(self._sits.values()) + with self._lock: + if now_ms is not None: + self._prune(now_ms) + else: + self._enforce_cap() + return list(self._sits.values()) def outcomes(self) -> list[dict]: - return list(self._outcomes) + with self._lock: + return list(self._outcomes) def situation(self, sid: str) -> dict | None: - s = self._sits.get(sid) - return dict(s) if s is not None else None + with self._lock: + s = self._sits.get(sid) + return dict(s) if s is not None else None def reset(self) -> None: - self._sits.clear() - self._outcomes.clear() - self._suppressed_count = 0 + with self._lock: + self._sits.clear() + self._outcomes.clear() + self._suppressed_count = 0 # needs_attention is outstanding work, so it counts as open even though it is # not in-flight. approvalsPending narrows to diagnosed/acting on its own. _OPEN: ClassVar[set[str]] = {"detected", "diagnosed", "acting", "needs_attention"} def metrics(self) -> dict: + with self._lock: + return self._metrics() + + def _metrics(self) -> dict: # Enforce the same cap situations() does. Without this the two endpoints # count different sets, and the console renders them side by side: the # noise-reduction tile said "N alerts -> 3 open" while the incident list diff --git a/services/read/tests/test_occurrences.py b/services/read/tests/test_occurrences.py new file mode 100644 index 0000000..6ad6f6e --- /dev/null +++ b/services/read/tests/test_occurrences.py @@ -0,0 +1,102 @@ +"""The same incident recurring under the same id, and events arriving out of order. + +A Situation's id is its signature, so a recurrence reuses the id; and the read +model consumes each topic on its own thread, so a detection can land after its +own diagnosis or outcome. +""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from common.contracts import ( + DiagnosedSituation, + RemediationOutcome, + RemediationResult, + RootCauseHypothesis, + Situation, + SituationStatus, +) +from services.read.projection import ReadModel + +T1 = datetime(2026, 9, 23, 10, tzinfo=UTC) +T2 = T1 + timedelta(minutes=20) + + +def _sit(first, status=SituationStatus.DETECTED): + return Situation( + id="sit-a", + status=status, + member_events=[], + severity="high", + first_seen=first, + last_seen=first, + signature="a", + ) + + +def _diag(first): + return DiagnosedSituation( + situation=_sit(first, SituationStatus.DIAGNOSED), + hypotheses=[ + RootCauseHypothesis( + situation_id="sit-a", + description="saturation", + confidence=0.6, + suggested_runbook_id="scale-service", + ) + ], + suggested_runbook_id="scale-service", + ) + + +def _outcome(ts, result=RemediationResult.SUCCESS): + return RemediationOutcome( + situation_id="sit-a", + playbook_id="scale-service", + result=result, + health_after="healthy", + ts=ts, + ) + + +def test_a_recurrence_starts_a_clean_card(): + rm = ReadModel() + rm.apply_detected(_sit(T1)) + rm.apply_diagnosed(_diag(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + assert rm.situation("sit-a")["status"] == "resolved" + + rm.apply_detected(_sit(T2)) + card = rm.situation("sit-a") + assert card["status"] == "detected" + assert "outcome" not in card # not the previous run's result + assert card["hypotheses"] == [] + assert set(card["stages"]) == {"detected"} + + +def test_a_late_detection_does_not_reopen_a_resolved_card(): + rm = ReadModel() + rm.apply_diagnosed(_diag(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + rm.apply_detected(_sit(T1)) # lost the race to its own outcome + assert rm.situation("sit-a")["status"] == "resolved" + + +def test_a_late_diagnosis_does_not_reopen_a_resolved_card(): + rm = ReadModel() + rm.apply_detected(_sit(T1)) + rm.apply_outcome(_outcome(T1 + timedelta(minutes=2))) + rm.apply_diagnosed(_diag(T1)) + card = rm.situation("sit-a") + assert card["status"] == "resolved" + assert card["suggested_runbook_id"] == "scale-service" # the data still lands + + +def test_an_event_from_an_earlier_occurrence_is_ignored(): + rm = ReadModel() + rm.apply_detected(_sit(T2)) + rm.apply_diagnosed(_diag(T1)) # replayed out of order during a rebuild + card = rm.situation("sit-a") + assert card["first_seen"] == int(T2.timestamp() * 1000) + assert card["hypotheses"] == [] diff --git a/tests/test_supervise.py b/tests/test_supervise.py new file mode 100644 index 0000000..f8682da --- /dev/null +++ b/tests/test_supervise.py @@ -0,0 +1,43 @@ +import threading + +from common.supervise import start_supervised, supervise + + +def test_restarts_after_a_crash_then_finishes(): + calls = [] + + def flaky(): + calls.append(1) + if len(calls) < 3: + raise RuntimeError("boom") + + supervise("t", flaky, threading.Event(), backoff_seconds=0.0) + assert len(calls) == 3 + + +def test_a_normal_return_is_not_restarted(): + calls = [] + supervise("t", lambda: calls.append(1), threading.Event(), backoff_seconds=0.0) + assert calls == [1] + + +def test_stop_event_ends_the_retry_loop(): + stop = threading.Event() + calls = [] + + def always_fails(): + calls.append(1) + stop.set() + raise RuntimeError("boom") + + supervise("t", always_fails, stop, backoff_seconds=0.0) + assert calls == [1] + + +def test_passes_args_and_kwargs_on_a_real_thread(): + got = [] + t = start_supervised( + "t", lambda a, b=None: got.append((a, b)), threading.Event(), (1,), {"b": 2} + ) + t.join(timeout=5) + assert got == [(1, 2)] From 10e745d435aed33e6e69033e26f385a88786c857 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 00:59:56 +0530 Subject: [PATCH 4/7] fix(learning): stable signatures, per-runbook RCA reliability, per-series detection switch - Signatures hashed the raw list of member fingerprints, so the same fault on the same service got a different signature depending on how many anomalous samples landed in the window (verified: 6 vs 7 samples -> two unrelated ids). Everything keyed on the signature - suppression, graduation evidence, RCA reliability, the author's past decisions - only matched by luck. It is now the hash of the SET of series involved. - Suppression cleared on a single success (reliability 1/1 = 1.0). It now needs RELIABILITY_SUPPRESS_MIN_SAMPLES (default 3) labelled outcomes. The service also never passed RELIABILITY_SUPPRESS_THRESHOLD to the engine - dead config, now wired. - RCA's reliability boost was one number per signature added equally to every runbook-bearing hypothesis: a uniform offset that could never change their order. It is now per (signature, runbook), so a proven runbook can overtake, with confidence as the tiebreak at the 1.0 clamp. It was also built once at boot; ReliabilityProvider re-reads the training store every RELIABILITY_REFRESH_SECONDS (default 60). - RobustCorrelator pooled every service's samples of a metric into one baseline, so a 10 -> 45 CPU jump on a quiet service scored ~0 beside one idling at 80. New CORRELATION_ROBUST_KEY_BY=series keeps one baseline per (metric, service). Opt-in, default unchanged: per-series windows fill ~5x slower, so the live warm-up needs re-tuning against a replay first. - Every top-of-hour opened an empty seasonal bucket and the robust detector scored 0 until it re-warmed. A cold bucket now scores against the fullest warm bucket of the same series. - CorrelationEngine.reset() rebuilt the correlator from three kwargs, silently resetting window size / seasonal buckets to defaults. Correlators now expose clone_empty() with their full configuration. Correlation's own reliability map (what drives suppression) is still loaded only at boot, deliberately - see the PR description: suppressing a situation drops it before RCA/action, so a live-refreshing map would stop remediating faults the system has learned to fix. Co-Authored-By: Claude Opus 5.5 --- common/config.py | 7 ++ deploy/k8s/platform/templates/configmap.yaml | 1 + deploy/k8s/platform/values.yaml | 4 + services/correlation/adapters/__init__.py | 1 + .../correlation/adapters/base_correlator.py | 36 ++++++++- .../correlation/adapters/robust_correlator.py | 68 +++++++++++++--- .../adapters/trained_correlator.py | 11 +++ services/correlation/app.py | 4 + services/correlation/engine.py | 18 +++-- services/correlation/tests/test_retrain.py | 11 +++ .../tests/test_robust_correlator.py | 67 +++++++++++++++ .../tests/test_suppressed_signal.py | 2 +- services/rca/app.py | 81 +++++++++++++------ services/rca/rank.py | 29 ++++--- services/rca/tests/test_rank.py | 20 ++++- .../rca/tests/test_reliability_provider.py | 78 ++++++++++++++++++ 16 files changed, 385 insertions(+), 53 deletions(-) create mode 100644 services/rca/tests/test_reliability_provider.py diff --git a/common/config.py b/common/config.py index 45ac1b4..9329287 100644 --- a/common/config.py +++ b/common/config.py @@ -20,6 +20,10 @@ class Settings(BaseSettings): hitl_poll_interval_seconds: float = 0.5 training_store_path: str = "data/training.jsonl" reliability_suppress_threshold: float = 0.8 + # Labelled outcomes a signature needs before it can be suppressed at all. + reliability_suppress_min_samples: int = 3 + # How often RCA re-reads the training store for its runbook track records. + reliability_refresh_seconds: float = 60.0 graduation_min_successes: int = 3 # Whether a dry-run SUCCESS counts toward graduating a playbook to AUTO. Off: # a simulation always "succeeds", so it is no evidence a fix is safe unattended. @@ -54,6 +58,9 @@ class Settings(BaseSettings): correlation_seasonal_buckets: int = 24 correlation_robust_window: int = 128 correlation_robust_warmup: int = 30 + # "metric" (default, historical) pools every service's samples of a metric into + # one baseline; "series" keeps one per (metric, service). See RobustCorrelator. + correlation_robust_key_by: str = "metric" # "metric" | "series" governance_mode: str = "in_process" # "in_process" | "http" governance_url: str = "http://localhost:8005" # read-service asks rca for the authoritative LLM config rather than diff --git a/deploy/k8s/platform/templates/configmap.yaml b/deploy/k8s/platform/templates/configmap.yaml index 62b807d..42b5e4c 100644 --- a/deploy/k8s/platform/templates/configmap.yaml +++ b/deploy/k8s/platform/templates/configmap.yaml @@ -41,6 +41,7 @@ data: # this correlator's name is meant to buy. INTELLIOPS_CORRELATION_ROBUST_WINDOW: {{ .Values.env.CORRELATION_ROBUST_WINDOW | default "128" | quote }} INTELLIOPS_CORRELATION_ROBUST_WARMUP: {{ .Values.env.CORRELATION_ROBUST_WARMUP | default "30" | quote }} + INTELLIOPS_CORRELATION_ROBUST_KEY_BY: {{ .Values.env.CORRELATION_ROBUST_KEY_BY | default "metric" | quote }} INTELLIOPS_DETECTION_POLICY: {{ .Values.env.DETECTION_POLICY | quote }} INTELLIOPS_RUNBOOK_SELECTOR_MODE: {{ .Values.env.RUNBOOK_SELECTOR_MODE | quote }} INTELLIOPS_REMEDIATOR_MODE: {{ .Values.env.REMEDIATOR_MODE | quote }} diff --git a/deploy/k8s/platform/values.yaml b/deploy/k8s/platform/values.yaml index a2722b7..0b73b45 100644 --- a/deploy/k8s/platform/values.yaml +++ b/deploy/k8s/platform/values.yaml @@ -75,6 +75,10 @@ env: CORRELATION_MIN_EVENTS: "1" # values-live raises this; 1 = historical behaviour CORRELATION_ROBUST_WINDOW: "128" CORRELATION_ROBUST_WARMUP: "30" # values-live requires a FULL window instead + # "series" gives each (metric, service) its own baseline instead of pooling + # every service into one per metric. Re-tune warm-up before flipping it: a + # per-series window fills ~5x slower (one sample per poll, not five). + CORRELATION_ROBUST_KEY_BY: "metric" GOVERNANCE_MODE: "in_process" # "http" in values-live — action + governance are GOVERNANCE_URL: "http://governance:8000" # separate pods, so approvals must go over HTTP HITL_POLL_TIMEOUT_SECONDS: "30" # values-live raises this so a human has time to click Approve diff --git a/services/correlation/adapters/__init__.py b/services/correlation/adapters/__init__.py index 68cf9f9..f006138 100644 --- a/services/correlation/adapters/__init__.py +++ b/services/correlation/adapters/__init__.py @@ -56,6 +56,7 @@ def make_correlator(settings: Settings) -> BaseCorrelator: seasonal_buckets=settings.correlation_seasonal_buckets, window_size=settings.correlation_robust_window, detection_policy=policy, + key_by=settings.correlation_robust_key_by, ) if kind == "trained": from services.correlation.adapters.trained_correlator import TrainedCorrelator diff --git a/services/correlation/adapters/base_correlator.py b/services/correlation/adapters/base_correlator.py index fa24f97..870d85a 100644 --- a/services/correlation/adapters/base_correlator.py +++ b/services/correlation/adapters/base_correlator.py @@ -30,6 +30,8 @@ def __init__( self._z_threshold = z_threshold self._warmup_samples = warmup_samples self._reliability: dict[str, float] = {} + # How many labelled outcomes each reliability figure rests on. + self._samples: dict[str, int] = {} self._policy = ( detection_policy if detection_policy is not None else DetectionPolicy(enabled=False) ) @@ -46,6 +48,23 @@ def snapshot(self) -> list[dict]: ... @abstractmethod def load(self, rows: list[dict]) -> None: ... + def _clone_kwargs(self) -> dict: + """Constructor kwargs that reproduce this correlator's configuration. + Subclasses with extra settings extend this.""" + return { + "z_threshold": self._z_threshold, + "warmup_samples": self._warmup_samples, + "detection_policy": self._policy, + } + + def clone_empty(self) -> BaseCorrelator: + """A fresh correlator with the same configuration and no learned state. + + CorrelationEngine.reset() used to rebuild via type(c)(z_threshold=, + warmup_samples=, detection_policy=), which quietly reset every other + setting - window size, seasonal buckets - to its default.""" + return type(self)(**self._clone_kwargs()) + def is_anomaly(self, event: TelemetryEvent) -> bool: return self.is_anomaly_scored(event, self.detect(event)) @@ -62,11 +81,16 @@ def retrain(self, training_data: list[dict]) -> None: if record.get("worked"): worked[sig] = worked.get(sig, 0) + 1 self._reliability = {sig: worked.get(sig, 0) / n for sig, n in total.items()} + self._samples = dict(total) def reliability(self, signature: str) -> float: return self._reliability.get(signature, 0.0) - def should_suppress(self, signature: str, threshold: float) -> bool: + def should_suppress(self, signature: str, threshold: float, min_samples: int = 1) -> bool: + # min_samples: a single success is reliability 1/1 = 1.0, which cleared any + # threshold. One data point is not a track record. + if self._samples.get(signature, 0) < min_samples: + return False return self.reliability(signature) >= threshold def _severity_band(self, score: float) -> str: @@ -78,5 +102,13 @@ def _severity_band(self, score: float) -> str: @staticmethod def _signature(events: list[TelemetryEvent]) -> str: - joined = "|".join(sorted(e.fingerprint for e in events)) + # The SET of series involved, not the multiset of samples. A fingerprint + # identifies a series (source + name + labels), so every sample of one + # series shares it; hashing the raw list made the signature depend on how + # many anomalous samples happened to land in the window - six vs seven + # samples of the same fault on the same service gave two unrelated + # signatures, and everything keyed on the signature (suppression, + # graduation evidence, RCA reliability, the author's past decisions) + # only matched by luck. + joined = "|".join(sorted({e.fingerprint for e in events})) return hashlib.sha1(joined.encode()).hexdigest()[:16] diff --git a/services/correlation/adapters/robust_correlator.py b/services/correlation/adapters/robust_correlator.py index aedce54..68e2905 100644 --- a/services/correlation/adapters/robust_correlator.py +++ b/services/correlation/adapters/robust_correlator.py @@ -53,25 +53,72 @@ def __init__( seasonal_buckets: int = 24, window_size: int = 128, detection_policy: DetectionPolicy | None = None, + key_by: str = "metric", ) -> None: # sets _z_threshold/_warmup_samples/_reliability/_policy super().__init__(z_threshold, warmup_samples, detection_policy=detection_policy) self._n_buckets = seasonal_buckets self._window_size = window_size - self._windows: dict[tuple[str, int], collections.deque] = {} + # "metric" (default, historical): one window per metric NAME, pooling every + # service's samples. A fault on a quiet service is then judged against a + # busier one's normal - a 10 -> 45 CPU jump scores ~0 beside a service + # idling at 80. "series": one window per (metric, service label). + # Per-series windows fill more slowly (one sample per poll instead of one + # per service per poll), so pair it with a smaller warm-up. + self._key_by = key_by + self._windows: dict[tuple, collections.deque] = {} + + def _clone_kwargs(self) -> dict: + return { + **super()._clone_kwargs(), + "seasonal_buckets": self._n_buckets, + "window_size": self._window_size, + "key_by": self._key_by, + } def _bucket(self, event: TelemetryEvent) -> int: return event.ts.hour % self._n_buckets + def _series(self, event: TelemetryEvent) -> str: + if self._key_by != "series": + return "" + return event.labels.get("service") or event.labels.get("job") or "" + + def _key(self, name: str, series: str, bucket: int) -> tuple: + # The default mode keeps the historical 2-tuple so snapshots and any + # external reader of _windows see exactly the old shape. + return (name, series, bucket) if self._key_by == "series" else (name, bucket) + + def _scoring_window(self, name: str, series: str, win) -> collections.deque: + """The window to score against: this hour's, or - while it is still + warming - the fullest warm window of the same series from another hour. + + Without the fallback every top-of-hour opened a fresh, empty bucket and the + detector scored 0 for everything until it re-warmed (~2 minutes under the + live posture) - a blackout on the hour, every hour.""" + if len(win) >= self._warmup_samples: + return win + best = win + for key, other in list(self._windows.items()): + if other is win or len(other) < self._warmup_samples: + continue + if key[0] != name or (self._key_by == "series" and key[1] != series): + continue + if len(other) > len(best): + best = other + return best + def detect(self, event: TelemetryEvent) -> float: if event.value is None: return 0.0 - key = (event.name, self._bucket(event)) + series = self._series(event) + key = self._key(event.name, series, self._bucket(event)) win = self._windows.setdefault(key, collections.deque(maxlen=self._window_size)) - if len(win) < self._warmup_samples: + ref = self._scoring_window(event.name, series, win) + if len(ref) < self._warmup_samples: score = 0.0 else: - arr = np.fromiter(win, dtype=float, count=len(win)) + arr = np.fromiter(ref, dtype=float, count=len(ref)) med = np.median(arr) mad = np.median(np.abs(arr - med)) deviation = abs(event.value - med) @@ -130,9 +177,9 @@ def baseline_snapshot(self) -> dict: produced the situation. """ pooled: dict[str, list[float]] = {} - for (name, _bucket), win in list(self._windows.items()): # list() = live-resize guard + for key, win in list(self._windows.items()): # list() = live-resize guard if win: - pooled.setdefault(name, []).extend(win) + pooled.setdefault(key[0], []).extend(win) out: dict = {} for name, samples in pooled.items(): @@ -147,13 +194,16 @@ def baseline_snapshot(self) -> dict: def snapshot(self) -> list[dict]: out: list[dict] = [] - for (name, bucket), win in list(self._windows.items()): # list() = live-resize guard - out.append({"metric_name": name, "bucket": bucket, "n": len(win), "window": list(win)}) + for key, win in list(self._windows.items()): # list() = live-resize guard + row = {"metric_name": key[0], "bucket": key[-1], "n": len(win), "window": list(win)} + if self._key_by == "series": + row["series"] = key[1] + out.append(row) return out def load(self, rows: list[dict]) -> None: for r in rows: - key = (r["metric_name"], int(r["bucket"])) + key = self._key(r["metric_name"], r.get("series", ""), int(r["bucket"])) self._windows[key] = collections.deque( (float(x) for x in r["window"]), maxlen=self._window_size ) diff --git a/services/correlation/adapters/trained_correlator.py b/services/correlation/adapters/trained_correlator.py index 71867e8..d36abfb 100644 --- a/services/correlation/adapters/trained_correlator.py +++ b/services/correlation/adapters/trained_correlator.py @@ -92,6 +92,8 @@ def __init__( super().__init__(z_threshold, warmup_samples, detection_policy=detection_policy) self._min_fit_samples = min_fit_samples self._contamination = contamination + self._seasonal_buckets = seasonal_buckets + self._window_size = window_size # The online path: a composed RobustCorrelator built with the same params. # Deliberately NOT given detection_policy: the engine decides anomaly-ness # using the OUTER (trained) correlator's _policy; the inner robust is used @@ -107,6 +109,15 @@ def __init__( self._features: collections.deque[list[float]] = collections.deque(maxlen=4096) self._model = None # sklearn IsolationForest once fitted, else None + def _clone_kwargs(self) -> dict: + return { + **super()._clone_kwargs(), + "seasonal_buckets": self._seasonal_buckets, + "window_size": self._window_size, + "min_fit_samples": self._min_fit_samples, + "contamination": self._contamination, + } + # --- featurization -------------------------------------------------------- @staticmethod diff --git a/services/correlation/app.py b/services/correlation/app.py index d418797..654eed4 100644 --- a/services/correlation/app.py +++ b/services/correlation/app.py @@ -106,6 +106,10 @@ async def lifespan(app: FastAPI): window_seconds=settings.correlation_window_seconds, group_by=settings.correlation_group_by, min_events=settings.correlation_min_events, + # Both were settings the service never passed: the threshold was dead + # config and the engine always used its 0.8 default. + suppress_threshold=settings.reliability_suppress_threshold, + suppress_min_samples=settings.reliability_suppress_min_samples, ) app.state.engine = engine # Reload-on-boot: restore the durable baseline + reliability BEFORE the diff --git a/services/correlation/engine.py b/services/correlation/engine.py index b2865cc..56fdf75 100644 --- a/services/correlation/engine.py +++ b/services/correlation/engine.py @@ -38,15 +38,21 @@ def __init__( suppress_threshold: float = 0.8, group_by: str = "window", min_events: int = 1, + suppress_min_samples: int = 1, ) -> None: self._correlator = correlator - self._correlator_factory = lambda: type(correlator)( - z_threshold=correlator._z_threshold, - warmup_samples=correlator._warmup_samples, - detection_policy=correlator._policy, + self._correlator_factory = ( + correlator.clone_empty + if hasattr(correlator, "clone_empty") + else lambda: type(correlator)( + z_threshold=correlator._z_threshold, + warmup_samples=correlator._warmup_samples, + detection_policy=correlator._policy, + ) ) self._window = window_seconds self._suppress_threshold = suppress_threshold + self._suppress_min_samples = max(1, int(suppress_min_samples)) self._group_by = group_by # How many anomalous events a window must hold before it is an incident. # @@ -157,7 +163,9 @@ def _correlate_buffer(self, key: str = _ALL) -> Situation | None: self._buffers.pop(key, None) self._max_scores.pop(key, None) # Closed loop: suppress a Situation whose signature reliably self-heals. - if self._correlator.should_suppress(sit.signature, self._suppress_threshold): + if self._correlator.should_suppress( + sit.signature, self._suppress_threshold, self._suppress_min_samples + ): self._suppressed.append(sit) return None return sit diff --git a/services/correlation/tests/test_retrain.py b/services/correlation/tests/test_retrain.py index 99e5794..c0cff70 100644 --- a/services/correlation/tests/test_retrain.py +++ b/services/correlation/tests/test_retrain.py @@ -43,3 +43,14 @@ def test_retrain_replaces_prior(): assert c.reliability("a") == 0.0 c.retrain([{"signature": "a", "worked": True}, {"signature": "a", "worked": True}]) assert c.reliability("a") == 1.0 + + +def test_one_success_is_not_a_track_record(): + # reliability 1/1 = 1.0 used to clear any threshold, so a single fixed + # incident was enough to silence that signature. + c = RiverCorrelator() + c.retrain([{"signature": "s", "worked": True}]) + assert c.should_suppress("s", 0.8) is True # historical default: min_samples=1 + assert c.should_suppress("s", 0.8, min_samples=3) is False + c.retrain([{"signature": "s", "worked": True}] * 3) + assert c.should_suppress("s", 0.8, min_samples=3) is True diff --git a/services/correlation/tests/test_robust_correlator.py b/services/correlation/tests/test_robust_correlator.py index 837e3a3..725b342 100644 --- a/services/correlation/tests/test_robust_correlator.py +++ b/services/correlation/tests/test_robust_correlator.py @@ -231,3 +231,70 @@ def test_baseline_snapshot_pools_across_seasonal_buckets(): def test_baseline_snapshot_is_empty_before_any_observation(): assert RobustCorrelator(window_size=50, warmup_samples=3).baseline_snapshot() == {} + + +# --- per-series baselines, the top-of-hour blackout, config-preserving reset --- + + +def _svc_event(svc, value, ts): + return TelemetryEvent( + source="prom", + kind=TelemetryKind.METRIC, + name="cpu_usage", + value=value, + labels={"service": svc}, + ts=ts, + fingerprint=f"fp-{svc}", + ) + + +def _two_services(c): + ts0 = datetime(2026, 8, 13, 0, 0, 0, tzinfo=UTC) + for i in range(40): # a idles near 80% cpu, b near 10% + c.detect(_svc_event("a", 80.0 + (i % 3), ts0 + timedelta(seconds=i))) + c.detect(_svc_event("b", 10.0 + (i % 3), ts0 + timedelta(seconds=i))) + return ts0 + timedelta(seconds=41) + + +def test_metric_keying_masks_a_fault_on_a_quiet_service(): + # The historical default, pinned so the difference below is visible. + c = RobustCorrelator(warmup_samples=40) + t = _two_services(c) + assert c.detect(_svc_event("b", 45.0, t)) < 3.0 + + +def test_series_keying_catches_it(): + c = RobustCorrelator(warmup_samples=40, key_by="series") + t = _two_services(c) + assert c.detect(_svc_event("b", 45.0, t)) > 3.0 + assert c.detect(_svc_event("a", 81.0, t)) < 3.0 # a's normal is still normal + + +def test_series_snapshot_round_trips(): + c = RobustCorrelator(warmup_samples=40, key_by="series") + t = _two_services(c) + rows = c.snapshot() + assert {r["series"] for r in rows} == {"a", "b"} + c2 = RobustCorrelator(warmup_samples=40, key_by="series") + c2.load(rows) + assert c2.detect(_svc_event("b", 45.0, t)) == c.detect(_svc_event("b", 45.0, t)) + + +def test_a_new_hour_scores_against_the_last_warm_hour(): + c = RobustCorrelator(z_threshold=3.0, warmup_samples=30) + _feed_flat(c, name="cpu", value=10.0, n=40, hour=0) + for i in range(40): # give the flat window a little spread + c.detect(_event(value=10.0 + (i % 3) * 0.1, ts=datetime(2026, 8, 13, 0, 1, i, tzinfo=UTC))) + one_am = datetime(2026, 8, 13, 1, 0, 0, tzinfo=UTC) + # the hour-1 bucket is empty; the spike used to score 0.0 here + assert c.detect(_event(value=500.0, ts=one_am)) > 3.0 + + +def test_reset_keeps_the_configuration(): + c = RobustCorrelator( + z_threshold=4.0, warmup_samples=7, seasonal_buckets=6, window_size=33, key_by="series" + ) + fresh = c.clone_empty() + assert (fresh._z_threshold, fresh._warmup_samples) == (4.0, 7) + assert (fresh._n_buckets, fresh._window_size, fresh._key_by) == (6, 33, "series") + assert fresh._windows == {} diff --git a/services/correlation/tests/test_suppressed_signal.py b/services/correlation/tests/test_suppressed_signal.py index 450347e..4f61d72 100644 --- a/services/correlation/tests/test_suppressed_signal.py +++ b/services/correlation/tests/test_suppressed_signal.py @@ -21,7 +21,7 @@ def _ev(v, i, t0): def test_pop_suppressed_returns_suppressed_situation(): # Force suppression: a correlator whose should_suppress is always True. class AlwaysSuppress(RiverCorrelator): - def should_suppress(self, signature, threshold): + def should_suppress(self, signature, threshold, min_samples=1): return True c = AlwaysSuppress(z_threshold=0.0, warmup_samples=0) diff --git a/services/rca/app.py b/services/rca/app.py index 151b691..6cba465 100644 --- a/services/rca/app.py +++ b/services/rca/app.py @@ -4,6 +4,7 @@ import logging import threading +import time from contextlib import asynccontextmanager from fastapi import FastAPI @@ -44,29 +45,59 @@ def _make_runbook_selector(settings): return NullRunbookSelector() -def _build_reliability_provider(training_store): - """Best-effort: per-signature worked/total from training records, same - math as RiverCorrelator/BaseCorrelator.retrain. Returns None if the read - fails, so RCA ranking degrades gracefully to rule-only behavior.""" - try: - records = training_store.read_all() - except Exception: - logger.exception("failed to read training store; ranking without reliability boost") - return None - - worked: dict[str, int] = {} - total: dict[str, int] = {} - for record in records: - sig = record.signature - total[sig] = total.get(sig, 0) + 1 - if record.worked: - worked[sig] = worked.get(sig, 0) + 1 - reliability = {sig: worked.get(sig, 0) / n for sig, n in total.items()} - - def _reliability_provider(signature: str) -> float: - return reliability.get(signature, 0.0) - - return _reliability_provider +class ReliabilityProvider: + """(signature, runbook_id) -> worked/total, re-read from the training store. + + Two things were wrong with the boot-time closure this replaces: + + - It was built once in the lifespan, so every outcome recorded after RCA + started was invisible to ranking until the pod restarted. + - It was keyed on signature alone, and rank_hypotheses added that one number + to every runbook-bearing hypothesis alike - which cannot change their order. + + Refreshes lazily, at most every `refresh_seconds`, on the consumer thread + that calls it. Best-effort: a failed read keeps the last good table (or + none), so ranking degrades to rule-only instead of raising. + """ + + def __init__(self, training_store, refresh_seconds: float = 60.0, clock=time.monotonic): + self._store = training_store + self._refresh = refresh_seconds + self._clock = clock + self._loaded_at: float | None = None + self._by_runbook: dict[tuple[str, str], float] = {} + self._by_signature: dict[str, float] = {} + + def _maybe_reload(self) -> None: + now = self._clock() + if self._loaded_at is not None and now - self._loaded_at < self._refresh: + return + self._loaded_at = now + try: + records = self._store.read_all() + except Exception: + logger.exception("failed to read training store; keeping the last reliability table") + return + worked: dict = {} + total: dict = {} + for r in records: + for key in ((r.signature, r.playbook_id), r.signature): + total[key] = total.get(key, 0) + 1 + if r.worked: + worked[key] = worked.get(key, 0) + 1 + ratio = {k: worked.get(k, 0) / n for k, n in total.items()} + self._by_runbook = {k: v for k, v in ratio.items() if isinstance(k, tuple)} + self._by_signature = {k: v for k, v in ratio.items() if isinstance(k, str)} + + def __call__(self, signature: str, runbook_id: str | None = None) -> float: + self._maybe_reload() + if runbook_id is None: + return self._by_signature.get(signature, 0.0) + return self._by_runbook.get((signature, runbook_id), 0.0) + + +def _build_reliability_provider(training_store, refresh_seconds: float = 60.0): + return ReliabilityProvider(training_store, refresh_seconds=refresh_seconds) @asynccontextmanager @@ -80,7 +111,9 @@ async def lifespan(app: FastAPI): audit_sink = stores.audit_sink holder = ProviderHolder(make_explanation_provider(settings)) app.state.provider_holder = holder - reliability_provider = _build_reliability_provider(stores.training_store) + reliability_provider = _build_reliability_provider( + stores.training_store, settings.reliability_refresh_seconds + ) selector = _make_runbook_selector(settings) thread = start_supervised( "rca-consumer", diff --git a/services/rca/rank.py b/services/rca/rank.py index ec51766..09ddcd5 100644 --- a/services/rca/rank.py +++ b/services/rca/rank.py @@ -4,10 +4,10 @@ sorted best-first. A low-confidence fallback guarantees a non-empty result so downstream always has something to act on (see flow.md 5.3). -An optional `reliability_provider` (situation.signature -> float in [0, 1], -e.g. the correlator's learned worked/total track record) can boost a -hypothesis whose suggested runbook has proven reliable for this signature. -Passing None preserves the original rule-only ranking exactly.""" +An optional `reliability_provider` ((signature, runbook_id) -> float in [0, 1], +the worked/total track record of THAT runbook on THAT signature) can boost a +hypothesis whose suggested runbook has proven reliable here. Passing None +preserves the original rule-only ranking exactly.""" from __future__ import annotations @@ -42,7 +42,7 @@ def _service_labels(situation: Situation) -> set[str]: def rank_hypotheses( situation: Situation, context: EnrichmentContext, - reliability_provider: Callable[[str], float] | None = None, + reliability_provider: Callable[[str, str | None], float] | None = None, store=None, selector=None, ) -> list[RootCauseHypothesis]: @@ -208,12 +208,21 @@ def rank_hypotheses( # runbook — the fallback (runbook_id=None) is never boosted, so the top # suggestion after ranking still resolves to a real playbook id whenever # any rule-based hypothesis fired. - reliability = reliability_provider(situation.signature) if situation.signature else 0.0 - reliability = max(0.0, min(1.0, reliability)) + # + # The track record is per (signature, runbook). It used to be one number per + # signature, added equally to every runbook-bearing hypothesis - a uniform + # offset that could never change their order, so the boost did nothing. + def _reliability(h: RootCauseHypothesis) -> float: + if h.suggested_runbook_id is None or not situation.signature: + return 0.0 + value = reliability_provider(situation.signature, h.suggested_runbook_id) + return max(0.0, min(1.0, value)) - def _score(h: RootCauseHypothesis) -> float: - boost = _RELIABILITY_WEIGHT * reliability if h.suggested_runbook_id is not None else 0.0 - return min(1.0, h.confidence + boost) + def _score(h: RootCauseHypothesis) -> tuple[float, float]: + boosted = min(1.0, h.confidence + _RELIABILITY_WEIGHT * _reliability(h)) + # Confidence breaks ties, so two hypotheses both clamped at 1.0 keep + # their confidence order instead of falling back to rule order. + return (boosted, h.confidence) hypotheses.sort(key=_score, reverse=True) return hypotheses diff --git a/services/rca/tests/test_rank.py b/services/rca/tests/test_rank.py index f9d66fb..4f2eaa5 100644 --- a/services/rca/tests/test_rank.py +++ b/services/rca/tests/test_rank.py @@ -184,7 +184,7 @@ def test_reliability_provider_boosts_proven_runbook(): ctx = EnrichmentContext() # no deploys -> only saturation rule fires situation = _situation(name="cpu_usage", labels={"service": "web"}) - def reliability(signature: str) -> float: + def reliability(signature: str, runbook_id: str | None = None) -> float: assert signature == situation.signature return 1.0 @@ -207,7 +207,7 @@ def test_reliability_provider_never_boosts_fallback_hypothesis(): recent_deploys=[{"service": "web", "version": "v2", "ts": NOW.isoformat()}] ) situation = _situation(name="cpu", labels={"service": "web"}) - hyps = rank_hypotheses(situation, ctx, lambda sig: 1.0) + hyps = rank_hypotheses(situation, ctx, lambda sig, rb=None: 1.0) assert hyps[0].suggested_runbook_id is not None @@ -471,3 +471,19 @@ def test_otel_duration_metrics_map_to_the_latency_rule(): for metric in ("otel_http_server_request_duration_ms", "otel_rpc_server_duration_ms"): hyps = rank_hypotheses(_situation(name=metric), ctx) assert hyps[0].suggested_runbook_id == "scale-service", metric + + +def test_reliability_is_per_runbook_and_can_reorder(): + # memory (restart-pod, 0.65) and cpu saturation (scale-service, 0.6) both + # fire. A proven track record for scale-service on THIS signature - and a + # poor one for restart-pod - must be able to put scale-service first. A + # per-signature figure applied to both equally never could. + situation = _situation_with_metric("memory_usage_mb cpu_usage", value=900.0) + ctx = EnrichmentContext() + record = {"scale-service": 1.0, "restart-pod": 0.0} + + unboosted = rank_hypotheses(situation, ctx, None) + boosted = rank_hypotheses(situation, ctx, lambda sig, rb=None: record.get(rb, 0.0)) + + assert unboosted[0].suggested_runbook_id == "restart-pod" + assert boosted[0].suggested_runbook_id == "scale-service" diff --git a/services/rca/tests/test_reliability_provider.py b/services/rca/tests/test_reliability_provider.py new file mode 100644 index 0000000..1e8309a --- /dev/null +++ b/services/rca/tests/test_reliability_provider.py @@ -0,0 +1,78 @@ +from datetime import UTC, datetime + +from common.contracts import RemediationResult, TrainingRecord +from services.rca.app import ReliabilityProvider + +NOW = datetime(2026, 9, 23, tzinfo=UTC) + + +def _rec(sig, pid, worked): + return TrainingRecord( + situation_id=f"sit-{sig}", + signature=sig, + playbook_id=pid, + result=RemediationResult.SUCCESS if worked else RemediationResult.ROLLED_BACK, + worked=worked, + ts=NOW, + ) + + +class _Store: + def __init__(self, records): + self.records = records + self.reads = 0 + + def read_all(self): + self.reads += 1 + return list(self.records) + + +class _Clock: + def __init__(self): + self.t = 0.0 + + def __call__(self): + return self.t + + +def test_track_record_is_per_runbook(): + store = _Store( + [ + _rec("s", "scale-service", True), + _rec("s", "restart-pod", False), + _rec("s", "restart-pod", True), + ] + ) + rel = ReliabilityProvider(store) + assert rel("s", "scale-service") == 1.0 + assert rel("s", "restart-pod") == 0.5 + assert rel("s") == 2 / 3 + assert rel("other", "scale-service") == 0.0 + + +def test_new_outcomes_are_seen_after_the_refresh_period(): + store = _Store([]) + clock = _Clock() + rel = ReliabilityProvider(store, refresh_seconds=60, clock=clock) + assert rel("s", "scale-service") == 0.0 + + store.records.append(_rec("s", "scale-service", True)) + clock.t = 30 + assert rel("s", "scale-service") == 0.0 # still cached + clock.t = 61 + assert rel("s", "scale-service") == 1.0 # the boot-time closure never got here + assert store.reads == 2 + + +def test_a_failed_read_keeps_the_last_table(): + store = _Store([_rec("s", "scale-service", True)]) + clock = _Clock() + rel = ReliabilityProvider(store, refresh_seconds=1, clock=clock) + assert rel("s", "scale-service") == 1.0 + + def boom(): + raise RuntimeError("db down") + + store.read_all = boom + clock.t = 5 + assert rel("s", "scale-service") == 1.0 From 1082416ef82b14374226cae646e5476b6a186526 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 01:01:58 +0530 Subject: [PATCH 5/7] fix(action): rollbacks that restore state, and outcomes that say what happened - Scale was undone with the inverse delta, which is wrong whenever the forward step was clamped: 9 replicas + 2 clamps to 10, and 10 - 2 = 8, one below the start. KubernetesRemediator now records the pre-scale count and rollback restores exactly that. - An unhealthy result was reported ROLLED_BACK even when the playbook had no rollback steps (restart-pod ships with none) or the rollback itself failed. Those are now FAILURE with "unhealthy:no-rollback" / "unhealthy:rollback-failed". - The post-remediation readiness check compared ready == replicas only, which the OLD ReplicaSet satisfies before the controller has observed the patch. It now also requires observedGeneration >= generation and updatedReplicas == replicas (both read defensively, so fakes/older clients are unaffected). Co-Authored-By: Claude Opus 5.5 --- services/action/adapters/k8s_health.py | 17 +++++--- services/action/adapters/k8s_remediator.py | 36 +++++++++++++--- services/action/remediate.py | 27 +++++++++++- services/action/tests/test_k8s_health.py | 43 ++++++++++++++++++++ services/action/tests/test_k8s_remediator.py | 21 ++++++++++ services/action/tests/test_remediate.py | 19 +++++++++ 6 files changed, 151 insertions(+), 12 deletions(-) diff --git a/services/action/adapters/k8s_health.py b/services/action/adapters/k8s_health.py index 6b5bd0f..abf9337 100644 --- a/services/action/adapters/k8s_health.py +++ b/services/action/adapters/k8s_health.py @@ -97,15 +97,22 @@ def _pod_ready(self, target: RemediationTarget) -> bool: # error would escape the try (Python does not consult sibling excepts for # an error raised while matching an except type). So: one catch, no gaps. try: - st = ( - self._api() - .read_namespaced_deployment_status(target.deployment, target.namespace) - .status - ) + dep = self._api().read_namespaced_deployment_status(target.deployment, target.namespace) + st = dep.status except Exception: # noqa: BLE001 — any client/config/connection error → not-yet-ready return False + # Right after the patch the controller may not have seen the new spec + # yet, and the status still describes the OLD ReplicaSet - fully ready. + # Checking ready == replicas alone could pass on the pre-restart pods. + generation = getattr(getattr(dep, "metadata", None), "generation", None) + observed = getattr(st, "observed_generation", None) + if generation is not None and observed is not None and observed < generation: + return False ready = st.ready_replicas or 0 desired = st.replicas or 0 + updated = getattr(st, "updated_replicas", None) + if updated is not None and updated < desired: + return False return desired > 0 and ready == desired def _safe_metric(self, predicate) -> bool: diff --git a/services/action/adapters/k8s_remediator.py b/services/action/adapters/k8s_remediator.py index d83de15..fca8aa4 100644 --- a/services/action/adapters/k8s_remediator.py +++ b/services/action/adapters/k8s_remediator.py @@ -37,6 +37,11 @@ def __init__(self, namespace_default: str, apps_v1=None, exc_type=None) -> None: self._ns_default = namespace_default self._apps_v1 = apps_v1 # injected in tests; loaded lazily if None self._exc_type = exc_type # the exception class to treat as a safe failure + # Replica count each deployment had before execute() first scaled it, so + # rollback() can put back exactly that. Undoing a scale with the inverse + # DELTA was wrong whenever the forward step was clamped: 9 replicas + 2 + # clamps to 10, and 10 - 2 = 8, one below where we started. + self._pre_scale: dict[tuple[str, str], int] = {} def _api(self): if self._apps_v1 is None: @@ -49,18 +54,25 @@ def _exc(self): return self._exc_type def execute(self, plan: RemediationPlan) -> bool: - return self._run(plan.target, plan.steps) + ns = plan.target.namespace or self._ns_default + self._pre_scale.pop((ns, plan.target.deployment), None) + return self._run(plan.target, plan.steps, rolling_back=False) def rollback(self, plan: RemediationPlan) -> bool: - return self._run(plan.target, plan.rollback_steps) - - def _run(self, target: RemediationTarget, steps: list[RemediationStep]) -> bool: + ok = self._run(plan.target, plan.rollback_steps, rolling_back=True) + ns = plan.target.namespace or self._ns_default + self._pre_scale.pop((ns, plan.target.deployment), None) + return ok + + def _run( + self, target: RemediationTarget, steps: list[RemediationStep], rolling_back: bool = False + ) -> bool: ns = target.namespace or self._ns_default try: exc_type = self._exc() api = self._api() for step in steps: - self._dispatch(api, ns, target.deployment, step) + self._dispatch(api, ns, target.deployment, step, rolling_back) except exc_type as exc: # any K8s API error → safe failure logger.warning("k8s remediation failed on %s/%s: %s", ns, target.deployment, exc) return False @@ -69,7 +81,9 @@ def _run(self, target: RemediationTarget, steps: list[RemediationStep]) -> bool: return False return True - def _dispatch(self, api, ns: str, deployment: str, step: RemediationStep) -> None: + def _dispatch( + self, api, ns: str, deployment: str, step: RemediationStep, rolling_back: bool = False + ) -> None: if step.action == "wait": return # readiness is the health checker's job if step.action == "restart": @@ -84,7 +98,17 @@ def _dispatch(self, api, ns: str, deployment: str, step: RemediationStep) -> Non api.patch_namespaced_deployment(deployment, ns, body) return if step.action == "scale": + key = (ns, deployment) + if rolling_back and key in self._pre_scale: + # Restore the recorded original; the step's delta is only a hint. + desired = self._pre_scale[key] + api.patch_namespaced_deployment_scale( + deployment, ns, {"spec": {"replicas": desired}} + ) + return current = api.read_namespaced_deployment(deployment, ns).spec.replicas or 1 + if not rolling_back: + self._pre_scale.setdefault(key, current) desired = max(_MIN_REPLICAS, min(_MAX_REPLICAS, current + (step.replicas or 0))) api.patch_namespaced_deployment_scale(deployment, ns, {"spec": {"replicas": desired}}) return diff --git a/services/action/remediate.py b/services/action/remediate.py index 885965a..2f39797 100644 --- a/services/action/remediate.py +++ b/services/action/remediate.py @@ -288,7 +288,32 @@ def execute_remediation( preflight=preflight, ) - remediator.rollback(plan) + # Unhealthy. Only report ROLLED_BACK when something was actually undone: a + # playbook with no rollback steps (restart-pod ships with none) used to be + # reported as rolled back when nothing ran, and a rollback that FAILED was + # reported the same way as one that worked. + if not plan.rollback_steps: + _audit(gate, situation, playbook, "unhealthy-no-rollback") + return _outcome( + situation, + playbook, + RemediationResult.FAILURE, + "unhealthy:no-rollback", + steps=steps, + mode=mode, + preflight=preflight, + ) + if not remediator.rollback(plan): + _audit(gate, situation, playbook, "rollback-failed") + return _outcome( + situation, + playbook, + RemediationResult.FAILURE, + "unhealthy:rollback-failed", + steps=steps, + mode=mode, + preflight=preflight, + ) _audit(gate, situation, playbook, "rolled-back") return _outcome( situation, diff --git a/services/action/tests/test_k8s_health.py b/services/action/tests/test_k8s_health.py index 0b208a7..14ec679 100644 --- a/services/action/tests/test_k8s_health.py +++ b/services/action/tests/test_k8s_health.py @@ -141,3 +141,46 @@ def test_back_compat_injected_metric_healthy_still_used(): exc_type=FakeExc, ) assert checker.check(_sit(), _tgt()) is False # injected predicate honored + + +class _RolloutApps: + """Status whose controller has not caught up with the latest spec yet.""" + + def __init__(self, generation, observed, updated, ready=1, desired=1): + self.generation, self.observed, self.updated = generation, observed, updated + self.ready, self.desired = ready, desired + + def read_namespaced_deployment_status(self, name, namespace): + outer = self + + class _Meta: + generation = outer.generation + + class _St: + observed_generation = outer.observed + updated_replicas = outer.updated + ready_replicas = outer.ready + replicas = outer.desired + + class _Dep: + metadata = _Meta() + status = _St() + + return _Dep() + + +def test_stale_status_from_before_the_patch_is_not_ready(): + # The old ReplicaSet is fully ready, but the controller has not observed + # the restart yet - that is not a recovered deployment. + apps = _RolloutApps(generation=5, observed=4, updated=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is False + + +def test_old_pods_still_serving_is_not_ready(): + apps = _RolloutApps(generation=5, observed=5, updated=0, ready=1, desired=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is False + + +def test_converged_rollout_is_ready(): + apps = _RolloutApps(generation=5, observed=5, updated=1, ready=1, desired=1) + assert _hc(apps, metric_ok=True).check(_sit(), _tgt()) is True diff --git a/services/action/tests/test_k8s_remediator.py b/services/action/tests/test_k8s_remediator.py index a86aab9..97a78ac 100644 --- a/services/action/tests/test_k8s_remediator.py +++ b/services/action/tests/test_k8s_remediator.py @@ -219,3 +219,24 @@ def list_namespaced_replica_set(self, namespace, **kwargs): step = RemediationStep(action="rollback_to_revision", revision=3) # Should fail-safe (revision not found) rather than patching the wrong template assert r.execute(_plan(step)) is False + + +class _StatefulApps(FakeAppsV1): + """Remembers the replica count it was scaled to.""" + + def patch_namespaced_deployment_scale(self, name, namespace, body): + super().patch_namespaced_deployment_scale(name, namespace, body) + self._replicas = body["spec"]["replicas"] + + +def test_rollback_restores_the_original_count_even_after_a_clamp(): + # 9 + 2 clamps to 10; undoing with the inverse delta gave 8. + apps = _StatefulApps(replicas=9) + r = KubernetesRemediator("ns", apps_v1=apps, exc_type=FakeApiException) + plan = RemediationPlan( + target=RemediationTarget(namespace="ns", deployment="demo-app"), + steps=[RemediationStep(action="scale", replicas=2)], + rollback_steps=[RemediationStep(action="scale", replicas=-2)], + ) + assert r.execute(plan) and apps._replicas == 10 + assert r.rollback(plan) and apps._replicas == 9 diff --git a/services/action/tests/test_remediate.py b/services/action/tests/test_remediate.py index b4450df..3adbcc4 100644 --- a/services/action/tests/test_remediate.py +++ b/services/action/tests/test_remediate.py @@ -247,3 +247,22 @@ def test_expire_failure_does_not_change_the_outcome(): gate = _ExpiringGate("pending", raises=True) out = _run(_playbook(hitl=HitlMode.HITL), gate, RecordingRemediator(), FixedHealthChecker(True)) assert out.health_after == "aborted:timeout" + + +# --- An unhealthy result is only "rolled back" when something was undone --- + + +def test_no_rollback_steps_is_a_failure_not_a_rollback(): + pb = _playbook().model_copy(update={"rollback_steps": []}) + r = RecordingRemediator() + out = _run(pb, FakeGate(), r, FixedHealthChecker(False)) + assert out.result == RemediationResult.FAILURE + assert out.health_after == "unhealthy:no-rollback" + assert r.rolled_back_plan is None + + +def test_a_failed_rollback_is_reported_as_such(): + r = RecordingRemediator(rollback_result=False) + out = _run(_playbook(), FakeGate(), r, FixedHealthChecker(False)) + assert out.result == RemediationResult.FAILURE + assert out.health_after == "unhealthy:rollback-failed" From 3fcd0ae2462a99f63891265916162298481810b4 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 01:04:51 +0530 Subject: [PATCH 6/7] chore: drop the unused geist package, dead proposePlaybook path, and uv:latest - geist was never imported (the fonts are vendored in frontend/src/fonts) but pulled in next + sharp - ~940 lockfile lines and the Dependabot "critical" alerts that never applied to this Vite app. - proposePlaybook (the sync draft call) had no caller left in the console since the draft-async + Agent Activity flow replaced it. The backend endpoint stays. - OutcomeReason claimed to be the exact health_after vocabulary but was missing preflight-failed, interrupted:unknown and the denied:unsafe-* reasons; add those and the two new unhealthy:* reasons. - Pin ghcr.io/astral-sh/uv to 0.11.28 in both Dockerfiles instead of :latest. Co-Authored-By: Claude Opus 5.5 --- deploy/Dockerfile | 3 +- deploy/Dockerfile.meridian | 3 +- frontend/package-lock.json | 948 +----------------------------------- frontend/package.json | 3 +- frontend/src/data/api.ts | 3 - frontend/src/data/source.ts | 27 - frontend/src/data/types.ts | 8 + 7 files changed, 19 insertions(+), 976 deletions(-) diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 07f35b9..c1cf955 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:1 FROM python:3.11-slim AS builder-base -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +# Pinned: :latest made every image build depend on whatever uv shipped that day. +COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv WORKDIR /app ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy diff --git a/deploy/Dockerfile.meridian b/deploy/Dockerfile.meridian index 3fa13be..b40ace6 100644 --- a/deploy/Dockerfile.meridian +++ b/deploy/Dockerfile.meridian @@ -13,7 +13,8 @@ RUN npm run build FROM python:3.11-slim -COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv +# Pinned: :latest made every image build depend on whatever uv shipped that day. +COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /usr/local/bin/uv WORKDIR /app ENV UV_COMPILE_BYTECODE=1 UV_LINK_MODE=copy diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ec992f4..bd6a070 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,7 +10,6 @@ "dependencies": { "@phosphor-icons/react": "^2.1.7", "framer-motion": "^11.11.17", - "geist": "^1.3.1", "react": "^18.3.1", "react-dom": "^18.3.1" }, @@ -38,593 +37,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", - "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", - "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", - "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "peer": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", - "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", - "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", - "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", - "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", - "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", - "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", - "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", - "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", - "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", - "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", - "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", - "cpu": [ - "arm" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", - "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", - "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", - "cpu": [ - "ppc64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", - "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", - "cpu": [ - "riscv64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", - "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", - "cpu": [ - "s390x" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", - "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", - "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", - "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", - "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/runtime": "^1.11.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", - "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", - "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", - "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": "^20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", - "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -664,161 +76,6 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, - "node_modules/@next/env": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.4.tgz", - "integrity": "sha512-cjWZnUUa6jZq2kFaNe/ZyJdZonOZ/QoN0Zka2nz/FLOrfx14pQuM9c5RaSVkWMqgdt4ksgPAMWPyHSs/CyV48Q==", - "license": "MIT", - "peer": true - }, - "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.4.tgz", - "integrity": "sha512-iBr3I5LZNk5/bgl5//iTgD2tcym14MX0Xo7fD//u9dYAEgGzza1y9oywluPtf74YnOswVdH1908aK9xVz7zQTw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-darwin-x64": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.4.tgz", - "integrity": "sha512-2dpiSyl2Jw/NrBPaU2MAKGSa+2MR82pJIn4Sm5Rjr+gxAeuh0z158Su3Z2O8zn7UNNq+ej4bToed6RcRN/Lydg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.4.tgz", - "integrity": "sha512-+t+U8HZT+fApePCS5h89CSH3datz29MkzyfCn+6fpsZBG/oiEOhINcb9rtkv6sdpToLGFn2e6146NzaKCXkqrA==", - "cpu": [ - "arm64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.4.tgz", - "integrity": "sha512-mx03GNs1ocQA5JQ4FxDMmIsNkdrZh8cuezKCrId28e5/gIPU/l7Kcy2+vmCCzdjnnmXJy+iOAu+7K0QppO6Urg==", - "cpu": [ - "arm64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.4.tgz", - "integrity": "sha512-YIhGY6fSMfha52bnVxnzc9zaVBzJg+cqQTOD8tXIBSx4fuv0pVMxQTE0PaS59YhnMOiYiG09IMwxJAf/CFm/Dw==", - "cpu": [ - "x64" - ], - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.4.tgz", - "integrity": "sha512-+eaaX6axpDb0yF1GCpiERe6njplvdC+nks/fKfcHu3XPGRrald8P3/X7yv7QLdjA51knnxwl9pxdIJsg+w1L+Q==", - "cpu": [ - "x64" - ], - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.4.tgz", - "integrity": "sha512-0jcXW7Xs/uzICrmgV3MhDYDeRy++1CqnpDIerlPIqYO4bhzB4WNbX/aRnQclustsAyTkFKB0z6rbcjmNg5tR8A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.4.tgz", - "integrity": "sha512-vvBzwu1pYQCp92maZCFCIw/XgOTMR5tur9GjakwIo2cmwRTMKajRZZDS9+e4KsUZWKu1E007WUeAFXRRjZeuzw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "peer": true, - "engines": { - "node": ">= 10" - } - }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -1160,16 +417,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@swc/helpers": { - "version": "0.5.23", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.23.tgz", - "integrity": "sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==", - "license": "Apache-2.0", - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/@types/prop-types": { "version": "15.7.15", "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", @@ -1297,6 +544,7 @@ "version": "2.11.14", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.14.tgz", "integrity": "sha512-JyJ954WzuIR8/FFzX0o5krdSTrBAkcCSRfWSleRsIHSWV+cZe2FI1PKggVkFke1hBldRs+LRxUczzE9iPmgZww==", + "dev": true, "license": "Apache-2.0", "bin": { "baseline-browser-mapping": "dist/cli.cjs" @@ -1379,6 +627,7 @@ "version": "1.0.30001809", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz", "integrity": "sha512-xxWVywk6a6Arlk+hymeycyn/VgqEfLDxupvhH/xiY5SJ/18kmi9o6MiO320DCUzypORHLtvh0I4i04tUhCNHNQ==", + "dev": true, "funding": [ { "type": "opencollective", @@ -1433,13 +682,6 @@ "node": ">= 6" } }, - "node_modules/client-only": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", - "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==", - "license": "MIT", - "peer": true - }, "node_modules/commander": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", @@ -1474,7 +716,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "engines": { "node": ">=8" @@ -1640,15 +882,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/geist": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/geist/-/geist-1.7.2.tgz", - "integrity": "sha512-Gu5lDFa3pLRyoBlBPf0QIFHVdWAnpco7fS1bJm41jyLPFoguBgiubseUN2oLXMgqZ7uxAxDoXcHMhCY/fOTTgg==", - "license": "SIL OPEN FONT LICENSE", - "peerDependencies": { - "next": ">=13.2.0" - } - }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2113,6 +1346,7 @@ "version": "3.3.18", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, "funding": [ { "type": "github", @@ -2127,89 +1361,6 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, - "node_modules/next": { - "version": "16.3.4", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.4.tgz", - "integrity": "sha512-/Ztf6CeRH+ejEXUrYtqI4gkS66eFIHuSwqi60RgcpWKodxFZx2/dqVCMKBwILfAHXQ+F1b1vAudgj3mnxqtoIA==", - "license": "MIT", - "peer": true, - "dependencies": { - "@next/env": "16.3.4", - "@swc/helpers": "0.5.23", - "baseline-browser-mapping": "^2.9.19", - "caniuse-lite": "^1.0.30001579", - "postcss": "8.5.23", - "styled-jsx": "5.1.6" - }, - "bin": { - "next": "dist/bin/next" - }, - "engines": { - "node": ">=20.9.0" - }, - "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.4", - "@next/swc-darwin-x64": "16.3.4", - "@next/swc-linux-arm64-gnu": "16.3.4", - "@next/swc-linux-arm64-musl": "16.3.4", - "@next/swc-linux-x64-gnu": "16.3.4", - "@next/swc-linux-x64-musl": "16.3.4", - "@next/swc-win32-arm64-msvc": "16.3.4", - "@next/swc-win32-x64-msvc": "16.3.4", - "sharp": "^0.35.4" - }, - "peerDependencies": { - "@opentelemetry/api": "^1.1.0", - "@playwright/test": "^1.51.1", - "babel-plugin-react-compiler": "*", - "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", - "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", - "sass": "^1.3.0" - }, - "peerDependenciesMeta": { - "@opentelemetry/api": { - "optional": true - }, - "@playwright/test": { - "optional": true - }, - "babel-plugin-react-compiler": { - "optional": true - }, - "sass": { - "optional": true - } - } - }, - "node_modules/next/node_modules/postcss": { - "version": "8.5.23", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.23.tgz", - "integrity": "sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "peer": true, - "dependencies": { - "nanoid": "^3.3.16", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, "node_modules/node-releases": { "version": "2.0.53", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.53.tgz", @@ -2261,6 +1412,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, "license": "ISC" }, "node_modules/picomatch": { @@ -2628,104 +1780,16 @@ "loose-envify": "^1.1.0" } }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "license": "ISC", - "optional": true, - "peer": true, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", - "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "@img/colour": "^1.1.0", - "detect-libc": "^2.1.2", - "semver": "^7.8.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.4", - "@img/sharp-darwin-x64": "0.35.4", - "@img/sharp-freebsd-wasm32": "0.35.4", - "@img/sharp-libvips-darwin-arm64": "1.3.3", - "@img/sharp-libvips-darwin-x64": "1.3.3", - "@img/sharp-libvips-linux-arm": "1.3.3", - "@img/sharp-libvips-linux-arm64": "1.3.3", - "@img/sharp-libvips-linux-ppc64": "1.3.3", - "@img/sharp-libvips-linux-riscv64": "1.3.3", - "@img/sharp-libvips-linux-s390x": "1.3.3", - "@img/sharp-libvips-linux-x64": "1.3.3", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", - "@img/sharp-libvips-linuxmusl-x64": "1.3.3", - "@img/sharp-linux-arm": "0.35.4", - "@img/sharp-linux-arm64": "0.35.4", - "@img/sharp-linux-ppc64": "0.35.4", - "@img/sharp-linux-riscv64": "0.35.4", - "@img/sharp-linux-s390x": "0.35.4", - "@img/sharp-linux-x64": "0.35.4", - "@img/sharp-linuxmusl-arm64": "0.35.4", - "@img/sharp-linuxmusl-x64": "0.35.4", - "@img/sharp-webcontainers-wasm32": "0.35.4", - "@img/sharp-win32-arm64": "0.35.4", - "@img/sharp-win32-ia32": "0.35.4", - "@img/sharp-win32-x64": "0.35.4" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, "license": "BSD-3-Clause", "engines": { "node": ">=0.10.0" } }, - "node_modules/styled-jsx": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.6.tgz", - "integrity": "sha512-qSVyDTeMotdvQYoHWLNGwRFJHC+i+ZvdBRYosOFgC+Wg1vx4frN2/RG/NA7SYqqvKNLf39P2LSRA2pu6n0XYZA==", - "license": "MIT", - "peer": true, - "dependencies": { - "client-only": "0.0.1" - }, - "engines": { - "node": ">= 12.0.0" - }, - "peerDependencies": { - "react": ">= 16.8.0 || 17.x.x || ^18.0.0-0 || ^19.0.0-0" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "babel-plugin-macros": { - "optional": true - } - } - }, "node_modules/sucrase": { "version": "3.35.1", "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 9e50660..0bc2b87 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -3,7 +3,7 @@ "private": true, "version": "1.0.0", "type": "module", - "description": "IntelliOps CoE \u2014 internal operator console (Overview \u00b7 Incidents \u00b7 Governance)", + "description": "IntelliOps CoE — internal operator console (Overview · Incidents · Governance)", "scripts": { "dev": "vite", "build": "node scripts/check-contrast.mjs && tsc -b && vite build", @@ -13,7 +13,6 @@ "dependencies": { "@phosphor-icons/react": "^2.1.7", "framer-motion": "^11.11.17", - "geist": "^1.3.1", "react": "^18.3.1", "react-dom": "^18.3.1" }, diff --git a/frontend/src/data/api.ts b/frontend/src/data/api.ts index 4832e8a..2db9afc 100644 --- a/frontend/src/data/api.ts +++ b/frontend/src/data/api.ts @@ -71,9 +71,6 @@ export async function decideApproval( export const loadProposals = () => getJSON(`${GOV}/playbooks/proposed`); -export const proposePlaybook = (situation: Situation, requestedBy: string) => - postJSON(`${GOV}/playbooks/proposed`, { situation, requested_by: requestedBy }); - export const approveProposal = (id: string, decidedBy: string) => postJSON(`${GOV}/playbooks/proposed/${id}/approve`, { decided_by: decidedBy }); diff --git a/frontend/src/data/source.ts b/frontend/src/data/source.ts index ffab9c0..83d04f8 100644 --- a/frontend/src/data/source.ts +++ b/frontend/src/data/source.ts @@ -61,33 +61,6 @@ export const testLlmConfig = LIVE export const loadProposals = LIVE ? api.loadProposals : async () => _mockProposals; -export const proposePlaybook = LIVE - ? api.proposePlaybook - : async (situation: Situation, _requestedBy: string): Promise => { - // mock mode: fabricate a proposal the same shape the server would return - // (server-assigned id, forced hitl) — honest about being a stub draft. - const proposal: ProposedPlaybook = { - id: `prop-mock-${Date.now().toString(36)}`, - playbook: { - id: `ai-${situation.signature}-mock`, - name: `Drafted fix · ${situation.service}`, - match_rule: situation.id, - steps: [{ action: "restart", note: "mock draft — no LLM configured" }], - hitl_mode: "hitl", - reversible: true, - rollback_steps: [], - }, - status: "proposed", - proposed_by: "runbook-author", - rationale: "mock mode: no LLM configured — this is a stub draft, not a real AI proposal.", - source_situation_id: situation.id, - decided_by: null, - ts: Date.now(), - }; - _mockProposals.push(proposal); - return proposal; - }; - export const approveProposal = LIVE ? api.approveProposal : async (id: string, decidedBy: string): Promise => { diff --git a/frontend/src/data/types.ts b/frontend/src/data/types.ts index eb95035..90549de 100644 --- a/frontend/src/data/types.ts +++ b/frontend/src/data/types.ts @@ -22,7 +22,15 @@ export type RemediationResult = "success" | "failure" | "rolled_back" | "escalat export type OutcomeReason = | "healthy" | "unhealthy:rolled-back" + | "unhealthy:no-rollback" + | "unhealthy:rollback-failed" | "execute-failed" + | "preflight-failed" + | "interrupted:unknown" + | "denied:unsafe-scale" + | "denied:unsafe-limits" + | "denied:unsafe-probe" + | "denied:unsafe-revision" | "denied:rbac" | "refused:not-reversible" | "aborted:rejected" From 802b1313335eab9547125fa418ee98aa4fdf9c73 Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 01:05:28 +0530 Subject: [PATCH 7/7] docs: governance API changes (register/decide/expire/graduate/demote) Co-Authored-By: Claude Opus 5.5 --- docs/OPERATIONS.md | 5 +++-- flow.md | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 7eec784..87a770e 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -29,7 +29,8 @@ need a token). In `token` mode **all** endpoints are gated — including the internal service-to-service paths on governance (`POST /audit`, `POST /rbac/check`, -`POST /approvals`, `GET /approvals/{id}`, `POST /playbooks/{id}/graduate`). +`POST /approvals`, `GET /approvals/{id}`, `POST /approvals/{id}/expire`, +`POST /playbooks/{id}/graduate`, `POST /playbooks/{id}/demote`). Internal callers (action's `HttpGovernanceGate`, feedback's graduator) attach the shared `Bearer` token to their requests automatically. @@ -65,7 +66,7 @@ that talk to governance over REST: | --- | --- | | `governance` | Validates incoming tokens on all endpoints. | | `action` | `HttpGovernanceGate` calls `POST /rbac/check`, `POST /audit`, `POST /approvals`, `GET /approvals/{id}`. | -| `feedback` | `_make_graduator` calls `POST /playbooks/{id}/graduate`. | +| `feedback` | `_make_graduator` / `_make_demoter` call `POST /playbooks/{id}/graduate` and `/demote`. | | `rca` | Uses the shared factory; gated if exposed. | Example compose environment block (add to each service above): diff --git a/flow.md b/flow.md index 6cbaafb..17303ee 100644 --- a/flow.md +++ b/flow.md @@ -243,10 +243,12 @@ the unmodified z-score (default `off`, byte-identical). See |----------|--------------|-----|-----------| | `check_rbac(actor, action, resource) → allow \| deny` | Decides whether an actor may perform an action. | Every automated action passes here — the enforceable RBAC guarantee. | RBAC policy | | `create_approval_request() → ApprovalRequest` | Opens a pending approval for a `hitl` playbook. | Materializes the human decision point. | `contracts.ApprovalRequest` | -| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. | The approval interface the React console's Approve/Reject buttons drive. | — | +| `decide(id, approve \| reject)` | REST endpoint (`POST /approvals/{id}/decide`) a human/console calls to approve or reject. Only `approved`/`rejected` are accepted (422 otherwise), the decider needs the matching `approve`/`reject` permission, a request that is no longer pending is a 409, and the decision is written to the audit log under the human's name. | The approval interface the React console's Approve/Reject buttons drive. | RBAC, `AuditSink` | +| `expire(id)` | `POST /approvals/{id}/expire` — the requester (action-service) marks a request it stopped waiting for as `expired` at its HITL timeout. | Without it a timed-out request stayed `pending` forever and the console offered an Approve that could no longer do anything. | `AuditSink` | | `get_approval(id)` / `list_approvals()` | REST reads (`GET /approvals/{id}`, `GET /approvals`) of the pending queue. | Lets the HTTP gate poll for a decision across containers, and the dashboard show what's pending. | — | | `write_audit(record)` | Appends an immutable `AuditRecord`. | The compliance backbone (NIST AI RMF / DORA / EU AI Act). | `AuditSink` | -| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. | Standardization — playbooks are shared, not reinvented per team. | playbook store | +| `register_playbook()` / `list_playbooks()` | Maintains the CoE playbook registry. `POST /playbooks?registered_by=` needs the `register` permission (coe-admin), refuses `hitl_mode=auto` unless the actor may also `graduate`, and is audited. | Standardization — playbooks are shared, not reinvented per team. A direct write bypasses the AI-proposal review, so it is an admin action, not an open one. | playbook store, RBAC | +| `graduate(id)` / `demote(id)` | `POST /playbooks/{id}/graduate` (hitl → auto) and `POST /playbooks/{id}/demote` (auto → hitl), both behind the `graduate` permission and audited. feedback-service calls graduate on clean **real** evidence (dry-run successes don't count unless `GRADUATION_COUNT_SIMULATED`) and demote when an auto playbook fails or rolls back. | Automation expands on evidence and contracts on failure. | RBAC, playbook store, `AuditSink` | | `propose_playbook(situation) → ProposedPlaybook` | `POST /playbooks/proposed`: calls a `RunbookAuthor` (LLM) to draft a typed runbook for a gap, forces `hitl_mode=HITL` + a server-assigned id, stores it as a **proposal** (not the live registry); 422 if the author declines. | The AI **proposes** a runbook for a gap; `model_validate` rejects any unsafe draft ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | `RunbookAuthor`, proposed store | | `approve_proposed(id)` / `reject_proposed(id)` | `POST /playbooks/proposed/{id}/approve\|reject` — RBAC-gated (reuses `approve`/`reject`), audited. **Approve registers the inner playbook into the live registry**; reject does not. | The human **disposes** — the only path from an AI draft to the live registry ([ADR-025](architectural.md#adr-025--ai-authored-runbooks-propose--approve)). | RBAC, playbook store, `AuditSink` |