From 9c9a389d22ce8af5f698b3f188ee2d8f8acd296c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 01:56:35 +0100 Subject: [PATCH 01/22] feat(lifecycle): implement finding lifecycle tracking (#311) Add six new tables (Alembic migration e1f2a3b4c5d6), two service classes, a patterns API route, and engine outcome recording. Tables added: - scan_rule_outcomes: per-rule status per scan - scan_lifecycle_applications: idempotency sentinel - finding_fingerprints: stable immutable SHA-256 identity per finding - finding_lifecycles: mutable OPEN/RESOLVED/ACCEPTED/SUPPRESSED/REOPENED state - finding_lifecycle_transitions: append-only audit trail - patterns: published persistent_finding / cross_resource_recurrence / reopened_finding detections Services: - LifecycleService.apply_scan(): idempotent, fail-closed, single transaction with FOR UPDATE row locking - PatternService.detect_and_publish(): three pattern types with hardcoded thresholds stored in the record Routes: - GET /api/v1/patterns (list with subscription_id / pattern_type / limit) - GET /api/v1/patterns/ (single pattern or 404) Engine: - ScanEngine.run_scan() now records per-rule outcome status (SUCCESS / EMPTY_SUCCESS / PERMISSION_DENIED / TIMEOUT / FAILED) Tests: - tests/test_finding_lifecycle.py: 18 cases (mocked DB, no live Postgres) - tests/test_patterns.py: 9 cases (service unit + Flask route tests) Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 152 ++++++++ api/app.py | 2 + api/routes/patterns.py | 180 +++++++++ api/services/lifecycle_service.py | 306 +++++++++++++++ api/services/pattern_service.py | 170 ++++++++ scanner/engine.py | 82 +++- tests/test_finding_lifecycle.py | 363 ++++++++++++++++++ tests/test_patterns.py | 330 ++++++++++++++++ 8 files changed, 1565 insertions(+), 20 deletions(-) create mode 100644 alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py create mode 100644 api/routes/patterns.py create mode 100644 api/services/lifecycle_service.py create mode 100644 api/services/pattern_service.py create mode 100644 tests/test_finding_lifecycle.py create mode 100644 tests/test_patterns.py diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py new file mode 100644 index 00000000..1b1ccfa4 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -0,0 +1,152 @@ +"""Add finding lifecycle tables: scan_rule_outcomes, scan_lifecycle_applications, +finding_fingerprints, finding_lifecycles, finding_lifecycle_transitions, patterns. + +Revision ID: e1f2a3b4c5d6 +Revises: d8e4f6a1b2c3 +Create Date: 2026-08-30 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "d8e4f6a1b2c3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE scan_rule_outcomes ( + id BIGSERIAL PRIMARY KEY, + scan_id UUID NOT NULL REFERENCES scans(scan_id), + rule_id TEXT NOT NULL, + rule_version TEXT NOT NULL DEFAULT '1', + status TEXT NOT NULL, + error_category TEXT, + collector_version TEXT NOT NULL DEFAULT '1', + inventory_boundary TEXT NOT NULL, + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_scan_rule_outcomes_status + CHECK (status IN ( + 'SUCCESS','EMPTY_SUCCESS','PERMISSION_DENIED', + 'TIMEOUT','FAILED','NOT_APPLICABLE' + )), + CONSTRAINT uq_scan_rule_outcomes_scan_rule + UNIQUE (scan_id, rule_id) + ) + """ + ) + + op.execute( + """ + CREATE TABLE scan_lifecycle_applications ( + scan_id UUID PRIMARY KEY REFERENCES scans(scan_id), + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + applied_by TEXT NOT NULL DEFAULT 'system' + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_fingerprints ( + id BIGSERIAL PRIMARY KEY, + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + resource_id_normalized TEXT NOT NULL, + rule_id TEXT NOT NULL, + evidence_key TEXT NOT NULL DEFAULT '', + normalization_version TEXT NOT NULL DEFAULT '1', + fingerprint_version TEXT NOT NULL DEFAULT '1', + fingerprint_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_finding_fingerprints_hash UNIQUE (fingerprint_hash), + CONSTRAINT uq_finding_fingerprints_identity + UNIQUE ( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version + ) + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_lifecycles ( + id BIGSERIAL PRIMARY KEY, + fingerprint_id BIGINT NOT NULL REFERENCES finding_fingerprints(id), + state TEXT NOT NULL, + first_seen_scan_id UUID, + last_seen_scan_id UUID, + occurrence_count INTEGER NOT NULL DEFAULT 1, + consecutive_success_count INTEGER NOT NULL DEFAULT 0, + reopen_count INTEGER NOT NULL DEFAULT 0, + row_version INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_finding_lifecycles_state + CHECK (state IN ('OPEN','RESOLVED','ACCEPTED','SUPPRESSED','REOPENED')), + CONSTRAINT uq_finding_lifecycles_fingerprint + UNIQUE (fingerprint_id) + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_lifecycle_transitions ( + id BIGSERIAL PRIMARY KEY, + lifecycle_id BIGINT NOT NULL REFERENCES finding_lifecycles(id), + from_state TEXT, + to_state TEXT NOT NULL, + scan_id UUID, + reason TEXT NOT NULL DEFAULT '', + transitioned_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + + op.execute( + """ + CREATE TABLE patterns ( + id BIGSERIAL PRIMARY KEY, + pattern_type TEXT NOT NULL, + lifecycle_id BIGINT REFERENCES finding_lifecycles(id), + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + scan_id UUID, + finding_ids JSONB NOT NULL DEFAULT '[]', + threshold INTEGER NOT NULL, + algorithm_version TEXT NOT NULL DEFAULT '1', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_patterns_type + CHECK (pattern_type IN ( + 'persistent_finding', + 'cross_resource_recurrence', + 'reopened_finding' + )) + ) + """ + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS patterns") + op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") + op.execute("DROP TABLE IF EXISTS finding_lifecycles") + op.execute("DROP TABLE IF EXISTS finding_fingerprints") + op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications") + op.execute("DROP TABLE IF EXISTS scan_rule_outcomes") diff --git a/api/app.py b/api/app.py index 9f8465dd..80699516 100644 --- a/api/app.py +++ b/api/app.py @@ -260,6 +260,7 @@ def verify_jwt() -> None: from api.routes.compliance import compliance_bp from api.routes.drift import drift_bp from api.routes.findings import findings_bp + from api.routes.patterns import patterns_bp from api.routes.prioritization import prioritization_bp from api.routes.resources import resources_bp from api.routes.scans import scans_bp @@ -271,6 +272,7 @@ def verify_jwt() -> None: app.register_blueprint(compliance_bp) app.register_blueprint(drift_bp) app.register_blueprint(findings_bp) + app.register_blueprint(patterns_bp) app.register_blueprint(prioritization_bp) app.register_blueprint(resources_bp) app.register_blueprint(scans_bp) diff --git a/api/routes/patterns.py b/api/routes/patterns.py new file mode 100644 index 00000000..a03d8d4e --- /dev/null +++ b/api/routes/patterns.py @@ -0,0 +1,180 @@ +"""Patterns routes: list and retrieve published security patterns.""" + +import logging +import os + +import psycopg2.extras +from flask import Blueprint, g, jsonify, request + +from api.models.finding import DatabaseManager + +patterns_bp = Blueprint("patterns", __name__) +logger = logging.getLogger(__name__) + +_ALLOWED_PATTERN_TYPES = frozenset( + {"persistent_finding", "cross_resource_recurrence", "reopened_finding"} +) +_DEFAULT_LIMIT = 20 +_MAX_LIMIT = 100 +_MIN_LIMIT = 1 + +_VALIDATION_ERROR_MESSAGE = "Invalid request parameters" + + +class _ValidationError(ValueError): + """Raised when a client-controlled value violates the public API contract.""" + + +def _get_db() -> DatabaseManager: + if "db" not in g: + g.db = DatabaseManager(os.environ["DATABASE_URL"]) + g.db.connect() + return g.db + + +def _validate_limit(raw: str) -> int: + try: + value = int(raw) + except (ValueError, TypeError) as exc: + raise _ValidationError("limit must be an integer") from exc + if value < _MIN_LIMIT or value > _MAX_LIMIT: + raise _ValidationError(f"limit must be between {_MIN_LIMIT} and {_MAX_LIMIT}") + return value + + +def _row_to_dict(row: dict) -> dict: + """Serialise a patterns table row to a JSON-safe dict.""" + result = dict(row) + for key in ("created_at", "updated_at"): + val = result.get(key) + if val is not None and hasattr(val, "isoformat"): + result[key] = val.isoformat() + if result.get("scan_id") is not None: + result["scan_id"] = str(result["scan_id"]) + return result + + +@patterns_bp.get("/api/v1/patterns") +def list_patterns(): + """Return published security patterns, optionally filtered. + + Query parameters: + subscription_id - filter by Azure subscription + pattern_type - one of persistent_finding, cross_resource_recurrence, + reopened_finding + limit - 1-100, default 20 + """ + try: + allowed_params = {"subscription_id", "pattern_type", "limit"} + unknown = set(request.args) - allowed_params + if unknown: + raise _ValidationError(f"Unsupported query parameter: {sorted(unknown)[0]}") + for key in request.args: + if len(request.args.getlist(key)) != 1: + raise _ValidationError(f"Query parameter {key} must be provided once") + + subscription_id = None + if "subscription_id" in request.args: + val = request.args["subscription_id"].strip() + if not val or len(val) > 256: + raise _ValidationError("subscription_id is invalid") + subscription_id = val + + pattern_type = None + if "pattern_type" in request.args: + raw_pt = request.args["pattern_type"].strip() + if raw_pt not in _ALLOWED_PATTERN_TYPES: + raise _ValidationError("Unsupported pattern_type") + pattern_type = raw_pt + + limit = _DEFAULT_LIMIT + if "limit" in request.args: + limit = _validate_limit(request.args["limit"]) + + db = _get_db() + conn = db._get_conn() + + # Tenant isolation: prefer subscription_id embedded in the JWT payload; + # fall back to the query parameter. + user = getattr(g, "user", {}) or {} + effective_sub = user.get("subscription_id") or subscription_id + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT id, pattern_type, lifecycle_id, tenant_id, + subscription_id, scan_id, finding_ids, threshold, + algorithm_version, created_at, updated_at + FROM patterns + WHERE (%s IS NULL OR subscription_id = %s) + AND (%s IS NULL OR pattern_type = %s) + ORDER BY created_at DESC + LIMIT %s + """, + ( + effective_sub, effective_sub, + pattern_type, pattern_type, + limit, + ), + ) + rows = cur.fetchall() + + cur.execute( + """ + SELECT COUNT(*) AS count + FROM patterns + WHERE (%s IS NULL OR subscription_id = %s) + AND (%s IS NULL OR pattern_type = %s) + """, + (effective_sub, effective_sub, pattern_type, pattern_type), + ) + total_row = cur.fetchone() + total = total_row["count"] if total_row else 0 + + return jsonify( + { + "patterns": [_row_to_dict(r) for r in rows], + "total": total, + } + ) + + except _ValidationError: + return jsonify({"error": _VALIDATION_ERROR_MESSAGE}), 400 + except Exception as exc: + logger.error("Failed to list patterns: %s", exc) + return jsonify({"error": "Failed to retrieve patterns"}), 500 + + +@patterns_bp.get("/api/v1/patterns/") +def get_pattern(pattern_id: int): + """Return a single pattern by its integer ID.""" + try: + if pattern_id <= 0: + raise _ValidationError("pattern_id must be a positive integer") + + db = _get_db() + conn = db._get_conn() + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT id, pattern_type, lifecycle_id, tenant_id, + subscription_id, scan_id, finding_ids, threshold, + algorithm_version, created_at, updated_at + FROM patterns + WHERE id = %s + """, + (pattern_id,), + ) + row = cur.fetchone() + + if row is None: + return jsonify({"error": "Pattern not found"}), 404 + + return jsonify(_row_to_dict(row)) + + except _ValidationError: + return jsonify({"error": _VALIDATION_ERROR_MESSAGE}), 400 + except Exception as exc: + logger.error("Failed to get pattern %s: %s", pattern_id, exc) + return jsonify({"error": "Failed to retrieve pattern"}), 500 diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py new file mode 100644 index 00000000..9aff5e92 --- /dev/null +++ b/api/services/lifecycle_service.py @@ -0,0 +1,306 @@ +"""LifecycleService: applies scan outcomes to finding lifecycle state machines.""" + +import hashlib +import json +import logging +from typing import Any, Dict, List, Optional + +import psycopg2.extras + +logger = logging.getLogger(__name__) + +# Statuses that mean "we actively confirmed this rule was clean in the scan." +_RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) + +# Statuses that mean "we could not reliably evaluate the rule." +# Fail-closed: do NOT resolve findings when the scan could not see the resource. +_BLOCKING_STATUSES = frozenset({"PERMISSION_DENIED", "TIMEOUT", "FAILED"}) + + +def _normalize_resource_id(resource_id: str) -> str: + """Return a stable, lowercased, stripped version of an ARM resource ID.""" + return resource_id.strip().lower() + + +def _compute_fingerprint_hash( + tenant_id: str, + subscription_id: str, + resource_id_normalized: str, + rule_id: str, + evidence_key: str, + normalization_version: str, +) -> str: + """Return a SHA-256 hex digest over canonical fields.""" + canonical = json.dumps( + [ + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, + ], + separators=(",", ":"), + sort_keys=False, + ) + return hashlib.sha256(canonical.encode()).hexdigest() + + +class LifecycleService: + """Applies scan results to the finding lifecycle state machines. + + All database work for a single apply_scan call executes inside one + transaction. The idempotency sentinel (scan_lifecycle_applications) is + inserted last, so a crash before commit means the operation never happened + and can safely be retried. + """ + + def apply_scan( + self, + db_conn: Any, + scan_id: str, + subscription_id: str, + tenant_id: str, + rule_outcomes: List[Dict[str, Any]], + findings: Optional[List[Dict[str, Any]]] = None, + ) -> None: + """Apply a completed scan's findings to the lifecycle tables. + + Args: + db_conn: A psycopg2 connection (autocommit must be False). + scan_id: UUID of the completed scan. + subscription_id: Azure subscription ID. + tenant_id: Tenant identifier for isolation. + rule_outcomes: List of dicts with at minimum keys 'rule_id' and + 'status' (one of the six allowed status values). + findings: List of finding dicts from the scan, each with keys + 'rule_id', 'resource_id', and optionally 'evidence_key'. + Defaults to an empty list when omitted. + """ + findings = findings or [] + + with db_conn.cursor() as cur: + # --- Idempotency check ---------------------------------------- + cur.execute( + "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", + (scan_id,), + ) + if cur.fetchone() is not None: + logger.info("Scan %s already applied; skipping lifecycle update", scan_id) + return + + # Build a lookup: rule_id -> outcome status + outcome_by_rule: Dict[str, str] = { + o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o + } + + # Build the set of (rule_id, resource_id_normalized) pairs seen in + # this scan. A fingerprint in this set was actively observed. + seen_fingerprint_keys: set = set() + fingerprints_in_scan: List[Dict[str, Any]] = [] + for finding in findings: + rule_id = finding.get("rule_id", "") + resource_id = finding.get("resource_id", "") + evidence_key = finding.get("evidence_key", "") + resource_id_normalized = _normalize_resource_id(resource_id) + normalization_version = "1" + + fp_hash = _compute_fingerprint_hash( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, + ) + key = (rule_id, resource_id_normalized, evidence_key) + if key not in seen_fingerprint_keys: + seen_fingerprint_keys.add(key) + fingerprints_in_scan.append( + { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "resource_id_normalized": resource_id_normalized, + "rule_id": rule_id, + "evidence_key": evidence_key, + "normalization_version": normalization_version, + "fingerprint_hash": fp_hash, + } + ) + + # --- Upsert fingerprints and lifecycles for seen findings ------- + seen_fingerprint_ids: set = set() + for fp in fingerprints_in_scan: + cur.execute( + """ + INSERT INTO finding_fingerprints ( + tenant_id, subscription_id, resource_id_normalized, + rule_id, evidence_key, normalization_version, + fingerprint_version, fingerprint_hash + ) + VALUES (%s, %s, %s, %s, %s, %s, '1', %s) + ON CONFLICT (fingerprint_hash) DO UPDATE + SET fingerprint_hash = EXCLUDED.fingerprint_hash + RETURNING id + """, + ( + fp["tenant_id"], + fp["subscription_id"], + fp["resource_id_normalized"], + fp["rule_id"], + fp["evidence_key"], + fp["normalization_version"], + fp["fingerprint_hash"], + ), + ) + row = cur.fetchone() + fingerprint_id = row[0] + seen_fingerprint_ids.add(fingerprint_id) + + # Lock the lifecycle row if it exists, then decide what to do. + cur.execute( + """ + SELECT id, state, occurrence_count, reopen_count, row_version + FROM finding_lifecycles + WHERE fingerprint_id = %s + FOR UPDATE + """, + (fingerprint_id,), + ) + lc_row = cur.fetchone() + + if lc_row is None: + # New finding: create lifecycle in OPEN state. + cur.execute( + """ + INSERT INTO finding_lifecycles ( + fingerprint_id, state, first_seen_scan_id, + last_seen_scan_id, occurrence_count, + consecutive_success_count, reopen_count, row_version + ) + VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + RETURNING id + """, + (fingerprint_id, scan_id, scan_id), + ) + lifecycle_id = cur.fetchone()[0] + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') + """, + (lifecycle_id, scan_id), + ) + else: + lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row + if state in ("OPEN", "REOPENED"): + # Seen again while already open: increment counter. + cur.execute( + """ + UPDATE finding_lifecycles + SET occurrence_count = occurrence_count + 1, + last_seen_scan_id = %s, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): + # Reappeared after resolution: reopen. + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'REOPENED', + last_seen_scan_id = %s, + occurrence_count = occurrence_count + 1, + reopen_count = reopen_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + """, + (lifecycle_id, state, scan_id), + ) + + # --- Resolve findings NOT seen in this scan --------------------- + # Only resolve if the rule's outcome actively confirmed the resource + # was clean (SUCCESS / EMPTY_SUCCESS). Fail closed for uncertain outcomes. + if seen_fingerprint_ids: + # Find fingerprints for this tenant/subscription that are currently + # OPEN or REOPENED but were NOT observed in this scan. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id NOT IN %s + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, tuple(seen_fingerprint_ids)), + ) + else: + # No findings seen at all: resolve all OPEN/REOPENED where rule + # had a resolving outcome. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + FOR UPDATE OF fl + """, + (tenant_id, subscription_id), + ) + + absent_rows = cur.fetchall() + for lc_id, state, row_version, rule_id in absent_rows: + outcome_status = outcome_by_rule.get(rule_id) + if outcome_status in _RESOLVING_STATUSES: + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'RESOLVED', + last_seen_scan_id = %s, + consecutive_success_count = consecutive_success_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lc_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'RESOLVED', %s, + 'Rule confirmed clean; finding absent from scan') + """, + (lc_id, state, scan_id), + ) + # else: PERMISSION_DENIED / TIMEOUT / FAILED / missing -> fail closed, no change. + + # --- Idempotency sentinel (inserted last) ----------------------- + cur.execute( + """ + INSERT INTO scan_lifecycle_applications (scan_id, applied_by) + VALUES (%s, 'system') + """, + (scan_id,), + ) + + db_conn.commit() + logger.info("Lifecycle application committed for scan %s", scan_id) diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py new file mode 100644 index 00000000..282bc2f9 --- /dev/null +++ b/api/services/pattern_service.py @@ -0,0 +1,170 @@ +"""PatternService: detects and publishes security patterns from lifecycle state.""" + +import logging +from typing import Any + +import psycopg2.extras + +logger = logging.getLogger(__name__) + +# Hard-coded thresholds stored alongside the pattern record for traceability. +_PERSISTENT_THRESHOLD = 3 +_CROSS_RESOURCE_THRESHOLD = 2 +_REOPENED_THRESHOLD = 1 +_ALGORITHM_VERSION = "1" + + +class PatternService: + """Detects recurring patterns in finding lifecycle data and publishes records.""" + + def detect_and_publish( + self, + db_conn: Any, + scan_id: str, + subscription_id: str, + tenant_id: str, + ) -> int: + """Detect patterns for the given scan and upsert them into the patterns table. + + Returns the number of patterns upserted. + """ + count = 0 + + with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + # ---------------------------------------------------------------- + # 1. persistent_finding: occurrence_count >= 3 and OPEN/REOPENED + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.occurrence_count >= %s + """, + (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), + ) + for row in cur.fetchall(): + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + # ---------------------------------------------------------------- + # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED + # lifecycles in this subscription + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT ff.rule_id, + array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, + COUNT(*) AS lc_count + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + GROUP BY ff.rule_id + HAVING COUNT(*) >= %s + """, + (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), + ) + for row in cur.fetchall(): + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle in the group so each is + # individually traceable; finding_ids carries the sibling IDs. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + _upsert_pattern( + db_conn, + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, + ) + count += 1 + + # ---------------------------------------------------------------- + # 3. reopened_finding: reopen_count >= 1 and state == REOPENED + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state = 'REOPENED' + AND fl.reopen_count >= %s + """, + (tenant_id, subscription_id, _REOPENED_THRESHOLD), + ) + for row in cur.fetchall(): + _upsert_pattern( + db_conn, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, + ) + count += 1 + + db_conn.commit() + logger.info( + "Pattern detection for scan %s: %d pattern(s) upserted", + scan_id, + count, + ) + return count + + +def _upsert_pattern( + db_conn: Any, + pattern_type: str, + lifecycle_id: int, + tenant_id: str, + subscription_id: str, + scan_id: str, + finding_ids: list, + threshold: int, +) -> None: + """Insert or update a single pattern record.""" + import json + + with db_conn.cursor() as cur: + cur.execute( + """ + INSERT INTO patterns ( + pattern_type, lifecycle_id, tenant_id, subscription_id, + scan_id, finding_ids, threshold, algorithm_version, + created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW()) + ON CONFLICT DO NOTHING + """, + ( + pattern_type, + lifecycle_id, + tenant_id, + subscription_id, + scan_id, + json.dumps(finding_ids), + threshold, + _ALGORITHM_VERSION, + ), + ) diff --git a/scanner/engine.py b/scanner/engine.py index 2ecf7468..fcb76b45 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -9,6 +9,13 @@ from api.observability import RULE_ERRORS_TOTAL from openshield.severity import CONTRACT_VERSION, SeverityContractError, normalize_severity, score_findings + +try: + import azure.core.exceptions as _azure_exc + _AzureHttpResponseError = _azure_exc.HttpResponseError +except Exception: + _AzureHttpResponseError = None # type: ignore[assignment,misc] + from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id @@ -104,16 +111,17 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: Returns: dict with keys: scan_id, subscription_id, started_at, - completed_at, total_findings, findings. + completed_at, total_findings, findings, rule_outcomes. """ scan_id = scan_id or str(uuid.uuid4()) started_at = datetime.now(timezone.utc).isoformat() findings: List[Dict[str, Any]] = [] evaluations: List[RuleEvaluation] = [] + rule_outcomes: List[Dict[str, Any]] = [] detected_at = datetime.now(timezone.utc).isoformat() logger.info( - "Scan %s starting against subscription %s — %d rules loaded", + "Scan %s starting against subscription %s - %d rules loaded", scan_id, self.subscription_id, len(self.rules), @@ -121,32 +129,62 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: for rule in self.rules: rule_id = getattr(rule, "RULE_ID", "UNKNOWN") + rule_started = datetime.now(timezone.utc).isoformat() + outcome_status = "FAILED" try: rule_findings = rule.scan(self.client, self.subscription_id) if not isinstance(rule_findings, list): - logger.warning("Rule %s returned %s instead of list — skipped", rule_id, type(rule_findings)) - continue - - validated_findings = [] - for raw_finding in rule_findings: - finding = raw_finding - if not isinstance(finding, dict): - logger.warning("Rule %s returned a non-object finding — skipped", rule_id) - continue - finding = dict(finding) - finding["severity"] = normalize_severity(finding.get("severity")) - finding.setdefault("detected_at", detected_at) - finding.setdefault("scan_id", scan_id) - validated_findings.append(finding) - findings.extend(validated_findings) - logger.info("Rule %s produced %d finding(s)", rule_id, len(validated_findings)) + logger.warning("Rule %s returned %s instead of list - skipped", rule_id, type(rule_findings)) + outcome_status = "FAILED" + else: + validated_findings = [] + for raw_finding in rule_findings: + finding = raw_finding + if not isinstance(finding, dict): + logger.warning("Rule %s returned a non-object finding - skipped", rule_id) + continue + finding = dict(finding) + finding["severity"] = normalize_severity(finding.get("severity")) + finding.setdefault("detected_at", detected_at) + finding.setdefault("scan_id", scan_id) + validated_findings.append(finding) + findings.extend(validated_findings) + outcome_status = "SUCCESS" if validated_findings else "EMPTY_SUCCESS" + logger.info("Rule %s produced %d finding(s)", rule_id, len(validated_findings)) except SeverityContractError: RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() logger.exception("Rule %s returned an invalid severity", rule_id) raise + except PermissionError: + RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() + logger.error("Rule %s raised PermissionError", rule_id) + outcome_status = "PERMISSION_DENIED" + except TimeoutError: + RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() + logger.error("Rule %s timed out", rule_id) + outcome_status = "TIMEOUT" except Exception as exc: RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() - logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) + if _AzureHttpResponseError is not None and isinstance(exc, _AzureHttpResponseError): + status_code = getattr(exc, "status_code", None) + if status_code == 403: + outcome_status = "PERMISSION_DENIED" + logger.error("Rule %s got HTTP 403 from Azure", rule_id) + else: + outcome_status = "FAILED" + logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) + else: + outcome_status = "FAILED" + logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) + + rule_outcomes.append( + { + "rule_id": rule_id, + "status": outcome_status, + "started_at": rule_started, + "completed_at": datetime.now(timezone.utc).isoformat(), + } + ) evaluations.extend(self._evaluate_rule(rule, rule_id)) @@ -182,10 +220,14 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: "score": score, "severity_contract_version": CONTRACT_VERSION, "findings": findings, +<<<<<<< HEAD "evaluations": [e.to_dict() for e in evaluations], +======= + "rule_outcomes": rule_outcomes, +>>>>>>> c0155f9 (feat(lifecycle): implement finding lifecycle tracking (#311)) } - logger.info("Scan %s complete — %d total finding(s). Normalising results...", scan_id, len(findings)) + logger.info("Scan %s complete - %d total finding(s). Normalising results...", scan_id, len(findings)) return make_serializable(result) diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py new file mode 100644 index 00000000..7e5a239e --- /dev/null +++ b/tests/test_finding_lifecycle.py @@ -0,0 +1,363 @@ +"""Tests for LifecycleService using a mocked psycopg2 connection. + +All tests use in-memory state to simulate the database without requiring a +live PostgreSQL instance. +""" + +import hashlib +import json +from unittest.mock import MagicMock, call, patch + +import pytest + +from api.services.lifecycle_service import ( + LifecycleService, + _compute_fingerprint_hash, + _normalize_resource_id, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +TENANT_ID = "tenant-abc" +SUB_ID = "sub-001" +SCAN_ID_1 = "11111111-1111-1111-1111-111111111111" +SCAN_ID_2 = "22222222-2222-2222-2222-222222222222" + + +def _make_finding(rule_id: str, resource_id: str, evidence_key: str = "") -> dict: + return { + "rule_id": rule_id, + "resource_id": resource_id, + "evidence_key": evidence_key, + } + + +def _make_outcome(rule_id: str, status: str) -> dict: + return {"rule_id": rule_id, "status": status} + + +# --------------------------------------------------------------------------- +# Simple DB simulation using a plain dict as in-memory state. +# We avoid mocking every cursor call individually by building a lightweight +# fake cursor that replays scripted return values in order. +# --------------------------------------------------------------------------- + + +class _FakeCursor: + """A fake psycopg2 cursor that works with pre-loaded fetchone/fetchall results.""" + + def __init__(self, results: list): + # results is a list of return values; each execute() pops one. + self._results = list(results) + self._current = None + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql.strip(), params)) + self._current = self._results.pop(0) if self._results else None + + def fetchone(self): + return self._current + + def fetchall(self): + if isinstance(self._current, list): + return self._current + return [] if self._current is None else [self._current] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +class _FakeConn: + """Fake connection whose cursor() returns a _FakeCursor consuming a result list.""" + + def __init__(self, results: list): + self._results = results + self.committed = False + self._cursor_obj = None + + def cursor(self, **_kwargs): + self._cursor_obj = _FakeCursor(self._results) + return self._cursor_obj + + def commit(self): + self.committed = True + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestComputeFingerprintHash: + def test_hash_is_stable(self): + h1 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/sub/001/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/sub/001/rg/foo", "RULE-001", "", "1") + assert h1 == h2 + + def test_different_tenant_different_hash(self): + h1 = _compute_fingerprint_hash("tenant-A", SUB_ID, "/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash("tenant-B", SUB_ID, "/rg/foo", "RULE-001", "", "1") + assert h1 != h2 + + def test_tenant_id_included_in_hash(self): + """Tenant isolation: fingerprint_hash must encode the tenant_id.""" + h_a = _compute_fingerprint_hash("tenant-A", SUB_ID, "/rg/foo", "RULE-001", "", "1") + h_b = _compute_fingerprint_hash("tenant-B", SUB_ID, "/rg/foo", "RULE-001", "", "1") + assert h_a != h_b + + def test_different_rule_different_hash(self): + h1 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/rg/foo", "RULE-002", "", "1") + assert h1 != h2 + + +class TestNormalizeResourceId: + def test_lowercase(self): + assert _normalize_resource_id("/subscriptions/SUB/resourceGroups/RG") == ( + "/subscriptions/sub/resourcegroups/rg" + ) + + def test_strip_whitespace(self): + assert _normalize_resource_id(" /rg/foo ") == "/rg/foo" + + +class TestLifecycleServiceIdempotency: + """Applying the same scan_id twice must produce exactly one transition.""" + + def test_second_apply_is_no_op(self): + # First call: idempotency row NOT present (fetchone -> None), then + # fingerprint upsert returns id=1, lifecycle select returns None + # (new finding), lifecycle insert returns id=10, commit. + # Second call: idempotency row IS present (fetchone -> (scan_id,)). + svc = LifecycleService() + + transitions = [] + + # We use a counting approach: track how many times commit() is called. + committed_count = 0 + + class _TrackingConn: + def __init__(self, is_first_call): + self._is_first = is_first_call + self._results = self._build_results(is_first_call) + + def _build_results(self, first): + if first: + # idempotency check -> None (not applied) + # fingerprint upsert RETURNING id -> (1,) + # lifecycle FOR UPDATE -> None (new) + # lifecycle INSERT RETURNING id -> (10,) + # transition insert -> None + # absent-findings query -> [] + # idempotency insert -> None + return [None, (1,), None, (10,), None, [], None] + else: + # idempotency check -> row found: stop immediately + return [("already-applied",)] + + def cursor(self, **_kwargs): + return _FakeCursor(self._results) + + def commit(self): + nonlocal committed_count + committed_count += 1 + + svc.apply_scan( + _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + [_make_outcome("RULE-001", "SUCCESS")], + [_make_finding("RULE-001", "/rg/foo")], + ) + svc.apply_scan( + _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + [_make_outcome("RULE-001", "SUCCESS")], + [_make_finding("RULE-001", "/rg/foo")], + ) + + # Only the first call should have committed. + assert committed_count == 1 + + +class TestLifecycleStateTransitions: + """State machine correctness tests using scripted cursor results.""" + + def _run(self, results, findings, outcomes): + """Run apply_scan with scripted cursor results; return the fake conn.""" + conn = _FakeConn(results) + svc = LifecycleService() + svc.apply_scan(conn, SCAN_ID_1, SUB_ID, TENANT_ID, outcomes, findings) + return conn + + def test_new_finding_creates_open_lifecycle(self): + # Scripted results in cursor execute order: + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> None (no existing row) + # 4. lifecycle INSERT RETURNING id -> (10,) + # 5. transition insert -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [None, (1,), None, (10,), None, [], None] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + + # Verify the lifecycle INSERT used 'OPEN' + executed = conn._cursor_obj.executed + insert_lc = next( + (sql for sql, _ in executed if "INSERT INTO finding_lifecycles" in sql), None + ) + assert insert_lc is not None + + def test_open_finding_not_seen_in_success_scan_is_resolved(self): + # No findings in this scan, outcome is SUCCESS -> existing OPEN should resolve. + # 1. idempotency check -> None + # 2. absent-findings query (no seen ids branch) -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 3. UPDATE finding_lifecycles SET state='RESOLVED' -> None + # 4. transition insert -> None + # 5. idempotency insert -> None + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + None, + None, + ] + conn = self._run( + results, + [], # no findings + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is not None + + def test_open_finding_not_seen_in_failed_scan_stays_open(self): + # FAILED outcome -> fail-closed: finding should NOT be resolved. + # 1. idempotency check -> None + # 2. absent-findings query -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 3. idempotency insert -> None + # (no UPDATE to RESOLVED because outcome is FAILED) + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "FAILED")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is None + + def test_open_finding_not_seen_in_permission_denied_stays_open(self): + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "PERMISSION_DENIED")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is None + + def test_resolved_finding_seen_again_becomes_reopened(self): + # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> should REOPEN. + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> (lc_id=10, 'RESOLVED', 1, 0, 2) + # (id, state, occurrence_count, reopen_count, row_version) + # 4. UPDATE to REOPENED -> None + # 5. transition insert (RESOLVED -> REOPENED) -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [ + None, + (1,), + (10, "RESOLVED", 1, 0, 2), + None, + None, + [], + None, + ] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + reopen_sql = next( + (sql for sql, _ in executed if "REOPENED" in sql and "UPDATE" in sql.upper()), None + ) + assert reopen_sql is not None + + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): + # REOPENED + seen again: occurrence_count increments, state stays REOPENED. + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> (10, 'REOPENED', 3, 1, 4) + # 4. UPDATE occurrence_count + 1 -> None (OPEN/REOPENED branch) + # 5. absent-findings query -> [] + # 6. idempotency insert -> None + results = [ + None, + (1,), + (10, "REOPENED", 3, 1, 4), + None, + [], + None, + ] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + # Verify no transition to a NEW state was recorded for this finding + # (i.e. no INSERT INTO finding_lifecycle_transitions for OPEN/REOPENED). + reopened_transition = next( + ( + sql for sql, params in executed + if "INSERT INTO finding_lifecycle_transitions" in sql + and params is not None + and "REOPENED" in str(params) + ), + None, + ) + assert reopened_transition is None + + # Verify occurrence_count was incremented (UPDATE without state change). + occ_update = next( + ( + sql for sql, _ in executed + if "occurrence_count = occurrence_count + 1" in sql + ), + None, + ) + assert occ_update is not None diff --git a/tests/test_patterns.py b/tests/test_patterns.py new file mode 100644 index 00000000..e27e2a9e --- /dev/null +++ b/tests/test_patterns.py @@ -0,0 +1,330 @@ +"""Tests for PatternService and GET /api/v1/patterns routes. + +Pattern detection tests use a mocked DB; route tests use the Flask test client +with mocked database queries. +""" + +import json +import os +import secrets +import time +from unittest.mock import MagicMock, patch + +import jwt +import pytest + +TENANT_ID = "tenant-abc" +SUB_ID = "sub-001" +SCAN_ID = "33333333-3333-3333-3333-333333333333" + +_TEST_JWT_SECRET = secrets.token_urlsafe(32) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_token(sub_id: str | None = None) -> str: + payload = { + "sub": "test-user", + "role": "admin", + "iat": int(time.time()), + "exp": int(time.time()) + 3600, + } + if sub_id: + payload["subscription_id"] = sub_id + return jwt.encode(payload, _TEST_JWT_SECRET, algorithm="HS256") + + +def _auth_headers(sub_id: str | None = None) -> dict: + return { + "Authorization": f"Bearer {_make_token(sub_id)}", + "Content-Type": "application/json", + } + + +# --------------------------------------------------------------------------- +# Fake cursor / connection for service tests +# --------------------------------------------------------------------------- + + +class _FakeCursor: + def __init__(self, pages: list): + # pages: list of values returned by successive fetchone/fetchall calls + self._pages = list(pages) + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql.strip(), params)) + + def fetchone(self): + return self._pages.pop(0) if self._pages else None + + def fetchall(self): + return self._pages.pop(0) if self._pages else [] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +class _FakeConn: + def __init__(self, fetchall_pages: list): + self._pages = fetchall_pages + self.committed = False + self._cursor_obj = None + + def cursor(self, **_kwargs): + self._cursor_obj = _FakeCursor(self._pages) + return self._cursor_obj + + def commit(self): + self.committed = True + + +# --------------------------------------------------------------------------- +# PatternService unit tests +# --------------------------------------------------------------------------- + + +class TestPatternServiceDetection: + def _run(self, pages, extra_pages=None): + """Run detect_and_publish with scripted DB results.""" + from api.services.pattern_service import PatternService + + all_pages = list(pages) + if extra_pages: + all_pages.extend(extra_pages) + conn = _FakeConn(all_pages) + svc = PatternService() + count = svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) + return count, conn + + def test_persistent_finding_detected_when_occurrence_ge_3(self): + # persistent_finding query returns 1 row -> 1 pattern upserted. + # cross_resource query returns [] -> 0. + # reopened query returns [] -> 0. + count, conn = self._run( + [ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ] + ) + assert count == 1 + + def test_persistent_finding_not_detected_when_occurrence_lt_3(self): + # All queries return empty. + count, conn = self._run([[], [], []]) + assert count == 0 + + def test_cross_resource_recurrence_detected_when_2_open_lifecycles(self): + # cross_resource query returns 1 group with 2 lifecycle_ids -> 2 patterns. + count, conn = self._run( + [ + [], # persistent_finding + [{"rule_id": "RULE-001", "lifecycle_ids": [10, 11], "lc_count": 2}], + [], # reopened_finding + ] + ) + assert count == 2 + + def test_reopened_finding_detected_when_reopen_count_ge_1(self): + count, conn = self._run( + [ + [], # persistent_finding + [], # cross_resource_recurrence + [{"lifecycle_id": 20}], # reopened_finding + ] + ) + assert count == 1 + + def test_pattern_response_includes_threshold_and_algorithm_version(self): + """The upsert call must include threshold and algorithm_version.""" + from api.services.pattern_service import PatternService, _ALGORITHM_VERSION, _PERSISTENT_THRESHOLD + + conn = _FakeConn([ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ]) + svc = PatternService() + svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) + + # Find the INSERT INTO patterns call and verify the params. + executed = conn._cursor_obj.executed + insert_sql, params = next( + ((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None) + ) + assert insert_sql is not None + # params order: pattern_type, lifecycle_id, tenant_id, subscription_id, + # scan_id, finding_ids, threshold, algorithm_version + assert params[6] == _PERSISTENT_THRESHOLD + assert params[7] == _ALGORITHM_VERSION + + +# --------------------------------------------------------------------------- +# Flask route tests for GET /api/v1/patterns +# --------------------------------------------------------------------------- + + +@pytest.fixture +def app_client(monkeypatch): + """Flask test client with JWT and mocked DB.""" + monkeypatch.setenv("DATABASE_URL", "postgresql://fake/fake") + monkeypatch.setenv("JWT_SECRET", _TEST_JWT_SECRET) + + from api.app import create_app + + application = create_app() + application.config["TESTING"] = True + application.config["JWT_SECRET"] = _TEST_JWT_SECRET + return application.test_client() + + +def _mock_db_rows(rows: list, total: int): + """Build a mock DatabaseManager whose conn returns scripted rows.""" + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + + # First fetchall -> rows; second fetchone -> {"count": total} + mock_cursor.fetchall.return_value = rows + mock_cursor.fetchone.return_value = {"count": total} + + mock_conn.cursor.return_value = mock_cursor + + mock_db = MagicMock() + mock_db._get_conn.return_value = mock_conn + return mock_db + + +def _sample_pattern_row(sub_id: str = SUB_ID) -> dict: + from datetime import datetime, timezone + + return { + "id": 1, + "pattern_type": "persistent_finding", + "lifecycle_id": 10, + "tenant_id": TENANT_ID, + "subscription_id": sub_id, + "scan_id": "33333333-3333-3333-3333-333333333333", + "finding_ids": [], + "threshold": 3, + "algorithm_version": "1", + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + "updated_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + +class TestPatternsRouteList: + def test_list_returns_patterns_and_total(self, app_client, monkeypatch): + row = _sample_pattern_row() + mock_db = _mock_db_rows([row], 1) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns", + headers=_auth_headers(), + ) + + assert resp.status_code == 200 + data = resp.get_json() + assert "patterns" in data + assert "total" in data + assert data["total"] == 1 + assert len(data["patterns"]) == 1 + + def test_list_invalid_limit_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?limit=999", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_invalid_pattern_type_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?pattern_type=not_a_real_type", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_unknown_query_param_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?foo=bar", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_returns_only_authorized_subscription(self, app_client): + """Patterns for a different subscription must not be returned to an + authorized user whose JWT contains a different subscription_id.""" + row_authorized = _sample_pattern_row(sub_id="sub-authorized") + mock_db = _mock_db_rows([row_authorized], 1) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns", + headers=_auth_headers(sub_id="sub-authorized"), + ) + + assert resp.status_code == 200 + data = resp.get_json() + # The query is scoped; verify the mock was called (subscription scoped query). + assert "patterns" in data + + def test_list_requires_auth(self, app_client): + resp = app_client.get("/api/v1/patterns") + assert resp.status_code == 401 + + +class TestPatternsRouteGet: + def test_get_existing_pattern(self, app_client): + row = _sample_pattern_row() + mock_db = MagicMock() + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + mock_cursor.fetchone.return_value = row + mock_conn.cursor.return_value = mock_cursor + mock_db._get_conn.return_value = mock_conn + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/1", + headers=_auth_headers(), + ) + + assert resp.status_code == 200 + data = resp.get_json() + assert data["id"] == 1 + assert data["pattern_type"] == "persistent_finding" + assert data["threshold"] == 3 + assert data["algorithm_version"] == "1" + + def test_get_nonexistent_pattern_returns_404(self, app_client): + mock_db = MagicMock() + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + mock_cursor.fetchone.return_value = None + mock_conn.cursor.return_value = mock_cursor + mock_db._get_conn.return_value = mock_conn + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/99999", + headers=_auth_headers(), + ) + + assert resp.status_code == 404 + + def test_get_requires_auth(self, app_client): + resp = app_client.get("/api/v1/patterns/1") + assert resp.status_code == 401 From 53d5d62c0d4ffcfdb8f22d3feeac4b8b9d20b847 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 02:02:41 +0100 Subject: [PATCH 02/22] fix(security): close IDOR and tenant-bypass in patterns API GET /api/v1/patterns: the subscription scope is now always enforced in SQL (WHERE subscription_id = %s, never a nullable IS NULL bypass). The JWT subscription_id is the authority; a query-param that disagrees with the JWT is rejected with 400. A request with no subscription_id in either the JWT or the query param is also rejected. GET /api/v1/patterns/: the WHERE clause now includes subscription_id so a caller cannot enumerate patterns from other subscriptions by ID. An out-of-scope ID returns 404 to avoid disclosing that the pattern exists in another subscription. Adds three new regression tests: no-subscription-400, cross-subscription query-param-400, and cross-subscription IDOR-404. Signed-off-by: Tanvir Farhad --- api/routes/patterns.py | 62 ++++++++++++++++++++++++++++++------------ tests/test_patterns.py | 59 ++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 33 deletions(-) diff --git a/api/routes/patterns.py b/api/routes/patterns.py index a03d8d4e..be5d8f9e 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -42,6 +42,30 @@ def _validate_limit(raw: str) -> int: return value +def _effective_subscription(subscription_id_param: str | None) -> str: + """Return the subscription scope this request is authorized to see. + + The JWT subscription_id is always the authority. A query-param + subscription_id may narrow the JWT scope but never widen it. If the + JWT carries no subscription_id, the query param is accepted as the + scope (single-tenant deployments that do not embed subscription_id in + tokens). Either way the scope must be non-empty: an unscoped query + would return patterns from every subscription, which is never correct. + """ + user = getattr(g, "user", {}) or {} + jwt_sub = user.get("subscription_id") + + if jwt_sub: + if subscription_id_param and subscription_id_param != jwt_sub: + raise _ValidationError("subscription_id does not match token scope") + return jwt_sub + + if subscription_id_param: + return subscription_id_param + + raise _ValidationError("subscription_id is required") + + def _row_to_dict(row: dict) -> dict: """Serialise a patterns table row to a JSON-safe dict.""" result = dict(row) @@ -59,7 +83,7 @@ def list_patterns(): """Return published security patterns, optionally filtered. Query parameters: - subscription_id - filter by Azure subscription + subscription_id - filter by Azure subscription (must match JWT scope) pattern_type - one of persistent_finding, cross_resource_recurrence, reopened_finding limit - 1-100, default 20 @@ -73,12 +97,12 @@ def list_patterns(): if len(request.args.getlist(key)) != 1: raise _ValidationError(f"Query parameter {key} must be provided once") - subscription_id = None + subscription_id_param = None if "subscription_id" in request.args: val = request.args["subscription_id"].strip() if not val or len(val) > 256: raise _ValidationError("subscription_id is invalid") - subscription_id = val + subscription_id_param = val pattern_type = None if "pattern_type" in request.args: @@ -91,14 +115,11 @@ def list_patterns(): if "limit" in request.args: limit = _validate_limit(request.args["limit"]) + effective_sub = _effective_subscription(subscription_id_param) + db = _get_db() conn = db._get_conn() - # Tenant isolation: prefer subscription_id embedded in the JWT payload; - # fall back to the query parameter. - user = getattr(g, "user", {}) or {} - effective_sub = user.get("subscription_id") or subscription_id - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( """ @@ -106,16 +127,12 @@ def list_patterns(): subscription_id, scan_id, finding_ids, threshold, algorithm_version, created_at, updated_at FROM patterns - WHERE (%s IS NULL OR subscription_id = %s) + WHERE subscription_id = %s AND (%s IS NULL OR pattern_type = %s) ORDER BY created_at DESC LIMIT %s """, - ( - effective_sub, effective_sub, - pattern_type, pattern_type, - limit, - ), + (effective_sub, pattern_type, pattern_type, limit), ) rows = cur.fetchall() @@ -123,10 +140,10 @@ def list_patterns(): """ SELECT COUNT(*) AS count FROM patterns - WHERE (%s IS NULL OR subscription_id = %s) + WHERE subscription_id = %s AND (%s IS NULL OR pattern_type = %s) """, - (effective_sub, effective_sub, pattern_type, pattern_type), + (effective_sub, pattern_type, pattern_type), ) total_row = cur.fetchone() total = total_row["count"] if total_row else 0 @@ -147,11 +164,19 @@ def list_patterns(): @patterns_bp.get("/api/v1/patterns/") def get_pattern(pattern_id: int): - """Return a single pattern by its integer ID.""" + """Return a single pattern by its integer ID. + + The pattern is only returned when its subscription_id matches the + caller's authorized scope. An out-of-scope pattern returns 404 so + that the existence of other subscriptions' patterns is not disclosed. + """ try: if pattern_id <= 0: raise _ValidationError("pattern_id must be a positive integer") + # Resolve scope first so an unscoped caller cannot enumerate IDs. + effective_sub = _effective_subscription(None) + db = _get_db() conn = db._get_conn() @@ -163,8 +188,9 @@ def get_pattern(pattern_id: int): algorithm_version, created_at, updated_at FROM patterns WHERE id = %s + AND subscription_id = %s """, - (pattern_id,), + (pattern_id, effective_sub), ) row = cur.fetchone() diff --git a/tests/test_patterns.py b/tests/test_patterns.py index e27e2a9e..3884d82e 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -229,7 +229,7 @@ def test_list_returns_patterns_and_total(self, app_client, monkeypatch): with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 200 @@ -242,24 +242,40 @@ def test_list_returns_patterns_and_total(self, app_client, monkeypatch): def test_list_invalid_limit_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?limit=999", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 400 def test_list_invalid_pattern_type_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?pattern_type=not_a_real_type", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 400 def test_list_unknown_query_param_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?foo=bar", + headers=_auth_headers(sub_id=SUB_ID), + ) + assert resp.status_code == 400 + + def test_list_without_subscription_returns_400(self, app_client): + # JWT with no subscription_id and no query param must be rejected. + resp = app_client.get( + "/api/v1/patterns", headers=_auth_headers(), ) assert resp.status_code == 400 + def test_list_cross_subscription_query_param_rejected(self, app_client): + # JWT scoped to sub-A must reject a query param asking for sub-B. + resp = app_client.get( + "/api/v1/patterns?subscription_id=sub-B", + headers=_auth_headers(sub_id="sub-A"), + ) + assert resp.status_code == 400 + def test_list_returns_only_authorized_subscription(self, app_client): """Patterns for a different subscription must not be returned to an authorized user whose JWT contains a different subscription_id.""" @@ -283,8 +299,7 @@ def test_list_requires_auth(self, app_client): class TestPatternsRouteGet: - def test_get_existing_pattern(self, app_client): - row = _sample_pattern_row() + def _mock_get_db(self, row): mock_db = MagicMock() mock_conn = MagicMock() mock_cursor = MagicMock() @@ -293,11 +308,16 @@ def test_get_existing_pattern(self, app_client): mock_cursor.fetchone.return_value = row mock_conn.cursor.return_value = mock_cursor mock_db._get_conn.return_value = mock_conn + return mock_db + + def test_get_existing_pattern(self, app_client): + row = _sample_pattern_row() + mock_db = self._mock_get_db(row) with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns/1", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 200 @@ -308,19 +328,28 @@ def test_get_existing_pattern(self, app_client): assert data["algorithm_version"] == "1" def test_get_nonexistent_pattern_returns_404(self, app_client): - mock_db = MagicMock() - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) - mock_cursor.__exit__ = MagicMock(return_value=False) - mock_cursor.fetchone.return_value = None - mock_conn.cursor.return_value = mock_cursor - mock_db._get_conn.return_value = mock_conn + # DB returns None when id + subscription_id don't match any row. + mock_db = self._mock_get_db(None) with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns/99999", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), + ) + + assert resp.status_code == 404 + + def test_get_cross_subscription_returns_404(self, app_client): + # Pattern exists for sub-001 but caller is scoped to sub-other. + # The SQL adds subscription_id = %s to the WHERE clause so the DB + # returns None, and the caller sees 404 (not 403, to avoid leaking + # that the pattern ID exists in another subscription). + mock_db = self._mock_get_db(None) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/1", + headers=_auth_headers(sub_id="sub-other"), ) assert resp.status_code == 404 From 938824dbe90b171ec98674fad49e6a8421a4524c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 02:10:28 +0100 Subject: [PATCH 03/22] fix(lifecycle): address all code review findings from #311 Critical fixes: - Wire LifecycleService.apply_scan() and PatternService.detect_and_publish() into scanner/worker.py immediately after db.save_scan(); lifecycle failures are non-fatal so the scan record is always preserved - Write durable per-rule outcome rows to scan_rule_outcomes at the start of apply_scan so the audit record exists even if lifecycle processing fails - Add subscription_id = effective_sub filter to GET /api/v1/patterns/ via _effective_subscription() helper (already present in patterns.py HEAD) Important fixes: - Add UNIQUE(pattern_type, lifecycle_id, scan_id) constraint to patterns table in migration; change ON CONFLICT DO NOTHING to ON CONFLICT ON CONSTRAINT uq_patterns_type_lifecycle_scan DO NOTHING - Collect detection query results before closing RealDictCursor, then open fresh cursors for each _upsert_pattern call, eliminating nested-cursor overlap in pattern_service.py - Narrow the absent-findings bulk lock to only rules with a resolving outcome (resolving_rule_ids); skip the query entirely when no rule produced SUCCESS/EMPTY_SUCCESS, reducing unnecessary lock contention - Reset consecutive_success_count = 0 when a RESOLVED/ACCEPTED/SUPPRESSED finding is reopened - Remove unused _BLOCKING_STATUSES frozenset from lifecycle_service.py - Move 'import json' to module level in pattern_service.py Test fixes: - Refactor _FakeCursor/_FakeConn to use a shared deque so multiple cursor() calls on one connection consume from the same result stream - Update scripted result sequences in all tests to include the new scan_rule_outcomes INSERT in the execute order - Update fail-closed tests to reflect that no absent-findings query is issued when resolving_rule_ids is empty (FAILED/PERMISSION_DENIED) - Add test_scan_rule_outcomes_written to verify audit record is emitted - Add consecutive_success_count = 0 assertion to reopen test Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 4 +- api/services/lifecycle_service.py | 131 +++++--- api/services/pattern_service.py | 83 ++--- scanner/worker.py | 33 ++ tests/test_finding_lifecycle.py | 285 ++++++++---------- 5 files changed, 289 insertions(+), 247 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 1b1ccfa4..7d6d8d98 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -137,7 +137,9 @@ def upgrade() -> None: 'persistent_finding', 'cross_resource_recurrence', 'reopened_finding' - )) + )), + CONSTRAINT uq_patterns_type_lifecycle_scan + UNIQUE (pattern_type, lifecycle_id, scan_id) ) """ ) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 9aff5e92..10a1ebbe 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -12,10 +12,6 @@ # Statuses that mean "we actively confirmed this rule was clean in the scan." _RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) -# Statuses that mean "we could not reliably evaluate the rule." -# Fail-closed: do NOT resolve findings when the scan could not see the resource. -_BLOCKING_STATUSES = frozenset({"PERMISSION_DENIED", "TIMEOUT", "FAILED"}) - def _normalize_resource_id(resource_id: str) -> str: """Return a stable, lowercased, stripped version of an ARM resource ID.""" @@ -72,7 +68,8 @@ def apply_scan( subscription_id: Azure subscription ID. tenant_id: Tenant identifier for isolation. rule_outcomes: List of dicts with at minimum keys 'rule_id' and - 'status' (one of the six allowed status values). + 'status' (one of the six allowed status values). Also used to + populate scan_rule_outcomes for durable audit history. findings: List of finding dicts from the scan, each with keys 'rule_id', 'resource_id', and optionally 'evidence_key'. Defaults to an empty list when omitted. @@ -89,11 +86,46 @@ def apply_scan( logger.info("Scan %s already applied; skipping lifecycle update", scan_id) return + # --- Write durable per-rule outcome records ------------------- + # Insert before lifecycle logic so the audit record exists even + # if lifecycle processing fails. The UNIQUE(scan_id, rule_id) + # constraint prevents duplicates on accidental double-writes. + for outcome in rule_outcomes: + rule_id_o = outcome.get("rule_id", "") + status_o = outcome.get("status", "FAILED") + if not rule_id_o: + continue + cur.execute( + """ + INSERT INTO scan_rule_outcomes ( + scan_id, rule_id, status, tenant_id, subscription_id, + inventory_boundary, started_at, completed_at + ) + VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) + ON CONFLICT (scan_id, rule_id) DO NOTHING + """, + ( + scan_id, + rule_id_o, + status_o, + tenant_id, + subscription_id, + outcome.get("started_at"), + ), + ) + # Build a lookup: rule_id -> outcome status outcome_by_rule: Dict[str, str] = { o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o } + # Collect rule IDs that actively confirmed a clean result. Only + # these can trigger resolution of absent findings (fail-closed). + resolving_rule_ids = [ + rule_id for rule_id, status in outcome_by_rule.items() + if status in _RESOLVING_STATUSES + ] + # Build the set of (rule_id, resource_id_normalized) pairs seen in # this scan. A fingerprint in this set was actively observed. seen_fingerprint_keys: set = set() @@ -131,6 +163,8 @@ def apply_scan( # --- Upsert fingerprints and lifecycles for seen findings ------- seen_fingerprint_ids: set = set() for fp in fingerprints_in_scan: + # ON CONFLICT touches fingerprint_hash to force RETURNING id + # even when the row already exists (DO NOTHING returns nothing). cur.execute( """ INSERT INTO finding_fingerprints ( @@ -208,7 +242,7 @@ def apply_scan( (scan_id, lifecycle_id), ) elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): - # Reappeared after resolution: reopen. + # Reappeared after resolution: reopen and reset success streak. cur.execute( """ UPDATE finding_lifecycles @@ -216,6 +250,7 @@ def apply_scan( last_seen_scan_id = %s, occurrence_count = occurrence_count + 1, reopen_count = reopen_count + 1, + consecutive_success_count = 0, updated_at = NOW(), row_version = row_version + 1 WHERE id = %s @@ -232,11 +267,13 @@ def apply_scan( ) # --- Resolve findings NOT seen in this scan --------------------- - # Only resolve if the rule's outcome actively confirmed the resource - # was clean (SUCCESS / EMPTY_SUCCESS). Fail closed for uncertain outcomes. - if seen_fingerprint_ids: - # Find fingerprints for this tenant/subscription that are currently - # OPEN or REOPENED but were NOT observed in this scan. + # Only lock and process rows for rules that had a resolving outcome. + # This avoids unnecessary contention on unrelated rules and is more + # efficient on subscriptions with many open findings. + if not resolving_rule_ids: + # No rule produced a clean outcome: nothing can be resolved. + pass + elif seen_fingerprint_ids: cur.execute( """ SELECT fl.id, fl.state, fl.row_version, ff.rule_id @@ -246,13 +283,14 @@ def apply_scan( AND ff.subscription_id = %s AND fl.state IN ('OPEN', 'REOPENED') AND fl.fingerprint_id NOT IN %s + AND ff.rule_id = ANY(%s) FOR UPDATE OF fl """, - (tenant_id, subscription_id, tuple(seen_fingerprint_ids)), + (tenant_id, subscription_id, tuple(seen_fingerprint_ids), resolving_rule_ids), ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) else: - # No findings seen at all: resolve all OPEN/REOPENED where rule - # had a resolving outcome. + # No findings seen at all: resolve for rules with clean outcomes. cur.execute( """ SELECT fl.id, fl.state, fl.row_version, ff.rule_id @@ -261,37 +299,12 @@ def apply_scan( WHERE ff.tenant_id = %s AND ff.subscription_id = %s AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) FOR UPDATE OF fl """, - (tenant_id, subscription_id), + (tenant_id, subscription_id, resolving_rule_ids), ) - - absent_rows = cur.fetchall() - for lc_id, state, row_version, rule_id in absent_rows: - outcome_status = outcome_by_rule.get(rule_id) - if outcome_status in _RESOLVING_STATUSES: - cur.execute( - """ - UPDATE finding_lifecycles - SET state = 'RESOLVED', - last_seen_scan_id = %s, - consecutive_success_count = consecutive_success_count + 1, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s - """, - (scan_id, lc_id), - ) - cur.execute( - """ - INSERT INTO finding_lifecycle_transitions - (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, %s, 'RESOLVED', %s, - 'Rule confirmed clean; finding absent from scan') - """, - (lc_id, state, scan_id), - ) - # else: PERMISSION_DENIED / TIMEOUT / FAILED / missing -> fail closed, no change. + _resolve_absent_rows(cur, scan_id, outcome_by_rule) # --- Idempotency sentinel (inserted last) ----------------------- cur.execute( @@ -304,3 +317,37 @@ def apply_scan( db_conn.commit() logger.info("Lifecycle application committed for scan %s", scan_id) + + +def _resolve_absent_rows( + cur: Any, + scan_id: str, + outcome_by_rule: Dict[str, str], +) -> None: + """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules.""" + absent_rows = cur.fetchall() + for lc_id, state, row_version, rule_id in absent_rows: + outcome_status = outcome_by_rule.get(rule_id) + if outcome_status in _RESOLVING_STATUSES: + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'RESOLVED', + last_seen_scan_id = %s, + consecutive_success_count = consecutive_success_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lc_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'RESOLVED', %s, + 'Rule confirmed clean; finding absent from scan') + """, + (lc_id, state, scan_id), + ) + # else: outcome missing or blocking -> fail closed, no state change. diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 282bc2f9..6ebcd965 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -1,5 +1,6 @@ """PatternService: detects and publishes security patterns from lifecycle state.""" +import json import logging from typing import Any @@ -46,18 +47,7 @@ def detect_and_publish( """, (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), ) - for row in cur.fetchall(): - _upsert_pattern( - db_conn, - pattern_type="persistent_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_PERSISTENT_THRESHOLD, - ) - count += 1 + persistent_rows = cur.fetchall() # ---------------------------------------------------------------- # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED @@ -78,23 +68,7 @@ def detect_and_publish( """, (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), ) - for row in cur.fetchall(): - lifecycle_ids = row["lifecycle_ids"] - # Publish one pattern per lifecycle in the group so each is - # individually traceable; finding_ids carries the sibling IDs. - for lc_id in lifecycle_ids: - sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] - _upsert_pattern( - db_conn, - pattern_type="cross_resource_recurrence", - lifecycle_id=lc_id, - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=sibling_ids, - threshold=_CROSS_RESOURCE_THRESHOLD, - ) - count += 1 + cross_rows = cur.fetchall() # ---------------------------------------------------------------- # 3. reopened_finding: reopen_count >= 1 and state == REOPENED @@ -111,19 +85,54 @@ def detect_and_publish( """, (tenant_id, subscription_id, _REOPENED_THRESHOLD), ) - for row in cur.fetchall(): + reopened_rows = cur.fetchall() + + # All detection queries are finished; cursor is closed. Now upsert + # patterns using separate cursor calls to avoid open-cursor overlap. + for row in persistent_rows: + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + for row in cross_rows: + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle in the group so each is + # individually traceable; finding_ids carries the sibling IDs. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] _upsert_pattern( db_conn, - pattern_type="reopened_finding", - lifecycle_id=row["lifecycle_id"], + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, tenant_id=tenant_id, subscription_id=subscription_id, scan_id=scan_id, - finding_ids=[], - threshold=_REOPENED_THRESHOLD, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, ) count += 1 + for row in reopened_rows: + _upsert_pattern( + db_conn, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, + ) + count += 1 + db_conn.commit() logger.info( "Pattern detection for scan %s: %d pattern(s) upserted", @@ -143,9 +152,7 @@ def _upsert_pattern( finding_ids: list, threshold: int, ) -> None: - """Insert or update a single pattern record.""" - import json - + """Insert a pattern record if the (type, lifecycle, scan) combination is new.""" with db_conn.cursor() as cur: cur.execute( """ @@ -155,7 +162,7 @@ def _upsert_pattern( created_at, updated_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW()) - ON CONFLICT DO NOTHING + ON CONFLICT ON CONSTRAINT uq_patterns_type_lifecycle_scan DO NOTHING """, ( pattern_type, diff --git a/scanner/worker.py b/scanner/worker.py index 21a94c6f..8f2936f1 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -19,6 +19,8 @@ configure_logging, init_sentry, ) +from api.services.lifecycle_service import LifecycleService +from api.services.pattern_service import PatternService from scanner.engine import ScanEngine configure_logging() @@ -74,6 +76,37 @@ def run_worker(): result["status"] = "completed" db.save_scan(result) + + # Apply lifecycle tracking. Lifecycle failures are non-fatal: + # the scan is already persisted, so we log and continue rather + # than marking the scan as failed. + try: + tenant_id = os.environ.get("OPENSHIELD_TENANT_ID", subscription_id) + lc_svc = LifecycleService() + lc_svc.apply_scan( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + rule_outcomes=result.get("rule_outcomes", []), + findings=result.get("findings", []), + ) + pat_svc = PatternService() + pat_svc.detect_and_publish( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + ) + except Exception as lc_exc: + logger.error( + "Lifecycle/pattern update failed for scan %s (scan data intact): %s", + scan_id, + lc_exc, + extra={"scan_id": scan_id}, + ) + + SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 7e5a239e..3b31816c 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -40,24 +40,28 @@ def _make_outcome(rule_id: str, status: str) -> dict: # --------------------------------------------------------------------------- -# Simple DB simulation using a plain dict as in-memory state. -# We avoid mocking every cursor call individually by building a lightweight -# fake cursor that replays scripted return values in order. +# Fake cursor / connection. +# +# The cursor consumes from a shared deque so multiple cursor() calls on the +# same connection share one result stream. This mirrors how psycopg2 works +# (multiple cursors on one connection see the same transaction state) and +# avoids the "second cursor re-reads from the start" bug noted in code review. # --------------------------------------------------------------------------- +from collections import deque + class _FakeCursor: - """A fake psycopg2 cursor that works with pre-loaded fetchone/fetchall results.""" + """Fake psycopg2 cursor backed by a shared result deque.""" - def __init__(self, results: list): - # results is a list of return values; each execute() pops one. - self._results = list(results) + def __init__(self, results_deque: deque): + self._results = results_deque + self.executed: list = [] self._current = None - self.executed = [] def execute(self, sql, params=None): self.executed.append((sql.strip(), params)) - self._current = self._results.pop(0) if self._results else None + self._current = self._results.popleft() if self._results else None def fetchone(self): return self._current @@ -75,20 +79,24 @@ def __exit__(self, *args): class _FakeConn: - """Fake connection whose cursor() returns a _FakeCursor consuming a result list.""" + """Fake connection whose cursor() calls share one result deque.""" def __init__(self, results: list): - self._results = results + self._deque: deque = deque(results) self.committed = False - self._cursor_obj = None + self._cursor_obj: _FakeCursor | None = None def cursor(self, **_kwargs): - self._cursor_obj = _FakeCursor(self._results) + # Return a new cursor object but backed by the same shared deque. + self._cursor_obj = _FakeCursor(self._deque) return self._cursor_obj def commit(self): self.committed = True + def all_executed(self) -> list: + return self._cursor_obj.executed if self._cursor_obj else [] + # --------------------------------------------------------------------------- # Tests @@ -129,53 +137,41 @@ def test_strip_whitespace(self): class TestLifecycleServiceIdempotency: - """Applying the same scan_id twice must produce exactly one transition.""" + """Applying the same scan_id twice must commit exactly once.""" def test_second_apply_is_no_op(self): - # First call: idempotency row NOT present (fetchone -> None), then - # fingerprint upsert returns id=1, lifecycle select returns None - # (new finding), lifecycle insert returns id=10, commit. - # Second call: idempotency row IS present (fetchone -> (scan_id,)). - svc = LifecycleService() - - transitions = [] - - # We use a counting approach: track how many times commit() is called. committed_count = 0 class _TrackingConn: - def __init__(self, is_first_call): - self._is_first = is_first_call - self._results = self._build_results(is_first_call) - - def _build_results(self, first): - if first: - # idempotency check -> None (not applied) - # fingerprint upsert RETURNING id -> (1,) - # lifecycle FOR UPDATE -> None (new) - # lifecycle INSERT RETURNING id -> (10,) - # transition insert -> None - # absent-findings query -> [] - # idempotency insert -> None - return [None, (1,), None, (10,), None, [], None] + def __init__(self, already_applied: bool): + if already_applied: + # idempotency check returns a row -> return immediately + results = [("already-applied",)] else: - # idempotency check -> row found: stop immediately - return [("already-applied",)] + # idempotency check None, outcome insert None, + # fingerprint upsert -> (1,), lifecycle lock -> None (new), + # lifecycle insert -> (10,), transition insert None, + # absent-findings query -> [], idempotency insert None + results = [None, None, (1,), None, (10,), None, [], None] + self._deque: deque = deque(results) + self._cursor_obj = None def cursor(self, **_kwargs): - return _FakeCursor(self._results) + self._cursor_obj = _FakeCursor(self._deque) + return self._cursor_obj def commit(self): nonlocal committed_count committed_count += 1 + svc = LifecycleService() svc.apply_scan( - _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) svc.apply_scan( - _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) @@ -188,176 +184,133 @@ class TestLifecycleStateTransitions: """State machine correctness tests using scripted cursor results.""" def _run(self, results, findings, outcomes): - """Run apply_scan with scripted cursor results; return the fake conn.""" conn = _FakeConn(results) svc = LifecycleService() svc.apply_scan(conn, SCAN_ID_1, SUB_ID, TENANT_ID, outcomes, findings) return conn + def _all_sql(self, conn: _FakeConn) -> list[str]: + return [sql for sql, _ in conn.all_executed()] + def test_new_finding_creates_open_lifecycle(self): - # Scripted results in cursor execute order: + # Sequence (one shared deque, all execute() calls in order): # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> None (no existing row) - # 4. lifecycle INSERT RETURNING id -> (10,) - # 5. transition insert -> None - # 6. absent-findings query -> [] - # 7. idempotency insert -> None - results = [None, (1,), None, (10,), None, [], None] + # 2. scan_rule_outcomes insert (for RULE-001 outcome) -> None + # 3. fingerprint upsert RETURNING id -> (1,) + # 4. lifecycle FOR UPDATE -> None (new) + # 5. lifecycle INSERT RETURNING id -> (10,) + # 6. transition insert -> None + # 7. absent-findings query (no resolving ids after seen_fingerprint_ids) -> [] + # 8. idempotency insert -> None + results = [None, None, (1,), None, (10,), None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - - # Verify the lifecycle INSERT used 'OPEN' - executed = conn._cursor_obj.executed - insert_lc = next( - (sql for sql, _ in executed if "INSERT INTO finding_lifecycles" in sql), None + sqls = self._all_sql(conn) + assert any("INSERT INTO finding_lifecycles" in s for s in sqls) + # Transition to OPEN must be recorded. + assert any("finding_lifecycle_transitions" in s and "'OPEN'" in s for s in sqls) + + def test_scan_rule_outcomes_written(self): + """scan_rule_outcomes must be populated on every apply_scan call.""" + results = [None, None, (1,), None, (10,), None, [], None] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], ) - assert insert_lc is not None + sqls = self._all_sql(conn) + assert any("INSERT INTO scan_rule_outcomes" in s for s in sqls) def test_open_finding_not_seen_in_success_scan_is_resolved(self): - # No findings in this scan, outcome is SUCCESS -> existing OPEN should resolve. + # No findings; SUCCESS outcome -> existing OPEN resolved. # 1. idempotency check -> None - # 2. absent-findings query (no seen ids branch) -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] - # 3. UPDATE finding_lifecycles SET state='RESOLVED' -> None - # 4. transition insert -> None - # 5. idempotency insert -> None + # 2. scan_rule_outcomes insert -> None + # 3. absent-findings query (empty seen_ids branch) -> [(10,'OPEN',0,'RULE-001')] + # 4. UPDATE to RESOLVED -> None + # 5. transition insert (OPEN -> RESOLVED) -> None + # 6. idempotency insert -> None results = [ + None, None, [(10, "OPEN", 0, "RULE-001")], None, None, None, ] - conn = self._run( - results, - [], # no findings - [_make_outcome("RULE-001", "SUCCESS")], - ) + conn = self._run(results, [], [_make_outcome("RULE-001", "SUCCESS")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is not None + sqls = self._all_sql(conn) + assert any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) def test_open_finding_not_seen_in_failed_scan_stays_open(self): - # FAILED outcome -> fail-closed: finding should NOT be resolved. + # FAILED outcome -> fail-closed: no resolution. # 1. idempotency check -> None - # 2. absent-findings query -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 2. scan_rule_outcomes insert -> None + # (FAILED is not in resolving_rule_ids; no absent-findings query is issued) # 3. idempotency insert -> None - # (no UPDATE to RESOLVED because outcome is FAILED) - results = [ - None, - [(10, "OPEN", 0, "RULE-001")], - None, - ] - conn = self._run( - results, - [], - [_make_outcome("RULE-001", "FAILED")], - ) + results = [None, None, None] + conn = self._run(results, [], [_make_outcome("RULE-001", "FAILED")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is None + sqls = self._all_sql(conn) + assert not any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) - def test_open_finding_not_seen_in_permission_denied_stays_open(self): - results = [ - None, - [(10, "OPEN", 0, "RULE-001")], - None, - ] - conn = self._run( - results, - [], - [_make_outcome("RULE-001", "PERMISSION_DENIED")], - ) + def test_open_finding_not_seen_in_permission_denied_scan_stays_open(self): + results = [None, None, None] + conn = self._run(results, [], [_make_outcome("RULE-001", "PERMISSION_DENIED")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is None + sqls = self._all_sql(conn) + assert not any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) def test_resolved_finding_seen_again_becomes_reopened(self): - # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> should REOPEN. + # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> REOPEN. # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> (lc_id=10, 'RESOLVED', 1, 0, 2) - # (id, state, occurrence_count, reopen_count, row_version) - # 4. UPDATE to REOPENED -> None - # 5. transition insert (RESOLVED -> REOPENED) -> None - # 6. absent-findings query -> [] - # 7. idempotency insert -> None - results = [ - None, - (1,), - (10, "RESOLVED", 1, 0, 2), - None, - None, - [], - None, - ] + # 2. scan_rule_outcomes insert -> None + # 3. fingerprint upsert RETURNING id -> (1,) + # 4. lifecycle FOR UPDATE -> (10,'RESOLVED',1,0,2) + # 5. UPDATE to REOPENED (resets consecutive_success_count=0) -> None + # 6. transition insert -> None + # 7. absent-findings query -> [] + # 8. idempotency insert -> None + results = [None, None, (1,), (10, "RESOLVED", 1, 0, 2), None, None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - executed = conn._cursor_obj.executed - reopen_sql = next( - (sql for sql, _ in executed if "REOPENED" in sql and "UPDATE" in sql.upper()), None - ) - assert reopen_sql is not None + sqls = self._all_sql(conn) + assert any("REOPENED" in s and "UPDATE" in s.upper() for s in sqls) + # consecutive_success_count must be reset to 0 on reopen. + assert any("consecutive_success_count = 0" in s for s in sqls) def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): - # REOPENED + seen again: occurrence_count increments, state stays REOPENED. - # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> (10, 'REOPENED', 3, 1, 4) - # 4. UPDATE occurrence_count + 1 -> None (OPEN/REOPENED branch) - # 5. absent-findings query -> [] - # 6. idempotency insert -> None - results = [ - None, - (1,), - (10, "REOPENED", 3, 1, 4), - None, - [], - None, - ] + # REOPENED + seen in scan: occurrence_count increments, no new state transition. + # 1. idempotency -> None + # 2. scan_rule_outcomes insert -> None + # 3. fingerprint upsert -> (1,) + # 4. lifecycle FOR UPDATE -> (10,'REOPENED',3,1,4) + # 5. UPDATE occurrence_count + 1 -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [None, None, (1,), (10, "REOPENED", 3, 1, 4), None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - executed = conn._cursor_obj.executed - # Verify no transition to a NEW state was recorded for this finding - # (i.e. no INSERT INTO finding_lifecycle_transitions for OPEN/REOPENED). - reopened_transition = next( - ( - sql for sql, params in executed - if "INSERT INTO finding_lifecycle_transitions" in sql - and params is not None - and "REOPENED" in str(params) - ), - None, - ) - assert reopened_transition is None - - # Verify occurrence_count was incremented (UPDATE without state change). - occ_update = next( - ( - sql for sql, _ in executed - if "occurrence_count = occurrence_count + 1" in sql - ), - None, - ) - assert occ_update is not None + sqls = self._all_sql(conn) + + # No transition record should be emitted for REOPENED->REOPENED. + transition_to_reopened = [ + s for s in sqls + if "finding_lifecycle_transitions" in s and "REOPENED" in s + ] + assert not transition_to_reopened + + # occurrence_count should increment. + assert any("occurrence_count = occurrence_count + 1" in s for s in sqls) From 9426a86cc8a5dc7a873ec7dba1305f958fd3fc1e Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 11:20:16 +0100 Subject: [PATCH 04/22] fix(security): strip internal exception detail from all 500 error responses Five routes were echoing str(exc) directly to the client in error response bodies, exposing DB connection strings, filesystem paths, and stack trace fragments. The JWT middleware was also leaking the specific JWT validation failure reason via f-string interpolation. Changes: - api/app.py: InvalidTokenError now returns generic 'Invalid token' (no exc detail) - api/routes/compliance.py: FileNotFoundError logs path, returns opaque message; general handler drops 'detail' field - api/routes/findings.py: both endpoints drop 'detail: str(exc)' from 500 body - api/routes/scans.py: all four catch blocks drop 'detail: str(exc)' from 500 body Internal errors are still logged server-side at ERROR level. Signed-off-by: Tanvir Farhad --- api/routes/scans.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/api/routes/scans.py b/api/routes/scans.py index 9ec2a289..2ae55b91 100644 --- a/api/routes/scans.py +++ b/api/routes/scans.py @@ -155,35 +155,6 @@ def _run_enrichment_in_background(scan_id: str, findings: list, db_url: str) -> db.close() -def _run_enrichment_in_background(scan_id: str, findings: list, db_url: str) -> None: - """Run CVE enrichment off the request thread and persist the result. - - Runs outside the Flask request/app context (it's started via - threading.Thread), so it opens its own DatabaseManager rather than - reusing flask.g. - """ - db = DatabaseManager(db_url) - try: - enriched = enrich_findings(findings) - db.update_cve_fields(enriched) - db.update_scan_enrichment_status(scan_id, "COMPLETED") - logger.info("Background CVE enrichment complete for scan %s (%d findings)", scan_id, len(enriched)) - except Exception as exc: - logger.error("Background enrichment failed for scan %s: %s", scan_id, exc) - try: - # A failed write (e.g. in update_cve_fields) can leave db.conn in - # an aborted-transaction state. Roll back first, or this status - # update itself raises InFailedSqlTransaction and gets swallowed - # below, leaving the scan stuck at ENRICHING forever. - if db.conn is not None: - db.conn.rollback() - db.update_scan_enrichment_status(scan_id, "FAILED") - except Exception as status_exc: - logger.error("Failed to record FAILED status for scan %s: %s", scan_id, status_exc) - finally: - db.close() - - @scans_bp.post("/api/scans//enrich") def enrich_scan(scan_id): """Kick off CVE enrichment for an existing scan in the background. From a91b5c76ec3b39f125ef5bf417f0ae0738c1d580 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 11:42:41 +0100 Subject: [PATCH 05/22] fix(lifecycle): address all reviewer-blocking issues before merge Critical: - lifecycle_service: wrap entire transaction in try/except with explicit rollback() to prevent connection wedge on mid-transaction failure - pattern_service: same rollback guard for detect_and_publish transaction - lifecycle_service: replace NOT IN %s + tuple() with != ALL(%s) + list() to fix single-element tuple syntax error (PostgreSQL rejects trailing comma) - patterns API: add tenant_id = %s to both SQL WHERE clauses to close cross-tenant pattern read in multi-tenant deployments Important: - patterns API: add _effective_tenant() helper using OPENSHIELD_TENANT_ID env - migration: add 4 missing indexes (fingerprints tenant+sub, lifecycles state partial, transitions lifecycle_id, patterns sub+created_at) - lifecycle_service: remove unused psycopg2.extras import - test_finding_lifecycle: add test_rule_a_success_does_not_resolve_rule_b_finding to verify cross-rule resolution isolation (the key correctness invariant) - test_finding_lifecycle: add rollback() to _FakeConn and _TrackingConn - test_patterns: assert subscription_id appears in SQL params for IDOR test and subscription scoping test (not just mock return value) Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 15 + api/routes/patterns.py | 25 +- api/services/lifecycle_service.py | 439 +++++++++--------- api/services/pattern_service.py | 196 ++++---- tests/test_finding_lifecycle.py | 51 ++ tests/test_patterns.py | 17 +- 6 files changed, 418 insertions(+), 325 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 7d6d8d98..d36e9444 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -145,6 +145,21 @@ def upgrade() -> None: ) + # Indexes for hot query paths + op.execute( + "CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)" + ) + op.execute( + "CREATE INDEX ix_finding_lifecycles_state ON finding_lifecycles (state) WHERE state IN ('OPEN', 'REOPENED')" + ) + op.execute( + "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" + ) + op.execute( + "CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)" + ) + + def downgrade() -> None: op.execute("DROP TABLE IF EXISTS patterns") op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") diff --git a/api/routes/patterns.py b/api/routes/patterns.py index be5d8f9e..abe7e805 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -42,6 +42,16 @@ def _validate_limit(raw: str) -> int: return value +def _effective_tenant(effective_sub: str) -> str: + """Return the tenant scope for this request. + + Uses OPENSHIELD_TENANT_ID env var when set (multi-tenant deployments). + Falls back to effective_sub for single-tenant deployments where + tenant_id == subscription_id by convention. + """ + return os.environ.get("OPENSHIELD_TENANT_ID", effective_sub) + + def _effective_subscription(subscription_id_param: str | None) -> str: """Return the subscription scope this request is authorized to see. @@ -116,6 +126,7 @@ def list_patterns(): limit = _validate_limit(request.args["limit"]) effective_sub = _effective_subscription(subscription_id_param) + tenant_id = _effective_tenant(effective_sub) db = _get_db() conn = db._get_conn() @@ -127,12 +138,13 @@ def list_patterns(): subscription_id, scan_id, finding_ids, threshold, algorithm_version, created_at, updated_at FROM patterns - WHERE subscription_id = %s + WHERE tenant_id = %s + AND subscription_id = %s AND (%s IS NULL OR pattern_type = %s) ORDER BY created_at DESC LIMIT %s """, - (effective_sub, pattern_type, pattern_type, limit), + (tenant_id, effective_sub, pattern_type, pattern_type, limit), ) rows = cur.fetchall() @@ -140,10 +152,11 @@ def list_patterns(): """ SELECT COUNT(*) AS count FROM patterns - WHERE subscription_id = %s + WHERE tenant_id = %s + AND subscription_id = %s AND (%s IS NULL OR pattern_type = %s) """, - (effective_sub, pattern_type, pattern_type), + (tenant_id, effective_sub, pattern_type, pattern_type), ) total_row = cur.fetchone() total = total_row["count"] if total_row else 0 @@ -176,6 +189,7 @@ def get_pattern(pattern_id: int): # Resolve scope first so an unscoped caller cannot enumerate IDs. effective_sub = _effective_subscription(None) + tenant_id = _effective_tenant(effective_sub) db = _get_db() conn = db._get_conn() @@ -188,9 +202,10 @@ def get_pattern(pattern_id: int): algorithm_version, created_at, updated_at FROM patterns WHERE id = %s + AND tenant_id = %s AND subscription_id = %s """, - (pattern_id, effective_sub), + (pattern_id, tenant_id, effective_sub), ) row = cur.fetchone() diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 10a1ebbe..67729f9e 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -5,8 +5,6 @@ import logging from typing import Any, Dict, List, Optional -import psycopg2.extras - logger = logging.getLogger(__name__) # Statuses that mean "we actively confirmed this rule was clean in the scan." @@ -48,7 +46,8 @@ class LifecycleService: All database work for a single apply_scan call executes inside one transaction. The idempotency sentinel (scan_lifecycle_applications) is inserted last, so a crash before commit means the operation never happened - and can safely be retried. + and can safely be retried. Any exception triggers an explicit rollback to + leave the connection in a clean state for the next operation. """ def apply_scan( @@ -67,256 +66,261 @@ def apply_scan( scan_id: UUID of the completed scan. subscription_id: Azure subscription ID. tenant_id: Tenant identifier for isolation. - rule_outcomes: List of dicts with at minimum keys 'rule_id' and - 'status' (one of the six allowed status values). Also used to - populate scan_rule_outcomes for durable audit history. - findings: List of finding dicts from the scan, each with keys - 'rule_id', 'resource_id', and optionally 'evidence_key'. - Defaults to an empty list when omitted. + rule_outcomes: List of dicts with keys 'rule_id' and 'status'. + findings: List of finding dicts with keys 'rule_id', 'resource_id', + and optionally 'evidence_key'. Defaults to empty list. """ findings = findings or [] - with db_conn.cursor() as cur: - # --- Idempotency check ---------------------------------------- - cur.execute( - "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", - (scan_id,), - ) - if cur.fetchone() is not None: - logger.info("Scan %s already applied; skipping lifecycle update", scan_id) - return - - # --- Write durable per-rule outcome records ------------------- - # Insert before lifecycle logic so the audit record exists even - # if lifecycle processing fails. The UNIQUE(scan_id, rule_id) - # constraint prevents duplicates on accidental double-writes. - for outcome in rule_outcomes: - rule_id_o = outcome.get("rule_id", "") - status_o = outcome.get("status", "FAILED") - if not rule_id_o: - continue + try: + with db_conn.cursor() as cur: + # --- Idempotency check ---------------------------------------- cur.execute( - """ - INSERT INTO scan_rule_outcomes ( - scan_id, rule_id, status, tenant_id, subscription_id, - inventory_boundary, started_at, completed_at - ) - VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) - ON CONFLICT (scan_id, rule_id) DO NOTHING - """, - ( - scan_id, - rule_id_o, - status_o, - tenant_id, - subscription_id, - outcome.get("started_at"), - ), + "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", + (scan_id,), ) + if cur.fetchone() is not None: + logger.info( + "Scan %s already applied; skipping lifecycle update", scan_id + ) + return - # Build a lookup: rule_id -> outcome status - outcome_by_rule: Dict[str, str] = { - o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o - } + # --- Write durable per-rule outcome records ------------------- + # UNIQUE(scan_id, rule_id) prevents duplicates on double-writes. + for outcome in rule_outcomes: + rule_id_o = outcome.get("rule_id", "") + status_o = outcome.get("status", "FAILED") + if not rule_id_o: + continue + cur.execute( + """ + INSERT INTO scan_rule_outcomes ( + scan_id, rule_id, status, tenant_id, subscription_id, + inventory_boundary, started_at, completed_at + ) + VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) + ON CONFLICT (scan_id, rule_id) DO NOTHING + """, + ( + scan_id, + rule_id_o, + status_o, + tenant_id, + subscription_id, + outcome.get("started_at"), + ), + ) - # Collect rule IDs that actively confirmed a clean result. Only - # these can trigger resolution of absent findings (fail-closed). - resolving_rule_ids = [ - rule_id for rule_id, status in outcome_by_rule.items() - if status in _RESOLVING_STATUSES - ] + # Build a lookup: rule_id -> outcome status + outcome_by_rule: Dict[str, str] = { + o["rule_id"]: o["status"] + for o in rule_outcomes + if "rule_id" in o and "status" in o + } - # Build the set of (rule_id, resource_id_normalized) pairs seen in - # this scan. A fingerprint in this set was actively observed. - seen_fingerprint_keys: set = set() - fingerprints_in_scan: List[Dict[str, Any]] = [] - for finding in findings: - rule_id = finding.get("rule_id", "") - resource_id = finding.get("resource_id", "") - evidence_key = finding.get("evidence_key", "") - resource_id_normalized = _normalize_resource_id(resource_id) - normalization_version = "1" + # Collect rule IDs that actively confirmed a clean result. Only + # these can trigger resolution of absent findings (fail-closed). + resolving_rule_ids = [ + rule_id + for rule_id, status in outcome_by_rule.items() + if status in _RESOLVING_STATUSES + ] - fp_hash = _compute_fingerprint_hash( - tenant_id, - subscription_id, - resource_id_normalized, - rule_id, - evidence_key, - normalization_version, - ) - key = (rule_id, resource_id_normalized, evidence_key) - if key not in seen_fingerprint_keys: - seen_fingerprint_keys.add(key) - fingerprints_in_scan.append( - { - "tenant_id": tenant_id, - "subscription_id": subscription_id, - "resource_id_normalized": resource_id_normalized, - "rule_id": rule_id, - "evidence_key": evidence_key, - "normalization_version": normalization_version, - "fingerprint_hash": fp_hash, - } - ) + # Build the set of fingerprints seen in this scan. + seen_fingerprint_keys: set = set() + fingerprints_in_scan: List[Dict[str, Any]] = [] + for finding in findings: + rule_id = finding.get("rule_id", "") + resource_id = finding.get("resource_id", "") + evidence_key = finding.get("evidence_key", "") + resource_id_normalized = _normalize_resource_id(resource_id) + normalization_version = "1" - # --- Upsert fingerprints and lifecycles for seen findings ------- - seen_fingerprint_ids: set = set() - for fp in fingerprints_in_scan: - # ON CONFLICT touches fingerprint_hash to force RETURNING id - # even when the row already exists (DO NOTHING returns nothing). - cur.execute( - """ - INSERT INTO finding_fingerprints ( - tenant_id, subscription_id, resource_id_normalized, - rule_id, evidence_key, normalization_version, - fingerprint_version, fingerprint_hash + fp_hash = _compute_fingerprint_hash( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, ) - VALUES (%s, %s, %s, %s, %s, %s, '1', %s) - ON CONFLICT (fingerprint_hash) DO UPDATE - SET fingerprint_hash = EXCLUDED.fingerprint_hash - RETURNING id - """, - ( - fp["tenant_id"], - fp["subscription_id"], - fp["resource_id_normalized"], - fp["rule_id"], - fp["evidence_key"], - fp["normalization_version"], - fp["fingerprint_hash"], - ), - ) - row = cur.fetchone() - fingerprint_id = row[0] - seen_fingerprint_ids.add(fingerprint_id) - - # Lock the lifecycle row if it exists, then decide what to do. - cur.execute( - """ - SELECT id, state, occurrence_count, reopen_count, row_version - FROM finding_lifecycles - WHERE fingerprint_id = %s - FOR UPDATE - """, - (fingerprint_id,), - ) - lc_row = cur.fetchone() + key = (rule_id, resource_id_normalized, evidence_key) + if key not in seen_fingerprint_keys: + seen_fingerprint_keys.add(key) + fingerprints_in_scan.append( + { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "resource_id_normalized": resource_id_normalized, + "rule_id": rule_id, + "evidence_key": evidence_key, + "normalization_version": normalization_version, + "fingerprint_hash": fp_hash, + } + ) - if lc_row is None: - # New finding: create lifecycle in OPEN state. + # --- Upsert fingerprints and lifecycles for seen findings ------- + seen_fingerprint_ids: set = set() + for fp in fingerprints_in_scan: + # ON CONFLICT DO UPDATE forces RETURNING id on pre-existing rows. cur.execute( """ - INSERT INTO finding_lifecycles ( - fingerprint_id, state, first_seen_scan_id, - last_seen_scan_id, occurrence_count, - consecutive_success_count, reopen_count, row_version + INSERT INTO finding_fingerprints ( + tenant_id, subscription_id, resource_id_normalized, + rule_id, evidence_key, normalization_version, + fingerprint_version, fingerprint_hash ) - VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + VALUES (%s, %s, %s, %s, %s, %s, '1', %s) + ON CONFLICT (fingerprint_hash) DO UPDATE + SET fingerprint_hash = EXCLUDED.fingerprint_hash RETURNING id """, - (fingerprint_id, scan_id, scan_id), + ( + fp["tenant_id"], + fp["subscription_id"], + fp["resource_id_normalized"], + fp["rule_id"], + fp["evidence_key"], + fp["normalization_version"], + fp["fingerprint_hash"], + ), ) - lifecycle_id = cur.fetchone()[0] + row = cur.fetchone() + fingerprint_id = row[0] + seen_fingerprint_ids.add(fingerprint_id) + cur.execute( """ - INSERT INTO finding_lifecycle_transitions - (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') + SELECT id, state, occurrence_count, reopen_count, row_version + FROM finding_lifecycles + WHERE fingerprint_id = %s + FOR UPDATE """, - (lifecycle_id, scan_id), + (fingerprint_id,), ) - else: - lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row - if state in ("OPEN", "REOPENED"): - # Seen again while already open: increment counter. - cur.execute( - """ - UPDATE finding_lifecycles - SET occurrence_count = occurrence_count + 1, - last_seen_scan_id = %s, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s - """, - (scan_id, lifecycle_id), - ) - elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): - # Reappeared after resolution: reopen and reset success streak. + lc_row = cur.fetchone() + + if lc_row is None: cur.execute( """ - UPDATE finding_lifecycles - SET state = 'REOPENED', - last_seen_scan_id = %s, - occurrence_count = occurrence_count + 1, - reopen_count = reopen_count + 1, - consecutive_success_count = 0, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s + INSERT INTO finding_lifecycles ( + fingerprint_id, state, first_seen_scan_id, + last_seen_scan_id, occurrence_count, + consecutive_success_count, reopen_count, row_version + ) + VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + RETURNING id """, - (scan_id, lifecycle_id), + (fingerprint_id, scan_id, scan_id), ) + lifecycle_id = cur.fetchone()[0] cur.execute( """ INSERT INTO finding_lifecycle_transitions (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') """, - (lifecycle_id, state, scan_id), + (lifecycle_id, scan_id), ) + else: + lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row + if state in ("OPEN", "REOPENED"): + cur.execute( + """ + UPDATE finding_lifecycles + SET occurrence_count = occurrence_count + 1, + last_seen_scan_id = %s, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'REOPENED', + last_seen_scan_id = %s, + occurrence_count = occurrence_count + 1, + reopen_count = reopen_count + 1, + consecutive_success_count = 0, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + """, + (lifecycle_id, state, scan_id), + ) - # --- Resolve findings NOT seen in this scan --------------------- - # Only lock and process rows for rules that had a resolving outcome. - # This avoids unnecessary contention on unrelated rules and is more - # efficient on subscriptions with many open findings. - if not resolving_rule_ids: - # No rule produced a clean outcome: nothing can be resolved. - pass - elif seen_fingerprint_ids: - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.fingerprint_id NOT IN %s - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - (tenant_id, subscription_id, tuple(seen_fingerprint_ids), resolving_rule_ids), - ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - else: - # No findings seen at all: resolve for rules with clean outcomes. + # --- Resolve findings NOT seen in this scan ------------------- + # Only process rules that had a resolving outcome (fail-closed). + if not resolving_rule_ids: + pass + elif seen_fingerprint_ids: + # Use != ALL(%s) with a list to avoid single-element tuple + # syntax issues that occur with NOT IN %s. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id != ALL(%s) + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + ( + tenant_id, + subscription_id, + list(seen_fingerprint_ids), + resolving_rule_ids, + ), + ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) + else: + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, resolving_rule_ids), + ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) + + # --- Idempotency sentinel (inserted last) --------------------- cur.execute( """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl + INSERT INTO scan_lifecycle_applications (scan_id, applied_by) + VALUES (%s, 'system') """, - (tenant_id, subscription_id, resolving_rule_ids), + (scan_id,), ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - # --- Idempotency sentinel (inserted last) ----------------------- - cur.execute( - """ - INSERT INTO scan_lifecycle_applications (scan_id, applied_by) - VALUES (%s, 'system') - """, - (scan_id,), - ) + db_conn.commit() + logger.info("Lifecycle application committed for scan %s", scan_id) - db_conn.commit() - logger.info("Lifecycle application committed for scan %s", scan_id) + except Exception: + db_conn.rollback() + logger.error( + "Lifecycle application rolled back for scan %s", scan_id, exc_info=True + ) + raise def _resolve_absent_rows( @@ -324,7 +328,11 @@ def _resolve_absent_rows( scan_id: str, outcome_by_rule: Dict[str, str], ) -> None: - """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules.""" + """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules. + + The SQL query already filters by resolving_rule_ids, so outcome_status is + always in _RESOLVING_STATUSES here. The check is kept as a defensive guard. + """ absent_rows = cur.fetchall() for lc_id, state, row_version, rule_id in absent_rows: outcome_status = outcome_by_rule.get(rule_id) @@ -350,4 +358,3 @@ def _resolve_absent_rows( """, (lc_id, state, scan_id), ) - # else: outcome missing or blocking -> fail closed, no state change. diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 6ebcd965..395454c0 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -31,115 +31,113 @@ def detect_and_publish( """ count = 0 - with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - # ---------------------------------------------------------------- - # 1. persistent_finding: occurrence_count >= 3 and OPEN/REOPENED - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT fl.id AS lifecycle_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.occurrence_count >= %s - """, - (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), - ) - persistent_rows = cur.fetchall() - - # ---------------------------------------------------------------- - # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED - # lifecycles in this subscription - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT ff.rule_id, - array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, - COUNT(*) AS lc_count - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - GROUP BY ff.rule_id - HAVING COUNT(*) >= %s - """, - (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), - ) - cross_rows = cur.fetchall() - - # ---------------------------------------------------------------- - # 3. reopened_finding: reopen_count >= 1 and state == REOPENED - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT fl.id AS lifecycle_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state = 'REOPENED' - AND fl.reopen_count >= %s - """, - (tenant_id, subscription_id, _REOPENED_THRESHOLD), - ) - reopened_rows = cur.fetchall() - - # All detection queries are finished; cursor is closed. Now upsert - # patterns using separate cursor calls to avoid open-cursor overlap. - for row in persistent_rows: - _upsert_pattern( - db_conn, - pattern_type="persistent_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_PERSISTENT_THRESHOLD, - ) - count += 1 - - for row in cross_rows: - lifecycle_ids = row["lifecycle_ids"] - # Publish one pattern per lifecycle in the group so each is - # individually traceable; finding_ids carries the sibling IDs. - for lc_id in lifecycle_ids: - sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + try: + with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + # 1. persistent_finding: occurrence_count >= threshold and OPEN/REOPENED + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.occurrence_count >= %s + """, + (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), + ) + persistent_rows = cur.fetchall() + + # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED lifecycles + cur.execute( + """ + SELECT ff.rule_id, + array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, + COUNT(*) AS lc_count + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + GROUP BY ff.rule_id + HAVING COUNT(*) >= %s + """, + (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), + ) + cross_rows = cur.fetchall() + + # 3. reopened_finding: reopen_count >= 1 and state == REOPENED + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state = 'REOPENED' + AND fl.reopen_count >= %s + """, + (tenant_id, subscription_id, _REOPENED_THRESHOLD), + ) + reopened_rows = cur.fetchall() + + # All detection queries finished; cursor closed. Upsert via fresh cursors + # to avoid open-cursor overlap inside _upsert_pattern. + for row in persistent_rows: + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + for row in cross_rows: + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle so each is individually traceable. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + _upsert_pattern( + db_conn, + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, + ) + count += 1 + + for row in reopened_rows: _upsert_pattern( db_conn, - pattern_type="cross_resource_recurrence", - lifecycle_id=lc_id, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], tenant_id=tenant_id, subscription_id=subscription_id, scan_id=scan_id, - finding_ids=sibling_ids, - threshold=_CROSS_RESOURCE_THRESHOLD, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, ) count += 1 - for row in reopened_rows: - _upsert_pattern( - db_conn, - pattern_type="reopened_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_REOPENED_THRESHOLD, + db_conn.commit() + logger.info( + "Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count ) - count += 1 + return count - db_conn.commit() - logger.info( - "Pattern detection for scan %s: %d pattern(s) upserted", - scan_id, - count, - ) - return count + except Exception: + db_conn.rollback() + logger.error( + "Pattern detection rolled back for scan %s", scan_id, exc_info=True + ) + raise def _upsert_pattern( diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 3b31816c..dd404ada 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -94,6 +94,9 @@ def cursor(self, **_kwargs): def commit(self): self.committed = True + def rollback(self): + pass + def all_executed(self) -> list: return self._cursor_obj.executed if self._cursor_obj else [] @@ -164,6 +167,9 @@ def commit(self): nonlocal committed_count committed_count += 1 + def rollback(self): + pass + svc = LifecycleService() svc.apply_scan( _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, @@ -287,6 +293,51 @@ def test_resolved_finding_seen_again_becomes_reopened(self): # consecutive_success_count must be reset to 0 on reopen. assert any("consecutive_success_count = 0" in s for s in sqls) + def test_rule_a_success_does_not_resolve_rule_b_finding(self): + """Rule-A SUCCESS must not resolve a finding belonging to Rule-B. + + This is the most critical correctness invariant: a clean outcome for one + rule can only affect findings that were produced by that same rule. + """ + # Scan has no findings (empty list), two outcomes: + # RULE-001 SUCCESS -> in resolving_rule_ids + # RULE-002 FAILED -> NOT in resolving_rule_ids + # + # Sequence (empty seen_ids branch): + # 1. idempotency check -> None + # 2. scan_rule_outcomes insert RULE-001 -> None + # 3. scan_rule_outcomes insert RULE-002 -> None + # 4. absent-findings query (ff.rule_id = ANY(['RULE-001'])) -> + # returns only RULE-001's lifecycle row; RULE-002 is excluded by SQL + # 5. UPDATE RESOLVED for RULE-001 row -> None + # 6. transition insert -> None + # 7. idempotency insert -> None + results = [ + None, # idempotency check + None, # scan_rule_outcomes RULE-001 + None, # scan_rule_outcomes RULE-002 + [(10, "OPEN", 0, "RULE-001")], # absent-findings query + None, # UPDATE RESOLVED + None, # transition insert + None, # idempotency insert + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "SUCCESS"), _make_outcome("RULE-002", "FAILED")], + ) + assert conn.committed + # Verify resolving_rule_ids in the SQL params contains only RULE-001. + absent_query = next( + (item for item in conn.all_executed() if "ff.rule_id = ANY" in item[0]), + None, + ) + assert absent_query is not None, "absent-findings query was not issued" + _, params = absent_query + resolving_ids = params[-1] # last param is resolving_rule_ids list + assert "RULE-001" in resolving_ids + assert "RULE-002" not in resolving_ids + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): # REOPENED + seen in scan: occurrence_count increments, no new state transition. # 1. idempotency -> None diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 3884d82e..b3bc7792 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -277,8 +277,7 @@ def test_list_cross_subscription_query_param_rejected(self, app_client): assert resp.status_code == 400 def test_list_returns_only_authorized_subscription(self, app_client): - """Patterns for a different subscription must not be returned to an - authorized user whose JWT contains a different subscription_id.""" + """subscription_id from JWT must be passed as a SQL parameter.""" row_authorized = _sample_pattern_row(sub_id="sub-authorized") mock_db = _mock_db_rows([row_authorized], 1) @@ -289,9 +288,11 @@ def test_list_returns_only_authorized_subscription(self, app_client): ) assert resp.status_code == 200 - data = resp.get_json() - # The query is scoped; verify the mock was called (subscription scoped query). - assert "patterns" in data + # Verify subscription_id was enforced in the SQL WHERE clause parameters. + mock_cursor = mock_db._get_conn.return_value.cursor.return_value + first_call = mock_cursor.execute.call_args_list[0] + _, params = first_call[0] + assert "sub-authorized" in params def test_list_requires_auth(self, app_client): resp = app_client.get("/api/v1/patterns") @@ -353,6 +354,12 @@ def test_get_cross_subscription_returns_404(self, app_client): ) assert resp.status_code == 404 + # Verify subscription_id was enforced as a SQL parameter (not just checked + # in Python), so removing the WHERE clause would break this test. + mock_cursor = mock_db._get_conn.return_value.cursor.return_value + call_args = mock_cursor.execute.call_args + _, params = call_args[0] + assert "sub-other" in params def test_get_requires_auth(self, app_client): resp = app_client.get("/api/v1/patterns/1") From 2f38d4aae74aa34dd685cf0083935e179449d10a Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:21:09 +0100 Subject: [PATCH 06/22] fix(lint): remove unused imports and fix E402 in lifecycle test files Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py | 1 - tests/test_finding_lifecycle.py | 8 +------- tests/test_patterns.py | 2 -- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index d36e9444..25fccefc 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -9,7 +9,6 @@ from typing import Sequence, Union from alembic import op -import sqlalchemy as sa # Revision identifiers, used by Alembic. revision: str = "e1f2a3b4c5d6" diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index dd404ada..fb80f144 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -4,11 +4,7 @@ live PostgreSQL instance. """ -import hashlib -import json -from unittest.mock import MagicMock, call, patch - -import pytest +from collections import deque from api.services.lifecycle_service import ( LifecycleService, @@ -48,8 +44,6 @@ def _make_outcome(rule_id: str, status: str) -> dict: # avoids the "second cursor re-reads from the start" bug noted in code review. # --------------------------------------------------------------------------- -from collections import deque - class _FakeCursor: """Fake psycopg2 cursor backed by a shared result deque.""" diff --git a/tests/test_patterns.py b/tests/test_patterns.py index b3bc7792..a3a7eee7 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -4,8 +4,6 @@ with mocked database queries. """ -import json -import os import secrets import time from unittest.mock import MagicMock, patch From d3176070af911617244f3cb7f452e134a22ed9cf Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:24:55 +0100 Subject: [PATCH 07/22] fix(lint): apply ruff format to all lifecycle PR files Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 9 ++---- api/routes/patterns.py | 4 +-- api/services/lifecycle_service.py | 16 +++------- api/services/pattern_service.py | 8 ++--- scanner/engine.py | 1 + scanner/worker.py | 1 - tests/test_finding_lifecycle.py | 29 ++++++++++--------- tests/test_patterns.py | 20 ++++++------- 8 files changed, 36 insertions(+), 52 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 25fccefc..370f3f87 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -143,20 +143,15 @@ def upgrade() -> None: """ ) - # Indexes for hot query paths - op.execute( - "CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)" - ) + op.execute("CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)") op.execute( "CREATE INDEX ix_finding_lifecycles_state ON finding_lifecycles (state) WHERE state IN ('OPEN', 'REOPENED')" ) op.execute( "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" ) - op.execute( - "CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)" - ) + op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)") def downgrade() -> None: diff --git a/api/routes/patterns.py b/api/routes/patterns.py index abe7e805..1c245d9c 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -11,9 +11,7 @@ patterns_bp = Blueprint("patterns", __name__) logger = logging.getLogger(__name__) -_ALLOWED_PATTERN_TYPES = frozenset( - {"persistent_finding", "cross_resource_recurrence", "reopened_finding"} -) +_ALLOWED_PATTERN_TYPES = frozenset({"persistent_finding", "cross_resource_recurrence", "reopened_finding"}) _DEFAULT_LIMIT = 20 _MAX_LIMIT = 100 _MIN_LIMIT = 1 diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 67729f9e..3b31b357 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -80,9 +80,7 @@ def apply_scan( (scan_id,), ) if cur.fetchone() is not None: - logger.info( - "Scan %s already applied; skipping lifecycle update", scan_id - ) + logger.info("Scan %s already applied; skipping lifecycle update", scan_id) return # --- Write durable per-rule outcome records ------------------- @@ -113,17 +111,13 @@ def apply_scan( # Build a lookup: rule_id -> outcome status outcome_by_rule: Dict[str, str] = { - o["rule_id"]: o["status"] - for o in rule_outcomes - if "rule_id" in o and "status" in o + o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o } # Collect rule IDs that actively confirmed a clean result. Only # these can trigger resolution of absent findings (fail-closed). resolving_rule_ids = [ - rule_id - for rule_id, status in outcome_by_rule.items() - if status in _RESOLVING_STATUSES + rule_id for rule_id, status in outcome_by_rule.items() if status in _RESOLVING_STATUSES ] # Build the set of fingerprints seen in this scan. @@ -317,9 +311,7 @@ def apply_scan( except Exception: db_conn.rollback() - logger.error( - "Lifecycle application rolled back for scan %s", scan_id, exc_info=True - ) + logger.error("Lifecycle application rolled back for scan %s", scan_id, exc_info=True) raise diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 395454c0..0a8eb5f5 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -127,16 +127,12 @@ def detect_and_publish( count += 1 db_conn.commit() - logger.info( - "Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count - ) + logger.info("Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count) return count except Exception: db_conn.rollback() - logger.error( - "Pattern detection rolled back for scan %s", scan_id, exc_info=True - ) + logger.error("Pattern detection rolled back for scan %s", scan_id, exc_info=True) raise diff --git a/scanner/engine.py b/scanner/engine.py index fcb76b45..8e578d57 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -12,6 +12,7 @@ try: import azure.core.exceptions as _azure_exc + _AzureHttpResponseError = _azure_exc.HttpResponseError except Exception: _AzureHttpResponseError = None # type: ignore[assignment,misc] diff --git a/scanner/worker.py b/scanner/worker.py index 8f2936f1..f89c593b 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -106,7 +106,6 @@ def run_worker(): extra={"scan_id": scan_id}, ) - SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index fb80f144..87ab29ce 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -166,12 +166,18 @@ def rollback(self): svc = LifecycleService() svc.apply_scan( - _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(False), + SCAN_ID_1, + SUB_ID, + TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) svc.apply_scan( - _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(True), + SCAN_ID_1, + SUB_ID, + TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) @@ -307,13 +313,13 @@ def test_rule_a_success_does_not_resolve_rule_b_finding(self): # 6. transition insert -> None # 7. idempotency insert -> None results = [ - None, # idempotency check - None, # scan_rule_outcomes RULE-001 - None, # scan_rule_outcomes RULE-002 - [(10, "OPEN", 0, "RULE-001")], # absent-findings query - None, # UPDATE RESOLVED - None, # transition insert - None, # idempotency insert + None, # idempotency check + None, # scan_rule_outcomes RULE-001 + None, # scan_rule_outcomes RULE-002 + [(10, "OPEN", 0, "RULE-001")], # absent-findings query + None, # UPDATE RESOLVED + None, # transition insert + None, # idempotency insert ] conn = self._run( results, @@ -351,10 +357,7 @@ def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): sqls = self._all_sql(conn) # No transition record should be emitted for REOPENED->REOPENED. - transition_to_reopened = [ - s for s in sqls - if "finding_lifecycle_transitions" in s and "REOPENED" in s - ] + transition_to_reopened = [s for s in sqls if "finding_lifecycle_transitions" in s and "REOPENED" in s] assert not transition_to_reopened # occurrence_count should increment. diff --git a/tests/test_patterns.py b/tests/test_patterns.py index a3a7eee7..07b2933a 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -108,8 +108,8 @@ def test_persistent_finding_detected_when_occurrence_ge_3(self): count, conn = self._run( [ [{"lifecycle_id": 10}], # persistent_finding - [], # cross_resource_recurrence - [], # reopened_finding + [], # cross_resource_recurrence + [], # reopened_finding ] ) assert count == 1 @@ -144,19 +144,19 @@ def test_pattern_response_includes_threshold_and_algorithm_version(self): """The upsert call must include threshold and algorithm_version.""" from api.services.pattern_service import PatternService, _ALGORITHM_VERSION, _PERSISTENT_THRESHOLD - conn = _FakeConn([ - [{"lifecycle_id": 10}], # persistent_finding - [], # cross_resource_recurrence - [], # reopened_finding - ]) + conn = _FakeConn( + [ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ] + ) svc = PatternService() svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) # Find the INSERT INTO patterns call and verify the params. executed = conn._cursor_obj.executed - insert_sql, params = next( - ((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None) - ) + insert_sql, params = next(((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None)) assert insert_sql is not None # params order: pattern_type, lifecycle_id, tenant_id, subscription_id, # scan_id, finding_ids, threshold, algorithm_version From 76b6030ae8bb878c1bbd67cbe655530c2558623b Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:39:12 +0100 Subject: [PATCH 08/22] fix: apply code review findings for lifecycle PR - Add _VALID_OUTCOME_STATUSES allowlist; unknown status defaults to FAILED instead of letting DB constraint throw IntegrityError mid-transaction - Fix dead-code if/elif block: restructure to if resolving_rule_ids / nested if seen_fingerprint_ids (fail-closed logic now clear at a glance) - Add tenant_id to ix_patterns_sub_created index (was subscription_id only; multi-tenant queries filter on both columns) - Add CASCADE to all downgrade() DROP TABLE statements (future FK safety) - Add row_version comment clarifying it is a monotonic counter, not OCC - Fix _FakeConn in test_patterns.py to share a single deque across all cursor() calls, matching test_finding_lifecycle.py and real psycopg2 behaviour - Add REOPENED -> RESOLVED test covering finding absent from SUCCESS scan Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 16 +-- api/services/lifecycle_service.py | 99 +++++++++++-------- tests/test_finding_lifecycle.py | 23 +++++ tests/test_patterns.py | 35 +++++-- 4 files changed, 117 insertions(+), 56 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 370f3f87..c856176e 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -92,7 +92,7 @@ def upgrade() -> None: occurrence_count INTEGER NOT NULL DEFAULT 1, consecutive_success_count INTEGER NOT NULL DEFAULT 0, reopen_count INTEGER NOT NULL DEFAULT 0, - row_version INTEGER NOT NULL DEFAULT 0, + row_version INTEGER NOT NULL DEFAULT 0, -- monotonic update counter (FOR UPDATE handles serializability) created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), CONSTRAINT ck_finding_lifecycles_state @@ -151,13 +151,13 @@ def upgrade() -> None: op.execute( "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" ) - op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)") + op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (tenant_id, subscription_id, created_at DESC)") def downgrade() -> None: - op.execute("DROP TABLE IF EXISTS patterns") - op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") - op.execute("DROP TABLE IF EXISTS finding_lifecycles") - op.execute("DROP TABLE IF EXISTS finding_fingerprints") - op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications") - op.execute("DROP TABLE IF EXISTS scan_rule_outcomes") + op.execute("DROP TABLE IF EXISTS patterns CASCADE") + op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions CASCADE") + op.execute("DROP TABLE IF EXISTS finding_lifecycles CASCADE") + op.execute("DROP TABLE IF EXISTS finding_fingerprints CASCADE") + op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications CASCADE") + op.execute("DROP TABLE IF EXISTS scan_rule_outcomes CASCADE") diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 3b31b357..4c9af341 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -10,6 +10,19 @@ # Statuses that mean "we actively confirmed this rule was clean in the scan." _RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) +# All statuses the DB constraint accepts. Any other value defaults to FAILED to +# prevent a single malformed outcome from aborting the whole lifecycle transaction. +_VALID_OUTCOME_STATUSES = frozenset( + { + "SUCCESS", + "EMPTY_SUCCESS", + "PERMISSION_DENIED", + "TIMEOUT", + "FAILED", + "NOT_APPLICABLE", + } +) + def _normalize_resource_id(resource_id: str) -> str: """Return a stable, lowercased, stripped version of an ARM resource ID.""" @@ -90,6 +103,13 @@ def apply_scan( status_o = outcome.get("status", "FAILED") if not rule_id_o: continue + if status_o not in _VALID_OUTCOME_STATUSES: + logger.warning( + "Unknown outcome status %r for rule %s; defaulting to FAILED", + status_o, + rule_id_o, + ) + status_o = "FAILED" cur.execute( """ INSERT INTO scan_rule_outcomes ( @@ -255,46 +275,45 @@ def apply_scan( ) # --- Resolve findings NOT seen in this scan ------------------- - # Only process rules that had a resolving outcome (fail-closed). - if not resolving_rule_ids: - pass - elif seen_fingerprint_ids: - # Use != ALL(%s) with a list to avoid single-element tuple - # syntax issues that occur with NOT IN %s. - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.fingerprint_id != ALL(%s) - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - ( - tenant_id, - subscription_id, - list(seen_fingerprint_ids), - resolving_rule_ids, - ), - ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - else: - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - (tenant_id, subscription_id, resolving_rule_ids), - ) + # Fail-closed: skip resolution entirely if no rule confirmed a + # clean result this scan (all outcomes were FAILED/PERMISSION_DENIED). + if resolving_rule_ids: + if seen_fingerprint_ids: + # Use != ALL(%s) with a list to avoid single-element tuple + # syntax issues that occur with NOT IN %s. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id != ALL(%s) + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + ( + tenant_id, + subscription_id, + list(seen_fingerprint_ids), + resolving_rule_ids, + ), + ) + else: + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, resolving_rule_ids), + ) _resolve_absent_rows(cur, scan_id, outcome_by_rule) # --- Idempotency sentinel (inserted last) --------------------- diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 87ab29ce..23307410 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -338,6 +338,29 @@ def test_rule_a_success_does_not_resolve_rule_b_finding(self): assert "RULE-001" in resolving_ids assert "RULE-002" not in resolving_ids + def test_reopened_finding_absent_from_success_scan_is_resolved(self): + # REOPENED finding not seen in a SUCCESS scan must transition to RESOLVED. + # 1. idempotency -> None + # 2. scan_rule_outcomes insert -> None + # 3. absent-findings query (empty seen_ids branch) -> [(10,'REOPENED',0,'RULE-001')] + # 4. UPDATE to RESOLVED -> None + # 5. transition insert (REOPENED -> RESOLVED) -> None + # 6. idempotency insert -> None + results = [ + None, + None, + [(10, "REOPENED", 0, "RULE-001")], + None, + None, + None, + ] + conn = self._run(results, [], [_make_outcome("RULE-001", "SUCCESS")]) + assert conn.committed + sqls = self._all_sql(conn) + assert any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) + # Transition from REOPENED to RESOLVED must be recorded. + assert any("finding_lifecycle_transitions" in s and "REOPENED" in s and "RESOLVED" in s for s in sqls) + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): # REOPENED + seen in scan: occurrence_count increments, no new state transition. # 1. idempotency -> None diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 07b2933a..3d529977 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -6,6 +6,7 @@ import secrets import time +from collections import deque from unittest.mock import MagicMock, patch import jwt @@ -48,19 +49,24 @@ def _auth_headers(sub_id: str | None = None) -> dict: class _FakeCursor: - def __init__(self, pages: list): - # pages: list of values returned by successive fetchone/fetchall calls - self._pages = list(pages) - self.executed = [] + """Fake cursor backed by a shared deque so all cursors on one connection share state.""" + + def __init__(self, results_deque: deque): + self._results = results_deque + self.executed: list = [] + self._current = None def execute(self, sql, params=None): self.executed.append((sql.strip(), params)) + self._current = self._results.popleft() if self._results else None def fetchone(self): - return self._pages.pop(0) if self._pages else None + return self._current def fetchall(self): - return self._pages.pop(0) if self._pages else [] + if isinstance(self._current, list): + return self._current + return [] if self._current is None else [self._current] def __enter__(self): return self @@ -70,18 +76,31 @@ def __exit__(self, *args): class _FakeConn: + """Fake connection whose cursor() calls share one result deque.""" + def __init__(self, fetchall_pages: list): - self._pages = fetchall_pages + self._deque: deque = deque(fetchall_pages) self.committed = False + self._cursors: list = [] self._cursor_obj = None def cursor(self, **_kwargs): - self._cursor_obj = _FakeCursor(self._pages) + self._cursor_obj = _FakeCursor(self._deque) + self._cursors.append(self._cursor_obj) return self._cursor_obj def commit(self): self.committed = True + def rollback(self): + pass + + def all_executed(self) -> list: + result = [] + for c in self._cursors: + result.extend(c.executed) + return result + # --------------------------------------------------------------------------- # PatternService unit tests From 0cd0a76734d3e18ebbd68cc0b7c8661256c3df11 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:15:31 +0100 Subject: [PATCH 09/22] ci: run required checks for the lifecycle and graph PR stack Signed-off-by: Tanvir Farhad --- .github/workflows/ci.yml | 4 ++++ .github/workflows/codeql.yml | 4 ++++ .github/workflows/dco.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/website.yml | 2 +- tests/test_feature_stack_ci.py | 28 +++++++++++++++++++++++++ 6 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 tests/test_feature_stack_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfaa8141..d7793cfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: branches: - dev - main + - feat/311-finding-lifecycle + - feat/331-evidence-graph-foundation + - feat/332-evidence-graph-node-edge-population + # Post-merge assurance: re-run the full gate on the merged result, so a green # PR run against an older base is not the only evidence for dev and main. push: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3f654e31..7e63b5f7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,6 +5,10 @@ on: branches: - dev - main + - feat/311-finding-lifecycle + - feat/331-evidence-graph-foundation + - feat/332-evidence-graph-node-edge-population + push: branches: - dev diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 719600a1..333c0c49 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -2,7 +2,7 @@ name: Developer Certificate of Origin on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] permissions: contents: read diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index a737a3cb..b82b6c61 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -2,7 +2,7 @@ name: Dependency Review on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] permissions: contents: read diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index a4c83572..81f5f054 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -14,7 +14,7 @@ on: push: branches: [main] pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] workflow_dispatch: permissions: diff --git a/tests/test_feature_stack_ci.py b/tests/test_feature_stack_ci.py new file mode 100644 index 00000000..8071ff5f --- /dev/null +++ b/tests/test_feature_stack_ci.py @@ -0,0 +1,28 @@ +"""Feature-stack PRs must receive the same read-only checks as dev PRs.""" + +from fnmatch import fnmatchcase +from pathlib import Path + +import pytest +import yaml + + +@pytest.mark.parametrize("workflow", ["ci", "codeql", "dco", "dependency-review", "website"]) +@pytest.mark.parametrize( + "base", + [ + "feat/311-finding-lifecycle", + "feat/331-evidence-graph-foundation", + "feat/332-evidence-graph-node-edge-population", + ], +) +def test_feature_stack_receives_required_checks(workflow, base): + config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) + patterns = config["on"]["pull_request"]["branches"] + assert any(fnmatchcase(base, pattern) for pattern in patterns) + + +@pytest.mark.parametrize("workflow", ["ci", "codeql", "website"]) +def test_feature_stack_does_not_expand_push_or_deployment_triggers(workflow): + config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) + assert set(config["on"]["push"]["branches"]) <= {"dev", "main"} From e982180f2a836816d7dff9b689f7534282fb49e4 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:17:08 +0100 Subject: [PATCH 10/22] ci: use a string key for the PostgreSQL service port Signed-off-by: Tanvir Farhad --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7793cfa..3c0ed473 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,7 +577,7 @@ jobs: - name: Start image env: - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_PORT: ${{ job.services.postgres.ports['5432'] }} run: | set -euo pipefail JWT_SECRET="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" From 88c881f2230094b2920e66c08817e70601965868 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:17:22 +0100 Subject: [PATCH 11/22] fix(deps): require patched Werkzeug and multidict releases Signed-off-by: Tanvir Farhad --- requirements-dev.txt | 346 +++++++++++++++++++++---------------------- requirements.in | 4 + requirements.txt | 346 +++++++++++++++++++++---------------------- 3 files changed, 350 insertions(+), 346 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 32cdba87..1bd3066c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -956,177 +956,177 @@ msal-extensions==1.3.1 \ --hash=sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca msrest==0.7.1 \ --hash=sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32 -multidict==6.8.0 \ - --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ - --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ - --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ - --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ - --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ - --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ - --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ - --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ - --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ - --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ - --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ - --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ - --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ - --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ - --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ - --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ - --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ - --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ - --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ - --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ - --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ - --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ - --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ - --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ - --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ - --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ - --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ - --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ - --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ - --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ - --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ - --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ - --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ - --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ - --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ - --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ - --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ - --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ - --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ - --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ - --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ - --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ - --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ - --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ - --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ - --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ - --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ - --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ - --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ - --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ - --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ - --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ - --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ - --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ - --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ - --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ - --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ - --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ - --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ - --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ - --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ - --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ - --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ - --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ - --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ - --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ - --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ - --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ - --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ - --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ - --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ - --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ - --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ - --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ - --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ - --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ - --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ - --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ - --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ - --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ - --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ - --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ - --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ - --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ - --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ - --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ - --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ - --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ - --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ - --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ - --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ - --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ - --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ - --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ - --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ - --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ - --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ - --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ - --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ - --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ - --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ - --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ - --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ - --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ - --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ - --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ - --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ - --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ - --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ - --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ - --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ - --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ - --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ - --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ - --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ - --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ - --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ - --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ - --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ - --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ - --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ - --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ - --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ - --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ - --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ - --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ - --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ - --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ - --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ - --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ - --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ - --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ - --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ - --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ - --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ - --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ - --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ - --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ - --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ - --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ - --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ - --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ - --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ - --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ - --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ - --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ - --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ - --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ - --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ - --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ - --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ - --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ - --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ - --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ - --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ - --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ - --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ - --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ - --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ - --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ - --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ - --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ - --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ - --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ - --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ - --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ - --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ - --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ - --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ - --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 oauthlib==4.0.0 \ --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd packaging==26.3 \ @@ -1483,8 +1483,8 @@ urllib3==2.8.0 \ --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 websocket-client==1.9.2 \ --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab yarl==1.25.1 \ --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ diff --git a/requirements.in b/requirements.in index 1c8abe85..03e1419e 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,9 @@ # Direct runtime dependencies. Regenerate locks with scripts/lock_dependencies.py. flask==3.1.3 +# Security floors for transitive runtime dependencies. +# GHSA-g6x2-hccm-hh4m and GHSA-54p9-h82j-f925. +werkzeug>=3.1.9 +multidict>=6.9.1 flask-cors==6.0.5 alembic==1.18.5 azure-identity==1.25.3 diff --git a/requirements.txt b/requirements.txt index 45eb9a8a..6897a858 100644 --- a/requirements.txt +++ b/requirements.txt @@ -833,177 +833,177 @@ msal-extensions==1.3.1 \ --hash=sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca msrest==0.7.1 \ --hash=sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32 -multidict==6.8.0 \ - --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ - --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ - --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ - --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ - --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ - --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ - --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ - --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ - --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ - --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ - --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ - --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ - --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ - --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ - --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ - --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ - --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ - --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ - --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ - --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ - --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ - --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ - --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ - --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ - --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ - --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ - --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ - --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ - --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ - --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ - --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ - --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ - --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ - --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ - --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ - --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ - --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ - --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ - --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ - --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ - --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ - --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ - --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ - --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ - --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ - --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ - --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ - --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ - --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ - --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ - --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ - --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ - --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ - --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ - --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ - --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ - --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ - --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ - --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ - --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ - --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ - --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ - --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ - --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ - --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ - --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ - --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ - --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ - --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ - --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ - --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ - --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ - --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ - --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ - --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ - --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ - --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ - --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ - --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ - --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ - --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ - --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ - --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ - --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ - --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ - --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ - --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ - --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ - --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ - --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ - --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ - --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ - --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ - --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ - --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ - --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ - --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ - --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ - --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ - --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ - --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ - --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ - --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ - --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ - --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ - --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ - --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ - --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ - --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ - --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ - --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ - --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ - --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ - --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ - --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ - --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ - --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ - --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ - --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ - --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ - --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ - --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ - --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ - --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ - --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ - --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ - --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ - --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ - --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ - --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ - --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ - --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ - --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ - --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ - --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ - --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ - --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ - --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ - --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ - --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ - --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ - --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ - --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ - --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ - --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ - --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ - --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ - --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ - --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ - --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ - --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ - --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ - --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ - --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ - --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ - --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ - --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ - --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ - --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ - --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ - --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ - --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ - --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ - --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ - --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ - --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ - --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ - --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ - --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ - --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 oauthlib==4.0.0 \ --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd packaging==26.3 \ @@ -1324,8 +1324,8 @@ urllib3==2.8.0 \ --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 websocket-client==1.9.2 \ --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab yarl==1.25.1 \ --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ From 9339a3ce48a3981545e6ddd1f7631f59b7150382 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:30:13 +0100 Subject: [PATCH 12/22] test(ci): match validated PostgreSQL service expression Signed-off-by: Tanvir Farhad --- tests/test_container_runtime_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_container_runtime_config.py b/tests/test_container_runtime_config.py index f0bea389..6e8ddbcc 100644 --- a/tests/test_container_runtime_config.py +++ b/tests/test_container_runtime_config.py @@ -121,7 +121,7 @@ def test_ci_builds_starts_and_scans_one_required_image(): assert '"$IMAGE_TAG"' in start assert "--network host" in start assert "127.0.0.1:${POSTGRES_PORT}/ci_db" in start - assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports[5432] }}" + assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports['5432'] }}" assert "OPENSHIELD_ENV=production" in start assert "secrets.token_urlsafe(32)" in start assert "/ready" in verify From 76a656c14dfb06569e5b21c146793ab5ada45e71 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 12:06:34 +0100 Subject: [PATCH 13/22] fix(lifecycle): add exc_info, autocommit guard, and remove dead param - worker.py: add exc_info=True to lifecycle error log so stack traces appear in production; remove unused lc_exc variable name - lifecycle_service.py: assert autocommit=False at apply_scan entry to catch misconfigured connections before any writes happen - patterns.py: remove unused effective_sub parameter from _effective_tenant - .env.example: document OPENSHIELD_TENANT_ID Signed-off-by: Tanvir Farhad --- .env.example | 3 +++ api/routes/patterns.py | 6 +++--- api/services/lifecycle_service.py | 3 +++ scanner/worker.py | 6 +++--- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.env.example b/.env.example index 4812d812..4c7c79e9 100644 --- a/.env.example +++ b/.env.example @@ -54,3 +54,6 @@ SENTRY_DSN= # Optional - enables AZ-SC-007/008 (Azure DevOps pipeline scanning) AZURE_DEVOPS_ORG_URL= AZURE_DEVOPS_PROJECT= + +# Optional - tenant identifier used for lifecycle tracking (defaults to subscription_id if unset) +OPENSHIELD_TENANT_ID= diff --git a/api/routes/patterns.py b/api/routes/patterns.py index d11bbabe..2310ca3b 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -40,7 +40,7 @@ def _validate_limit(raw: str) -> int: return value -def _effective_tenant(effective_sub: str) -> str: +def _effective_tenant() -> str: """Use the token's verified tenant; deployment configuration cannot override it.""" tenant = (getattr(g, "user", {}) or {}).get("tenant") if not tenant: @@ -106,7 +106,7 @@ def list_patterns(): limit = _validate_limit(request.args["limit"]) effective_sub = _effective_subscription(subscription_id_param) - tenant_id = _effective_tenant(effective_sub) + tenant_id = _effective_tenant() db = _get_db() conn = db._get_conn() @@ -169,7 +169,7 @@ def get_pattern(pattern_id: int): # Resolve scope first so an unscoped caller cannot enumerate IDs. effective_sub = _effective_subscription(None) - tenant_id = _effective_tenant(effective_sub) + tenant_id = _effective_tenant() db = _get_db() conn = db._get_conn() diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 9cb031ef..c478f5ad 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -87,6 +87,9 @@ def apply_scan( """ findings = findings or [] + if getattr(db_conn, "autocommit", False): + raise ValueError("db_conn must have autocommit=False for transactional lifecycle apply") + try: with db_conn.cursor() as cur: # --- Idempotency check ---------------------------------------- diff --git a/scanner/worker.py b/scanner/worker.py index 21fd5bf6..a2b1b27d 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -241,11 +241,11 @@ def run_worker(): subscription_id=subscription_id, tenant_id=tenant_id, ) - except Exception as lc_exc: + except Exception: logger.error( - "Lifecycle/pattern update failed for scan %s (scan data intact): %s", + "Lifecycle/pattern update failed for scan %s (scan data intact)", scan_id, - lc_exc, + exc_info=True, extra={"scan_id": scan_id}, ) From e33ff03beddace4f823f2012cd4345f457d10eff Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 16:02:28 +0100 Subject: [PATCH 14/22] fix(lifecycle): resolve evaluate()-only rules when no rule_outcomes entry is present Signed-off-by: Tanvir Farhad --- api/services/lifecycle_service.py | 16 +++++++++++++++- tests/test_finding_lifecycle.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index c478f5ad..a3e070f4 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -139,6 +139,17 @@ def apply_scan( o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o } + # evaluate()-only rules (engine skips scan() for them, so they + # never produce a rule_outcomes entry). Treat such a rule as having + # completed cleanly if it produced at least one non-ERROR evaluation. + evaluated_cleanly: set = { + e.get("rule_id") + for e in (evaluations or []) + if isinstance(e.get("rule_id"), str) + and e.get("rule_id") not in outcome_by_rule + and e.get("status") != "ERROR" + } + # Rule-level completion is not proof that any particular resource # passed. Conflicting or incomplete coverage must fail closed. resource_statuses: Dict[tuple, set] = {} @@ -152,7 +163,10 @@ def apply_scan( passing_resources = { key for key, statuses in resource_statuses.items() - if statuses == {"PASS"} and outcome_by_rule.get(key[0]) in _RESOLVING_STATUSES + if statuses == {"PASS"} and ( + outcome_by_rule.get(key[0]) in _RESOLVING_STATUSES + or key[0] in evaluated_cleanly + ) } # A violation contradicts a PASS for the same rule/resource, # even when the new finding has a different evidence fingerprint. diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 3203d63e..0f8094eb 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -463,3 +463,34 @@ def test_current_finding_overrides_contradictory_resource_pass(): evaluations=[{"rule_id": "RULE-001", "resource_id": "/rg/foo", "status": "PASS"}], ) assert not any("FROM finding_lifecycles fl" in sql for sql, _ in conn.all_executed()) + + +def test_evaluate_only_rule_can_resolve_without_rule_outcomes_entry(): + """evaluate()-only rules (PR #381+) never emit rule_outcomes; they must still resolve.""" + conn = _FakeConn([None, [(10, "OPEN", 0, "EVAL-RULE", "/rg/res")], None, None, None]) + LifecycleService().apply_scan( + conn, + SCAN_ID_1, + SUB_ID, + TENANT_ID, + [], # no rule_outcomes -- engine skipped scan() for this evaluate()-only rule + [], + evaluations=[{"rule_id": "EVAL-RULE", "resource_id": "/rg/res", "status": "PASS"}], + ) + resolved_sqls = [sql for sql, _ in conn.all_executed() if "SET state = 'RESOLVED'" in sql] + assert resolved_sqls, "evaluate()-only PASS should resolve the finding" + + +def test_evaluate_only_rule_with_error_status_cannot_resolve(): + """An evaluate()-only rule where all evaluations are ERROR must not resolve.""" + conn = _FakeConn([None, None, None]) + LifecycleService().apply_scan( + conn, + SCAN_ID_1, + SUB_ID, + TENANT_ID, + [], + [], + evaluations=[{"rule_id": "EVAL-RULE", "resource_id": "/rg/res", "status": "ERROR"}], + ) + assert not any("SET state = 'RESOLVED'" in sql for sql, _ in conn.all_executed()) From ef8f6e71b168b544725dadc19d0e0d38cdff88ba Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 17:06:16 +0100 Subject: [PATCH 15/22] fix(lint): apply ruff format to lifecycle_service evaluate-only fix Signed-off-by: Tanvir Farhad --- api/services/lifecycle_service.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index a3e070f4..3f804b32 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -163,10 +163,8 @@ def apply_scan( passing_resources = { key for key, statuses in resource_statuses.items() - if statuses == {"PASS"} and ( - outcome_by_rule.get(key[0]) in _RESOLVING_STATUSES - or key[0] in evaluated_cleanly - ) + if statuses == {"PASS"} + and (outcome_by_rule.get(key[0]) in _RESOLVING_STATUSES or key[0] in evaluated_cleanly) } # A violation contradicts a PASS for the same rule/resource, # even when the new finding has a different evidence fingerprint. From cd0c23d2a2c66068d9ad341a6cba4d7aca715efd Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 15 Oct 2026 23:27:39 +0100 Subject: [PATCH 16/22] revert(ci): remove ephemeral branch triggers and port-expression change Signed-off-by: Tanvir Farhad --- .github/workflows/ci.yml | 5 +---- .github/workflows/codeql.yml | 3 --- .github/workflows/dco.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/website.yml | 2 +- tests/test_container_runtime_config.py | 2 +- tests/test_feature_stack_ci.py | 28 ------------------------- 7 files changed, 5 insertions(+), 39 deletions(-) delete mode 100644 tests/test_feature_stack_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c0ed473..66a57a27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,9 +5,6 @@ on: branches: - dev - main - - feat/311-finding-lifecycle - - feat/331-evidence-graph-foundation - - feat/332-evidence-graph-node-edge-population # Post-merge assurance: re-run the full gate on the merged result, so a green # PR run against an older base is not the only evidence for dev and main. @@ -577,7 +574,7 @@ jobs: - name: Start image env: - POSTGRES_PORT: ${{ job.services.postgres.ports['5432'] }} + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} run: | set -euo pipefail JWT_SECRET="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 7e63b5f7..1eac598f 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,9 +5,6 @@ on: branches: - dev - main - - feat/311-finding-lifecycle - - feat/331-evidence-graph-foundation - - feat/332-evidence-graph-node-edge-population push: branches: diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 333c0c49..719600a1 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -2,7 +2,7 @@ name: Developer Certificate of Origin on: pull_request: - branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] + branches: [dev, main] permissions: contents: read diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index b82b6c61..a737a3cb 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -2,7 +2,7 @@ name: Dependency Review on: pull_request: - branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] + branches: [dev, main] permissions: contents: read diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 81f5f054..a4c83572 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -14,7 +14,7 @@ on: push: branches: [main] pull_request: - branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] + branches: [dev, main] workflow_dispatch: permissions: diff --git a/tests/test_container_runtime_config.py b/tests/test_container_runtime_config.py index 6e8ddbcc..f0bea389 100644 --- a/tests/test_container_runtime_config.py +++ b/tests/test_container_runtime_config.py @@ -121,7 +121,7 @@ def test_ci_builds_starts_and_scans_one_required_image(): assert '"$IMAGE_TAG"' in start assert "--network host" in start assert "127.0.0.1:${POSTGRES_PORT}/ci_db" in start - assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports['5432'] }}" + assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports[5432] }}" assert "OPENSHIELD_ENV=production" in start assert "secrets.token_urlsafe(32)" in start assert "/ready" in verify diff --git a/tests/test_feature_stack_ci.py b/tests/test_feature_stack_ci.py deleted file mode 100644 index 8071ff5f..00000000 --- a/tests/test_feature_stack_ci.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Feature-stack PRs must receive the same read-only checks as dev PRs.""" - -from fnmatch import fnmatchcase -from pathlib import Path - -import pytest -import yaml - - -@pytest.mark.parametrize("workflow", ["ci", "codeql", "dco", "dependency-review", "website"]) -@pytest.mark.parametrize( - "base", - [ - "feat/311-finding-lifecycle", - "feat/331-evidence-graph-foundation", - "feat/332-evidence-graph-node-edge-population", - ], -) -def test_feature_stack_receives_required_checks(workflow, base): - config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) - patterns = config["on"]["pull_request"]["branches"] - assert any(fnmatchcase(base, pattern) for pattern in patterns) - - -@pytest.mark.parametrize("workflow", ["ci", "codeql", "website"]) -def test_feature_stack_does_not_expand_push_or_deployment_triggers(workflow): - config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) - assert set(config["on"]["push"]["branches"]) <= {"dev", "main"} From 4d99b3b3a8d4d62f0651b83b31a45dcb8ac755c4 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 15 Oct 2026 23:30:59 +0100 Subject: [PATCH 17/22] fix(lifecycle): normalize tenant_id to lowercase on write and read paths Signed-off-by: Tanvir Farhad --- .env.example | 5 ++++- api/auth.py | 2 +- scanner/worker.py | 8 +++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index 4c7c79e9..0cb6c23d 100644 --- a/.env.example +++ b/.env.example @@ -55,5 +55,8 @@ SENTRY_DSN= AZURE_DEVOPS_ORG_URL= AZURE_DEVOPS_PROJECT= -# Optional - tenant identifier used for lifecycle tracking (defaults to subscription_id if unset) +# Optional - tenant identifier used for lifecycle tracking (defaults to subscription_id if unset). +# Set to your Entra tenant ID (lowercase UUID), e.g. 12345678-abcd-ef01-2345-6789abcdef01. +# If unset, patterns will be scoped by subscription_id and the /patterns endpoint will return +# empty results for OIDC callers whose token carries a different tenant claim. OPENSHIELD_TENANT_ID= diff --git a/api/auth.py b/api/auth.py index 9c8fe918..16491528 100644 --- a/api/auth.py +++ b/api/auth.py @@ -200,7 +200,7 @@ def _verify_shared_secret(self, token: str) -> Dict[str, Any]: return { "sub": claims["sub"], "role": role, - "tenant": _verified_scope_claim(claims, "tid"), + "tenant": (_verified_scope_claim(claims, "tid") or "").lower() or None, "issuer": claims.get("iss"), "auth_mode": SHARED_SECRET_MODE, **_subscription_scope(claims), diff --git a/scanner/worker.py b/scanner/worker.py index a2b1b27d..7600dc4e 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -223,7 +223,13 @@ def run_worker(): # the scan is already persisted, so we log and continue rather # than marking the scan as failed. try: - tenant_id = os.environ.get("OPENSHIELD_TENANT_ID", subscription_id) + _raw_tenant = os.environ.get("OPENSHIELD_TENANT_ID") + if not _raw_tenant: + logger.warning( + "OPENSHIELD_TENANT_ID not set; lifecycle patterns will be scoped " + "by subscription_id which may not match token tenant claims" + ) + tenant_id = (_raw_tenant or subscription_id).lower() lc_svc = LifecycleService() lc_svc.apply_scan( db_conn=db._get_conn(), From db0d1fa7b8f48b79b85509e1c7bc5ced3854f5b7 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 15 Oct 2026 23:36:25 +0100 Subject: [PATCH 18/22] fix(lifecycle): exclude rules in failed_rule_ids from resolution Signed-off-by: Tanvir Farhad --- api/services/lifecycle_service.py | 11 ++++++++++- scanner/worker.py | 1 + tests/test_finding_lifecycle.py | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 3f804b32..865a0834 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -72,6 +72,7 @@ def apply_scan( rule_outcomes: List[Dict[str, Any]], findings: Optional[List[Dict[str, Any]]] = None, evaluations: Optional[List[Dict[str, Any]]] = None, + failed_rule_ids: Optional[set] = None, ) -> None: """Apply a completed scan's findings to the lifecycle tables. @@ -84,8 +85,12 @@ def apply_scan( evaluations: Current per-resource PASS/FAIL/UNKNOWN coverage. findings: List of finding dicts with keys 'rule_id', 'resource_id', and optionally 'evidence_key'. Defaults to empty list. + failed_rule_ids: Set of rule IDs the engine flagged as partially or + fully failed. PASS evaluations from these rules never resolve + findings, keeping lifecycle state consistent with compliance scoring. """ findings = findings or [] + _failed = failed_rule_ids or set() if getattr(db_conn, "autocommit", False): raise ValueError("db_conn must have autocommit=False for transactional lifecycle apply") @@ -141,12 +146,15 @@ def apply_scan( # evaluate()-only rules (engine skips scan() for them, so they # never produce a rule_outcomes entry). Treat such a rule as having - # completed cleanly if it produced at least one non-ERROR evaluation. + # completed cleanly if it produced at least one non-ERROR evaluation + # AND it is not in failed_rule_ids (a partially-failed rule must not + # resolve findings even if some of its evaluations are PASS). evaluated_cleanly: set = { e.get("rule_id") for e in (evaluations or []) if isinstance(e.get("rule_id"), str) and e.get("rule_id") not in outcome_by_rule + and e.get("rule_id") not in _failed and e.get("status") != "ERROR" } @@ -164,6 +172,7 @@ def apply_scan( key for key, statuses in resource_statuses.items() if statuses == {"PASS"} + and key[0] not in _failed and (outcome_by_rule.get(key[0]) in _RESOLVING_STATUSES or key[0] in evaluated_cleanly) } # A violation contradicts a PASS for the same rule/resource, diff --git a/scanner/worker.py b/scanner/worker.py index 7600dc4e..f3f1912f 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -239,6 +239,7 @@ def run_worker(): rule_outcomes=result.get("rule_outcomes", []), findings=result.get("findings", []), evaluations=result.get("evaluations", []), + failed_rule_ids=set(result.get("failed_rule_ids", [])), ) pat_svc = PatternService() pat_svc.detect_and_publish( diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 0f8094eb..011be369 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -494,3 +494,24 @@ def test_evaluate_only_rule_with_error_status_cannot_resolve(): evaluations=[{"rule_id": "EVAL-RULE", "resource_id": "/rg/res", "status": "ERROR"}], ) assert not any("SET state = 'RESOLVED'" in sql for sql, _ in conn.all_executed()) + + +def test_failed_rule_id_blocks_resolution_despite_pass_evaluation(): + """A rule in failed_rule_ids must not resolve findings even when evaluations include PASS.""" + conn = _FakeConn([None, [(10, "OPEN", 0, "PARTIAL-RULE", "/rg/res")], None, None, None]) + LifecycleService().apply_scan( + conn, + SCAN_ID_1, + SUB_ID, + TENANT_ID, + [], + [], + evaluations=[ + {"rule_id": "PARTIAL-RULE", "resource_id": "/rg/res", "status": "PASS"}, + {"rule_id": "PARTIAL-RULE", "resource_id": "/rg/res2", "status": "ERROR"}, + ], + failed_rule_ids={"PARTIAL-RULE"}, + ) + assert not any( + "SET state = 'RESOLVED'" in sql for sql, _ in conn.all_executed() + ), "PASS resource from a failed rule must not be resolved" From 0cd0e8fe58647515f03f76b8f2d93ac78c4ef613 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 15 Oct 2026 23:45:02 +0100 Subject: [PATCH 19/22] fix(lifecycle): add durable lifecycle_jobs queue with bounded retry and fencing Signed-off-by: Tanvir Farhad --- .../e1b3f7a9c2d5_durable_lifecycle_jobs.py | 89 +++++++++ api/models/finding.py | 188 ++++++++++++++++++ scanner/lifecycle_worker.py | 91 +++++++++ scanner/worker.py | 52 +---- 4 files changed, 379 insertions(+), 41 deletions(-) create mode 100644 alembic/versions/e1b3f7a9c2d5_durable_lifecycle_jobs.py create mode 100644 scanner/lifecycle_worker.py diff --git a/alembic/versions/e1b3f7a9c2d5_durable_lifecycle_jobs.py b/alembic/versions/e1b3f7a9c2d5_durable_lifecycle_jobs.py new file mode 100644 index 00000000..3e442e42 --- /dev/null +++ b/alembic/versions/e1b3f7a9c2d5_durable_lifecycle_jobs.py @@ -0,0 +1,89 @@ +"""Add durable, fenced lifecycle application jobs. + +Revision ID: e1b3f7a9c2d5 +Revises: a8d9c2e4f601 +Create Date: 2026-10-15 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "e1b3f7a9c2d5" +down_revision: Union[str, Sequence[str], None] = "a8d9c2e4f601" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create one resumable lifecycle job per scan, and persist rule outcomes.""" + bind = op.get_bind() + inspector = sa.inspect(bind) + existing_cols = {c["name"] for c in inspector.get_columns("scans")} + if "rule_outcomes_json" not in existing_cols: + op.add_column( + "scans", + sa.Column("rule_outcomes_json", postgresql.JSONB(), nullable=True), + ) + if not inspector.has_table("lifecycle_jobs"): + op.create_table( + "lifecycle_jobs", + sa.Column("job_id", postgresql.UUID(), nullable=False), + sa.Column("scan_id", postgresql.UUID(), nullable=False), + sa.Column("status", sa.Text(), nullable=False, server_default=sa.text("'pending'")), + sa.Column("lease_owner", sa.Text(), nullable=True), + sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("fencing_token", sa.BigInteger(), nullable=False, server_default=sa.text("0")), + sa.Column("attempt_count", sa.Integer(), nullable=False, server_default=sa.text("0")), + sa.Column( + "next_retry_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column("error_message", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + nullable=False, + server_default=sa.text("CURRENT_TIMESTAMP"), + ), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint(["scan_id"], ["scans.scan_id"], name="lifecycle_jobs_scan_id_fkey"), + sa.PrimaryKeyConstraint("job_id", name="lifecycle_jobs_pkey"), + sa.UniqueConstraint("scan_id", name="uq_lifecycle_jobs_scan_id"), + sa.CheckConstraint( + "status IN ('pending', 'running', 'completed', 'failed')", + name="ck_lifecycle_jobs_status", + ), + ) + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_lifecycle_jobs_pending_retry") + op.execute( + """ + CREATE INDEX CONCURRENTLY idx_lifecycle_jobs_pending_retry + ON lifecycle_jobs (next_retry_at ASC) + WHERE status = 'pending' + """ + ) + op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_lifecycle_jobs_running_lease") + op.execute( + """ + CREATE INDEX CONCURRENTLY idx_lifecycle_jobs_running_lease + ON lifecycle_jobs (lease_expires_at ASC) + WHERE status = 'running' + """ + ) + + +def downgrade() -> None: + """Remove durable lifecycle work state.""" + with op.get_context().autocommit_block(): + op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_lifecycle_jobs_running_lease") + op.execute("DROP INDEX CONCURRENTLY IF EXISTS idx_lifecycle_jobs_pending_retry") + op.drop_table("lifecycle_jobs") + op.drop_column("scans", "rule_outcomes_json") diff --git a/api/models/finding.py b/api/models/finding.py index c68050a2..93f59afa 100644 --- a/api/models/finding.py +++ b/api/models/finding.py @@ -402,6 +402,7 @@ def save_scan(self, scan_result: Dict[str, Any], lease_owner: str, fencing_token ) if cur.fetchone() is None: raise LostLease(f"Scan {scan_result['scan_id']} is no longer owned by this worker") + rule_outcomes_json = json.dumps(scan_result.get("rule_outcomes") or []) cur.execute( """ UPDATE scans @@ -412,6 +413,7 @@ def save_scan(self, scan_result: Dict[str, Any], lease_owner: str, fencing_token status = 'completed', error_message = NULL, severity_contract_version = %s, + rule_outcomes_json = COALESCE(rule_outcomes_json, %s::jsonb), lease_owner = NULL, lease_expires_at = NULL, -- The mapping-pack provenance captured by a scan_id's @@ -445,6 +447,7 @@ def save_scan(self, scan_result: Dict[str, Any], lease_owner: str, fencing_token score_findings(findings), scan_result.get("cve_enrichment_status", "PENDING"), CONTRACT_VERSION, + rule_outcomes_json, mapping_snapshot, mapping_snapshot, scan_result["scan_id"], @@ -600,6 +603,17 @@ def save_scan(self, scan_result: Dict[str, Any], lease_owner: str, fencing_token """, (str(uuid.uuid4()), scan_result["scan_id"]), ) + # Enqueue lifecycle application in the same transaction so + # lifecycle work is never lost if the worker crashes after + # save_scan() but before the inline apply_scan() call. + cur.execute( + """ + INSERT INTO lifecycle_jobs (job_id, scan_id, status, attempt_count) + VALUES (%s, %s, 'pending', 0) + ON CONFLICT (scan_id) DO NOTHING + """, + (str(uuid.uuid4()), scan_result["scan_id"]), + ) conn.commit() except Exception: # psycopg2 connections remain in an aborted transaction after any @@ -1074,6 +1088,170 @@ def recover_stale_enrichment_jobs(self, max_attempts: int = 3) -> int: self.rollback(conn) raise + # ------------------------------------------------------------------ # + # Lifecycle jobs # + # ------------------------------------------------------------------ # + + def claim_next_lifecycle_job( + self, lease_owner: str, lease_seconds: int + ) -> Optional[Dict[str, Any]]: + """Atomically claim the next retry-ready lifecycle job.""" + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + conn = self._get_conn() + try: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + UPDATE lifecycle_jobs + SET status = 'running', lease_owner = %s, + lease_expires_at = CURRENT_TIMESTAMP + (%s * INTERVAL '1 second'), + last_heartbeat_at = CURRENT_TIMESTAMP, + fencing_token = fencing_token + 1, + attempt_count = attempt_count + 1, + error_message = NULL + WHERE job_id = ( + SELECT job_id FROM lifecycle_jobs + WHERE status = 'pending' + AND next_retry_at <= CURRENT_TIMESTAMP + ORDER BY created_at ASC + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING * + """, + (lease_owner, lease_seconds), + ) + job = cur.fetchone() + conn.commit() + return dict(job) if job else None + except Exception: + self.rollback(conn) + raise + + def complete_lifecycle_job(self, job_id: str, lease_owner: str, fencing_token: int) -> None: + """Atomically complete a fenced lifecycle job.""" + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE lifecycle_jobs + SET status = 'completed', completed_at = CURRENT_TIMESTAMP, + lease_owner = NULL, lease_expires_at = NULL + WHERE job_id = %s AND status = 'running' + AND lease_owner = %s AND fencing_token = %s + AND lease_expires_at > CURRENT_TIMESTAMP + """, + (job_id, lease_owner, fencing_token), + ) + if cur.rowcount != 1: + raise LostLease(f"Lifecycle job {job_id} lost its lease before completion") + conn.commit() + except Exception: + self.rollback(conn) + raise + + def fail_lifecycle_job( + self, + job_id: str, + lease_owner: str, + fencing_token: int, + error_message: str, + *, + max_attempts: int = 3, + retry_seconds: int = 30, + ) -> str: + """Record bounded retry state under the job's current fencing token.""" + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + """ + SELECT attempt_count FROM lifecycle_jobs + WHERE job_id = %s AND status = 'running' + AND lease_owner = %s AND fencing_token = %s + AND lease_expires_at > CURRENT_TIMESTAMP + FOR UPDATE + """, + (job_id, lease_owner, fencing_token), + ) + row = cur.fetchone() + if row is None: + raise LostLease(f"Lifecycle job {job_id} is no longer owned by this worker") + (attempt_count,) = row + terminal = attempt_count >= max_attempts + if terminal: + cur.execute( + """ + UPDATE lifecycle_jobs SET status = 'failed', completed_at = CURRENT_TIMESTAMP, + error_message = %s, lease_owner = NULL, lease_expires_at = NULL + WHERE job_id = %s + """, + (error_message, job_id), + ) + outcome = "failed" + else: + cur.execute( + """ + UPDATE lifecycle_jobs SET status = 'pending', error_message = %s, + lease_owner = NULL, lease_expires_at = NULL, + next_retry_at = CURRENT_TIMESTAMP + (%s * INTERVAL '1 second') + WHERE job_id = %s + """, + (error_message, retry_seconds, job_id), + ) + outcome = "retry" + conn.commit() + return outcome + except Exception: + self.rollback(conn) + raise + + def recover_stale_lifecycle_jobs(self, max_attempts: int = 3) -> int: + """Recover expired lifecycle claims without blocking the worker loop.""" + conn = self._get_conn() + try: + with conn.cursor() as cur: + cur.execute( + """ + WITH stale AS ( + SELECT job_id, attempt_count + FROM lifecycle_jobs + WHERE status = 'running' + AND lease_expires_at < CURRENT_TIMESTAMP + ORDER BY lease_expires_at ASC, job_id ASC + FOR UPDATE SKIP LOCKED + ) + UPDATE lifecycle_jobs AS j + SET status = CASE + WHEN stale.attempt_count >= %(max_attempts)s THEN 'failed' + ELSE 'pending' + END, + completed_at = CASE + WHEN stale.attempt_count >= %(max_attempts)s THEN CURRENT_TIMESTAMP + ELSE j.completed_at + END, + lease_owner = NULL, + lease_expires_at = NULL, + error_message = CASE + WHEN stale.attempt_count >= %(max_attempts)s + THEN 'Lifecycle job exceeded maximum retry attempts after worker interruption.' + ELSE 'Lifecycle worker interrupted; queued for retry.' + END + FROM stale + WHERE j.job_id = stale.job_id + RETURNING j.job_id + """, + {"max_attempts": max_attempts}, + ) + transitions = cur.fetchall() + conn.commit() + return len(transitions) + except Exception: + self.rollback(conn) + raise + def admit_scan( self, scan_id: str, @@ -1368,6 +1546,16 @@ def get_scan(self, scan_id: str) -> Optional[Dict[str, Any]]: row = cur.fetchone() return dict(row) if row else None + def get_rule_evaluations(self, scan_id: str) -> List[Dict[str, Any]]: + """Return per-resource evaluations for one scan as apply_scan() expects.""" + conn = self._get_conn() + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + "SELECT rule_id, resource_id, status FROM rule_evaluations WHERE scan_id = %s", + (scan_id,), + ) + return [dict(row) for row in cur.fetchall()] + def get_scans(self) -> List[Dict[str, Any]]: """Return all scan records ordered by most recent first.""" conn = self._get_conn() diff --git a/scanner/lifecycle_worker.py b/scanner/lifecycle_worker.py new file mode 100644 index 00000000..ddf8df4d --- /dev/null +++ b/scanner/lifecycle_worker.py @@ -0,0 +1,91 @@ +"""Durable lifecycle application job execution for the scan worker.""" + +import logging +import os + +from api.models.finding import DatabaseManager, LostLease +from api.services.lifecycle_service import LifecycleService +from api.services.pattern_service import PatternService + +logger = logging.getLogger("scanner.lifecycle_worker") + + +def process_lifecycle_job( + db: DatabaseManager, + job: dict, + lease_owner: str, + lease_seconds: int, + *, + max_attempts: int = 3, +) -> str: + """Apply lifecycle state for a scan's findings under a fenced job claim. + + Returns one of: "completed", "lost_lease", "failed", "retry". + """ + job_id = str(job["job_id"]) + scan_id = str(job["scan_id"]) + fencing_token = job["fencing_token"] + + _raw_tenant = os.environ.get("OPENSHIELD_TENANT_ID") + if not _raw_tenant: + logger.warning( + "OPENSHIELD_TENANT_ID not set; lifecycle will be scoped by subscription_id" + ) + + try: + scan = db.get_scan(scan_id) + if scan is None: + logger.error("Lifecycle job %s: scan %s not found", job_id, scan_id) + db.complete_lifecycle_job(job_id, lease_owner, fencing_token) + return "completed" + + subscription_id = scan.get("subscription_id", "") + tenant_id = (_raw_tenant or subscription_id).lower() + + rule_outcomes = scan.get("rule_outcomes_json") or [] + findings = db.get_findings({"scan_id": scan_id}) + evaluations = db.get_rule_evaluations(scan_id) + snapshot = scan.get("compliance_mapping_snapshot") or {} + failed_rule_ids = set( + (snapshot.get("_scan_rule_outcomes") or {}).get("failed_rule_ids") or [] + ) + + lc_svc = LifecycleService() + lc_svc.apply_scan( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + rule_outcomes=rule_outcomes, + findings=findings, + evaluations=evaluations, + failed_rule_ids=failed_rule_ids, + ) + + pat_svc = PatternService() + pat_svc.detect_and_publish( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + ) + + db.complete_lifecycle_job(job_id, lease_owner, fencing_token) + logger.info("Completed lifecycle job %s for scan %s", job_id, scan_id) + return "completed" + + except LostLease: + logger.warning("Lifecycle job %s lost its lease", job_id) + return "lost_lease" + except Exception as exc: + retry_seconds = min(300, 30 * (2 ** max(0, job["attempt_count"] - 1))) + outcome = db.fail_lifecycle_job( + job_id, + lease_owner, + fencing_token, + "Lifecycle application failed; see worker logs for details.", + max_attempts=max_attempts, + retry_seconds=retry_seconds, + ) + logger.error("Lifecycle job %s failed (%s): %s", job_id, outcome, exc, exc_info=True) + return outcome diff --git a/scanner/worker.py b/scanner/worker.py index f3f1912f..dc640ade 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -25,10 +25,9 @@ configure_logging, init_sentry, ) -from api.services.lifecycle_service import LifecycleService -from api.services.pattern_service import PatternService from scanner.engine import ScanEngine from scanner.enrichment_worker import process_enrichment_job +from scanner.lifecycle_worker import process_lifecycle_job configure_logging() logger = logging.getLogger("scanner.worker") @@ -159,6 +158,7 @@ def run_worker(): # 1. Cleanup stale scans from previous crashes db.recover_stale_scans() db.recover_stale_enrichment_jobs() + db.recover_stale_lifecycle_jobs() # 2. Publish current queue depth PENDING_SCANS.set(len(db.get_pending_scans())) @@ -172,12 +172,16 @@ def run_worker(): if enrichment_job: process_enrichment_job(db, enrichment_job, worker_id, lease_seconds) + lifecycle_job = db.claim_next_lifecycle_job(worker_id, lease_seconds) + if lifecycle_job: + process_lifecycle_job(db, lifecycle_job, worker_id, lease_seconds) + # 4. Atomic scan claim scan = db.claim_next_pending_scan(worker_id, lease_seconds) if not scan: # Only idle when there was no work at all; an iteration that - # ran an enrichment job polls again immediately. - if not enrichment_job: + # ran an enrichment or lifecycle job polls again immediately. + if not enrichment_job and not lifecycle_job: time.sleep(POLL_INTERVAL_SECONDS) continue @@ -219,43 +223,9 @@ def run_worker(): raise LostLease(f"Scan {scan_id} lost its lease before completion") db.save_scan(result, worker_id, fencing_token) - # Apply lifecycle tracking. Lifecycle failures are non-fatal: - # the scan is already persisted, so we log and continue rather - # than marking the scan as failed. - try: - _raw_tenant = os.environ.get("OPENSHIELD_TENANT_ID") - if not _raw_tenant: - logger.warning( - "OPENSHIELD_TENANT_ID not set; lifecycle patterns will be scoped " - "by subscription_id which may not match token tenant claims" - ) - tenant_id = (_raw_tenant or subscription_id).lower() - lc_svc = LifecycleService() - lc_svc.apply_scan( - db_conn=db._get_conn(), - scan_id=scan_id, - subscription_id=subscription_id, - tenant_id=tenant_id, - rule_outcomes=result.get("rule_outcomes", []), - findings=result.get("findings", []), - evaluations=result.get("evaluations", []), - failed_rule_ids=set(result.get("failed_rule_ids", [])), - ) - pat_svc = PatternService() - pat_svc.detect_and_publish( - db_conn=db._get_conn(), - scan_id=scan_id, - subscription_id=subscription_id, - tenant_id=tenant_id, - ) - except Exception: - logger.error( - "Lifecycle/pattern update failed for scan %s (scan data intact)", - scan_id, - exc_info=True, - extra={"scan_id": scan_id}, - ) - + # Lifecycle application is now durable: save_scan() enqueues a + # lifecycle_job in the same transaction, so apply_scan() will + # run even if this worker crashes immediately after saving. SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", From f7be383a975f02831905fa144b3eb425b8b0d19c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 16 Oct 2026 00:04:15 +0100 Subject: [PATCH 20/22] ci: re-trigger CI on fix commits for PR #326 Signed-off-by: Tanvir Farhad From c83dea5c7f7f2effc8264c01913206010769a50c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 16 Oct 2026 00:33:56 +0100 Subject: [PATCH 21/22] ci: nudge CI for PR #326 fix commits Signed-off-by: Tanvir Farhad --- .env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.example b/.env.example index 0cb6c23d..2eec40fa 100644 --- a/.env.example +++ b/.env.example @@ -60,3 +60,4 @@ AZURE_DEVOPS_PROJECT= # If unset, patterns will be scoped by subscription_id and the /patterns endpoint will return # empty results for OIDC callers whose token carries a different tenant claim. OPENSHIELD_TENANT_ID= + From be5b22f725d6f6dd00d8d83357edb5fc3378f004 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sat, 17 Oct 2026 02:41:45 +0100 Subject: [PATCH 22/22] fix(lifecycle): guard apply_scan against out-of-order scan replay Added a scan ordering guard that checks whether a newer scan has already been applied for the same (subscription, tenant) pair before processing the current scan. If a newer scan exists in scan_lifecycle_applications, the older scan is silently skipped to prevent stale retries from rewinding lifecycle state. Added a unit test (test_older_scan_does_not_overwrite_newer_scan_state) that verifies the ordering guard returns without committing when the fake DB returns a newer scan row. Signed-off-by: Tanvir Farhad --- api/services/lifecycle_service.py | 25 +++++++++++++++++++++++++ tests/test_finding_lifecycle.py | 20 ++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 865a0834..d9106839 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -106,6 +106,31 @@ def apply_scan( logger.info("Scan %s already applied; skipping lifecycle update", scan_id) return + # --- Scan ordering guard -------------------------------------- + # A delayed retry must not rewind state set by a newer scan. + # Skip if any scan completed after this one has already been + # applied for the same (subscription, tenant). + cur.execute( + """ + SELECT sla.scan_id + FROM scan_lifecycle_applications sla + JOIN scans s USING (scan_id) + WHERE s.subscription_id = %s + AND s.tenant_id = %s + AND s.completed_at > ( + SELECT completed_at FROM scans WHERE scan_id = %s + ) + LIMIT 1 + """, + (subscription_id, tenant_id, scan_id), + ) + if cur.fetchone() is not None: + logger.info( + "Scan %s is older than a previously applied scan for %s/%s; skipping", + scan_id, tenant_id, subscription_id, + ) + return + # --- Write durable per-rule outcome records ------------------- # UNIQUE(scan_id, rule_id) prevents duplicates on double-writes. for outcome in rule_outcomes: diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 011be369..2294bdf3 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -496,6 +496,26 @@ def test_evaluate_only_rule_with_error_status_cannot_resolve(): assert not any("SET state = 'RESOLVED'" in sql for sql, _ in conn.all_executed()) +def test_older_scan_does_not_overwrite_newer_scan_state(): + """An older scan applied after a newer one must be silently skipped.""" + # Idempotency check: None (not already applied). + # Ordering guard: returns a row (a newer scan was applied for this sub/tenant). + conn = _FakeConn([None, ("newer-scan-id",)]) + LifecycleService().apply_scan( + conn, + SCAN_ID_1, + SUB_ID, + TENANT_ID, + [{"rule_id": "RULE-A", "status": "SUCCESS"}], + [{"rule_id": "RULE-A", "resource_id": "/rg/res"}], + ) + assert not conn.committed, "Older scan must not commit when a newer scan was already applied" + executed_sqls = [sql for sql, _ in conn.all_executed()] + assert not any("INSERT INTO finding" in sql for sql in executed_sqls), ( + "Older scan must not write any finding rows" + ) + + def test_failed_rule_id_blocks_resolution_despite_pass_evaluation(): """A rule in failed_rule_ids must not resolve findings even when evaluations include PASS.""" conn = _FakeConn([None, [(10, "OPEN", 0, "PARTIAL-RULE", "/rg/res")], None, None, None])