From 0d4461f78399935f586487652dec5a81f3541e5b Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:34:31 +0100 Subject: [PATCH 01/67] feat(graph): add graph_nodes, graph_edges, finding_graph_nodes migration Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 101 ++++++++++++++++++ tests/test_graph_migration.py | 74 +++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 alembic/versions/e1f2a3b4c5d6_graph_schema.py create mode 100644 tests/test_graph_migration.py diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py new file mode 100644 index 00000000..eb55ce5d --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -0,0 +1,101 @@ +"""Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. + +Revision ID: e1f2a3b4c5d6 +Revises: 3f59f83a5253 +Create Date: 2026-09-24 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "3f59f83a5253" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "graph_nodes", + sa.Column("node_id", postgresql.UUID(), nullable=False), + sa.Column("tenant_id", sa.Text(), nullable=False), + sa.Column("subscription_id", sa.Text(), nullable=False), + sa.Column("resource_id", sa.Text(), nullable=False), + sa.Column("resource_type", sa.Text(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("location", sa.Text(), nullable=False, server_default=sa.text("''")), + sa.Column("resource_group", sa.Text(), nullable=False, server_default=sa.text("''")), + sa.Column("snapshot_id", sa.Text(), nullable=False), + sa.Column("properties", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.text("now()")), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.text("now()")), + sa.PrimaryKeyConstraint("node_id", name="graph_nodes_pkey"), + ) + op.create_index("idx_graph_nodes_resource_id", "graph_nodes", ["resource_id"]) + op.create_index("idx_graph_nodes_tenant_subscription", "graph_nodes", + ["tenant_id", "subscription_id"]) + op.create_index( + "uq_graph_nodes_tenant_resource", + "graph_nodes", + ["tenant_id", "resource_id"], + unique=True, + ) + + op.create_table( + "graph_edges", + sa.Column("edge_id", postgresql.UUID(), nullable=False), + sa.Column("source_node_id", postgresql.UUID(), nullable=False), + sa.Column("target_node_id", postgresql.UUID(), nullable=False), + sa.Column("relationship_type", sa.Text(), nullable=False), + sa.Column("evidence_source", sa.Text(), nullable=False), + sa.Column("evidence_snapshot_id", sa.Text(), nullable=False), + sa.Column("confidence", sa.Float(), nullable=False, server_default=sa.text("1.0")), + sa.Column("collected_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.text("now()")), + sa.Column("properties", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), + sa.ForeignKeyConstraint(["source_node_id"], ["graph_nodes.node_id"], + name="graph_edges_source_fkey", ondelete="CASCADE"), + sa.ForeignKeyConstraint(["target_node_id"], ["graph_nodes.node_id"], + name="graph_edges_target_fkey", ondelete="CASCADE"), + sa.PrimaryKeyConstraint("edge_id", name="graph_edges_pkey"), + ) + op.create_index("idx_graph_edges_source", "graph_edges", ["source_node_id"]) + op.create_index("idx_graph_edges_target", "graph_edges", ["target_node_id"]) + op.create_index( + "uq_graph_edges_source_target_type", + "graph_edges", + ["source_node_id", "target_node_id", "relationship_type"], + unique=True, + ) + + op.create_table( + "finding_graph_nodes", + sa.Column("finding_id", sa.Integer(), nullable=False), + sa.Column("node_id", postgresql.UUID(), nullable=False), + sa.ForeignKeyConstraint(["finding_id"], ["findings.id"], + name="finding_graph_nodes_finding_fkey", ondelete="CASCADE"), + sa.ForeignKeyConstraint(["node_id"], ["graph_nodes.node_id"], + name="finding_graph_nodes_node_fkey", ondelete="CASCADE"), + sa.PrimaryKeyConstraint("finding_id", "node_id", name="finding_graph_nodes_pkey"), + ) + op.create_index("idx_finding_graph_nodes_node_id", "finding_graph_nodes", ["node_id"]) + + +def downgrade() -> None: + op.drop_index("idx_finding_graph_nodes_node_id", table_name="finding_graph_nodes") + op.drop_table("finding_graph_nodes") + + op.drop_index("uq_graph_edges_source_target_type", table_name="graph_edges") + op.drop_index("idx_graph_edges_target", table_name="graph_edges") + op.drop_index("idx_graph_edges_source", table_name="graph_edges") + op.drop_table("graph_edges") + + op.drop_index("uq_graph_nodes_tenant_resource", table_name="graph_nodes") + op.drop_index("idx_graph_nodes_tenant_subscription", table_name="graph_nodes") + op.drop_index("idx_graph_nodes_resource_id", table_name="graph_nodes") + op.drop_table("graph_nodes") diff --git a/tests/test_graph_migration.py b/tests/test_graph_migration.py new file mode 100644 index 00000000..21382e4a --- /dev/null +++ b/tests/test_graph_migration.py @@ -0,0 +1,74 @@ +"""Verify the graph schema migration applies and rolls back cleanly.""" +import os +import pytest +from alembic.config import Config +from alembic import command +from sqlalchemy import create_engine, inspect + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/openshield_test") + + +@pytest.fixture(scope="module") +def engine(): + return create_engine(DATABASE_URL) + + +@pytest.fixture(scope="module", autouse=True) +def run_migrations(engine): + cfg = Config("alembic.ini") + command.upgrade(cfg, "head") + yield + command.downgrade(cfg, "e1f2a3b4c5d6-1") # one step back + + +def test_graph_nodes_table_exists(engine): + inspector = inspect(engine) + assert "graph_nodes" in inspector.get_table_names() + + +def test_graph_edges_table_exists(engine): + inspector = inspect(engine) + assert "graph_edges" in inspector.get_table_names() + + +def test_finding_graph_nodes_table_exists(engine): + inspector = inspect(engine) + assert "finding_graph_nodes" in inspector.get_table_names() + + +def test_graph_nodes_columns(engine): + inspector = inspect(engine) + cols = {c["name"] for c in inspector.get_columns("graph_nodes")} + assert cols >= { + "node_id", "tenant_id", "subscription_id", "resource_id", + "resource_type", "name", "location", "resource_group", + "snapshot_id", "properties", "created_at", "updated_at", + } + + +def test_graph_edges_columns(engine): + inspector = inspect(engine) + cols = {c["name"] for c in inspector.get_columns("graph_edges")} + assert cols >= { + "edge_id", "source_node_id", "target_node_id", + "relationship_type", "evidence_source", "evidence_snapshot_id", + "confidence", "collected_at", "properties", + } + + +def test_finding_graph_nodes_columns(engine): + inspector = inspect(engine) + cols = {c["name"] for c in inspector.get_columns("finding_graph_nodes")} + assert cols >= {"finding_id", "node_id"} + + +def test_downgrade_removes_tables(engine): + cfg = Config("alembic.ini") + command.downgrade(cfg, "3f59f83a5253") + inspector = inspect(engine) + tables = inspector.get_table_names() + assert "graph_nodes" not in tables + assert "graph_edges" not in tables + assert "finding_graph_nodes" not in tables + # Restore for subsequent tests + command.upgrade(cfg, "head") From 1db1436ff6d54813b578d4324bd1c40493cd3ced Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:37:58 +0100 Subject: [PATCH 02/67] feat(graph): add snapshot_bridge to collect ARG inventory inside ScanEngine Signed-off-by: Tanvir Farhad --- scanner/graph/__init__.py | 0 scanner/graph/snapshot_bridge.py | 55 +++++++++++++++++++ tests/test_graph_snapshot_bridge.py | 85 +++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+) create mode 100644 scanner/graph/__init__.py create mode 100644 scanner/graph/snapshot_bridge.py create mode 100644 tests/test_graph_snapshot_bridge.py diff --git a/scanner/graph/__init__.py b/scanner/graph/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scanner/graph/snapshot_bridge.py b/scanner/graph/snapshot_bridge.py new file mode 100644 index 00000000..7d079de8 --- /dev/null +++ b/scanner/graph/snapshot_bridge.py @@ -0,0 +1,55 @@ +"""Thin wrapper that collects an ARG InventorySnapshot for use inside ScanEngine.""" +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from scanner.arg_inventory import ArgInventoryClient, InventorySnapshot, InventoryStatus + +if TYPE_CHECKING: + from scanner.azure_client import AzureClient + +logger = logging.getLogger(__name__) + + +def collect_snapshot(client: AzureClient, subscription_id: str) -> InventorySnapshot | None: + """Collect one ARG snapshot for the given subscription. + + Returns None if the collection fails entirely or the ARG service is + unavailable. A PARTIAL snapshot (some pages succeeded) is returned as-is + so callers can still use the resources that were collected. + """ + tenant_id = getattr(client, "tenant_id", None) + credential = getattr(client, "credential", None) + + if not tenant_id or not credential: + logger.warning( + "snapshot_bridge: AzureClient missing tenant_id or credential — skipping ARG collection" + ) + return None + + try: + with ArgInventoryClient(credential=credential) as arg: + snapshot = arg.collect( + tenant_id=tenant_id, + subscription_ids=[subscription_id], + ) + except Exception as exc: + logger.warning("snapshot_bridge: ARG collection failed — %s", exc) + return None + + if snapshot.status == InventoryStatus.FAILED: + logger.warning( + "snapshot_bridge: ARG collection returned FAILED status (errors: %s)", + snapshot.errors, + ) + return None + + if snapshot.status == InventoryStatus.PARTIAL: + logger.warning( + "snapshot_bridge: ARG collection is PARTIAL (%d errors) — continuing with %d resources", + len(snapshot.errors), + len(snapshot.resources), + ) + + return snapshot diff --git a/tests/test_graph_snapshot_bridge.py b/tests/test_graph_snapshot_bridge.py new file mode 100644 index 00000000..e7b94168 --- /dev/null +++ b/tests/test_graph_snapshot_bridge.py @@ -0,0 +1,85 @@ +"""Tests for the snapshot bridge that wraps ArgInventoryClient for use in ScanEngine.""" +from unittest.mock import MagicMock, patch +import pytest + +from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource +from scanner.graph.snapshot_bridge import collect_snapshot + + +def _make_snapshot(status=InventoryStatus.COMPLETE, errors=()): + resource = InventoryResource( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + subscription_id="00000000-0000-0000-0000-000000000002", + resource_id="/subscriptions/00000000-0000-0000-0000-000000000002/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1", + resource_type="microsoft.network/virtualnetworks", + name="vnet1", + location="eastus", + resource_group="rg", + tags={}, + properties={}, + ) + return InventorySnapshot( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + requested_subscriptions=("00000000-0000-0000-0000-000000000002",), + status=status, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=100, + pages=1, + resources=(resource,), + errors=errors, + ) + + +def test_collect_snapshot_returns_snapshot_on_success(): + mock_client = MagicMock() + mock_client.subscription_id = "00000000-0000-0000-0000-000000000002" + mock_client.credential = MagicMock() + mock_client.tenant_id = "00000000-0000-0000-0000-000000000001" + + expected = _make_snapshot() + with patch("scanner.graph.snapshot_bridge.ArgInventoryClient") as MockArg: + instance = MockArg.return_value.__enter__.return_value + instance.collect.return_value = expected + result = collect_snapshot(mock_client, "00000000-0000-0000-0000-000000000002") + + assert result is expected + + +def test_collect_snapshot_returns_none_on_exception(): + mock_client = MagicMock() + mock_client.tenant_id = "00000000-0000-0000-0000-000000000001" + + with patch("scanner.graph.snapshot_bridge.ArgInventoryClient") as MockArg: + MockArg.return_value.__enter__.side_effect = Exception("ARG unavailable") + result = collect_snapshot(mock_client, "00000000-0000-0000-0000-000000000002") + + assert result is None + + +def test_collect_snapshot_returns_none_when_status_failed(): + mock_client = MagicMock() + mock_client.tenant_id = "00000000-0000-0000-0000-000000000001" + + failed = _make_snapshot(status=InventoryStatus.FAILED) + with patch("scanner.graph.snapshot_bridge.ArgInventoryClient") as MockArg: + instance = MockArg.return_value.__enter__.return_value + instance.collect.return_value = failed + result = collect_snapshot(mock_client, "00000000-0000-0000-0000-000000000002") + + assert result is None + + +def test_collect_snapshot_returns_partial_snapshot(): + mock_client = MagicMock() + mock_client.tenant_id = "00000000-0000-0000-0000-000000000001" + + partial = _make_snapshot(status=InventoryStatus.PARTIAL, errors=("one page failed",)) + with patch("scanner.graph.snapshot_bridge.ArgInventoryClient") as MockArg: + instance = MockArg.return_value.__enter__.return_value + instance.collect.return_value = partial + result = collect_snapshot(mock_client, "00000000-0000-0000-0000-000000000002") + + assert result is not None + assert result.status == InventoryStatus.PARTIAL From 079af34db8cba6fdb90efece358748a49cafc750 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:43:09 +0100 Subject: [PATCH 03/67] feat(graph): wire InventorySnapshot into ScanEngine.run_scan() Signed-off-by: Tanvir Farhad --- scanner/engine.py | 13 +++- tests/test_graph_engine_integration.py | 82 ++++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/test_graph_engine_integration.py diff --git a/scanner/engine.py b/scanner/engine.py index 1fef83df..54bc2cfa 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -11,6 +11,7 @@ from openshield.severity import CONTRACT_VERSION, SeverityContractError, normalize_severity, score_findings from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id +from scanner.graph.snapshot_bridge import collect_snapshot logger = logging.getLogger(__name__) @@ -117,6 +118,10 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: evaluations: List[RuleEvaluation] = [] detected_at = datetime.now(timezone.utc).isoformat() + # Collect an ARG inventory snapshot for graph population and rule enrichment. + # Failure is non-fatal: rules fall back to direct SDK calls. + snapshot = collect_snapshot(self.client, self.subscription_id) + logger.info( "Scan %s starting against subscription %s — %d rules loaded", scan_id, @@ -152,7 +157,11 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: continue try: - rule_findings = rule.scan(self.client, self.subscription_id) + try: + rule_findings = rule.scan(self.client, self.subscription_id, snapshot) + except TypeError: + # Legacy rule does not accept snapshot parameter. + rule_findings = rule.scan(self.client, self.subscription_id) if not isinstance(rule_findings, list): logger.warning("Rule %s returned %s instead of list — skipped", rule_id, type(rule_findings)) failed_rule_ids.append(rule_id) @@ -222,6 +231,8 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: "total_findings": len(findings), "score": score, "severity_contract_version": CONTRACT_VERSION, + "snapshot_id": snapshot.snapshot_id if snapshot else None, + "snapshot_status": snapshot.status.value if snapshot else None, "findings": findings, "evaluations": [e.to_dict() for e in evaluations], "failed_rule_ids": failed_rule_ids, diff --git a/tests/test_graph_engine_integration.py b/tests/test_graph_engine_integration.py new file mode 100644 index 00000000..92fc6d4e --- /dev/null +++ b/tests/test_graph_engine_integration.py @@ -0,0 +1,82 @@ +"""Verify ScanEngine collects a snapshot and passes it to rules.""" +from unittest.mock import MagicMock, patch +import pytest + +from scanner.arg_inventory import InventorySnapshot, InventoryStatus +from scanner.engine import ScanEngine + + +def _make_snapshot(): + return InventorySnapshot( + snapshot_id="snap-abc", + tenant_id="00000000-0000-0000-0000-000000000001", + requested_subscriptions=("00000000-0000-0000-0000-000000000002",), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=50, + pages=1, + resources=(), + errors=(), + ) + + +@pytest.fixture +def engine(): + with patch("scanner.engine.AzureClient"), patch("scanner.engine.ScanEngine.load_rules"): + eng = ScanEngine("00000000-0000-0000-0000-000000000002") + eng.rules = [] + return eng + + +def test_run_scan_result_contains_snapshot_id(engine): + snapshot = _make_snapshot() + with patch("scanner.engine.collect_snapshot", return_value=snapshot): + result = engine.run_scan() + assert result["snapshot_id"] == "snap-abc" + assert result["snapshot_status"] == "COMPLETE" + + +def test_run_scan_snapshot_none_when_collection_fails(engine): + with patch("scanner.engine.collect_snapshot", return_value=None): + result = engine.run_scan() + assert result["snapshot_id"] is None + assert result["snapshot_status"] is None + + +def test_run_scan_passes_snapshot_to_rule_scan(engine): + snapshot = _make_snapshot() + received = {} + + def fake_scan(client, subscription_id, snapshot=None): + received["snapshot"] = snapshot + return [] + + mock_rule = MagicMock() + mock_rule.RULE_ID = "AZ-TEST-001" + mock_rule.SEVERITY = "HIGH" + mock_rule.scan = fake_scan + engine.rules = [mock_rule] + + with patch("scanner.engine.collect_snapshot", return_value=snapshot): + engine.run_scan() + + assert received["snapshot"] is snapshot + + +def test_run_scan_existing_rule_without_snapshot_param_still_works(engine): + snapshot = _make_snapshot() + + def legacy_scan(client, subscription_id): + return [{"rule_id": "AZ-LEGACY-001", "severity": "LOW", "resource_id": "r1", + "rule_name": "Legacy", "description": "d", "remediation": "r"}] + + mock_rule = MagicMock() + mock_rule.RULE_ID = "AZ-LEGACY-001" + mock_rule.SEVERITY = "LOW" + mock_rule.scan = legacy_scan + engine.rules = [mock_rule] + + with patch("scanner.engine.collect_snapshot", return_value=snapshot): + result = engine.run_scan() + + assert result["total_findings"] == 1 From 95e0b5d69e69f218758f7c844f3a462c35720f7a Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:45:47 +0100 Subject: [PATCH 04/67] fix(graph): remove unused pytest import from snapshot_bridge tests Signed-off-by: Tanvir Farhad --- tests/test_graph_snapshot_bridge.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_graph_snapshot_bridge.py b/tests/test_graph_snapshot_bridge.py index e7b94168..344cb792 100644 --- a/tests/test_graph_snapshot_bridge.py +++ b/tests/test_graph_snapshot_bridge.py @@ -1,6 +1,5 @@ """Tests for the snapshot bridge that wraps ArgInventoryClient for use in ScanEngine.""" from unittest.mock import MagicMock, patch -import pytest from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource from scanner.graph.snapshot_bridge import collect_snapshot From 5d6676556c87d8d350c63a5626d7e482d8d49dce Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 00:42:33 +0100 Subject: [PATCH 05/67] fix(lint): apply ruff format to graph foundation files Signed-off-by: Tanvir Farhad --- scanner/graph/snapshot_bridge.py | 5 ++--- tests/test_graph_engine_integration.py | 13 ++++++++++-- tests/test_graph_migration.py | 28 ++++++++++++++++++++------ tests/test_graph_snapshot_bridge.py | 1 + 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/scanner/graph/snapshot_bridge.py b/scanner/graph/snapshot_bridge.py index 7d079de8..08a724b3 100644 --- a/scanner/graph/snapshot_bridge.py +++ b/scanner/graph/snapshot_bridge.py @@ -1,4 +1,5 @@ """Thin wrapper that collects an ARG InventorySnapshot for use inside ScanEngine.""" + from __future__ import annotations import logging @@ -23,9 +24,7 @@ def collect_snapshot(client: AzureClient, subscription_id: str) -> InventorySnap credential = getattr(client, "credential", None) if not tenant_id or not credential: - logger.warning( - "snapshot_bridge: AzureClient missing tenant_id or credential — skipping ARG collection" - ) + logger.warning("snapshot_bridge: AzureClient missing tenant_id or credential — skipping ARG collection") return None try: diff --git a/tests/test_graph_engine_integration.py b/tests/test_graph_engine_integration.py index 92fc6d4e..d68869f0 100644 --- a/tests/test_graph_engine_integration.py +++ b/tests/test_graph_engine_integration.py @@ -1,4 +1,5 @@ """Verify ScanEngine collects a snapshot and passes it to rules.""" + from unittest.mock import MagicMock, patch import pytest @@ -67,8 +68,16 @@ def test_run_scan_existing_rule_without_snapshot_param_still_works(engine): snapshot = _make_snapshot() def legacy_scan(client, subscription_id): - return [{"rule_id": "AZ-LEGACY-001", "severity": "LOW", "resource_id": "r1", - "rule_name": "Legacy", "description": "d", "remediation": "r"}] + return [ + { + "rule_id": "AZ-LEGACY-001", + "severity": "LOW", + "resource_id": "r1", + "rule_name": "Legacy", + "description": "d", + "remediation": "r", + } + ] mock_rule = MagicMock() mock_rule.RULE_ID = "AZ-LEGACY-001" diff --git a/tests/test_graph_migration.py b/tests/test_graph_migration.py index 21382e4a..dfe31b37 100644 --- a/tests/test_graph_migration.py +++ b/tests/test_graph_migration.py @@ -1,4 +1,5 @@ """Verify the graph schema migration applies and rolls back cleanly.""" + import os import pytest from alembic.config import Config @@ -40,9 +41,18 @@ def test_graph_nodes_columns(engine): inspector = inspect(engine) cols = {c["name"] for c in inspector.get_columns("graph_nodes")} assert cols >= { - "node_id", "tenant_id", "subscription_id", "resource_id", - "resource_type", "name", "location", "resource_group", - "snapshot_id", "properties", "created_at", "updated_at", + "node_id", + "tenant_id", + "subscription_id", + "resource_id", + "resource_type", + "name", + "location", + "resource_group", + "snapshot_id", + "properties", + "created_at", + "updated_at", } @@ -50,9 +60,15 @@ def test_graph_edges_columns(engine): inspector = inspect(engine) cols = {c["name"] for c in inspector.get_columns("graph_edges")} assert cols >= { - "edge_id", "source_node_id", "target_node_id", - "relationship_type", "evidence_source", "evidence_snapshot_id", - "confidence", "collected_at", "properties", + "edge_id", + "source_node_id", + "target_node_id", + "relationship_type", + "evidence_source", + "evidence_snapshot_id", + "confidence", + "collected_at", + "properties", } diff --git a/tests/test_graph_snapshot_bridge.py b/tests/test_graph_snapshot_bridge.py index 344cb792..a18f230c 100644 --- a/tests/test_graph_snapshot_bridge.py +++ b/tests/test_graph_snapshot_bridge.py @@ -1,4 +1,5 @@ """Tests for the snapshot bridge that wraps ArgInventoryClient for use in ScanEngine.""" + from unittest.mock import MagicMock, patch from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource From 691c265b5714dd2818060b33557e5ca1801c7d19 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 00:56:22 +0100 Subject: [PATCH 06/67] fix(lint): ruff 0.16.6 format for graph schema migration Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index eb55ce5d..65df6098 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -30,15 +30,12 @@ def upgrade() -> None: sa.Column("resource_group", sa.Text(), nullable=False, server_default=sa.text("''")), sa.Column("snapshot_id", sa.Text(), nullable=False), sa.Column("properties", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.text("now()")), - sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.text("now()")), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), sa.PrimaryKeyConstraint("node_id", name="graph_nodes_pkey"), ) op.create_index("idx_graph_nodes_resource_id", "graph_nodes", ["resource_id"]) - op.create_index("idx_graph_nodes_tenant_subscription", "graph_nodes", - ["tenant_id", "subscription_id"]) + op.create_index("idx_graph_nodes_tenant_subscription", "graph_nodes", ["tenant_id", "subscription_id"]) op.create_index( "uq_graph_nodes_tenant_resource", "graph_nodes", @@ -55,13 +52,14 @@ def upgrade() -> None: sa.Column("evidence_source", sa.Text(), nullable=False), sa.Column("evidence_snapshot_id", sa.Text(), nullable=False), sa.Column("confidence", sa.Float(), nullable=False, server_default=sa.text("1.0")), - sa.Column("collected_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.text("now()")), + sa.Column("collected_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), sa.Column("properties", postgresql.JSONB(), nullable=False, server_default=sa.text("'{}'::jsonb")), - sa.ForeignKeyConstraint(["source_node_id"], ["graph_nodes.node_id"], - name="graph_edges_source_fkey", ondelete="CASCADE"), - sa.ForeignKeyConstraint(["target_node_id"], ["graph_nodes.node_id"], - name="graph_edges_target_fkey", ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["source_node_id"], ["graph_nodes.node_id"], name="graph_edges_source_fkey", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["target_node_id"], ["graph_nodes.node_id"], name="graph_edges_target_fkey", ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("edge_id", name="graph_edges_pkey"), ) op.create_index("idx_graph_edges_source", "graph_edges", ["source_node_id"]) @@ -77,10 +75,12 @@ def upgrade() -> None: "finding_graph_nodes", sa.Column("finding_id", sa.Integer(), nullable=False), sa.Column("node_id", postgresql.UUID(), nullable=False), - sa.ForeignKeyConstraint(["finding_id"], ["findings.id"], - name="finding_graph_nodes_finding_fkey", ondelete="CASCADE"), - sa.ForeignKeyConstraint(["node_id"], ["graph_nodes.node_id"], - name="finding_graph_nodes_node_fkey", ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["finding_id"], ["findings.id"], name="finding_graph_nodes_finding_fkey", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["node_id"], ["graph_nodes.node_id"], name="finding_graph_nodes_node_fkey", ondelete="CASCADE" + ), sa.PrimaryKeyConstraint("finding_id", "node_id", name="finding_graph_nodes_pkey"), ) op.create_index("idx_finding_graph_nodes_node_id", "finding_graph_nodes", ["node_id"]) From a92c2aa3fac7fa97c24308c30c397b915b0b190a Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 01:23:14 +0100 Subject: [PATCH 07/67] fix(logging): preserve pytest caplog handler in configure_logging Replace the full root.handlers reset with a marker-based selective removal so configure_logging() only evicts the handler it previously installed. This prevents test framework handlers (pytest LogCaptureHandler) from being wiped when configure_logging() is called at module level in api/app.py or scanner/worker.py, fixing test_startup_warns_when_allowlist_is_unset across the full test suite. Signed-off-by: Tanvir Farhad --- api/observability.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/api/observability.py b/api/observability.py index 19f2263c..0ec5e134 100644 --- a/api/observability.py +++ b/api/observability.py @@ -158,18 +158,23 @@ def _refresh_pool_metrics() -> None: # --------------------------------------------------------------------------- # # Structured logging # # --------------------------------------------------------------------------- # +_HANDLER_MARKER = "_openshield_json_handler" + + def configure_logging(level: int = logging.INFO) -> None: """Configure the root logger to emit structured JSON to stderr. - Safe to call multiple times — it replaces the root handlers each call so - repeated invocations (API import, worker start, tests) do not stack - duplicate handlers. Any ``extra={...}`` fields passed to log calls are - included as top-level JSON keys. + Safe to call multiple times — repeated invocations replace only the + handler installed by a previous call, preserving any handlers added by + the test framework (e.g. pytest's LogCaptureHandler). Any ``extra={...}`` + fields passed to log calls are included as top-level JSON keys. """ handler = logging.StreamHandler() handler.setFormatter(JsonFormatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + setattr(handler, _HANDLER_MARKER, True) root = logging.getLogger() - root.handlers = [handler] + root.handlers = [h for h in root.handlers if not getattr(h, _HANDLER_MARKER, False)] + root.handlers.append(handler) root.setLevel(level) From 5270edfeeb727a5cf95410cf65b66ac06229fe79 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 01:54:28 +0100 Subject: [PATCH 08/67] fix(alembic): preserve existing loggers when fileConfig runs Pass disable_existing_loggers=False to fileConfig so Alembic's ini-driven logging setup does not disable application loggers (e.g. api.app) that were created before the migration run. The Python default of True causes any logger not listed in alembic.ini to be silenced for the rest of the process, which broke test_startup_warns_when_allowlist_is_unset when test_graph_migration.py ran first in the full suite. Signed-off-by: Tanvir Farhad --- alembic/env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/env.py b/alembic/env.py index f1ce2f8e..68c08d09 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -9,7 +9,7 @@ config = context.config if config.config_file_name is not None: - fileConfig(config.config_file_name) + fileConfig(config.config_file_name, disable_existing_loggers=False) # OpenShield intentionally uses raw SQL rather than ORM metadata. Migration # revisions therefore define every schema operation explicitly. From 204a74096364b41a296126fd96edcf4669ffb401 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 11:29:24 +0100 Subject: [PATCH 09/67] fix(graph): add tenant_id to AzureClient, merge alembic heads, replace TypeError guard with inspect.signature Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 +-- scanner/azure_client.py | 10 +++++- scanner/engine.py | 6 ++-- tests/test_graph_snapshot_bridge.py | 31 +++++++++++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index 65df6098..d2dfca10 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: 3f59f83a5253 +Revises: 3f59f83a5253, d4a8c1e6b2f9 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = "3f59f83a5253" +down_revision: Union[str, Sequence[str], None] = ("3f59f83a5253", "d4a8c1e6b2f9") branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/scanner/azure_client.py b/scanner/azure_client.py index 315ac571..3dd21277 100644 --- a/scanner/azure_client.py +++ b/scanner/azure_client.py @@ -48,9 +48,17 @@ class AzureClient: crash the scan engine. """ - def __init__(self, subscription_id: str, credential: Optional[Any] = None) -> None: + def __init__( + self, + subscription_id: str, + credential: Optional[Any] = None, + tenant_id: Optional[str] = None, + ) -> None: + import os + self.subscription_id = subscription_id self.credential = credential or DefaultAzureCredential() + self.tenant_id = tenant_id or os.environ.get("AZURE_TENANT_ID") self._managed_clusters_cache: Any = _UNSET self._aks_security_posture_cache: Any = _UNSET self._function_apps_cache: Any = _UNSET diff --git a/scanner/engine.py b/scanner/engine.py index 54bc2cfa..fd3855f3 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -1,6 +1,7 @@ """Scan engine: loads rules dynamically and orchestrates a full subscription scan.""" import importlib.util +import inspect import logging import uuid from datetime import datetime, timezone @@ -157,10 +158,9 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: continue try: - try: + if len(inspect.signature(rule.scan).parameters) >= 3: rule_findings = rule.scan(self.client, self.subscription_id, snapshot) - except TypeError: - # Legacy rule does not accept snapshot parameter. + else: rule_findings = rule.scan(self.client, self.subscription_id) if not isinstance(rule_findings, list): logger.warning("Rule %s returned %s instead of list — skipped", rule_id, type(rule_findings)) diff --git a/tests/test_graph_snapshot_bridge.py b/tests/test_graph_snapshot_bridge.py index a18f230c..952b9088 100644 --- a/tests/test_graph_snapshot_bridge.py +++ b/tests/test_graph_snapshot_bridge.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock, patch from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource +from scanner.azure_client import AzureClient from scanner.graph.snapshot_bridge import collect_snapshot @@ -83,3 +84,33 @@ def test_collect_snapshot_returns_partial_snapshot(): assert result is not None assert result.status == InventoryStatus.PARTIAL + + +def test_azure_client_tenant_id_from_env(monkeypatch): + monkeypatch.setenv("AZURE_TENANT_ID", "env-tenant-abc") + with patch("scanner.azure_client.DefaultAzureCredential"): + client = AzureClient(subscription_id="sub-123") + assert client.tenant_id == "env-tenant-abc" + + +def test_azure_client_explicit_tenant_id_overrides_env(monkeypatch): + monkeypatch.setenv("AZURE_TENANT_ID", "env-tenant") + with patch("scanner.azure_client.DefaultAzureCredential"): + client = AzureClient(subscription_id="sub-123", tenant_id="explicit-tenant") + assert client.tenant_id == "explicit-tenant" + + +def test_azure_client_tenant_id_none_when_env_unset(monkeypatch): + monkeypatch.delenv("AZURE_TENANT_ID", raising=False) + with patch("scanner.azure_client.DefaultAzureCredential"): + client = AzureClient(subscription_id="sub-123") + assert client.tenant_id is None + + +def test_collect_snapshot_skips_when_no_tenant_id(monkeypatch): + monkeypatch.delenv("AZURE_TENANT_ID", raising=False) + with patch("scanner.azure_client.DefaultAzureCredential"): + client = AzureClient(subscription_id="sub-123") + assert client.tenant_id is None + result = collect_snapshot(client, "sub-123") + assert result is None From 923c60f5b0ee430c007d3bb4417a0c1ddcfd4f1f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 22:59:36 +0100 Subject: [PATCH 10/67] fix(alembic): use single-parent down_revision for graph schema migration 3f59f83a5253 is a linear ancestor of d4a8c1e6b2f9 on the dev chain, so the two-parent tuple created a spurious merge root. Single-parent down_revision d4a8c1e6b2f9 is the correct head this migration extends. Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index d2dfca10..d23a0f87 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: 3f59f83a5253, d4a8c1e6b2f9 +Revises: d4a8c1e6b2f9 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = ("3f59f83a5253", "d4a8c1e6b2f9") +down_revision: Union[str, Sequence[str], None] = "d4a8c1e6b2f9" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 4a55acb6575fba2c24f1b35e9ec791ac8652fba6 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Mon, 28 Sep 2026 00:02:56 +0100 Subject: [PATCH 11/67] fix(graph): resolve alembic head dynamically in admission test, move import os to module level - test_scan_admission_migration_postgres.py: replace hardcoded _HEAD with ScriptDirectory.from_config().get_current_head() so the test stays valid when new migrations are added without manual updates to the pinned revision. - scanner/azure_client.py: move inline 'import os' from __init__ and _build_devops_client to module-level import. Signed-off-by: Tanvir Farhad --- scanner/azure_client.py | 5 +---- tests/test_scan_admission_migration_postgres.py | 7 +++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/scanner/azure_client.py b/scanner/azure_client.py index 3dd21277..0a041b1d 100644 --- a/scanner/azure_client.py +++ b/scanner/azure_client.py @@ -1,6 +1,7 @@ """Azure SDK wrapper providing typed accessors for all CSPM scan operations.""" import logging +import os from datetime import datetime, timezone from typing import Any, Dict, List, Optional @@ -54,8 +55,6 @@ def __init__( credential: Optional[Any] = None, tenant_id: Optional[str] = None, ) -> None: - import os - self.subscription_id = subscription_id self.credential = credential or DefaultAzureCredential() self.tenant_id = tenant_id or os.environ.get("AZURE_TENANT_ID") @@ -82,8 +81,6 @@ def _build_devops_client(self) -> Optional[Any]: subscription has an associated Azure DevOps organization), so rules that depend on this must treat None as "not configured, skip" rather than "unknown, indeterminate".""" - import os - org_url = os.environ.get("AZURE_DEVOPS_ORG_URL") project = os.environ.get("AZURE_DEVOPS_PROJECT") if not org_url or not project: diff --git a/tests/test_scan_admission_migration_postgres.py b/tests/test_scan_admission_migration_postgres.py index 55de7053..831e9649 100644 --- a/tests/test_scan_admission_migration_postgres.py +++ b/tests/test_scan_admission_migration_postgres.py @@ -30,6 +30,13 @@ _ACTIVE_INDEX = "uq_scans_one_active_per_subscription" _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +def _resolve_head() -> str: + cfg = Config(os.path.join(_REPO_ROOT, "alembic.ini")) + cfg.set_main_option("script_location", os.path.join(_REPO_ROOT, "alembic")) + return ScriptDirectory.from_config(cfg).get_current_head() + +_HEAD = _resolve_head() + @contextmanager def _scratch_database(): From cc6dfaf171e1f31f62bc1ab226b0dd811b845a62 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Mon, 28 Sep 2026 01:11:13 +0100 Subject: [PATCH 12/67] fix(lint): add blank lines around _resolve_head for ruff format compliance Signed-off-by: Tanvir Farhad --- tests/test_scan_admission_migration_postgres.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_scan_admission_migration_postgres.py b/tests/test_scan_admission_migration_postgres.py index 831e9649..a08255d2 100644 --- a/tests/test_scan_admission_migration_postgres.py +++ b/tests/test_scan_admission_migration_postgres.py @@ -30,11 +30,13 @@ _ACTIVE_INDEX = "uq_scans_one_active_per_subscription" _REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + def _resolve_head() -> str: cfg = Config(os.path.join(_REPO_ROOT, "alembic.ini")) cfg.set_main_option("script_location", os.path.join(_REPO_ROOT, "alembic")) return ScriptDirectory.from_config(cfg).get_current_head() + _HEAD = _resolve_head() From 583aa4729e17299de52fd9d7fa85b2af0b1c5360 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 1 Oct 2026 00:58:45 +0100 Subject: [PATCH 13/67] fix(graph): repoint migration down_revision to 3a76ff935bf6 (dev head) After #310 merged, dev's Alembic head moved from d4a8c1e6b2f9 to 3a76ff935bf6. Update down_revision and Revises docstring to match so the graph schema migration chains cleanly from the new single head. Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index d23a0f87..73190d5f 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: d4a8c1e6b2f9 +Revises: 3a76ff935bf6 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = "d4a8c1e6b2f9" +down_revision: Union[str, Sequence[str], None] = "3a76ff935bf6" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From a9271878822df3d221c99bdea1cdbe69a0276989 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:52:14 +0100 Subject: [PATCH 14/67] feat(graph): add node_service to upsert graph nodes and link findings Signed-off-by: Tanvir Farhad --- scanner/graph/node_service.py | 85 +++++++++++++++++++++++ tests/test_graph_node_service.py | 111 +++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 scanner/graph/node_service.py create mode 100644 tests/test_graph_node_service.py diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py new file mode 100644 index 00000000..c1154c1a --- /dev/null +++ b/scanner/graph/node_service.py @@ -0,0 +1,85 @@ +"""Upsert graph nodes from an InventorySnapshot and link findings to nodes.""" +from __future__ import annotations + +import json +import logging +import uuid +from typing import TYPE_CHECKING + +import psycopg2 +import psycopg2.extras + +if TYPE_CHECKING: + from scanner.arg_inventory import InventorySnapshot + +logger = logging.getLogger(__name__) + +_UPSERT_NODE_SQL = """ +INSERT INTO graph_nodes ( + node_id, tenant_id, subscription_id, resource_id, resource_type, + name, location, resource_group, snapshot_id, properties, created_at, updated_at +) +VALUES ( + %(node_id)s, %(tenant_id)s, %(subscription_id)s, %(resource_id)s, %(resource_type)s, + %(name)s, %(location)s, %(resource_group)s, %(snapshot_id)s, %(properties)s, + now(), now() +) +ON CONFLICT (tenant_id, resource_id) +DO UPDATE SET + resource_type = EXCLUDED.resource_type, + name = EXCLUDED.name, + location = EXCLUDED.location, + resource_group = EXCLUDED.resource_group, + snapshot_id = EXCLUDED.snapshot_id, + properties = EXCLUDED.properties, + updated_at = now() +""" + +_LINK_FINDINGS_SQL = """ +INSERT INTO finding_graph_nodes (finding_id, node_id) +SELECT f.id, n.node_id +FROM findings f +JOIN graph_nodes n ON lower(f.resource_id) = lower(n.resource_id) + AND n.tenant_id = ( + SELECT tenant_id FROM graph_nodes + WHERE snapshot_id = ( + SELECT snapshot_id FROM graph_nodes ORDER BY updated_at DESC LIMIT 1 + ) + LIMIT 1 + ) +WHERE f.scan_id = %(scan_id)s +ON CONFLICT DO NOTHING +""" + + +def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: + """Upsert graph_nodes from snapshot resources. Returns count of rows written.""" + if not snapshot.resources: + return 0 + + written = 0 + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + for resource in snapshot.resources: + cur.execute(_UPSERT_NODE_SQL, { + "node_id": str(uuid.uuid4()), + "tenant_id": resource.tenant_id, + "subscription_id": resource.subscription_id, + "resource_id": resource.resource_id, + "resource_type": resource.resource_type, + "name": resource.name, + "location": resource.location, + "resource_group": resource.resource_group, + "snapshot_id": resource.snapshot_id, + "properties": json.dumps(resource.properties), + }) + written += 1 + return written + + +def link_findings_to_nodes(scan_id: str, dsn: str) -> int: + """Link findings from this scan to their graph nodes by resource_id.""" + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id}) + return cur.rowcount if cur.rowcount >= 0 else 0 diff --git a/tests/test_graph_node_service.py b/tests/test_graph_node_service.py new file mode 100644 index 00000000..12f70b8b --- /dev/null +++ b/tests/test_graph_node_service.py @@ -0,0 +1,111 @@ +"""Tests for graph node upsert and finding-to-node linking.""" +from unittest.mock import MagicMock, patch + +from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource +from scanner.graph.node_service import populate_nodes, link_findings_to_nodes + + +def _make_resource(resource_id: str, resource_type: str = "microsoft.network/virtualnetworks", + subscription_id: str = "00000000-0000-0000-0000-000000000002") -> InventoryResource: + return InventoryResource( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + subscription_id=subscription_id, + resource_id=resource_id, + resource_type=resource_type, + name=resource_id.split("/")[-1], + location="eastus", + resource_group="rg", + tags={}, + properties={"key": "val"}, + ) + + +def _make_snapshot(resources=()): + return InventorySnapshot( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + requested_subscriptions=("00000000-0000-0000-0000-000000000002",), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=10, + pages=1, + resources=tuple(resources), + errors=(), + ) + + +def test_populate_nodes_upserts_each_resource(): + resources = [ + _make_resource("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1"), + _make_resource("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet2"), + ] + snapshot = _make_snapshot(resources) + + mock_conn = MagicMock() + mock_cur = MagicMock() + mock_conn.__enter__ = MagicMock(return_value=mock_conn) + mock_conn.__exit__ = MagicMock(return_value=False) + mock_cur.__enter__ = MagicMock(return_value=mock_cur) + mock_cur.__exit__ = MagicMock(return_value=False) + mock_conn.cursor.return_value = mock_cur + + with patch("scanner.graph.node_service.psycopg2.connect", return_value=mock_conn): + count = populate_nodes(snapshot, "postgresql://test/db") + + assert count == 2 + assert mock_cur.execute.call_count == 2 + + +def test_populate_nodes_empty_snapshot_returns_zero(): + snapshot = _make_snapshot(resources=[]) + + with patch("scanner.graph.node_service.psycopg2.connect") as mock_connect: + count = populate_nodes(snapshot, "postgresql://test/db") + + assert count == 0 + mock_connect.assert_not_called() + + +def test_populate_nodes_does_not_write_cross_tenant_resources(): + resource = _make_resource("/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1") + snapshot = _make_snapshot([resource]) + + captured_args = [] + mock_conn = MagicMock() + mock_cur = MagicMock() + mock_conn.__enter__ = MagicMock(return_value=mock_conn) + mock_conn.__exit__ = MagicMock(return_value=False) + mock_cur.__enter__ = MagicMock(return_value=mock_cur) + mock_cur.__exit__ = MagicMock(return_value=False) + mock_conn.cursor.return_value = mock_cur + + def capture_execute(sql, params=None): + if params: + captured_args.append(params) + + mock_cur.execute.side_effect = capture_execute + + with patch("scanner.graph.node_service.psycopg2.connect", return_value=mock_conn): + populate_nodes(snapshot, "postgresql://test/db") + + for params in captured_args: + assert "00000000-0000-0000-0000-000000000001" in str(params), "tenant_id must be in every write" + + +def test_link_findings_to_nodes_executes_insert(): + mock_conn = MagicMock() + mock_cur = MagicMock() + mock_conn.__enter__ = MagicMock(return_value=mock_conn) + mock_conn.__exit__ = MagicMock(return_value=False) + mock_cur.__enter__ = MagicMock(return_value=mock_cur) + mock_cur.__exit__ = MagicMock(return_value=False) + mock_conn.cursor.return_value = mock_cur + mock_cur.rowcount = 3 + + with patch("scanner.graph.node_service.psycopg2.connect", return_value=mock_conn): + link_findings_to_nodes("scan-uuid-1", "postgresql://test/db") + + assert mock_cur.execute.called + sql_called = mock_cur.execute.call_args[0][0] + assert "finding_graph_nodes" in sql_called From fbcf09466c46121d5596ed3dcd8141a9e8c08b49 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:54:17 +0100 Subject: [PATCH 15/67] feat(graph): add edge detectors for NSG, subnet, public IP, identity, and storage Signed-off-by: Tanvir Farhad --- scanner/graph/edge_detector.py | 173 ++++++++++++++++++++++++++++++ tests/test_graph_edge_detector.py | 120 +++++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 scanner/graph/edge_detector.py create mode 100644 tests/test_graph_edge_detector.py diff --git a/scanner/graph/edge_detector.py b/scanner/graph/edge_detector.py new file mode 100644 index 00000000..61cf0ad5 --- /dev/null +++ b/scanner/graph/edge_detector.py @@ -0,0 +1,173 @@ +"""Typed edge detectors that infer relationships between Azure resources in an InventorySnapshot.""" +from __future__ import annotations + +import logging +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scanner.arg_inventory import InventorySnapshot + +logger = logging.getLogger(__name__) + +_NSG_TYPE = "microsoft.network/networksecuritygroups" +_PUBLIC_IP_TYPE = "microsoft.network/publicipaddresses" +_STORAGE_TYPE = "microsoft.storage/storageaccounts" + + +@dataclass +class GraphEdge: + """A directed relationship between two Azure resources.""" + + source_resource_id: str + target_resource_id: str + relationship_type: str + evidence_source: str + confidence: float + + +class EdgeDetector(ABC): + """Base class for relationship detectors.""" + + @abstractmethod + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + """Return edges inferred from this snapshot.""" + + +class NsgToSubnetDetector(EdgeDetector): + """NSG -> Subnet: PROTECTS (ARG-confirmed, confidence 1.0).""" + + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + edges = [] + for resource in snapshot.resources: + if resource.resource_type.lower() != _NSG_TYPE: + continue + subnets = resource.properties.get("subnets") or [] + for subnet in subnets: + subnet_id = subnet.get("id") if isinstance(subnet, dict) else None + if not subnet_id: + continue + edges.append(GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=subnet_id, + relationship_type="PROTECTS", + evidence_source="arg:properties.subnets", + confidence=1.0, + )) + return edges + + +class SubnetToResourceDetector(EdgeDetector): + """Resource -> Subnet: MEMBER_OF (inferred from properties, confidence 0.8).""" + + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + edges = [] + resource_ids = {r.resource_id.lower() for r in snapshot.resources} + for resource in snapshot.resources: + subnet_id = (resource.properties.get("subnet") or {}).get("id") + if not subnet_id: + continue + if subnet_id.lower() not in resource_ids: + continue + edges.append(GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=subnet_id, + relationship_type="MEMBER_OF", + evidence_source="arg:properties.subnet.id", + confidence=0.8, + )) + return edges + + +class PublicIpToResourceDetector(EdgeDetector): + """PublicIP -> Resource: EXPOSES (ARG-confirmed, confidence 1.0).""" + + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + edges = [] + for resource in snapshot.resources: + if resource.resource_type.lower() != _PUBLIC_IP_TYPE: + continue + ip_config = resource.properties.get("ipConfiguration") or {} + target_id = ip_config.get("id") + if not target_id: + continue + # Strip the NIC sub-path to get the parent VM resource ID (4 provider path segments) + parts = target_id.split("/providers/") + if len(parts) >= 2: + provider_path = parts[-1].split("/") + if len(provider_path) >= 4: + target_id = "/providers/".join(parts[:-1]) + "/providers/" + "/".join(provider_path[:4]) + edges.append(GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=target_id, + relationship_type="EXPOSES", + evidence_source="arg:properties.ipConfiguration.id", + confidence=1.0, + )) + return edges + + +class IdentityToResourceDetector(EdgeDetector): + """Identity -> Resource: HAS_IDENTITY (inferred from properties, confidence 0.8).""" + + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + edges = [] + for resource in snapshot.resources: + identity = resource.properties.get("identity") or {} + user_assigned = identity.get("userAssignedIdentities") or {} + for identity_id in user_assigned: + if not identity_id: + continue + edges.append(GraphEdge( + source_resource_id=identity_id, + target_resource_id=resource.resource_id, + relationship_type="HAS_IDENTITY", + evidence_source="arg:properties.identity.userAssignedIdentities", + confidence=0.8, + )) + return edges + + +class StoragePrivateEndpointDetector(EdgeDetector): + """Storage -> PrivateEndpoint: REACHABLE_VIA (ARG-confirmed, confidence 1.0).""" + + def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: + edges = [] + for resource in snapshot.resources: + if resource.resource_type.lower() != _STORAGE_TYPE: + continue + connections = resource.properties.get("privateEndpointConnections") or [] + for conn in connections: + pe_id = ((conn.get("properties") or {}).get("privateEndpoint") or {}).get("id") + if not pe_id: + continue + edges.append(GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=pe_id, + relationship_type="REACHABLE_VIA", + evidence_source="arg:properties.privateEndpointConnections", + confidence=1.0, + )) + return edges + + +def detect_all_edges(snapshot: InventorySnapshot) -> list[GraphEdge]: + """Run all detectors and return the combined edge list. + + Per-detector exceptions are caught and logged; the function never raises. + """ + detectors: list[EdgeDetector] = [ + NsgToSubnetDetector(), + SubnetToResourceDetector(), + PublicIpToResourceDetector(), + IdentityToResourceDetector(), + StoragePrivateEndpointDetector(), + ] + edges: list[GraphEdge] = [] + for detector in detectors: + try: + edges.extend(detector.detect(snapshot)) + except Exception as exc: + logger.warning("edge_detector: %s failed: %s", type(detector).__name__, exc) + return edges diff --git a/tests/test_graph_edge_detector.py b/tests/test_graph_edge_detector.py new file mode 100644 index 00000000..247e09d9 --- /dev/null +++ b/tests/test_graph_edge_detector.py @@ -0,0 +1,120 @@ +"""Tests for graph edge detectors.""" +from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource +from scanner.graph.edge_detector import ( + GraphEdge, # noqa: F401 — imported to verify public API surface + NsgToSubnetDetector, + SubnetToResourceDetector, # noqa: F401 — imported to verify public API surface + PublicIpToResourceDetector, + IdentityToResourceDetector, + StoragePrivateEndpointDetector, + detect_all_edges, +) + + +def _resource(resource_id, resource_type, properties=None, subscription_id="00000000-0000-0000-0000-000000000002"): + return InventoryResource( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + subscription_id=subscription_id, + resource_id=resource_id, + resource_type=resource_type, + name=resource_id.split("/")[-1], + location="eastus", + resource_group="rg", + tags={}, + properties=properties or {}, + ) + + +def _snapshot(*resources): + return InventorySnapshot( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + requested_subscriptions=("00000000-0000-0000-0000-000000000002",), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=10, + pages=1, + resources=tuple(resources), + errors=(), + ) + + +def test_nsg_to_subnet_detects_protects_edge(): + subnet_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/default" + nsg_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkSecurityGroups/nsg1" + nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", { + "subnets": [{"id": subnet_id}] + }) + snapshot = _snapshot(nsg) + edges = NsgToSubnetDetector().detect(snapshot) + assert len(edges) == 1 + assert edges[0].relationship_type == "PROTECTS" + assert edges[0].source_resource_id == nsg_id + assert edges[0].target_resource_id == subnet_id + assert edges[0].confidence == 1.0 + + +def test_nsg_to_subnet_no_subnets_returns_empty(): + nsg = _resource("/nsg1", "microsoft.network/networksecuritygroups", {"subnets": []}) + snapshot = _snapshot(nsg) + assert NsgToSubnetDetector().detect(snapshot) == [] + + +def test_public_ip_to_resource_detects_exposes_edge(): + vm_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + pip_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/pip1" + pip = _resource(pip_id, "microsoft.network/publicipaddresses", { + "ipConfiguration": {"id": vm_id + "/networkInterfaces/nic1/ipConfigurations/ipconfig1"} + }) + snapshot = _snapshot(pip) + edges = PublicIpToResourceDetector().detect(snapshot) + assert len(edges) == 1 + assert edges[0].relationship_type == "EXPOSES" + assert edges[0].source_resource_id == pip_id + assert edges[0].confidence == 1.0 + + +def test_identity_to_resource_detects_has_identity_edge(): + vm_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + identity_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1" + vm = _resource(vm_id, "microsoft.compute/virtualmachines", { + "identity": { + "userAssignedIdentities": {identity_id: {}} + } + }) + snapshot = _snapshot(vm) + edges = IdentityToResourceDetector().detect(snapshot) + assert len(edges) == 1 + assert edges[0].relationship_type == "HAS_IDENTITY" + assert edges[0].target_resource_id == vm_id + assert edges[0].confidence == 0.8 + + +def test_storage_private_endpoint_detects_reachable_via_edge(): + storage_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sa1" + pe_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/privateEndpoints/pe1" + storage = _resource(storage_id, "microsoft.storage/storageaccounts", { + "privateEndpointConnections": [{"properties": {"privateEndpoint": {"id": pe_id}}}] + }) + snapshot = _snapshot(storage) + edges = StoragePrivateEndpointDetector().detect(snapshot) + assert len(edges) == 1 + assert edges[0].relationship_type == "REACHABLE_VIA" + assert edges[0].confidence == 1.0 + + +def test_detect_all_edges_runs_all_detectors(): + nsg_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkSecurityGroups/nsg1" + subnet_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/default" + nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", { + "subnets": [{"id": subnet_id}] + }) + snapshot = _snapshot(nsg) + edges = detect_all_edges(snapshot) + assert any(e.relationship_type == "PROTECTS" for e in edges) + + +def test_empty_snapshot_returns_no_edges(): + snapshot = _snapshot() + assert detect_all_edges(snapshot) == [] From 5c662ff07668565fe55bbc9febe873ce27fa339a Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:55:25 +0100 Subject: [PATCH 16/67] fix(graph): pass tenant_id explicitly to link_findings_to_nodes Replaces ORDER BY updated_at DESC LIMIT 1 subquery with a direct bound parameter to prevent cross-tenant finding linkage under concurrent scans. Signed-off-by: Tanvir Farhad --- scanner/graph/node_service.py | 12 +++--------- tests/test_graph_node_service.py | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index c1154c1a..3f466218 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -40,13 +40,7 @@ SELECT f.id, n.node_id FROM findings f JOIN graph_nodes n ON lower(f.resource_id) = lower(n.resource_id) - AND n.tenant_id = ( - SELECT tenant_id FROM graph_nodes - WHERE snapshot_id = ( - SELECT snapshot_id FROM graph_nodes ORDER BY updated_at DESC LIMIT 1 - ) - LIMIT 1 - ) + AND n.tenant_id = %(tenant_id)s WHERE f.scan_id = %(scan_id)s ON CONFLICT DO NOTHING """ @@ -77,9 +71,9 @@ def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: return written -def link_findings_to_nodes(scan_id: str, dsn: str) -> int: +def link_findings_to_nodes(scan_id: str, tenant_id: str, dsn: str) -> int: """Link findings from this scan to their graph nodes by resource_id.""" with psycopg2.connect(dsn) as conn: with conn.cursor() as cur: - cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id}) + cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id, "tenant_id": tenant_id}) return cur.rowcount if cur.rowcount >= 0 else 0 diff --git a/tests/test_graph_node_service.py b/tests/test_graph_node_service.py index 12f70b8b..c643344e 100644 --- a/tests/test_graph_node_service.py +++ b/tests/test_graph_node_service.py @@ -104,7 +104,7 @@ def test_link_findings_to_nodes_executes_insert(): mock_cur.rowcount = 3 with patch("scanner.graph.node_service.psycopg2.connect", return_value=mock_conn): - link_findings_to_nodes("scan-uuid-1", "postgresql://test/db") + link_findings_to_nodes("scan-uuid-1", "00000000-0000-0000-0000-000000000001", "postgresql://test/db") assert mock_cur.execute.called sql_called = mock_cur.execute.call_args[0][0] From dbbba38db6f5ba885affc0f1d95ee0ddbb1deb6e Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 18:01:39 +0100 Subject: [PATCH 17/67] feat(graph): wire post-scan node and edge population into ScanEngine Signed-off-by: Tanvir Farhad --- scanner/engine.py | 10 ++++ scanner/graph/graph_populator.py | 85 ++++++++++++++++++++++++++++ tests/test_graph_engine_post_scan.py | 63 +++++++++++++++++++++ 3 files changed, 158 insertions(+) create mode 100644 scanner/graph/graph_populator.py create mode 100644 tests/test_graph_engine_post_scan.py diff --git a/scanner/engine.py b/scanner/engine.py index fd3855f3..a2558ebe 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -3,6 +3,7 @@ import importlib.util import inspect import logging +import os import uuid from datetime import datetime, timezone from pathlib import Path @@ -12,6 +13,7 @@ from openshield.severity import CONTRACT_VERSION, SeverityContractError, normalize_severity, score_findings from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id +from scanner.graph.graph_populator import populate_graph from scanner.graph.snapshot_bridge import collect_snapshot logger = logging.getLogger(__name__) @@ -240,6 +242,14 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: logger.info("Scan %s complete — %d total finding(s). Normalising results...", scan_id, len(findings)) + # Populate the attack graph as a non-fatal post-scan step. + dsn = os.environ.get("DATABASE_URL") + if snapshot and dsn: + try: + populate_graph(scan_id, snapshot, dsn) + except Exception as exc: + logger.warning("run_scan: graph population raised unexpectedly: %s", exc) + return make_serializable(result) def _legacy_placeholder(self, rule_id: str) -> RuleEvaluation: diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py new file mode 100644 index 00000000..540bb821 --- /dev/null +++ b/scanner/graph/graph_populator.py @@ -0,0 +1,85 @@ +"""Orchestrate post-scan graph population: nodes, edges, finding links.""" +from __future__ import annotations + +import logging +import uuid +from typing import TYPE_CHECKING + +import psycopg2 +import psycopg2.extras + +from scanner.graph.node_service import link_findings_to_nodes, populate_nodes +from scanner.graph.edge_detector import detect_all_edges + +if TYPE_CHECKING: + from scanner.arg_inventory import InventorySnapshot + +logger = logging.getLogger(__name__) + +_UPSERT_EDGE_SQL = """ +INSERT INTO graph_edges ( + edge_id, source_node_id, target_node_id, relationship_type, + evidence_source, evidence_snapshot_id, confidence, collected_at, properties +) +SELECT + %(edge_id)s, + src.node_id, + tgt.node_id, + %(relationship_type)s, + %(evidence_source)s, + %(evidence_snapshot_id)s, + %(confidence)s, + now(), + '{}'::jsonb +FROM graph_nodes src, graph_nodes tgt +WHERE lower(src.resource_id) = lower(%(source_resource_id)s) + AND lower(tgt.resource_id) = lower(%(target_resource_id)s) +ON CONFLICT (source_node_id, target_node_id, relationship_type) DO UPDATE SET + confidence = EXCLUDED.confidence, + evidence_source = EXCLUDED.evidence_source, + evidence_snapshot_id = EXCLUDED.evidence_snapshot_id, + collected_at = now() +""" + + +def _write_edges(edges: list, snapshot_id: str, dsn: str) -> int: + if not edges: + return 0 + written = 0 + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + for edge in edges: + cur.execute(_UPSERT_EDGE_SQL, { + "edge_id": str(uuid.uuid4()), + "source_resource_id": edge.source_resource_id, + "target_resource_id": edge.target_resource_id, + "relationship_type": edge.relationship_type, + "evidence_source": edge.evidence_source, + "evidence_snapshot_id": snapshot_id, + "confidence": edge.confidence, + }) + written += 1 + return written + + +def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: + """Populate nodes, edges, and finding links for one scan. Failure is non-fatal.""" + try: + node_count = populate_nodes(snapshot, dsn) + logger.info("graph: upserted %d nodes for scan %s", node_count, scan_id) + except Exception as exc: + logger.warning("graph: node population failed for scan %s: %s", scan_id, exc) + return + + try: + edges = detect_all_edges(snapshot) + edge_count = _write_edges(edges, snapshot.snapshot_id, dsn) + logger.info("graph: wrote %d edges for scan %s", edge_count, scan_id) + except Exception as exc: + logger.warning("graph: edge population failed for scan %s: %s", scan_id, exc) + + try: + link_count = link_findings_to_nodes(scan_id, snapshot.tenant_id, dsn) + logger.info("graph: linked %d findings to nodes for scan %s", link_count, scan_id) + except Exception as exc: + logger.warning("graph: finding link failed for scan %s: %s", scan_id, exc) diff --git a/tests/test_graph_engine_post_scan.py b/tests/test_graph_engine_post_scan.py new file mode 100644 index 00000000..71bc51b6 --- /dev/null +++ b/tests/test_graph_engine_post_scan.py @@ -0,0 +1,63 @@ +"""Tests for post-scan graph population wiring in ScanEngine.""" +from unittest.mock import patch +import pytest + +from scanner.engine import ScanEngine +from scanner.arg_inventory import InventorySnapshot, InventoryStatus + + +def _make_snapshot(): + return InventorySnapshot( + snapshot_id="snap-1", + tenant_id="00000000-0000-0000-0000-000000000001", + requested_subscriptions=("00000000-0000-0000-0000-000000000002",), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=10, + pages=1, + resources=(), + errors=(), + ) + + +@pytest.fixture +def engine(): + with patch("scanner.engine.AzureClient"), patch("scanner.engine.ScanEngine.load_rules"): + eng = ScanEngine("00000000-0000-0000-0000-000000000002") + eng.rules = [] + return eng + + +def test_populate_graph_called_when_snapshot_and_dsn_available(engine): + snapshot = _make_snapshot() + with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ + patch("scanner.engine.populate_graph") as mock_populate, \ + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + engine.run_scan() + mock_populate.assert_called_once() + + +def test_populate_graph_not_called_when_snapshot_none(engine): + with patch("scanner.engine.collect_snapshot", return_value=None), \ + patch("scanner.engine.populate_graph") as mock_populate, \ + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + engine.run_scan() + mock_populate.assert_not_called() + + +def test_populate_graph_not_called_when_no_database_url(engine): + snapshot = _make_snapshot() + with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ + patch("scanner.engine.populate_graph") as mock_populate, \ + patch.dict("os.environ", {}, clear=True): + engine.run_scan() + mock_populate.assert_not_called() + + +def test_scan_succeeds_even_if_populate_graph_raises(engine): + snapshot = _make_snapshot() + with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ + patch("scanner.engine.populate_graph", side_effect=Exception("DB down")), \ + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + result = engine.run_scan() + assert result["status"] == "completed" From 149deca19df77ec887bc25504da30894e59fc27c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 00:45:04 +0100 Subject: [PATCH 18/67] fix(lint): apply ruff format to node/edge population files Signed-off-by: Tanvir Farhad --- scanner/graph/edge_detector.py | 81 ++++++++++++++++------------ scanner/graph/graph_populator.py | 22 ++++---- scanner/graph/node_service.py | 28 +++++----- tests/test_graph_edge_detector.py | 33 ++++++------ tests/test_graph_engine_post_scan.py | 33 +++++++----- tests/test_graph_node_service.py | 8 ++- 6 files changed, 118 insertions(+), 87 deletions(-) diff --git a/scanner/graph/edge_detector.py b/scanner/graph/edge_detector.py index 61cf0ad5..ae07eae8 100644 --- a/scanner/graph/edge_detector.py +++ b/scanner/graph/edge_detector.py @@ -1,4 +1,5 @@ """Typed edge detectors that infer relationships between Azure resources in an InventorySnapshot.""" + from __future__ import annotations import logging @@ -48,13 +49,15 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: subnet_id = subnet.get("id") if isinstance(subnet, dict) else None if not subnet_id: continue - edges.append(GraphEdge( - source_resource_id=resource.resource_id, - target_resource_id=subnet_id, - relationship_type="PROTECTS", - evidence_source="arg:properties.subnets", - confidence=1.0, - )) + edges.append( + GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=subnet_id, + relationship_type="PROTECTS", + evidence_source="arg:properties.subnets", + confidence=1.0, + ) + ) return edges @@ -70,13 +73,15 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: continue if subnet_id.lower() not in resource_ids: continue - edges.append(GraphEdge( - source_resource_id=resource.resource_id, - target_resource_id=subnet_id, - relationship_type="MEMBER_OF", - evidence_source="arg:properties.subnet.id", - confidence=0.8, - )) + edges.append( + GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=subnet_id, + relationship_type="MEMBER_OF", + evidence_source="arg:properties.subnet.id", + confidence=0.8, + ) + ) return edges @@ -98,13 +103,15 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: provider_path = parts[-1].split("/") if len(provider_path) >= 4: target_id = "/providers/".join(parts[:-1]) + "/providers/" + "/".join(provider_path[:4]) - edges.append(GraphEdge( - source_resource_id=resource.resource_id, - target_resource_id=target_id, - relationship_type="EXPOSES", - evidence_source="arg:properties.ipConfiguration.id", - confidence=1.0, - )) + edges.append( + GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=target_id, + relationship_type="EXPOSES", + evidence_source="arg:properties.ipConfiguration.id", + confidence=1.0, + ) + ) return edges @@ -119,13 +126,15 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: for identity_id in user_assigned: if not identity_id: continue - edges.append(GraphEdge( - source_resource_id=identity_id, - target_resource_id=resource.resource_id, - relationship_type="HAS_IDENTITY", - evidence_source="arg:properties.identity.userAssignedIdentities", - confidence=0.8, - )) + edges.append( + GraphEdge( + source_resource_id=identity_id, + target_resource_id=resource.resource_id, + relationship_type="HAS_IDENTITY", + evidence_source="arg:properties.identity.userAssignedIdentities", + confidence=0.8, + ) + ) return edges @@ -142,13 +151,15 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: pe_id = ((conn.get("properties") or {}).get("privateEndpoint") or {}).get("id") if not pe_id: continue - edges.append(GraphEdge( - source_resource_id=resource.resource_id, - target_resource_id=pe_id, - relationship_type="REACHABLE_VIA", - evidence_source="arg:properties.privateEndpointConnections", - confidence=1.0, - )) + edges.append( + GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=pe_id, + relationship_type="REACHABLE_VIA", + evidence_source="arg:properties.privateEndpointConnections", + confidence=1.0, + ) + ) return edges diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index 540bb821..c5c47565 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -1,4 +1,5 @@ """Orchestrate post-scan graph population: nodes, edges, finding links.""" + from __future__ import annotations import logging @@ -49,15 +50,18 @@ def _write_edges(edges: list, snapshot_id: str, dsn: str) -> int: with psycopg2.connect(dsn) as conn: with conn.cursor() as cur: for edge in edges: - cur.execute(_UPSERT_EDGE_SQL, { - "edge_id": str(uuid.uuid4()), - "source_resource_id": edge.source_resource_id, - "target_resource_id": edge.target_resource_id, - "relationship_type": edge.relationship_type, - "evidence_source": edge.evidence_source, - "evidence_snapshot_id": snapshot_id, - "confidence": edge.confidence, - }) + cur.execute( + _UPSERT_EDGE_SQL, + { + "edge_id": str(uuid.uuid4()), + "source_resource_id": edge.source_resource_id, + "target_resource_id": edge.target_resource_id, + "relationship_type": edge.relationship_type, + "evidence_source": edge.evidence_source, + "evidence_snapshot_id": snapshot_id, + "confidence": edge.confidence, + }, + ) written += 1 return written diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index 3f466218..2b571e7f 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -1,4 +1,5 @@ """Upsert graph nodes from an InventorySnapshot and link findings to nodes.""" + from __future__ import annotations import json @@ -55,18 +56,21 @@ def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: with psycopg2.connect(dsn) as conn: with conn.cursor() as cur: for resource in snapshot.resources: - cur.execute(_UPSERT_NODE_SQL, { - "node_id": str(uuid.uuid4()), - "tenant_id": resource.tenant_id, - "subscription_id": resource.subscription_id, - "resource_id": resource.resource_id, - "resource_type": resource.resource_type, - "name": resource.name, - "location": resource.location, - "resource_group": resource.resource_group, - "snapshot_id": resource.snapshot_id, - "properties": json.dumps(resource.properties), - }) + cur.execute( + _UPSERT_NODE_SQL, + { + "node_id": str(uuid.uuid4()), + "tenant_id": resource.tenant_id, + "subscription_id": resource.subscription_id, + "resource_id": resource.resource_id, + "resource_type": resource.resource_type, + "name": resource.name, + "location": resource.location, + "resource_group": resource.resource_group, + "snapshot_id": resource.snapshot_id, + "properties": json.dumps(resource.properties), + }, + ) written += 1 return written diff --git a/tests/test_graph_edge_detector.py b/tests/test_graph_edge_detector.py index 247e09d9..4649cb7d 100644 --- a/tests/test_graph_edge_detector.py +++ b/tests/test_graph_edge_detector.py @@ -1,4 +1,5 @@ """Tests for graph edge detectors.""" + from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource from scanner.graph.edge_detector import ( GraphEdge, # noqa: F401 — imported to verify public API surface @@ -43,9 +44,7 @@ def _snapshot(*resources): def test_nsg_to_subnet_detects_protects_edge(): subnet_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/default" nsg_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkSecurityGroups/nsg1" - nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", { - "subnets": [{"id": subnet_id}] - }) + nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", {"subnets": [{"id": subnet_id}]}) snapshot = _snapshot(nsg) edges = NsgToSubnetDetector().detect(snapshot) assert len(edges) == 1 @@ -64,9 +63,11 @@ def test_nsg_to_subnet_no_subnets_returns_empty(): def test_public_ip_to_resource_detects_exposes_edge(): vm_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" pip_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/publicIPAddresses/pip1" - pip = _resource(pip_id, "microsoft.network/publicipaddresses", { - "ipConfiguration": {"id": vm_id + "/networkInterfaces/nic1/ipConfigurations/ipconfig1"} - }) + pip = _resource( + pip_id, + "microsoft.network/publicipaddresses", + {"ipConfiguration": {"id": vm_id + "/networkInterfaces/nic1/ipConfigurations/ipconfig1"}}, + ) snapshot = _snapshot(pip) edges = PublicIpToResourceDetector().detect(snapshot) assert len(edges) == 1 @@ -78,11 +79,9 @@ def test_public_ip_to_resource_detects_exposes_edge(): def test_identity_to_resource_detects_has_identity_edge(): vm_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" identity_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id1" - vm = _resource(vm_id, "microsoft.compute/virtualmachines", { - "identity": { - "userAssignedIdentities": {identity_id: {}} - } - }) + vm = _resource( + vm_id, "microsoft.compute/virtualmachines", {"identity": {"userAssignedIdentities": {identity_id: {}}}} + ) snapshot = _snapshot(vm) edges = IdentityToResourceDetector().detect(snapshot) assert len(edges) == 1 @@ -94,9 +93,11 @@ def test_identity_to_resource_detects_has_identity_edge(): def test_storage_private_endpoint_detects_reachable_via_edge(): storage_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sa1" pe_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/privateEndpoints/pe1" - storage = _resource(storage_id, "microsoft.storage/storageaccounts", { - "privateEndpointConnections": [{"properties": {"privateEndpoint": {"id": pe_id}}}] - }) + storage = _resource( + storage_id, + "microsoft.storage/storageaccounts", + {"privateEndpointConnections": [{"properties": {"privateEndpoint": {"id": pe_id}}}]}, + ) snapshot = _snapshot(storage) edges = StoragePrivateEndpointDetector().detect(snapshot) assert len(edges) == 1 @@ -107,9 +108,7 @@ def test_storage_private_endpoint_detects_reachable_via_edge(): def test_detect_all_edges_runs_all_detectors(): nsg_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/networkSecurityGroups/nsg1" subnet_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Network/virtualNetworks/vnet1/subnets/default" - nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", { - "subnets": [{"id": subnet_id}] - }) + nsg = _resource(nsg_id, "microsoft.network/networksecuritygroups", {"subnets": [{"id": subnet_id}]}) snapshot = _snapshot(nsg) edges = detect_all_edges(snapshot) assert any(e.relationship_type == "PROTECTS" for e in edges) diff --git a/tests/test_graph_engine_post_scan.py b/tests/test_graph_engine_post_scan.py index 71bc51b6..8c4fe17c 100644 --- a/tests/test_graph_engine_post_scan.py +++ b/tests/test_graph_engine_post_scan.py @@ -1,4 +1,5 @@ """Tests for post-scan graph population wiring in ScanEngine.""" + from unittest.mock import patch import pytest @@ -30,34 +31,42 @@ def engine(): def test_populate_graph_called_when_snapshot_and_dsn_available(engine): snapshot = _make_snapshot() - with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ - patch("scanner.engine.populate_graph") as mock_populate, \ - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + with ( + patch("scanner.engine.collect_snapshot", return_value=snapshot), + patch("scanner.engine.populate_graph") as mock_populate, + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), + ): engine.run_scan() mock_populate.assert_called_once() def test_populate_graph_not_called_when_snapshot_none(engine): - with patch("scanner.engine.collect_snapshot", return_value=None), \ - patch("scanner.engine.populate_graph") as mock_populate, \ - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + with ( + patch("scanner.engine.collect_snapshot", return_value=None), + patch("scanner.engine.populate_graph") as mock_populate, + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), + ): engine.run_scan() mock_populate.assert_not_called() def test_populate_graph_not_called_when_no_database_url(engine): snapshot = _make_snapshot() - with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ - patch("scanner.engine.populate_graph") as mock_populate, \ - patch.dict("os.environ", {}, clear=True): + with ( + patch("scanner.engine.collect_snapshot", return_value=snapshot), + patch("scanner.engine.populate_graph") as mock_populate, + patch.dict("os.environ", {}, clear=True), + ): engine.run_scan() mock_populate.assert_not_called() def test_scan_succeeds_even_if_populate_graph_raises(engine): snapshot = _make_snapshot() - with patch("scanner.engine.collect_snapshot", return_value=snapshot), \ - patch("scanner.engine.populate_graph", side_effect=Exception("DB down")), \ - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}): + with ( + patch("scanner.engine.collect_snapshot", return_value=snapshot), + patch("scanner.engine.populate_graph", side_effect=Exception("DB down")), + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), + ): result = engine.run_scan() assert result["status"] == "completed" diff --git a/tests/test_graph_node_service.py b/tests/test_graph_node_service.py index c643344e..f4204001 100644 --- a/tests/test_graph_node_service.py +++ b/tests/test_graph_node_service.py @@ -1,12 +1,16 @@ """Tests for graph node upsert and finding-to-node linking.""" + from unittest.mock import MagicMock, patch from scanner.arg_inventory import InventorySnapshot, InventoryStatus, InventoryResource from scanner.graph.node_service import populate_nodes, link_findings_to_nodes -def _make_resource(resource_id: str, resource_type: str = "microsoft.network/virtualnetworks", - subscription_id: str = "00000000-0000-0000-0000-000000000002") -> InventoryResource: +def _make_resource( + resource_id: str, + resource_type: str = "microsoft.network/virtualnetworks", + subscription_id: str = "00000000-0000-0000-0000-000000000002", +) -> InventoryResource: return InventoryResource( snapshot_id="snap-1", tenant_id="00000000-0000-0000-0000-000000000001", From d861b4eb1f2508f5e9ee279fafd1d6bf35626fd1 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 11:35:15 +0100 Subject: [PATCH 19/67] fix(graph): post-save graph population, fix detectors for ARG schema, tenant filter on edge upsert Signed-off-by: Tanvir Farhad --- scanner/arg_inventory.py | 1 + scanner/engine.py | 12 ++------ scanner/graph/edge_detector.py | 37 ++++++++++++++-------- scanner/graph/graph_populator.py | 7 +++-- scanner/worker.py | 15 +++++++++ tests/test_graph_engine_post_scan.py | 46 +++++++++++++++------------- 6 files changed, 71 insertions(+), 47 deletions(-) diff --git a/scanner/arg_inventory.py b/scanner/arg_inventory.py index feaefb54..92574470 100644 --- a/scanner/arg_inventory.py +++ b/scanner/arg_inventory.py @@ -17,6 +17,7 @@ DEFAULT_QUERY = """ Resources +| extend properties = bag_merge(properties, pack('identity', identity)) | project id, name, type, location, subscriptionId, resourceGroup, tenantId, tags, properties | order by id asc """.strip() diff --git a/scanner/engine.py b/scanner/engine.py index a2558ebe..723b1dfd 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -3,7 +3,6 @@ import importlib.util import inspect import logging -import os import uuid from datetime import datetime, timezone from pathlib import Path @@ -13,7 +12,6 @@ from openshield.severity import CONTRACT_VERSION, SeverityContractError, normalize_severity, score_findings from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id -from scanner.graph.graph_populator import populate_graph from scanner.graph.snapshot_bridge import collect_snapshot logger = logging.getLogger(__name__) @@ -65,6 +63,7 @@ def __init__(self, subscription_id: str) -> None: self.subscription_id = subscription_id self.client = AzureClient(subscription_id) self.rules: List[Any] = [] + self.snapshot: Optional[Any] = None self.load_rules() # ------------------------------------------------------------------ # @@ -124,6 +123,7 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: # Collect an ARG inventory snapshot for graph population and rule enrichment. # Failure is non-fatal: rules fall back to direct SDK calls. snapshot = collect_snapshot(self.client, self.subscription_id) + self.snapshot = snapshot logger.info( "Scan %s starting against subscription %s — %d rules loaded", @@ -242,14 +242,6 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: logger.info("Scan %s complete — %d total finding(s). Normalising results...", scan_id, len(findings)) - # Populate the attack graph as a non-fatal post-scan step. - dsn = os.environ.get("DATABASE_URL") - if snapshot and dsn: - try: - populate_graph(scan_id, snapshot, dsn) - except Exception as exc: - logger.warning("run_scan: graph population raised unexpectedly: %s", exc) - return make_serializable(result) def _legacy_placeholder(self, rule_id: str) -> RuleEvaluation: diff --git a/scanner/graph/edge_detector.py b/scanner/graph/edge_detector.py index ae07eae8..ce5878d1 100644 --- a/scanner/graph/edge_detector.py +++ b/scanner/graph/edge_detector.py @@ -68,20 +68,31 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: edges = [] resource_ids = {r.resource_id.lower() for r in snapshot.resources} for resource in snapshot.resources: - subnet_id = (resource.properties.get("subnet") or {}).get("id") - if not subnet_id: - continue - if subnet_id.lower() not in resource_ids: - continue - edges.append( - GraphEdge( - source_resource_id=resource.resource_id, - target_resource_id=subnet_id, - relationship_type="MEMBER_OF", - evidence_source="arg:properties.subnet.id", - confidence=0.8, + subnet_ids: list[str] = [] + + # Direct subnet property (VMs attached directly) + direct = (resource.properties.get("subnet") or {}).get("id") + if direct: + subnet_ids.append(direct) + + # NIC ipConfigurations pattern (most common path for NICs) + for ip_cfg in resource.properties.get("ipConfigurations", []): + sid = (ip_cfg.get("properties", {}).get("subnet") or {}).get("id") + if sid: + subnet_ids.append(sid) + + for subnet_id in subnet_ids: + if not subnet_id or subnet_id.lower() not in resource_ids: + continue + edges.append( + GraphEdge( + source_resource_id=resource.resource_id, + target_resource_id=subnet_id, + relationship_type="MEMBER_OF", + evidence_source="arg:properties.subnet.id", + confidence=0.8, + ) ) - ) return edges diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index c5c47565..2da6479e 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -34,7 +34,9 @@ '{}'::jsonb FROM graph_nodes src, graph_nodes tgt WHERE lower(src.resource_id) = lower(%(source_resource_id)s) + AND src.tenant_id = %(tenant_id)s AND lower(tgt.resource_id) = lower(%(target_resource_id)s) + AND tgt.tenant_id = %(tenant_id)s ON CONFLICT (source_node_id, target_node_id, relationship_type) DO UPDATE SET confidence = EXCLUDED.confidence, evidence_source = EXCLUDED.evidence_source, @@ -43,7 +45,7 @@ """ -def _write_edges(edges: list, snapshot_id: str, dsn: str) -> int: +def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int: if not edges: return 0 written = 0 @@ -59,6 +61,7 @@ def _write_edges(edges: list, snapshot_id: str, dsn: str) -> int: "relationship_type": edge.relationship_type, "evidence_source": edge.evidence_source, "evidence_snapshot_id": snapshot_id, + "tenant_id": tenant_id, "confidence": edge.confidence, }, ) @@ -77,7 +80,7 @@ def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: try: edges = detect_all_edges(snapshot) - edge_count = _write_edges(edges, snapshot.snapshot_id, dsn) + edge_count = _write_edges(edges, snapshot.snapshot_id, snapshot.tenant_id, dsn) logger.info("graph: wrote %d edges for scan %s", edge_count, scan_id) except Exception as exc: logger.warning("graph: edge population failed for scan %s: %s", scan_id, exc) diff --git a/scanner/worker.py b/scanner/worker.py index 2ae431ae..5c0d388a 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -27,6 +27,7 @@ ) from scanner.engine import ScanEngine from scanner.enrichment_worker import process_enrichment_job +from scanner.graph.graph_populator import populate_graph configure_logging() logger = logging.getLogger("scanner.worker") @@ -216,6 +217,20 @@ def run_worker(): if heartbeat.lost.is_set(): raise LostLease(f"Scan {scan_id} lost its lease before completion") db.save_scan(result, worker_id, fencing_token) + + # Graph population runs after findings are persisted so + # link_findings_to_nodes can join against the saved rows. + dsn = os.environ.get("DATABASE_URL") + if engine.snapshot and dsn: + try: + populate_graph(scan_id, engine.snapshot, dsn) + except Exception as exc: + logger.warning( + "worker: graph population raised unexpectedly for scan %s: %s", + scan_id, + exc, + ) + SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", diff --git a/tests/test_graph_engine_post_scan.py b/tests/test_graph_engine_post_scan.py index 8c4fe17c..dd4dc643 100644 --- a/tests/test_graph_engine_post_scan.py +++ b/tests/test_graph_engine_post_scan.py @@ -1,6 +1,6 @@ -"""Tests for post-scan graph population wiring in ScanEngine.""" +"""Tests for post-scan graph population wiring in ScanEngine and worker.""" -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest from scanner.engine import ScanEngine @@ -29,44 +29,46 @@ def engine(): return eng -def test_populate_graph_called_when_snapshot_and_dsn_available(engine): +def test_snapshot_stored_on_engine_after_run_scan(engine): snapshot = _make_snapshot() - with ( - patch("scanner.engine.collect_snapshot", return_value=snapshot), - patch("scanner.engine.populate_graph") as mock_populate, - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), - ): + with patch("scanner.engine.collect_snapshot", return_value=snapshot): engine.run_scan() - mock_populate.assert_called_once() + assert engine.snapshot is snapshot -def test_populate_graph_not_called_when_snapshot_none(engine): - with ( - patch("scanner.engine.collect_snapshot", return_value=None), - patch("scanner.engine.populate_graph") as mock_populate, - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), - ): +def test_snapshot_is_none_on_engine_when_collection_fails(engine): + with patch("scanner.engine.collect_snapshot", return_value=None): engine.run_scan() - mock_populate.assert_not_called() + assert engine.snapshot is None -def test_populate_graph_not_called_when_no_database_url(engine): +def test_scan_result_does_not_call_populate_graph(engine): + """populate_graph must NOT be called inside run_scan — it belongs in worker.""" snapshot = _make_snapshot() with ( patch("scanner.engine.collect_snapshot", return_value=snapshot), - patch("scanner.engine.populate_graph") as mock_populate, - patch.dict("os.environ", {}, clear=True), + patch("scanner.graph.graph_populator.populate_graph") as mock_populate, + patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), ): engine.run_scan() mock_populate.assert_not_called() -def test_scan_succeeds_even_if_populate_graph_raises(engine): +def test_scan_succeeds_and_stores_snapshot_even_without_database_url(engine): snapshot = _make_snapshot() with ( patch("scanner.engine.collect_snapshot", return_value=snapshot), - patch("scanner.engine.populate_graph", side_effect=Exception("DB down")), - patch.dict("os.environ", {"DATABASE_URL": "postgresql://test/db"}), + patch.dict("os.environ", {}, clear=True), ): result = engine.run_scan() assert result["status"] == "completed" + assert engine.snapshot is snapshot + + +def test_worker_imports_populate_graph(): + """Sanity-check that worker imports populate_graph for post-save invocation.""" + import scanner.worker as worker_module # noqa: PLC0415 + + assert hasattr(worker_module, "populate_graph"), ( + "worker.py must import populate_graph so it can call it after db.save_scan" + ) From 06db8ceb700b0910c35d137c122694d54993806b Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 23:05:13 +0100 Subject: [PATCH 20/67] fix(lint): remove unused MagicMock import from post-scan graph test Signed-off-by: Tanvir Farhad --- tests/test_graph_engine_post_scan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_graph_engine_post_scan.py b/tests/test_graph_engine_post_scan.py index dd4dc643..520614f0 100644 --- a/tests/test_graph_engine_post_scan.py +++ b/tests/test_graph_engine_post_scan.py @@ -1,6 +1,6 @@ """Tests for post-scan graph population wiring in ScanEngine and worker.""" -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest from scanner.engine import ScanEngine From 2478c87e5228bf301a2dec12c4d5375c1917fc79 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 1 Oct 2026 00:37:39 +0100 Subject: [PATCH 21/67] fix(graph): synthesise subnet nodes from VNet properties.subnets before edge detection ARG Resources has no top-level rows for subnets; they are nested inside the parent VNet's properties.subnets. Without this, SubnetToResourceDetector and NsgToSubnetDetector produce edges whose target has no graph_node row, and _UPSERT_EDGE_SQL silently drops them (INSERT ... SELECT JOIN graph_nodes). _synthesise_subnet_resources() walks VNet resources, extracts each entry in properties.subnets, and returns synthetic InventoryResource objects inheriting the VNet's tenant_id, subscription_id, location and resource_group. populate_graph builds an augmented snapshot (dataclasses.replace on the frozen dataclass) before calling populate_nodes and detect_all_edges, so subnets get graph_nodes and the NSG->subnet->NIC path is traversable by BFS in #354. 7 new tests in test_graph_populator_subnet_synthesis.py use realistic ARG response fixtures (VNet with subnet, NSG, NIC, VM with user-assigned identity) and assert both the synthesis behaviour and the original bug (edge dropped without synthesis). Signed-off-by: Tanvir Farhad --- scanner/graph/graph_populator.py | 53 ++++- .../test_graph_populator_subnet_synthesis.py | 209 ++++++++++++++++++ 2 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 tests/test_graph_populator_subnet_synthesis.py diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index 2da6479e..c3a073dc 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -4,11 +4,13 @@ import logging import uuid +from dataclasses import replace as dc_replace from typing import TYPE_CHECKING import psycopg2 import psycopg2.extras +from scanner.arg_inventory import InventoryResource from scanner.graph.node_service import link_findings_to_nodes, populate_nodes from scanner.graph.edge_detector import detect_all_edges @@ -69,17 +71,64 @@ def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int return written +_VNET_TYPE = "microsoft.network/virtualnetworks" +_SUBNET_TYPE = "microsoft.network/virtualnetworks/subnets" + + +def _synthesise_subnet_resources(snapshot: InventorySnapshot) -> list[InventoryResource]: + """Return synthetic InventoryResource entries for subnets nested inside VNets. + + ARG Resources has no top-level rows for subnets; they appear only as + properties.subnets on the parent VNet. Without this step, SubnetToResourceDetector + and NsgToSubnetDetector produce edges whose target has no matching graph_node, + and _UPSERT_EDGE_SQL silently drops them (INSERT ... SELECT JOIN graph_nodes). + """ + subnets: list[InventoryResource] = [] + for resource in snapshot.resources: + if resource.resource_type.lower() != _VNET_TYPE: + continue + for subnet in resource.properties.get("subnets") or []: + subnet_id = subnet.get("id") if isinstance(subnet, dict) else None + if not subnet_id: + continue + subnet_name = subnet.get("name", subnet_id.split("/")[-1]) + subnets.append( + InventoryResource( + snapshot_id=resource.snapshot_id, + tenant_id=resource.tenant_id, + subscription_id=resource.subscription_id, + resource_id=subnet_id, + resource_type=_SUBNET_TYPE, + name=subnet_name, + location=resource.location, + resource_group=resource.resource_group, + tags={}, + properties=subnet.get("properties") or {}, + ) + ) + return subnets + + def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: """Populate nodes, edges, and finding links for one scan. Failure is non-fatal.""" + subnet_resources = _synthesise_subnet_resources(snapshot) + if subnet_resources: + logger.debug("graph: synthesised %d subnet nodes from VNet properties", len(subnet_resources)) + + augmented_snapshot = dc_replace( + snapshot, + resources=snapshot.resources + tuple(subnet_resources), + ) + try: - node_count = populate_nodes(snapshot, dsn) + node_count = populate_nodes(augmented_snapshot, dsn) logger.info("graph: upserted %d nodes for scan %s", node_count, scan_id) except Exception as exc: logger.warning("graph: node population failed for scan %s: %s", scan_id, exc) return try: - edges = detect_all_edges(snapshot) + edges = detect_all_edges(augmented_snapshot) edge_count = _write_edges(edges, snapshot.snapshot_id, snapshot.tenant_id, dsn) logger.info("graph: wrote %d edges for scan %s", edge_count, scan_id) except Exception as exc: diff --git a/tests/test_graph_populator_subnet_synthesis.py b/tests/test_graph_populator_subnet_synthesis.py new file mode 100644 index 00000000..06688896 --- /dev/null +++ b/tests/test_graph_populator_subnet_synthesis.py @@ -0,0 +1,209 @@ +"""Tests for subnet node synthesis in graph_populator. + +ARG Resources has no top-level rows for subnets — they are nested under the +parent VNet's properties.subnets. _synthesise_subnet_resources extracts them +so populate_graph can write graph_nodes for subnets and edge upserts succeed. +""" + +from scanner.arg_inventory import InventoryResource, InventorySnapshot, InventoryStatus +from scanner.graph.graph_populator import _synthesise_subnet_resources + +# Realistic ARG response shapes ------------------------------------------------- +# These mirror what Azure Resource Graph returns for a VNet with one subnet, +# a NIC attached to that subnet, and a VM with a user-assigned identity. + +_TENANT = "00000000-0000-0000-0000-000000000001" +_SUB = "00000000-0000-0000-0000-000000000002" +_SNAP = "snap-test-1" + +_VNET_ID = ( + "/subscriptions/00000000-0000-0000-0000-000000000002" + "/resourceGroups/rg-prod" + "/providers/Microsoft.Network/virtualNetworks/vnet-prod" +) +_SUBNET_ID = _VNET_ID + "/subnets/default" +_NSG_ID = ( + "/subscriptions/00000000-0000-0000-0000-000000000002" + "/resourceGroups/rg-prod" + "/providers/Microsoft.Network/networkSecurityGroups/nsg-prod" +) +_NIC_ID = ( + "/subscriptions/00000000-0000-0000-0000-000000000002" + "/resourceGroups/rg-prod" + "/providers/Microsoft.Network/networkInterfaces/nic-prod" +) +_VM_ID = ( + "/subscriptions/00000000-0000-0000-0000-000000000002" + "/resourceGroups/rg-prod" + "/providers/Microsoft.Compute/virtualMachines/vm-prod" +) +_IDENTITY_ID = ( + "/subscriptions/00000000-0000-0000-0000-000000000002" + "/resourceGroups/rg-prod" + "/providers/Microsoft.ManagedIdentity/userAssignedIdentities/id-prod" +) + + +def _resource(resource_id, resource_type, properties=None): + return InventoryResource( + snapshot_id=_SNAP, + tenant_id=_TENANT, + subscription_id=_SUB, + resource_id=resource_id, + resource_type=resource_type, + name=resource_id.split("/")[-1], + location="uksouth", + resource_group="rg-prod", + tags={}, + properties=properties or {}, + ) + + +# ARG returns subnets nested inside the VNet properties.subnets array +_VNET = _resource( + _VNET_ID, + "microsoft.network/virtualnetworks", + { + "subnets": [ + { + "id": _SUBNET_ID, + "name": "default", + "properties": { + "addressPrefix": "10.0.0.0/24", + "networkSecurityGroup": {"id": _NSG_ID}, + }, + } + ], + "addressSpace": {"addressPrefixes": ["10.0.0.0/16"]}, + }, +) + +_NSG = _resource( + _NSG_ID, + "microsoft.network/networksecuritygroups", + {"subnets": [{"id": _SUBNET_ID}]}, +) + +# ARG returns NIC with ipConfigurations.properties.subnet.id +_NIC = _resource( + _NIC_ID, + "microsoft.network/networkinterfaces", + { + "ipConfigurations": [ + { + "properties": { + "subnet": {"id": _SUBNET_ID}, + "privateIPAddress": "10.0.0.4", + } + } + ] + }, +) + +_VM = _resource( + _VM_ID, + "microsoft.compute/virtualmachines", + { + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": {_IDENTITY_ID: {}}, + } + }, +) + + +def _snapshot(*resources): + return InventorySnapshot( + snapshot_id=_SNAP, + tenant_id=_TENANT, + requested_subscriptions=(_SUB,), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-30T00:00:00+00:00", + duration_ms=42, + pages=1, + resources=tuple(resources), + errors=(), + ) + + +# Tests ------------------------------------------------------------------------- + + +def test_synthesise_extracts_subnet_from_vnet(): + """Subnet nested in VNet properties.subnets must produce a synthetic node.""" + snapshot = _snapshot(_VNET, _NSG, _NIC, _VM) + subnets = _synthesise_subnet_resources(snapshot) + assert len(subnets) == 1 + subnet = subnets[0] + assert subnet.resource_id == _SUBNET_ID + assert subnet.resource_type == "microsoft.network/virtualnetworks/subnets" + assert subnet.tenant_id == _TENANT + assert subnet.subscription_id == _SUB + assert subnet.name == "default" + assert subnet.location == "uksouth" + + +def test_synthesise_subnet_inherits_vnet_metadata(): + """Synthesised subnet must carry VNet's tenant_id, subscription_id and location.""" + snapshot = _snapshot(_VNET) + subnets = _synthesise_subnet_resources(snapshot) + assert subnets[0].snapshot_id == _SNAP + assert subnets[0].resource_group == "rg-prod" + + +def test_synthesise_subnet_properties_from_arg(): + """Subnet properties block from ARG (addressPrefix, NSG link) must be preserved.""" + snapshot = _snapshot(_VNET) + subnets = _synthesise_subnet_resources(snapshot) + assert "addressPrefix" in subnets[0].properties + assert subnets[0].properties["addressPrefix"] == "10.0.0.0/24" + + +def test_synthesise_no_subnets_returns_empty(): + """VNet with empty subnets list must not produce any synthetic nodes.""" + vnet_no_subnets = _resource(_VNET_ID, "microsoft.network/virtualnetworks", {"subnets": []}) + snapshot = _snapshot(vnet_no_subnets) + assert _synthesise_subnet_resources(snapshot) == [] + + +def test_synthesise_non_vnet_resources_ignored(): + """NSGs, NICs, and VMs must not produce subnet synthetics.""" + snapshot = _snapshot(_NSG, _NIC, _VM) + assert _synthesise_subnet_resources(snapshot) == [] + + +def test_synthesise_missing_subnet_id_skipped(): + """Subnet entry with no id field must be silently skipped.""" + vnet = _resource( + _VNET_ID, + "microsoft.network/virtualnetworks", + {"subnets": [{"name": "broken", "properties": {}}]}, + ) + snapshot = _snapshot(vnet) + assert _synthesise_subnet_resources(snapshot) == [] + + +def test_nsg_to_subnet_edge_requires_subnet_in_resource_ids(): + """Confirm the original problem: SubnetToResourceDetector drops NIC->subnet + when subnet is not in resource_ids. After synthesis it must be present.""" + from scanner.graph.edge_detector import SubnetToResourceDetector + + # Without synthesis: NIC->subnet edge is dropped + snapshot_no_subnet = _snapshot(_NSG, _NIC, _VM) + edges_without = SubnetToResourceDetector().detect(snapshot_no_subnet) + assert not any(e.target_resource_id == _SUBNET_ID for e in edges_without), ( + "subnet should not appear in edges when it has no node" + ) + + # With synthesis: subnet is in resource_ids, edge is emitted + subnets = _synthesise_subnet_resources(_snapshot(_VNET, _NSG, _NIC, _VM)) + from dataclasses import replace as dc_replace + + augmented = dc_replace( + snapshot_no_subnet, + resources=snapshot_no_subnet.resources + tuple(subnets), + ) + edges_with = SubnetToResourceDetector().detect(augmented) + assert any(e.target_resource_id == _SUBNET_ID for e in edges_with), ( + "NIC->subnet MEMBER_OF edge must be present after subnet synthesis" + ) From ae4d441eb209c2d0904432003ddfe32aa86f1f66 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 02:42:29 +0100 Subject: [PATCH 22/67] test(graph): verify DEFAULT_QUERY bag_merge and identity field parsing by IdentityToResourceDetector Signed-off-by: Tanvir Farhad --- tests/test_arg_inventory.py | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/test_arg_inventory.py b/tests/test_arg_inventory.py index 17fe54ec..55f95f56 100644 --- a/tests/test_arg_inventory.py +++ b/tests/test_arg_inventory.py @@ -227,3 +227,63 @@ def test_context_manager_closes_sdk_client(): pass client.close.assert_called_once_with() + + +def test_default_query_contains_bag_merge_and_identity(): + """DEFAULT_QUERY must merge the top-level identity column into properties. + + ARG does not include the identity field in the properties column by default. + The bag_merge call ensures it is accessible as properties.identity so that + IdentityToResourceDetector can read it without special-casing the query caller. + """ + from scanner.arg_inventory import DEFAULT_QUERY + + assert "bag_merge" in DEFAULT_QUERY + assert "identity" in DEFAULT_QUERY + + +def test_identity_field_merged_into_properties_is_parsed_by_identity_detector(): + """A row whose identity is merged into properties by bag_merge is detected correctly. + + Simulates the ARG response shape produced by DEFAULT_QUERY: the top-level + identity column is merged into the properties dict before ArgInventoryClient + returns it, so IdentityToResourceDetector should find userAssignedIdentities there. + """ + from scanner.arg_inventory import InventoryResource, InventorySnapshot, InventoryStatus + from scanner.graph.edge_detector import IdentityToResourceDetector + + vm_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.Compute/virtualMachines/vm1" + identity_id = "/subscriptions/sub/resourceGroups/rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/mid1" + + # Simulate what ARG returns after bag_merge(properties, pack('identity', identity)): + # the resource row has properties.identity set with userAssignedIdentities. + resource = InventoryResource( + snapshot_id="snap-kql", + tenant_id=TENANT_ID, + subscription_id=SUBSCRIPTION_A, + resource_id=vm_id, + resource_type="microsoft.compute/virtualmachines", + name="vm1", + location="eastus", + resource_group="rg", + tags={}, + properties={"identity": {"userAssignedIdentities": {identity_id: {}}}}, + ) + snapshot = InventorySnapshot( + snapshot_id="snap-kql", + tenant_id=TENANT_ID, + requested_subscriptions=(SUBSCRIPTION_A,), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-24T00:00:00+00:00", + duration_ms=1, + pages=1, + resources=(resource,), + errors=(), + ) + + edges = IdentityToResourceDetector().detect(snapshot) + + assert len(edges) == 1 + assert edges[0].relationship_type == "HAS_IDENTITY" + assert edges[0].source_resource_id == identity_id + assert edges[0].target_resource_id == vm_id From b959ba9b4bd9d1f40ee47e1b4354c0713bb1ef08 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 14:26:46 +0100 Subject: [PATCH 23/67] fix: close psycopg2 connections explicitly, fix edge rowcount, correct PublicIp path trimming to 3 segments Signed-off-by: Tanvir Farhad --- scanner/graph/edge_detector.py | 6 +++--- scanner/graph/graph_populator.py | 13 ++++++++++--- scanner/graph/node_service.py | 25 ++++++++++++++++++++----- 3 files changed, 33 insertions(+), 11 deletions(-) diff --git a/scanner/graph/edge_detector.py b/scanner/graph/edge_detector.py index ce5878d1..2c494a20 100644 --- a/scanner/graph/edge_detector.py +++ b/scanner/graph/edge_detector.py @@ -108,12 +108,12 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: target_id = ip_config.get("id") if not target_id: continue - # Strip the NIC sub-path to get the parent VM resource ID (4 provider path segments) + # Trim to the NIC resource ID (namespace/type/name = 3 segments). parts = target_id.split("/providers/") if len(parts) >= 2: provider_path = parts[-1].split("/") - if len(provider_path) >= 4: - target_id = "/providers/".join(parts[:-1]) + "/providers/" + "/".join(provider_path[:4]) + if len(provider_path) >= 3: + target_id = "/providers/".join(parts[:-1]) + "/providers/" + "/".join(provider_path[:3]) edges.append( GraphEdge( source_resource_id=resource.resource_id, diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index c3a073dc..1c62ef87 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING import psycopg2 -import psycopg2.extras + from scanner.arg_inventory import InventoryResource from scanner.graph.node_service import link_findings_to_nodes, populate_nodes @@ -51,7 +51,8 @@ def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int if not edges: return 0 written = 0 - with psycopg2.connect(dsn) as conn: + conn = psycopg2.connect(dsn) + try: with conn.cursor() as cur: for edge in edges: cur.execute( @@ -67,7 +68,13 @@ def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int "confidence": edge.confidence, }, ) - written += 1 + written += max(cur.rowcount, 0) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() return written diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index 2b571e7f..50355b8c 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING import psycopg2 -import psycopg2.extras + if TYPE_CHECKING: from scanner.arg_inventory import InventorySnapshot @@ -53,7 +53,8 @@ def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: return 0 written = 0 - with psycopg2.connect(dsn) as conn: + conn = psycopg2.connect(dsn) + try: with conn.cursor() as cur: for resource in snapshot.resources: cur.execute( @@ -71,13 +72,27 @@ def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: "properties": json.dumps(resource.properties), }, ) - written += 1 + written += max(cur.rowcount, 0) + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() return written def link_findings_to_nodes(scan_id: str, tenant_id: str, dsn: str) -> int: """Link findings from this scan to their graph nodes by resource_id.""" - with psycopg2.connect(dsn) as conn: + conn = psycopg2.connect(dsn) + try: with conn.cursor() as cur: cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id, "tenant_id": tenant_id}) - return cur.rowcount if cur.rowcount >= 0 else 0 + result = cur.rowcount if cur.rowcount >= 0 else 0 + conn.commit() + return result + except Exception: + conn.rollback() + raise + finally: + conn.close() From 54e25821ccf917e4fc87083e65c2435037d07185 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 14:26:40 +0100 Subject: [PATCH 24/67] fix: add DATABASE_URL guard to migration test, use parameter name for snapshot dispatch, exc_info on ARG failure, tighten down_revision type Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 2 +- scanner/engine.py | 2 +- scanner/graph/snapshot_bridge.py | 4 ++-- tests/test_graph_migration.py | 7 ++++++- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index 73190d5f..3150a294 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = "3a76ff935bf6" +down_revision: str = "3a76ff935bf6" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/scanner/engine.py b/scanner/engine.py index 723b1dfd..205361c6 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -160,7 +160,7 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: continue try: - if len(inspect.signature(rule.scan).parameters) >= 3: + if "snapshot" in inspect.signature(rule.scan).parameters: rule_findings = rule.scan(self.client, self.subscription_id, snapshot) else: rule_findings = rule.scan(self.client, self.subscription_id) diff --git a/scanner/graph/snapshot_bridge.py b/scanner/graph/snapshot_bridge.py index 08a724b3..e5d7311e 100644 --- a/scanner/graph/snapshot_bridge.py +++ b/scanner/graph/snapshot_bridge.py @@ -33,8 +33,8 @@ def collect_snapshot(client: AzureClient, subscription_id: str) -> InventorySnap tenant_id=tenant_id, subscription_ids=[subscription_id], ) - except Exception as exc: - logger.warning("snapshot_bridge: ARG collection failed — %s", exc) + except Exception: + logger.warning("snapshot_bridge: ARG collection failed", exc_info=True) return None if snapshot.status == InventoryStatus.FAILED: diff --git a/tests/test_graph_migration.py b/tests/test_graph_migration.py index dfe31b37..9a5a620f 100644 --- a/tests/test_graph_migration.py +++ b/tests/test_graph_migration.py @@ -6,7 +6,12 @@ from alembic import command from sqlalchemy import create_engine, inspect -DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/openshield_test") +DATABASE_URL = os.environ.get("DATABASE_URL") + +pytestmark = pytest.mark.skipif( + not DATABASE_URL, + reason="DATABASE_URL is required for PostgreSQL migration tests", +) @pytest.fixture(scope="module") From 6f461df48a0f4467890fac73cb75ae8d2c9a7d98 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 13:53:43 +0100 Subject: [PATCH 25/67] fix(graph): chain foundation on current dev and isolate migration tests Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 +- tests/test_graph_migration.py | 46 +++++++++++++++++-- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index 3150a294..761e20e9 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: 3a76ff935bf6 +Revises: b6d2f8a4c1e7 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: str = "3a76ff935bf6" +down_revision: str = "b6d2f8a4c1e7" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/tests/test_graph_migration.py b/tests/test_graph_migration.py index 9a5a620f..e9613f4d 100644 --- a/tests/test_graph_migration.py +++ b/tests/test_graph_migration.py @@ -1,6 +1,9 @@ """Verify the graph schema migration applies and rolls back cleanly.""" import os +import uuid + +import psycopg2 import pytest from alembic.config import Config from alembic import command @@ -15,8 +18,37 @@ @pytest.fixture(scope="module") -def engine(): - return create_engine(DATABASE_URL) +def scratch_database(): + """Each module owns its database, so downgrade never changes shared state.""" + base = DATABASE_URL.rsplit("/", 1)[0] + name = f"openshield_graph_mig_{uuid.uuid4().hex[:12]}" + admin = psycopg2.connect(f"{base}/postgres") + admin.autocommit = True + previous = os.environ.get("DATABASE_URL") + try: + with admin.cursor() as cur: + cur.execute(f'CREATE DATABASE "{name}"') + os.environ["DATABASE_URL"] = f"{base}/{name}" + yield os.environ["DATABASE_URL"] + finally: + if previous is None: + os.environ.pop("DATABASE_URL", None) + else: + os.environ["DATABASE_URL"] = previous + with admin.cursor() as cur: + cur.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %s AND pid <> pg_backend_pid()", + (name,), + ) + cur.execute(f'DROP DATABASE IF EXISTS "{name}"') + admin.close() + + +@pytest.fixture(scope="module") +def engine(scratch_database): + engine = create_engine(scratch_database) + yield engine + engine.dispose() @pytest.fixture(scope="module", autouse=True) @@ -24,7 +56,6 @@ def run_migrations(engine): cfg = Config("alembic.ini") command.upgrade(cfg, "head") yield - command.downgrade(cfg, "e1f2a3b4c5d6-1") # one step back def test_graph_nodes_table_exists(engine): @@ -85,7 +116,7 @@ def test_finding_graph_nodes_columns(engine): def test_downgrade_removes_tables(engine): cfg = Config("alembic.ini") - command.downgrade(cfg, "3f59f83a5253") + command.downgrade(cfg, "b6d2f8a4c1e7") inspector = inspect(engine) tables = inspector.get_table_names() assert "graph_nodes" not in tables @@ -93,3 +124,10 @@ def test_downgrade_removes_tables(engine): assert "finding_graph_nodes" not in tables # Restore for subsequent tests command.upgrade(cfg, "head") + + +def test_upgrade_from_current_dev(engine): + cfg = Config("alembic.ini") + command.downgrade(cfg, "b6d2f8a4c1e7") + command.upgrade(cfg, "head") + assert "graph_nodes" in inspect(engine).get_table_names() From 0592c667ecd01ddf7e1a5db8e38d19f46dd9920f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:10:16 +0100 Subject: [PATCH 26/67] fix(graph): publish atomic current scopes and expire complete snapshots Signed-off-by: Tanvir Farhad --- .../e2c4a6b8d013_graph_current_scope.py | 48 ++++++ scanner/graph/graph_populator.py | 93 ++++++++---- scanner/graph/node_service.py | 54 ++++--- tests/test_graph_freshness_postgres.py | 139 ++++++++++++++++++ tests/test_graph_freshness_schema.py | 17 +++ tests/test_graph_node_service.py | 13 ++ 6 files changed, 313 insertions(+), 51 deletions(-) create mode 100644 alembic/versions/e2c4a6b8d013_graph_current_scope.py create mode 100644 tests/test_graph_freshness_postgres.py create mode 100644 tests/test_graph_freshness_schema.py diff --git a/alembic/versions/e2c4a6b8d013_graph_current_scope.py b/alembic/versions/e2c4a6b8d013_graph_current_scope.py new file mode 100644 index 00000000..cf4ca8d2 --- /dev/null +++ b/alembic/versions/e2c4a6b8d013_graph_current_scope.py @@ -0,0 +1,48 @@ +"""Track the explicit current snapshot for each tenant/subscription graph. + +Revision ID: e2c4a6b8d013 +Revises: e1f2a3b4c5d6 +""" + +from typing import Sequence, Union +from alembic import op + +revision: str = "e2c4a6b8d013" +down_revision: str = "e1f2a3b4c5d6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute(""" + CREATE TABLE graph_snapshot_scopes ( + tenant_id text NOT NULL, + subscription_id text NOT NULL, + snapshot_id text NOT NULL, + status text NOT NULL CHECK (status IN ('COMPLETE', 'PARTIAL')), + collected_at timestamptz NOT NULL, + PRIMARY KEY (tenant_id, subscription_id) + ) + """) + op.execute(""" + CREATE VIEW current_graph_nodes AS + SELECT n.* FROM graph_nodes n + JOIN graph_snapshot_scopes s + ON s.tenant_id = n.tenant_id AND s.subscription_id = n.subscription_id + AND s.snapshot_id = n.snapshot_id + """) + op.execute(""" + CREATE VIEW current_graph_edges AS + SELECT e.* FROM graph_edges e + JOIN current_graph_nodes src ON src.node_id = e.source_node_id + JOIN current_graph_nodes tgt ON tgt.node_id = e.target_node_id + WHERE src.tenant_id = tgt.tenant_id + AND e.evidence_snapshot_id = src.snapshot_id + AND e.evidence_snapshot_id = tgt.snapshot_id + """) + + +def downgrade() -> None: + op.execute("DROP VIEW current_graph_edges") + op.execute("DROP VIEW current_graph_nodes") + op.execute("DROP TABLE graph_snapshot_scopes") diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index 1c62ef87..eb8e4095 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -7,11 +7,9 @@ from dataclasses import replace as dc_replace from typing import TYPE_CHECKING -import psycopg2 - -from scanner.arg_inventory import InventoryResource -from scanner.graph.node_service import link_findings_to_nodes, populate_nodes +from scanner.arg_inventory import InventoryResource, InventoryStatus +from scanner.graph.node_service import graph_connection, link_findings_to_nodes, populate_nodes from scanner.graph.edge_detector import detect_all_edges if TYPE_CHECKING: @@ -39,6 +37,8 @@ AND src.tenant_id = %(tenant_id)s AND lower(tgt.resource_id) = lower(%(target_resource_id)s) AND tgt.tenant_id = %(tenant_id)s + AND src.snapshot_id = %(evidence_snapshot_id)s + AND tgt.snapshot_id = %(evidence_snapshot_id)s ON CONFLICT (source_node_id, target_node_id, relationship_type) DO UPDATE SET confidence = EXCLUDED.confidence, evidence_source = EXCLUDED.evidence_source, @@ -47,12 +47,11 @@ """ -def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int: +def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str, *, connection=None) -> int: if not edges: return 0 written = 0 - conn = psycopg2.connect(dsn) - try: + with graph_connection(dsn, connection) as conn: with conn.cursor() as cur: for edge in edges: cur.execute( @@ -69,12 +68,6 @@ def _write_edges(edges: list, snapshot_id: str, tenant_id: str, dsn: str) -> int }, ) written += max(cur.rowcount, 0) - conn.commit() - except Exception: - conn.rollback() - raise - finally: - conn.close() return written @@ -127,22 +120,62 @@ def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: resources=snapshot.resources + tuple(subnet_resources), ) - try: - node_count = populate_nodes(augmented_snapshot, dsn) - logger.info("graph: upserted %d nodes for scan %s", node_count, scan_id) - except Exception as exc: - logger.warning("graph: node population failed for scan %s: %s", scan_id, exc) + if snapshot.status == InventoryStatus.FAILED: return - - try: - edges = detect_all_edges(augmented_snapshot) - edge_count = _write_edges(edges, snapshot.snapshot_id, snapshot.tenant_id, dsn) - logger.info("graph: wrote %d edges for scan %s", edge_count, scan_id) - except Exception as exc: - logger.warning("graph: edge population failed for scan %s: %s", scan_id, exc) - try: - link_count = link_findings_to_nodes(scan_id, snapshot.tenant_id, dsn) - logger.info("graph: linked %d findings to nodes for scan %s", link_count, scan_id) - except Exception as exc: - logger.warning("graph: finding link failed for scan %s: %s", scan_id, exc) + # Publish the new scope only after every write succeeds. Partial snapshots + # retain historical rows, but the views expose only explicit current evidence. + with graph_connection(dsn) as conn: + node_count = populate_nodes(augmented_snapshot, dsn, connection=conn) + edges = detect_all_edges(augmented_snapshot) + edge_count = _write_edges(edges, snapshot.snapshot_id, snapshot.tenant_id, dsn, connection=conn) + with conn.cursor() as cur: + params = { + "tenant": snapshot.tenant_id, + "subscriptions": list(snapshot.requested_subscriptions), + "snapshot": snapshot.snapshot_id, + } + if snapshot.status == InventoryStatus.COMPLETE: + cur.execute( + """ + DELETE FROM graph_edges e USING graph_nodes src, graph_nodes tgt + WHERE src.node_id = e.source_node_id AND tgt.node_id = e.target_node_id + AND src.tenant_id = %(tenant)s AND tgt.tenant_id = %(tenant)s + AND (src.subscription_id = ANY(%(subscriptions)s) + OR tgt.subscription_id = ANY(%(subscriptions)s)) + AND e.evidence_snapshot_id <> %(snapshot)s + """, + params, + ) + cur.execute( + """ + DELETE FROM graph_nodes + WHERE tenant_id = %(tenant)s AND subscription_id = ANY(%(subscriptions)s) + AND snapshot_id <> %(snapshot)s + """, + params, + ) + for subscription in snapshot.requested_subscriptions: + cur.execute( + """ + INSERT INTO graph_snapshot_scopes + (tenant_id, subscription_id, snapshot_id, status, collected_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (tenant_id, subscription_id) DO UPDATE SET + snapshot_id = EXCLUDED.snapshot_id, status = EXCLUDED.status, + collected_at = EXCLUDED.collected_at + """, + ( + snapshot.tenant_id, + subscription, + snapshot.snapshot_id, + snapshot.status.value, + snapshot.collected_at, + ), + ) + link_count = link_findings_to_nodes(scan_id, snapshot.tenant_id, dsn, connection=conn) + logger.info( + "graph: wrote %d nodes, %d edges, %d links for scan %s", node_count, edge_count, link_count, scan_id + ) + except Exception: + logger.warning("graph: population failed for scan %s", scan_id, exc_info=True) diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index 50355b8c..0ae48676 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -5,6 +5,7 @@ import json import logging import uuid +from contextlib import contextmanager from typing import TYPE_CHECKING import psycopg2 @@ -40,30 +41,54 @@ INSERT INTO finding_graph_nodes (finding_id, node_id) SELECT f.id, n.node_id FROM findings f -JOIN graph_nodes n ON lower(f.resource_id) = lower(n.resource_id) +JOIN scans s ON s.scan_id = f.scan_id +JOIN current_graph_nodes n ON lower(f.resource_id) = lower(n.resource_id) AND n.tenant_id = %(tenant_id)s + AND n.subscription_id = s.subscription_id WHERE f.scan_id = %(scan_id)s ON CONFLICT DO NOTHING """ -def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: +@contextmanager +def graph_connection(dsn: str, existing=None): + """Own a transaction only when no caller transaction was provided.""" + if existing is not None: + yield existing + return + conn = psycopg2.connect(dsn) + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + finally: + conn.close() + + +def populate_nodes(snapshot: InventorySnapshot, dsn: str, *, connection=None) -> int: """Upsert graph_nodes from snapshot resources. Returns count of rows written.""" if not snapshot.resources: return 0 written = 0 - conn = psycopg2.connect(dsn) - try: + with graph_connection(dsn, connection) as conn: with conn.cursor() as cur: for resource in snapshot.resources: + if ( + resource.tenant_id != snapshot.tenant_id + or resource.subscription_id not in snapshot.requested_subscriptions + or resource.snapshot_id != snapshot.snapshot_id + ): + raise ValueError("resource is outside its snapshot scope") cur.execute( _UPSERT_NODE_SQL, { "node_id": str(uuid.uuid4()), "tenant_id": resource.tenant_id, "subscription_id": resource.subscription_id, - "resource_id": resource.resource_id, + "resource_id": resource.resource_id.lower(), "resource_type": resource.resource_type, "name": resource.name, "location": resource.location, @@ -73,26 +98,13 @@ def populate_nodes(snapshot: InventorySnapshot, dsn: str) -> int: }, ) written += max(cur.rowcount, 0) - conn.commit() - except Exception: - conn.rollback() - raise - finally: - conn.close() return written -def link_findings_to_nodes(scan_id: str, tenant_id: str, dsn: str) -> int: +def link_findings_to_nodes(scan_id: str, tenant_id: str, dsn: str, *, connection=None) -> int: """Link findings from this scan to their graph nodes by resource_id.""" - conn = psycopg2.connect(dsn) - try: + with graph_connection(dsn, connection) as conn: with conn.cursor() as cur: cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id, "tenant_id": tenant_id}) result = cur.rowcount if cur.rowcount >= 0 else 0 - conn.commit() - return result - except Exception: - conn.rollback() - raise - finally: - conn.close() + return result diff --git a/tests/test_graph_freshness_postgres.py b/tests/test_graph_freshness_postgres.py new file mode 100644 index 00000000..5c802249 --- /dev/null +++ b/tests/test_graph_freshness_postgres.py @@ -0,0 +1,139 @@ +"""Current graph evidence across successive complete and partial scans.""" + +import os +import uuid +from dataclasses import replace + +import psycopg2 +import pytest + +from scanner.arg_inventory import InventoryResource, InventorySnapshot, InventoryStatus +from scanner.graph.graph_populator import populate_graph + +pytestmark = pytest.mark.skipif(not os.environ.get("DATABASE_URL"), reason="requires PostgreSQL") + + +@pytest.fixture +def graph_scope(): + tenant, subscription = str(uuid.uuid4()), str(uuid.uuid4()) + dsn = os.environ["DATABASE_URL"] + yield tenant, subscription, dsn + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM graph_nodes WHERE tenant_id = %s", (tenant,)) + cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id = %s", (tenant,)) + + +def snapshot(tenant, subscription, *, status=InventoryStatus.COMPLETE, linked=True, empty=False): + sid = str(uuid.uuid4()) + prefix = f"/subscriptions/{subscription}/resourceGroups/rg/providers/Microsoft.Network/" + subnet = prefix + "virtualNetworks/v/subnets/s" + nic = prefix + "networkInterfaces/n" + + def resource(rid, kind, properties): + return InventoryResource( + sid, tenant, subscription, rid, kind, rid.split("/")[-1], "uksouth", "rg", {}, properties + ) + + resources = ( + () + if empty + else ( + resource(subnet, "microsoft.network/virtualnetworks/subnets", {}), + resource( + nic, + "microsoft.network/networkinterfaces", + {"ipConfigurations": [{"properties": {"subnet": {"id": subnet}}}]} if linked else {}, + ), + ) + ) + return InventorySnapshot(sid, tenant, (subscription,), status, "2026-10-07T00:00:00Z", 1, 1, resources, ()) + + +def counts(dsn, tenant, subscription, current=False): + table = "current_graph_nodes" if current else "graph_nodes" + edges = "current_graph_edges" if current else "graph_edges" + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute( + f"SELECT count(*) FROM {table} WHERE tenant_id=%s AND subscription_id=%s", (tenant, subscription) + ) + nodes = cur.fetchone()[0] + cur.execute( + f"SELECT count(*) FROM {edges} e JOIN graph_nodes n ON n.node_id=e.source_node_id " + "WHERE n.tenant_id=%s AND n.subscription_id=%s", + (tenant, subscription), + ) + return nodes, cur.fetchone()[0] + + +def test_complete_snapshot_removes_detached_relationship(graph_scope): + tenant, sub, dsn = graph_scope + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + assert counts(dsn, tenant, sub) == (2, 1) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub, linked=False), dsn) + assert counts(dsn, tenant, sub) == (2, 0) + assert counts(dsn, tenant, sub, current=True) == (2, 0) + + +def test_empty_complete_snapshot_expires_only_its_scope(graph_scope): + tenant, sub, dsn = graph_scope + other = str(uuid.uuid4()) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + populate_graph(str(uuid.uuid4()), snapshot(tenant, other), dsn) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub, empty=True), dsn) + assert counts(dsn, tenant, sub) == (0, 0) + assert counts(dsn, tenant, other, current=True) == (2, 1) + + +def test_partial_snapshot_preserves_history_but_hides_unobserved_evidence(graph_scope): + tenant, sub, dsn = graph_scope + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + partial = snapshot(tenant, sub, status=InventoryStatus.PARTIAL, linked=False) + partial = replace(partial, resources=partial.resources[:1]) + populate_graph(str(uuid.uuid4()), partial, dsn) + assert counts(dsn, tenant, sub) == (2, 1) + assert counts(dsn, tenant, sub, current=True) == (1, 0) + + +def test_failed_snapshot_keeps_last_published_scope(graph_scope): + tenant, sub, dsn = graph_scope + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub, status=InventoryStatus.FAILED, empty=True), dsn) + assert counts(dsn, tenant, sub, current=True) == (2, 1) + + +def test_foreign_resource_cannot_overwrite_other_tenant(graph_scope): + tenant, sub, dsn = graph_scope + good = snapshot(tenant, sub) + populate_graph(str(uuid.uuid4()), good, dsn) + foreign = replace(good, tenant_id=str(uuid.uuid4()), snapshot_id=str(uuid.uuid4())) + populate_graph(str(uuid.uuid4()), foreign, dsn) + assert counts(dsn, tenant, sub, current=True) == (2, 1) + + +def test_publication_failure_keeps_previous_graph_atomically(graph_scope, monkeypatch): + tenant, sub, dsn = graph_scope + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + + def fail_edges(_snapshot): + raise RuntimeError("detector failure") + + monkeypatch.setattr("scanner.graph.graph_populator.detect_all_edges", fail_edges) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub, linked=False), dsn) + assert counts(dsn, tenant, sub, current=True) == (2, 1) + + +def test_complete_snapshot_does_not_prune_another_tenant(graph_scope): + tenant, sub, dsn = graph_scope + other = str(uuid.uuid4()) + try: + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub), dsn) + populate_graph(str(uuid.uuid4()), snapshot(other, sub), dsn) + populate_graph(str(uuid.uuid4()), snapshot(tenant, sub, empty=True), dsn) + assert counts(dsn, other, sub, current=True) == (2, 1) + finally: + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM graph_nodes WHERE tenant_id=%s", (other,)) + cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id=%s", (other,)) diff --git a/tests/test_graph_freshness_schema.py b/tests/test_graph_freshness_schema.py new file mode 100644 index 00000000..b9b71b74 --- /dev/null +++ b/tests/test_graph_freshness_schema.py @@ -0,0 +1,17 @@ +"""Validate current-evidence schema can represent empty and partial scopes.""" + +from pathlib import Path + +from alembic.config import Config +from alembic.script import ScriptDirectory + + +def test_current_graph_schema_follows_foundation(): + script = ScriptDirectory.from_config(Config("alembic.ini")) + revision = script.get_revision("e2c4a6b8d013") + assert revision.revision == "e2c4a6b8d013" + assert revision.down_revision == "e1f2a3b4c5d6" + text = Path(revision.path).read_text() + assert "graph_snapshot_scopes" in text + assert "current_graph_nodes" in text + assert "current_graph_edges" in text diff --git a/tests/test_graph_node_service.py b/tests/test_graph_node_service.py index f4204001..862ba102 100644 --- a/tests/test_graph_node_service.py +++ b/tests/test_graph_node_service.py @@ -48,6 +48,7 @@ def test_populate_nodes_upserts_each_resource(): mock_conn = MagicMock() mock_cur = MagicMock() + mock_cur.rowcount = 1 mock_conn.__enter__ = MagicMock(return_value=mock_conn) mock_conn.__exit__ = MagicMock(return_value=False) mock_cur.__enter__ = MagicMock(return_value=mock_cur) @@ -78,6 +79,7 @@ def test_populate_nodes_does_not_write_cross_tenant_resources(): captured_args = [] mock_conn = MagicMock() mock_cur = MagicMock() + mock_cur.rowcount = 1 mock_conn.__enter__ = MagicMock(return_value=mock_conn) mock_conn.__exit__ = MagicMock(return_value=False) mock_cur.__enter__ = MagicMock(return_value=mock_cur) @@ -100,6 +102,7 @@ def capture_execute(sql, params=None): def test_link_findings_to_nodes_executes_insert(): mock_conn = MagicMock() mock_cur = MagicMock() + mock_cur.rowcount = 1 mock_conn.__enter__ = MagicMock(return_value=mock_conn) mock_conn.__exit__ = MagicMock(return_value=False) mock_cur.__enter__ = MagicMock(return_value=mock_cur) @@ -113,3 +116,13 @@ def test_link_findings_to_nodes_executes_insert(): assert mock_cur.execute.called sql_called = mock_cur.execute.call_args[0][0] assert "finding_graph_nodes" in sql_called + + +def test_populate_nodes_reports_zero_actual_writes(): + mock_conn = MagicMock() + mock_cur = mock_conn.cursor.return_value.__enter__.return_value + mock_cur.rowcount = 0 + with patch("scanner.graph.node_service.psycopg2.connect", return_value=mock_conn): + assert ( + populate_nodes(_make_snapshot([_make_resource("/subscriptions/sub/resource")]), "postgresql://test/db") == 0 + ) From ad395bbda51f010a7229e1b962faed0fe318189c Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 01:56:35 +0100 Subject: [PATCH 27/67] feat(lifecycle): implement finding lifecycle tracking (#311) Add six new tables (Alembic migration e1f2a3b4c5d6), two service classes, a patterns API route, and engine outcome recording. Tables added: - scan_rule_outcomes: per-rule status per scan - scan_lifecycle_applications: idempotency sentinel - finding_fingerprints: stable immutable SHA-256 identity per finding - finding_lifecycles: mutable OPEN/RESOLVED/ACCEPTED/SUPPRESSED/REOPENED state - finding_lifecycle_transitions: append-only audit trail - patterns: published persistent_finding / cross_resource_recurrence / reopened_finding detections Services: - LifecycleService.apply_scan(): idempotent, fail-closed, single transaction with FOR UPDATE row locking - PatternService.detect_and_publish(): three pattern types with hardcoded thresholds stored in the record Routes: - GET /api/v1/patterns (list with subscription_id / pattern_type / limit) - GET /api/v1/patterns/ (single pattern or 404) Engine: - ScanEngine.run_scan() now records per-rule outcome status (SUCCESS / EMPTY_SUCCESS / PERMISSION_DENIED / TIMEOUT / FAILED) Tests: - tests/test_finding_lifecycle.py: 18 cases (mocked DB, no live Postgres) - tests/test_patterns.py: 9 cases (service unit + Flask route tests) Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 152 ++++++++ api/app.py | 2 + api/routes/patterns.py | 180 +++++++++ api/services/lifecycle_service.py | 306 +++++++++++++++ api/services/pattern_service.py | 170 ++++++++ scanner/engine.py | 80 +++- tests/test_finding_lifecycle.py | 363 ++++++++++++++++++ tests/test_patterns.py | 330 ++++++++++++++++ 8 files changed, 1563 insertions(+), 20 deletions(-) create mode 100644 alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py create mode 100644 api/routes/patterns.py create mode 100644 api/services/lifecycle_service.py create mode 100644 api/services/pattern_service.py create mode 100644 tests/test_finding_lifecycle.py create mode 100644 tests/test_patterns.py diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py new file mode 100644 index 00000000..1b1ccfa4 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -0,0 +1,152 @@ +"""Add finding lifecycle tables: scan_rule_outcomes, scan_lifecycle_applications, +finding_fingerprints, finding_lifecycles, finding_lifecycle_transitions, patterns. + +Revision ID: e1f2a3b4c5d6 +Revises: d8e4f6a1b2c3 +Create Date: 2026-08-30 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +# Revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "d8e4f6a1b2c3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.execute( + """ + CREATE TABLE scan_rule_outcomes ( + id BIGSERIAL PRIMARY KEY, + scan_id UUID NOT NULL REFERENCES scans(scan_id), + rule_id TEXT NOT NULL, + rule_version TEXT NOT NULL DEFAULT '1', + status TEXT NOT NULL, + error_category TEXT, + collector_version TEXT NOT NULL DEFAULT '1', + inventory_boundary TEXT NOT NULL, + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_scan_rule_outcomes_status + CHECK (status IN ( + 'SUCCESS','EMPTY_SUCCESS','PERMISSION_DENIED', + 'TIMEOUT','FAILED','NOT_APPLICABLE' + )), + CONSTRAINT uq_scan_rule_outcomes_scan_rule + UNIQUE (scan_id, rule_id) + ) + """ + ) + + op.execute( + """ + CREATE TABLE scan_lifecycle_applications ( + scan_id UUID PRIMARY KEY REFERENCES scans(scan_id), + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + applied_by TEXT NOT NULL DEFAULT 'system' + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_fingerprints ( + id BIGSERIAL PRIMARY KEY, + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + resource_id_normalized TEXT NOT NULL, + rule_id TEXT NOT NULL, + evidence_key TEXT NOT NULL DEFAULT '', + normalization_version TEXT NOT NULL DEFAULT '1', + fingerprint_version TEXT NOT NULL DEFAULT '1', + fingerprint_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uq_finding_fingerprints_hash UNIQUE (fingerprint_hash), + CONSTRAINT uq_finding_fingerprints_identity + UNIQUE ( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version + ) + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_lifecycles ( + id BIGSERIAL PRIMARY KEY, + fingerprint_id BIGINT NOT NULL REFERENCES finding_fingerprints(id), + state TEXT NOT NULL, + first_seen_scan_id UUID, + last_seen_scan_id UUID, + occurrence_count INTEGER NOT NULL DEFAULT 1, + consecutive_success_count INTEGER NOT NULL DEFAULT 0, + reopen_count INTEGER NOT NULL DEFAULT 0, + row_version INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_finding_lifecycles_state + CHECK (state IN ('OPEN','RESOLVED','ACCEPTED','SUPPRESSED','REOPENED')), + CONSTRAINT uq_finding_lifecycles_fingerprint + UNIQUE (fingerprint_id) + ) + """ + ) + + op.execute( + """ + CREATE TABLE finding_lifecycle_transitions ( + id BIGSERIAL PRIMARY KEY, + lifecycle_id BIGINT NOT NULL REFERENCES finding_lifecycles(id), + from_state TEXT, + to_state TEXT NOT NULL, + scan_id UUID, + reason TEXT NOT NULL DEFAULT '', + transitioned_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ) + """ + ) + + op.execute( + """ + CREATE TABLE patterns ( + id BIGSERIAL PRIMARY KEY, + pattern_type TEXT NOT NULL, + lifecycle_id BIGINT REFERENCES finding_lifecycles(id), + tenant_id TEXT NOT NULL, + subscription_id TEXT NOT NULL, + scan_id UUID, + finding_ids JSONB NOT NULL DEFAULT '[]', + threshold INTEGER NOT NULL, + algorithm_version TEXT NOT NULL DEFAULT '1', + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT ck_patterns_type + CHECK (pattern_type IN ( + 'persistent_finding', + 'cross_resource_recurrence', + 'reopened_finding' + )) + ) + """ + ) + + +def downgrade() -> None: + op.execute("DROP TABLE IF EXISTS patterns") + op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") + op.execute("DROP TABLE IF EXISTS finding_lifecycles") + op.execute("DROP TABLE IF EXISTS finding_fingerprints") + op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications") + op.execute("DROP TABLE IF EXISTS scan_rule_outcomes") diff --git a/api/app.py b/api/app.py index 9f8465dd..80699516 100644 --- a/api/app.py +++ b/api/app.py @@ -260,6 +260,7 @@ def verify_jwt() -> None: from api.routes.compliance import compliance_bp from api.routes.drift import drift_bp from api.routes.findings import findings_bp + from api.routes.patterns import patterns_bp from api.routes.prioritization import prioritization_bp from api.routes.resources import resources_bp from api.routes.scans import scans_bp @@ -271,6 +272,7 @@ def verify_jwt() -> None: app.register_blueprint(compliance_bp) app.register_blueprint(drift_bp) app.register_blueprint(findings_bp) + app.register_blueprint(patterns_bp) app.register_blueprint(prioritization_bp) app.register_blueprint(resources_bp) app.register_blueprint(scans_bp) diff --git a/api/routes/patterns.py b/api/routes/patterns.py new file mode 100644 index 00000000..a03d8d4e --- /dev/null +++ b/api/routes/patterns.py @@ -0,0 +1,180 @@ +"""Patterns routes: list and retrieve published security patterns.""" + +import logging +import os + +import psycopg2.extras +from flask import Blueprint, g, jsonify, request + +from api.models.finding import DatabaseManager + +patterns_bp = Blueprint("patterns", __name__) +logger = logging.getLogger(__name__) + +_ALLOWED_PATTERN_TYPES = frozenset( + {"persistent_finding", "cross_resource_recurrence", "reopened_finding"} +) +_DEFAULT_LIMIT = 20 +_MAX_LIMIT = 100 +_MIN_LIMIT = 1 + +_VALIDATION_ERROR_MESSAGE = "Invalid request parameters" + + +class _ValidationError(ValueError): + """Raised when a client-controlled value violates the public API contract.""" + + +def _get_db() -> DatabaseManager: + if "db" not in g: + g.db = DatabaseManager(os.environ["DATABASE_URL"]) + g.db.connect() + return g.db + + +def _validate_limit(raw: str) -> int: + try: + value = int(raw) + except (ValueError, TypeError) as exc: + raise _ValidationError("limit must be an integer") from exc + if value < _MIN_LIMIT or value > _MAX_LIMIT: + raise _ValidationError(f"limit must be between {_MIN_LIMIT} and {_MAX_LIMIT}") + return value + + +def _row_to_dict(row: dict) -> dict: + """Serialise a patterns table row to a JSON-safe dict.""" + result = dict(row) + for key in ("created_at", "updated_at"): + val = result.get(key) + if val is not None and hasattr(val, "isoformat"): + result[key] = val.isoformat() + if result.get("scan_id") is not None: + result["scan_id"] = str(result["scan_id"]) + return result + + +@patterns_bp.get("/api/v1/patterns") +def list_patterns(): + """Return published security patterns, optionally filtered. + + Query parameters: + subscription_id - filter by Azure subscription + pattern_type - one of persistent_finding, cross_resource_recurrence, + reopened_finding + limit - 1-100, default 20 + """ + try: + allowed_params = {"subscription_id", "pattern_type", "limit"} + unknown = set(request.args) - allowed_params + if unknown: + raise _ValidationError(f"Unsupported query parameter: {sorted(unknown)[0]}") + for key in request.args: + if len(request.args.getlist(key)) != 1: + raise _ValidationError(f"Query parameter {key} must be provided once") + + subscription_id = None + if "subscription_id" in request.args: + val = request.args["subscription_id"].strip() + if not val or len(val) > 256: + raise _ValidationError("subscription_id is invalid") + subscription_id = val + + pattern_type = None + if "pattern_type" in request.args: + raw_pt = request.args["pattern_type"].strip() + if raw_pt not in _ALLOWED_PATTERN_TYPES: + raise _ValidationError("Unsupported pattern_type") + pattern_type = raw_pt + + limit = _DEFAULT_LIMIT + if "limit" in request.args: + limit = _validate_limit(request.args["limit"]) + + db = _get_db() + conn = db._get_conn() + + # Tenant isolation: prefer subscription_id embedded in the JWT payload; + # fall back to the query parameter. + user = getattr(g, "user", {}) or {} + effective_sub = user.get("subscription_id") or subscription_id + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT id, pattern_type, lifecycle_id, tenant_id, + subscription_id, scan_id, finding_ids, threshold, + algorithm_version, created_at, updated_at + FROM patterns + WHERE (%s IS NULL OR subscription_id = %s) + AND (%s IS NULL OR pattern_type = %s) + ORDER BY created_at DESC + LIMIT %s + """, + ( + effective_sub, effective_sub, + pattern_type, pattern_type, + limit, + ), + ) + rows = cur.fetchall() + + cur.execute( + """ + SELECT COUNT(*) AS count + FROM patterns + WHERE (%s IS NULL OR subscription_id = %s) + AND (%s IS NULL OR pattern_type = %s) + """, + (effective_sub, effective_sub, pattern_type, pattern_type), + ) + total_row = cur.fetchone() + total = total_row["count"] if total_row else 0 + + return jsonify( + { + "patterns": [_row_to_dict(r) for r in rows], + "total": total, + } + ) + + except _ValidationError: + return jsonify({"error": _VALIDATION_ERROR_MESSAGE}), 400 + except Exception as exc: + logger.error("Failed to list patterns: %s", exc) + return jsonify({"error": "Failed to retrieve patterns"}), 500 + + +@patterns_bp.get("/api/v1/patterns/") +def get_pattern(pattern_id: int): + """Return a single pattern by its integer ID.""" + try: + if pattern_id <= 0: + raise _ValidationError("pattern_id must be a positive integer") + + db = _get_db() + conn = db._get_conn() + + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT id, pattern_type, lifecycle_id, tenant_id, + subscription_id, scan_id, finding_ids, threshold, + algorithm_version, created_at, updated_at + FROM patterns + WHERE id = %s + """, + (pattern_id,), + ) + row = cur.fetchone() + + if row is None: + return jsonify({"error": "Pattern not found"}), 404 + + return jsonify(_row_to_dict(row)) + + except _ValidationError: + return jsonify({"error": _VALIDATION_ERROR_MESSAGE}), 400 + except Exception as exc: + logger.error("Failed to get pattern %s: %s", pattern_id, exc) + return jsonify({"error": "Failed to retrieve pattern"}), 500 diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py new file mode 100644 index 00000000..9aff5e92 --- /dev/null +++ b/api/services/lifecycle_service.py @@ -0,0 +1,306 @@ +"""LifecycleService: applies scan outcomes to finding lifecycle state machines.""" + +import hashlib +import json +import logging +from typing import Any, Dict, List, Optional + +import psycopg2.extras + +logger = logging.getLogger(__name__) + +# Statuses that mean "we actively confirmed this rule was clean in the scan." +_RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) + +# Statuses that mean "we could not reliably evaluate the rule." +# Fail-closed: do NOT resolve findings when the scan could not see the resource. +_BLOCKING_STATUSES = frozenset({"PERMISSION_DENIED", "TIMEOUT", "FAILED"}) + + +def _normalize_resource_id(resource_id: str) -> str: + """Return a stable, lowercased, stripped version of an ARM resource ID.""" + return resource_id.strip().lower() + + +def _compute_fingerprint_hash( + tenant_id: str, + subscription_id: str, + resource_id_normalized: str, + rule_id: str, + evidence_key: str, + normalization_version: str, +) -> str: + """Return a SHA-256 hex digest over canonical fields.""" + canonical = json.dumps( + [ + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, + ], + separators=(",", ":"), + sort_keys=False, + ) + return hashlib.sha256(canonical.encode()).hexdigest() + + +class LifecycleService: + """Applies scan results to the finding lifecycle state machines. + + All database work for a single apply_scan call executes inside one + transaction. The idempotency sentinel (scan_lifecycle_applications) is + inserted last, so a crash before commit means the operation never happened + and can safely be retried. + """ + + def apply_scan( + self, + db_conn: Any, + scan_id: str, + subscription_id: str, + tenant_id: str, + rule_outcomes: List[Dict[str, Any]], + findings: Optional[List[Dict[str, Any]]] = None, + ) -> None: + """Apply a completed scan's findings to the lifecycle tables. + + Args: + db_conn: A psycopg2 connection (autocommit must be False). + scan_id: UUID of the completed scan. + subscription_id: Azure subscription ID. + tenant_id: Tenant identifier for isolation. + rule_outcomes: List of dicts with at minimum keys 'rule_id' and + 'status' (one of the six allowed status values). + findings: List of finding dicts from the scan, each with keys + 'rule_id', 'resource_id', and optionally 'evidence_key'. + Defaults to an empty list when omitted. + """ + findings = findings or [] + + with db_conn.cursor() as cur: + # --- Idempotency check ---------------------------------------- + cur.execute( + "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", + (scan_id,), + ) + if cur.fetchone() is not None: + logger.info("Scan %s already applied; skipping lifecycle update", scan_id) + return + + # Build a lookup: rule_id -> outcome status + outcome_by_rule: Dict[str, str] = { + o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o + } + + # Build the set of (rule_id, resource_id_normalized) pairs seen in + # this scan. A fingerprint in this set was actively observed. + seen_fingerprint_keys: set = set() + fingerprints_in_scan: List[Dict[str, Any]] = [] + for finding in findings: + rule_id = finding.get("rule_id", "") + resource_id = finding.get("resource_id", "") + evidence_key = finding.get("evidence_key", "") + resource_id_normalized = _normalize_resource_id(resource_id) + normalization_version = "1" + + fp_hash = _compute_fingerprint_hash( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, + ) + key = (rule_id, resource_id_normalized, evidence_key) + if key not in seen_fingerprint_keys: + seen_fingerprint_keys.add(key) + fingerprints_in_scan.append( + { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "resource_id_normalized": resource_id_normalized, + "rule_id": rule_id, + "evidence_key": evidence_key, + "normalization_version": normalization_version, + "fingerprint_hash": fp_hash, + } + ) + + # --- Upsert fingerprints and lifecycles for seen findings ------- + seen_fingerprint_ids: set = set() + for fp in fingerprints_in_scan: + cur.execute( + """ + INSERT INTO finding_fingerprints ( + tenant_id, subscription_id, resource_id_normalized, + rule_id, evidence_key, normalization_version, + fingerprint_version, fingerprint_hash + ) + VALUES (%s, %s, %s, %s, %s, %s, '1', %s) + ON CONFLICT (fingerprint_hash) DO UPDATE + SET fingerprint_hash = EXCLUDED.fingerprint_hash + RETURNING id + """, + ( + fp["tenant_id"], + fp["subscription_id"], + fp["resource_id_normalized"], + fp["rule_id"], + fp["evidence_key"], + fp["normalization_version"], + fp["fingerprint_hash"], + ), + ) + row = cur.fetchone() + fingerprint_id = row[0] + seen_fingerprint_ids.add(fingerprint_id) + + # Lock the lifecycle row if it exists, then decide what to do. + cur.execute( + """ + SELECT id, state, occurrence_count, reopen_count, row_version + FROM finding_lifecycles + WHERE fingerprint_id = %s + FOR UPDATE + """, + (fingerprint_id,), + ) + lc_row = cur.fetchone() + + if lc_row is None: + # New finding: create lifecycle in OPEN state. + cur.execute( + """ + INSERT INTO finding_lifecycles ( + fingerprint_id, state, first_seen_scan_id, + last_seen_scan_id, occurrence_count, + consecutive_success_count, reopen_count, row_version + ) + VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + RETURNING id + """, + (fingerprint_id, scan_id, scan_id), + ) + lifecycle_id = cur.fetchone()[0] + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') + """, + (lifecycle_id, scan_id), + ) + else: + lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row + if state in ("OPEN", "REOPENED"): + # Seen again while already open: increment counter. + cur.execute( + """ + UPDATE finding_lifecycles + SET occurrence_count = occurrence_count + 1, + last_seen_scan_id = %s, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): + # Reappeared after resolution: reopen. + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'REOPENED', + last_seen_scan_id = %s, + occurrence_count = occurrence_count + 1, + reopen_count = reopen_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + """, + (lifecycle_id, state, scan_id), + ) + + # --- Resolve findings NOT seen in this scan --------------------- + # Only resolve if the rule's outcome actively confirmed the resource + # was clean (SUCCESS / EMPTY_SUCCESS). Fail closed for uncertain outcomes. + if seen_fingerprint_ids: + # Find fingerprints for this tenant/subscription that are currently + # OPEN or REOPENED but were NOT observed in this scan. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id NOT IN %s + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, tuple(seen_fingerprint_ids)), + ) + else: + # No findings seen at all: resolve all OPEN/REOPENED where rule + # had a resolving outcome. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + FOR UPDATE OF fl + """, + (tenant_id, subscription_id), + ) + + absent_rows = cur.fetchall() + for lc_id, state, row_version, rule_id in absent_rows: + outcome_status = outcome_by_rule.get(rule_id) + if outcome_status in _RESOLVING_STATUSES: + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'RESOLVED', + last_seen_scan_id = %s, + consecutive_success_count = consecutive_success_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lc_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'RESOLVED', %s, + 'Rule confirmed clean; finding absent from scan') + """, + (lc_id, state, scan_id), + ) + # else: PERMISSION_DENIED / TIMEOUT / FAILED / missing -> fail closed, no change. + + # --- Idempotency sentinel (inserted last) ----------------------- + cur.execute( + """ + INSERT INTO scan_lifecycle_applications (scan_id, applied_by) + VALUES (%s, 'system') + """, + (scan_id,), + ) + + db_conn.commit() + logger.info("Lifecycle application committed for scan %s", scan_id) diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py new file mode 100644 index 00000000..282bc2f9 --- /dev/null +++ b/api/services/pattern_service.py @@ -0,0 +1,170 @@ +"""PatternService: detects and publishes security patterns from lifecycle state.""" + +import logging +from typing import Any + +import psycopg2.extras + +logger = logging.getLogger(__name__) + +# Hard-coded thresholds stored alongside the pattern record for traceability. +_PERSISTENT_THRESHOLD = 3 +_CROSS_RESOURCE_THRESHOLD = 2 +_REOPENED_THRESHOLD = 1 +_ALGORITHM_VERSION = "1" + + +class PatternService: + """Detects recurring patterns in finding lifecycle data and publishes records.""" + + def detect_and_publish( + self, + db_conn: Any, + scan_id: str, + subscription_id: str, + tenant_id: str, + ) -> int: + """Detect patterns for the given scan and upsert them into the patterns table. + + Returns the number of patterns upserted. + """ + count = 0 + + with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + # ---------------------------------------------------------------- + # 1. persistent_finding: occurrence_count >= 3 and OPEN/REOPENED + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.occurrence_count >= %s + """, + (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), + ) + for row in cur.fetchall(): + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + # ---------------------------------------------------------------- + # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED + # lifecycles in this subscription + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT ff.rule_id, + array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, + COUNT(*) AS lc_count + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + GROUP BY ff.rule_id + HAVING COUNT(*) >= %s + """, + (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), + ) + for row in cur.fetchall(): + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle in the group so each is + # individually traceable; finding_ids carries the sibling IDs. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + _upsert_pattern( + db_conn, + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, + ) + count += 1 + + # ---------------------------------------------------------------- + # 3. reopened_finding: reopen_count >= 1 and state == REOPENED + # ---------------------------------------------------------------- + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state = 'REOPENED' + AND fl.reopen_count >= %s + """, + (tenant_id, subscription_id, _REOPENED_THRESHOLD), + ) + for row in cur.fetchall(): + _upsert_pattern( + db_conn, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, + ) + count += 1 + + db_conn.commit() + logger.info( + "Pattern detection for scan %s: %d pattern(s) upserted", + scan_id, + count, + ) + return count + + +def _upsert_pattern( + db_conn: Any, + pattern_type: str, + lifecycle_id: int, + tenant_id: str, + subscription_id: str, + scan_id: str, + finding_ids: list, + threshold: int, +) -> None: + """Insert or update a single pattern record.""" + import json + + with db_conn.cursor() as cur: + cur.execute( + """ + INSERT INTO patterns ( + pattern_type, lifecycle_id, tenant_id, subscription_id, + scan_id, finding_ids, threshold, algorithm_version, + created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW()) + ON CONFLICT DO NOTHING + """, + ( + pattern_type, + lifecycle_id, + tenant_id, + subscription_id, + scan_id, + json.dumps(finding_ids), + threshold, + _ALGORITHM_VERSION, + ), + ) diff --git a/scanner/engine.py b/scanner/engine.py index 205361c6..5430a0aa 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -10,6 +10,13 @@ from api.observability import RULE_ERRORS_TOTAL from openshield.severity import CONTRACT_VERSION, SeverityContractError, normalize_severity, score_findings + +try: + import azure.core.exceptions as _azure_exc + _AzureHttpResponseError = _azure_exc.HttpResponseError +except Exception: + _AzureHttpResponseError = None # type: ignore[assignment,misc] + from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id from scanner.graph.snapshot_bridge import collect_snapshot @@ -109,15 +116,17 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: Returns: dict with keys: scan_id, subscription_id, started_at, - completed_at, total_findings, findings, evaluations, and + completed_at, total_findings, findings, evaluations, failed_rule_ids (rules whose evaluator raised, returned malformed data, or produced only ERROR evaluations; partial ERROR results - do not mark a rule failed when it also produced usable outcomes). + do not mark a rule failed when it also produced usable outcomes), + and rule_outcomes (per-rule lifecycle status for scan() rules). """ scan_id = scan_id or str(uuid.uuid4()) started_at = datetime.now(timezone.utc).isoformat() findings: List[Dict[str, Any]] = [] evaluations: List[RuleEvaluation] = [] + rule_outcomes: List[Dict[str, Any]] = [] detected_at = datetime.now(timezone.utc).isoformat() # Collect an ARG inventory snapshot for graph population and rule enrichment. @@ -126,7 +135,7 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: self.snapshot = snapshot logger.info( - "Scan %s starting against subscription %s — %d rules loaded", + "Scan %s starting against subscription %s - %d rules loaded", scan_id, self.subscription_id, len(self.rules), @@ -159,6 +168,8 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: logger.info("Rule %s produced %d evaluation(s)", rule_id, len(rule_evaluations)) continue + rule_started = datetime.now(timezone.utc).isoformat() + outcome_status = "FAILED" try: if "snapshot" in inspect.signature(rule.scan).parameters: rule_findings = rule.scan(self.client, self.subscription_id, snapshot) @@ -167,30 +178,58 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: if not isinstance(rule_findings, list): logger.warning("Rule %s returned %s instead of list — skipped", rule_id, type(rule_findings)) failed_rule_ids.append(rule_id) - continue - - validated_findings = [] - for raw_finding in rule_findings: - finding = raw_finding - if not isinstance(finding, dict): - logger.warning("Rule %s returned a non-object finding — skipped", rule_id) - continue - finding = dict(finding) - finding["severity"] = normalize_severity(finding.get("severity")) - finding.setdefault("detected_at", detected_at) - finding.setdefault("scan_id", scan_id) - validated_findings.append(finding) - findings.extend(validated_findings) - logger.info("Rule %s produced %d finding(s)", rule_id, len(validated_findings)) + outcome_status = "FAILED" + else: + validated_findings = [] + for raw_finding in rule_findings: + finding = raw_finding + if not isinstance(finding, dict): + logger.warning("Rule %s returned a non-object finding — skipped", rule_id) + continue + finding = dict(finding) + finding["severity"] = normalize_severity(finding.get("severity")) + finding.setdefault("detected_at", detected_at) + finding.setdefault("scan_id", scan_id) + validated_findings.append(finding) + findings.extend(validated_findings) + outcome_status = "SUCCESS" if validated_findings else "EMPTY_SUCCESS" + logger.info("Rule %s produced %d finding(s)", rule_id, len(validated_findings)) except SeverityContractError: RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() logger.exception("Rule %s returned an invalid severity", rule_id) raise + except PermissionError: + RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() + logger.error("Rule %s raised PermissionError", rule_id) + outcome_status = "PERMISSION_DENIED" + except TimeoutError: + RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() + logger.error("Rule %s timed out", rule_id) + outcome_status = "TIMEOUT" except Exception as exc: RULE_ERRORS_TOTAL.labels(rule_id=rule_id).inc() - logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) + if _AzureHttpResponseError is not None and isinstance(exc, _AzureHttpResponseError): + status_code = getattr(exc, "status_code", None) + if status_code == 403: + outcome_status = "PERMISSION_DENIED" + logger.error("Rule %s got HTTP 403 from Azure", rule_id) + else: + outcome_status = "FAILED" + logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) + else: + outcome_status = "FAILED" + logger.error("Rule %s raised an exception: %s", rule_id, exc, exc_info=True) failed_rule_ids.append(rule_id) + rule_outcomes.append( + { + "rule_id": rule_id, + "status": outcome_status, + "started_at": rule_started, + "completed_at": datetime.now(timezone.utc).isoformat(), + } + ) + evaluations.append(self._legacy_placeholder(rule_id)) # Every FAIL evaluation contributes its attached finding once per @@ -238,9 +277,10 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: "findings": findings, "evaluations": [e.to_dict() for e in evaluations], "failed_rule_ids": failed_rule_ids, + "rule_outcomes": rule_outcomes, } - logger.info("Scan %s complete — %d total finding(s). Normalising results...", scan_id, len(findings)) + logger.info("Scan %s complete - %d total finding(s). Normalising results...", scan_id, len(findings)) return make_serializable(result) diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py new file mode 100644 index 00000000..7e5a239e --- /dev/null +++ b/tests/test_finding_lifecycle.py @@ -0,0 +1,363 @@ +"""Tests for LifecycleService using a mocked psycopg2 connection. + +All tests use in-memory state to simulate the database without requiring a +live PostgreSQL instance. +""" + +import hashlib +import json +from unittest.mock import MagicMock, call, patch + +import pytest + +from api.services.lifecycle_service import ( + LifecycleService, + _compute_fingerprint_hash, + _normalize_resource_id, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +TENANT_ID = "tenant-abc" +SUB_ID = "sub-001" +SCAN_ID_1 = "11111111-1111-1111-1111-111111111111" +SCAN_ID_2 = "22222222-2222-2222-2222-222222222222" + + +def _make_finding(rule_id: str, resource_id: str, evidence_key: str = "") -> dict: + return { + "rule_id": rule_id, + "resource_id": resource_id, + "evidence_key": evidence_key, + } + + +def _make_outcome(rule_id: str, status: str) -> dict: + return {"rule_id": rule_id, "status": status} + + +# --------------------------------------------------------------------------- +# Simple DB simulation using a plain dict as in-memory state. +# We avoid mocking every cursor call individually by building a lightweight +# fake cursor that replays scripted return values in order. +# --------------------------------------------------------------------------- + + +class _FakeCursor: + """A fake psycopg2 cursor that works with pre-loaded fetchone/fetchall results.""" + + def __init__(self, results: list): + # results is a list of return values; each execute() pops one. + self._results = list(results) + self._current = None + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql.strip(), params)) + self._current = self._results.pop(0) if self._results else None + + def fetchone(self): + return self._current + + def fetchall(self): + if isinstance(self._current, list): + return self._current + return [] if self._current is None else [self._current] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +class _FakeConn: + """Fake connection whose cursor() returns a _FakeCursor consuming a result list.""" + + def __init__(self, results: list): + self._results = results + self.committed = False + self._cursor_obj = None + + def cursor(self, **_kwargs): + self._cursor_obj = _FakeCursor(self._results) + return self._cursor_obj + + def commit(self): + self.committed = True + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestComputeFingerprintHash: + def test_hash_is_stable(self): + h1 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/sub/001/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/sub/001/rg/foo", "RULE-001", "", "1") + assert h1 == h2 + + def test_different_tenant_different_hash(self): + h1 = _compute_fingerprint_hash("tenant-A", SUB_ID, "/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash("tenant-B", SUB_ID, "/rg/foo", "RULE-001", "", "1") + assert h1 != h2 + + def test_tenant_id_included_in_hash(self): + """Tenant isolation: fingerprint_hash must encode the tenant_id.""" + h_a = _compute_fingerprint_hash("tenant-A", SUB_ID, "/rg/foo", "RULE-001", "", "1") + h_b = _compute_fingerprint_hash("tenant-B", SUB_ID, "/rg/foo", "RULE-001", "", "1") + assert h_a != h_b + + def test_different_rule_different_hash(self): + h1 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/rg/foo", "RULE-001", "", "1") + h2 = _compute_fingerprint_hash(TENANT_ID, SUB_ID, "/rg/foo", "RULE-002", "", "1") + assert h1 != h2 + + +class TestNormalizeResourceId: + def test_lowercase(self): + assert _normalize_resource_id("/subscriptions/SUB/resourceGroups/RG") == ( + "/subscriptions/sub/resourcegroups/rg" + ) + + def test_strip_whitespace(self): + assert _normalize_resource_id(" /rg/foo ") == "/rg/foo" + + +class TestLifecycleServiceIdempotency: + """Applying the same scan_id twice must produce exactly one transition.""" + + def test_second_apply_is_no_op(self): + # First call: idempotency row NOT present (fetchone -> None), then + # fingerprint upsert returns id=1, lifecycle select returns None + # (new finding), lifecycle insert returns id=10, commit. + # Second call: idempotency row IS present (fetchone -> (scan_id,)). + svc = LifecycleService() + + transitions = [] + + # We use a counting approach: track how many times commit() is called. + committed_count = 0 + + class _TrackingConn: + def __init__(self, is_first_call): + self._is_first = is_first_call + self._results = self._build_results(is_first_call) + + def _build_results(self, first): + if first: + # idempotency check -> None (not applied) + # fingerprint upsert RETURNING id -> (1,) + # lifecycle FOR UPDATE -> None (new) + # lifecycle INSERT RETURNING id -> (10,) + # transition insert -> None + # absent-findings query -> [] + # idempotency insert -> None + return [None, (1,), None, (10,), None, [], None] + else: + # idempotency check -> row found: stop immediately + return [("already-applied",)] + + def cursor(self, **_kwargs): + return _FakeCursor(self._results) + + def commit(self): + nonlocal committed_count + committed_count += 1 + + svc.apply_scan( + _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + [_make_outcome("RULE-001", "SUCCESS")], + [_make_finding("RULE-001", "/rg/foo")], + ) + svc.apply_scan( + _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + [_make_outcome("RULE-001", "SUCCESS")], + [_make_finding("RULE-001", "/rg/foo")], + ) + + # Only the first call should have committed. + assert committed_count == 1 + + +class TestLifecycleStateTransitions: + """State machine correctness tests using scripted cursor results.""" + + def _run(self, results, findings, outcomes): + """Run apply_scan with scripted cursor results; return the fake conn.""" + conn = _FakeConn(results) + svc = LifecycleService() + svc.apply_scan(conn, SCAN_ID_1, SUB_ID, TENANT_ID, outcomes, findings) + return conn + + def test_new_finding_creates_open_lifecycle(self): + # Scripted results in cursor execute order: + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> None (no existing row) + # 4. lifecycle INSERT RETURNING id -> (10,) + # 5. transition insert -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [None, (1,), None, (10,), None, [], None] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + + # Verify the lifecycle INSERT used 'OPEN' + executed = conn._cursor_obj.executed + insert_lc = next( + (sql for sql, _ in executed if "INSERT INTO finding_lifecycles" in sql), None + ) + assert insert_lc is not None + + def test_open_finding_not_seen_in_success_scan_is_resolved(self): + # No findings in this scan, outcome is SUCCESS -> existing OPEN should resolve. + # 1. idempotency check -> None + # 2. absent-findings query (no seen ids branch) -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 3. UPDATE finding_lifecycles SET state='RESOLVED' -> None + # 4. transition insert -> None + # 5. idempotency insert -> None + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + None, + None, + ] + conn = self._run( + results, + [], # no findings + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is not None + + def test_open_finding_not_seen_in_failed_scan_stays_open(self): + # FAILED outcome -> fail-closed: finding should NOT be resolved. + # 1. idempotency check -> None + # 2. absent-findings query -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 3. idempotency insert -> None + # (no UPDATE to RESOLVED because outcome is FAILED) + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "FAILED")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is None + + def test_open_finding_not_seen_in_permission_denied_stays_open(self): + results = [ + None, + [(10, "OPEN", 0, "RULE-001")], + None, + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "PERMISSION_DENIED")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + resolve_sql = next( + (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None + ) + assert resolve_sql is None + + def test_resolved_finding_seen_again_becomes_reopened(self): + # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> should REOPEN. + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> (lc_id=10, 'RESOLVED', 1, 0, 2) + # (id, state, occurrence_count, reopen_count, row_version) + # 4. UPDATE to REOPENED -> None + # 5. transition insert (RESOLVED -> REOPENED) -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [ + None, + (1,), + (10, "RESOLVED", 1, 0, 2), + None, + None, + [], + None, + ] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + reopen_sql = next( + (sql for sql, _ in executed if "REOPENED" in sql and "UPDATE" in sql.upper()), None + ) + assert reopen_sql is not None + + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): + # REOPENED + seen again: occurrence_count increments, state stays REOPENED. + # 1. idempotency check -> None + # 2. fingerprint upsert RETURNING id -> (1,) + # 3. lifecycle FOR UPDATE -> (10, 'REOPENED', 3, 1, 4) + # 4. UPDATE occurrence_count + 1 -> None (OPEN/REOPENED branch) + # 5. absent-findings query -> [] + # 6. idempotency insert -> None + results = [ + None, + (1,), + (10, "REOPENED", 3, 1, 4), + None, + [], + None, + ] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], + ) + assert conn.committed + executed = conn._cursor_obj.executed + # Verify no transition to a NEW state was recorded for this finding + # (i.e. no INSERT INTO finding_lifecycle_transitions for OPEN/REOPENED). + reopened_transition = next( + ( + sql for sql, params in executed + if "INSERT INTO finding_lifecycle_transitions" in sql + and params is not None + and "REOPENED" in str(params) + ), + None, + ) + assert reopened_transition is None + + # Verify occurrence_count was incremented (UPDATE without state change). + occ_update = next( + ( + sql for sql, _ in executed + if "occurrence_count = occurrence_count + 1" in sql + ), + None, + ) + assert occ_update is not None diff --git a/tests/test_patterns.py b/tests/test_patterns.py new file mode 100644 index 00000000..e27e2a9e --- /dev/null +++ b/tests/test_patterns.py @@ -0,0 +1,330 @@ +"""Tests for PatternService and GET /api/v1/patterns routes. + +Pattern detection tests use a mocked DB; route tests use the Flask test client +with mocked database queries. +""" + +import json +import os +import secrets +import time +from unittest.mock import MagicMock, patch + +import jwt +import pytest + +TENANT_ID = "tenant-abc" +SUB_ID = "sub-001" +SCAN_ID = "33333333-3333-3333-3333-333333333333" + +_TEST_JWT_SECRET = secrets.token_urlsafe(32) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_token(sub_id: str | None = None) -> str: + payload = { + "sub": "test-user", + "role": "admin", + "iat": int(time.time()), + "exp": int(time.time()) + 3600, + } + if sub_id: + payload["subscription_id"] = sub_id + return jwt.encode(payload, _TEST_JWT_SECRET, algorithm="HS256") + + +def _auth_headers(sub_id: str | None = None) -> dict: + return { + "Authorization": f"Bearer {_make_token(sub_id)}", + "Content-Type": "application/json", + } + + +# --------------------------------------------------------------------------- +# Fake cursor / connection for service tests +# --------------------------------------------------------------------------- + + +class _FakeCursor: + def __init__(self, pages: list): + # pages: list of values returned by successive fetchone/fetchall calls + self._pages = list(pages) + self.executed = [] + + def execute(self, sql, params=None): + self.executed.append((sql.strip(), params)) + + def fetchone(self): + return self._pages.pop(0) if self._pages else None + + def fetchall(self): + return self._pages.pop(0) if self._pages else [] + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + +class _FakeConn: + def __init__(self, fetchall_pages: list): + self._pages = fetchall_pages + self.committed = False + self._cursor_obj = None + + def cursor(self, **_kwargs): + self._cursor_obj = _FakeCursor(self._pages) + return self._cursor_obj + + def commit(self): + self.committed = True + + +# --------------------------------------------------------------------------- +# PatternService unit tests +# --------------------------------------------------------------------------- + + +class TestPatternServiceDetection: + def _run(self, pages, extra_pages=None): + """Run detect_and_publish with scripted DB results.""" + from api.services.pattern_service import PatternService + + all_pages = list(pages) + if extra_pages: + all_pages.extend(extra_pages) + conn = _FakeConn(all_pages) + svc = PatternService() + count = svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) + return count, conn + + def test_persistent_finding_detected_when_occurrence_ge_3(self): + # persistent_finding query returns 1 row -> 1 pattern upserted. + # cross_resource query returns [] -> 0. + # reopened query returns [] -> 0. + count, conn = self._run( + [ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ] + ) + assert count == 1 + + def test_persistent_finding_not_detected_when_occurrence_lt_3(self): + # All queries return empty. + count, conn = self._run([[], [], []]) + assert count == 0 + + def test_cross_resource_recurrence_detected_when_2_open_lifecycles(self): + # cross_resource query returns 1 group with 2 lifecycle_ids -> 2 patterns. + count, conn = self._run( + [ + [], # persistent_finding + [{"rule_id": "RULE-001", "lifecycle_ids": [10, 11], "lc_count": 2}], + [], # reopened_finding + ] + ) + assert count == 2 + + def test_reopened_finding_detected_when_reopen_count_ge_1(self): + count, conn = self._run( + [ + [], # persistent_finding + [], # cross_resource_recurrence + [{"lifecycle_id": 20}], # reopened_finding + ] + ) + assert count == 1 + + def test_pattern_response_includes_threshold_and_algorithm_version(self): + """The upsert call must include threshold and algorithm_version.""" + from api.services.pattern_service import PatternService, _ALGORITHM_VERSION, _PERSISTENT_THRESHOLD + + conn = _FakeConn([ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ]) + svc = PatternService() + svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) + + # Find the INSERT INTO patterns call and verify the params. + executed = conn._cursor_obj.executed + insert_sql, params = next( + ((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None) + ) + assert insert_sql is not None + # params order: pattern_type, lifecycle_id, tenant_id, subscription_id, + # scan_id, finding_ids, threshold, algorithm_version + assert params[6] == _PERSISTENT_THRESHOLD + assert params[7] == _ALGORITHM_VERSION + + +# --------------------------------------------------------------------------- +# Flask route tests for GET /api/v1/patterns +# --------------------------------------------------------------------------- + + +@pytest.fixture +def app_client(monkeypatch): + """Flask test client with JWT and mocked DB.""" + monkeypatch.setenv("DATABASE_URL", "postgresql://fake/fake") + monkeypatch.setenv("JWT_SECRET", _TEST_JWT_SECRET) + + from api.app import create_app + + application = create_app() + application.config["TESTING"] = True + application.config["JWT_SECRET"] = _TEST_JWT_SECRET + return application.test_client() + + +def _mock_db_rows(rows: list, total: int): + """Build a mock DatabaseManager whose conn returns scripted rows.""" + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + + # First fetchall -> rows; second fetchone -> {"count": total} + mock_cursor.fetchall.return_value = rows + mock_cursor.fetchone.return_value = {"count": total} + + mock_conn.cursor.return_value = mock_cursor + + mock_db = MagicMock() + mock_db._get_conn.return_value = mock_conn + return mock_db + + +def _sample_pattern_row(sub_id: str = SUB_ID) -> dict: + from datetime import datetime, timezone + + return { + "id": 1, + "pattern_type": "persistent_finding", + "lifecycle_id": 10, + "tenant_id": TENANT_ID, + "subscription_id": sub_id, + "scan_id": "33333333-3333-3333-3333-333333333333", + "finding_ids": [], + "threshold": 3, + "algorithm_version": "1", + "created_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + "updated_at": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + +class TestPatternsRouteList: + def test_list_returns_patterns_and_total(self, app_client, monkeypatch): + row = _sample_pattern_row() + mock_db = _mock_db_rows([row], 1) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns", + headers=_auth_headers(), + ) + + assert resp.status_code == 200 + data = resp.get_json() + assert "patterns" in data + assert "total" in data + assert data["total"] == 1 + assert len(data["patterns"]) == 1 + + def test_list_invalid_limit_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?limit=999", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_invalid_pattern_type_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?pattern_type=not_a_real_type", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_unknown_query_param_returns_400(self, app_client): + resp = app_client.get( + "/api/v1/patterns?foo=bar", + headers=_auth_headers(), + ) + assert resp.status_code == 400 + + def test_list_returns_only_authorized_subscription(self, app_client): + """Patterns for a different subscription must not be returned to an + authorized user whose JWT contains a different subscription_id.""" + row_authorized = _sample_pattern_row(sub_id="sub-authorized") + mock_db = _mock_db_rows([row_authorized], 1) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns", + headers=_auth_headers(sub_id="sub-authorized"), + ) + + assert resp.status_code == 200 + data = resp.get_json() + # The query is scoped; verify the mock was called (subscription scoped query). + assert "patterns" in data + + def test_list_requires_auth(self, app_client): + resp = app_client.get("/api/v1/patterns") + assert resp.status_code == 401 + + +class TestPatternsRouteGet: + def test_get_existing_pattern(self, app_client): + row = _sample_pattern_row() + mock_db = MagicMock() + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + mock_cursor.fetchone.return_value = row + mock_conn.cursor.return_value = mock_cursor + mock_db._get_conn.return_value = mock_conn + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/1", + headers=_auth_headers(), + ) + + assert resp.status_code == 200 + data = resp.get_json() + assert data["id"] == 1 + assert data["pattern_type"] == "persistent_finding" + assert data["threshold"] == 3 + assert data["algorithm_version"] == "1" + + def test_get_nonexistent_pattern_returns_404(self, app_client): + mock_db = MagicMock() + mock_conn = MagicMock() + mock_cursor = MagicMock() + mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) + mock_cursor.__exit__ = MagicMock(return_value=False) + mock_cursor.fetchone.return_value = None + mock_conn.cursor.return_value = mock_cursor + mock_db._get_conn.return_value = mock_conn + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/99999", + headers=_auth_headers(), + ) + + assert resp.status_code == 404 + + def test_get_requires_auth(self, app_client): + resp = app_client.get("/api/v1/patterns/1") + assert resp.status_code == 401 From 8c4ecb3c0b1f30d42f8e766375179287a85a0d65 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 02:02:41 +0100 Subject: [PATCH 28/67] fix(security): close IDOR and tenant-bypass in patterns API GET /api/v1/patterns: the subscription scope is now always enforced in SQL (WHERE subscription_id = %s, never a nullable IS NULL bypass). The JWT subscription_id is the authority; a query-param that disagrees with the JWT is rejected with 400. A request with no subscription_id in either the JWT or the query param is also rejected. GET /api/v1/patterns/: the WHERE clause now includes subscription_id so a caller cannot enumerate patterns from other subscriptions by ID. An out-of-scope ID returns 404 to avoid disclosing that the pattern exists in another subscription. Adds three new regression tests: no-subscription-400, cross-subscription query-param-400, and cross-subscription IDOR-404. Signed-off-by: Tanvir Farhad --- api/routes/patterns.py | 62 ++++++++++++++++++++++++++++++------------ tests/test_patterns.py | 59 ++++++++++++++++++++++++++++++---------- 2 files changed, 88 insertions(+), 33 deletions(-) diff --git a/api/routes/patterns.py b/api/routes/patterns.py index a03d8d4e..be5d8f9e 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -42,6 +42,30 @@ def _validate_limit(raw: str) -> int: return value +def _effective_subscription(subscription_id_param: str | None) -> str: + """Return the subscription scope this request is authorized to see. + + The JWT subscription_id is always the authority. A query-param + subscription_id may narrow the JWT scope but never widen it. If the + JWT carries no subscription_id, the query param is accepted as the + scope (single-tenant deployments that do not embed subscription_id in + tokens). Either way the scope must be non-empty: an unscoped query + would return patterns from every subscription, which is never correct. + """ + user = getattr(g, "user", {}) or {} + jwt_sub = user.get("subscription_id") + + if jwt_sub: + if subscription_id_param and subscription_id_param != jwt_sub: + raise _ValidationError("subscription_id does not match token scope") + return jwt_sub + + if subscription_id_param: + return subscription_id_param + + raise _ValidationError("subscription_id is required") + + def _row_to_dict(row: dict) -> dict: """Serialise a patterns table row to a JSON-safe dict.""" result = dict(row) @@ -59,7 +83,7 @@ def list_patterns(): """Return published security patterns, optionally filtered. Query parameters: - subscription_id - filter by Azure subscription + subscription_id - filter by Azure subscription (must match JWT scope) pattern_type - one of persistent_finding, cross_resource_recurrence, reopened_finding limit - 1-100, default 20 @@ -73,12 +97,12 @@ def list_patterns(): if len(request.args.getlist(key)) != 1: raise _ValidationError(f"Query parameter {key} must be provided once") - subscription_id = None + subscription_id_param = None if "subscription_id" in request.args: val = request.args["subscription_id"].strip() if not val or len(val) > 256: raise _ValidationError("subscription_id is invalid") - subscription_id = val + subscription_id_param = val pattern_type = None if "pattern_type" in request.args: @@ -91,14 +115,11 @@ def list_patterns(): if "limit" in request.args: limit = _validate_limit(request.args["limit"]) + effective_sub = _effective_subscription(subscription_id_param) + db = _get_db() conn = db._get_conn() - # Tenant isolation: prefer subscription_id embedded in the JWT payload; - # fall back to the query parameter. - user = getattr(g, "user", {}) or {} - effective_sub = user.get("subscription_id") or subscription_id - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( """ @@ -106,16 +127,12 @@ def list_patterns(): subscription_id, scan_id, finding_ids, threshold, algorithm_version, created_at, updated_at FROM patterns - WHERE (%s IS NULL OR subscription_id = %s) + WHERE subscription_id = %s AND (%s IS NULL OR pattern_type = %s) ORDER BY created_at DESC LIMIT %s """, - ( - effective_sub, effective_sub, - pattern_type, pattern_type, - limit, - ), + (effective_sub, pattern_type, pattern_type, limit), ) rows = cur.fetchall() @@ -123,10 +140,10 @@ def list_patterns(): """ SELECT COUNT(*) AS count FROM patterns - WHERE (%s IS NULL OR subscription_id = %s) + WHERE subscription_id = %s AND (%s IS NULL OR pattern_type = %s) """, - (effective_sub, effective_sub, pattern_type, pattern_type), + (effective_sub, pattern_type, pattern_type), ) total_row = cur.fetchone() total = total_row["count"] if total_row else 0 @@ -147,11 +164,19 @@ def list_patterns(): @patterns_bp.get("/api/v1/patterns/") def get_pattern(pattern_id: int): - """Return a single pattern by its integer ID.""" + """Return a single pattern by its integer ID. + + The pattern is only returned when its subscription_id matches the + caller's authorized scope. An out-of-scope pattern returns 404 so + that the existence of other subscriptions' patterns is not disclosed. + """ try: if pattern_id <= 0: raise _ValidationError("pattern_id must be a positive integer") + # Resolve scope first so an unscoped caller cannot enumerate IDs. + effective_sub = _effective_subscription(None) + db = _get_db() conn = db._get_conn() @@ -163,8 +188,9 @@ def get_pattern(pattern_id: int): algorithm_version, created_at, updated_at FROM patterns WHERE id = %s + AND subscription_id = %s """, - (pattern_id,), + (pattern_id, effective_sub), ) row = cur.fetchone() diff --git a/tests/test_patterns.py b/tests/test_patterns.py index e27e2a9e..3884d82e 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -229,7 +229,7 @@ def test_list_returns_patterns_and_total(self, app_client, monkeypatch): with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 200 @@ -242,24 +242,40 @@ def test_list_returns_patterns_and_total(self, app_client, monkeypatch): def test_list_invalid_limit_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?limit=999", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 400 def test_list_invalid_pattern_type_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?pattern_type=not_a_real_type", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 400 def test_list_unknown_query_param_returns_400(self, app_client): resp = app_client.get( "/api/v1/patterns?foo=bar", + headers=_auth_headers(sub_id=SUB_ID), + ) + assert resp.status_code == 400 + + def test_list_without_subscription_returns_400(self, app_client): + # JWT with no subscription_id and no query param must be rejected. + resp = app_client.get( + "/api/v1/patterns", headers=_auth_headers(), ) assert resp.status_code == 400 + def test_list_cross_subscription_query_param_rejected(self, app_client): + # JWT scoped to sub-A must reject a query param asking for sub-B. + resp = app_client.get( + "/api/v1/patterns?subscription_id=sub-B", + headers=_auth_headers(sub_id="sub-A"), + ) + assert resp.status_code == 400 + def test_list_returns_only_authorized_subscription(self, app_client): """Patterns for a different subscription must not be returned to an authorized user whose JWT contains a different subscription_id.""" @@ -283,8 +299,7 @@ def test_list_requires_auth(self, app_client): class TestPatternsRouteGet: - def test_get_existing_pattern(self, app_client): - row = _sample_pattern_row() + def _mock_get_db(self, row): mock_db = MagicMock() mock_conn = MagicMock() mock_cursor = MagicMock() @@ -293,11 +308,16 @@ def test_get_existing_pattern(self, app_client): mock_cursor.fetchone.return_value = row mock_conn.cursor.return_value = mock_cursor mock_db._get_conn.return_value = mock_conn + return mock_db + + def test_get_existing_pattern(self, app_client): + row = _sample_pattern_row() + mock_db = self._mock_get_db(row) with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns/1", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), ) assert resp.status_code == 200 @@ -308,19 +328,28 @@ def test_get_existing_pattern(self, app_client): assert data["algorithm_version"] == "1" def test_get_nonexistent_pattern_returns_404(self, app_client): - mock_db = MagicMock() - mock_conn = MagicMock() - mock_cursor = MagicMock() - mock_cursor.__enter__ = MagicMock(return_value=mock_cursor) - mock_cursor.__exit__ = MagicMock(return_value=False) - mock_cursor.fetchone.return_value = None - mock_conn.cursor.return_value = mock_cursor - mock_db._get_conn.return_value = mock_conn + # DB returns None when id + subscription_id don't match any row. + mock_db = self._mock_get_db(None) with patch("api.routes.patterns._get_db", return_value=mock_db): resp = app_client.get( "/api/v1/patterns/99999", - headers=_auth_headers(), + headers=_auth_headers(sub_id=SUB_ID), + ) + + assert resp.status_code == 404 + + def test_get_cross_subscription_returns_404(self, app_client): + # Pattern exists for sub-001 but caller is scoped to sub-other. + # The SQL adds subscription_id = %s to the WHERE clause so the DB + # returns None, and the caller sees 404 (not 403, to avoid leaking + # that the pattern ID exists in another subscription). + mock_db = self._mock_get_db(None) + + with patch("api.routes.patterns._get_db", return_value=mock_db): + resp = app_client.get( + "/api/v1/patterns/1", + headers=_auth_headers(sub_id="sub-other"), ) assert resp.status_code == 404 From 46002c7917100d6a931473bac9d0c34b4d431ab4 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 02:10:28 +0100 Subject: [PATCH 29/67] fix(lifecycle): address all code review findings from #311 Critical fixes: - Wire LifecycleService.apply_scan() and PatternService.detect_and_publish() into scanner/worker.py immediately after db.save_scan(); lifecycle failures are non-fatal so the scan record is always preserved - Write durable per-rule outcome rows to scan_rule_outcomes at the start of apply_scan so the audit record exists even if lifecycle processing fails - Add subscription_id = effective_sub filter to GET /api/v1/patterns/ via _effective_subscription() helper (already present in patterns.py HEAD) Important fixes: - Add UNIQUE(pattern_type, lifecycle_id, scan_id) constraint to patterns table in migration; change ON CONFLICT DO NOTHING to ON CONFLICT ON CONSTRAINT uq_patterns_type_lifecycle_scan DO NOTHING - Collect detection query results before closing RealDictCursor, then open fresh cursors for each _upsert_pattern call, eliminating nested-cursor overlap in pattern_service.py - Narrow the absent-findings bulk lock to only rules with a resolving outcome (resolving_rule_ids); skip the query entirely when no rule produced SUCCESS/EMPTY_SUCCESS, reducing unnecessary lock contention - Reset consecutive_success_count = 0 when a RESOLVED/ACCEPTED/SUPPRESSED finding is reopened - Remove unused _BLOCKING_STATUSES frozenset from lifecycle_service.py - Move 'import json' to module level in pattern_service.py Test fixes: - Refactor _FakeCursor/_FakeConn to use a shared deque so multiple cursor() calls on one connection consume from the same result stream - Update scripted result sequences in all tests to include the new scan_rule_outcomes INSERT in the execute order - Update fail-closed tests to reflect that no absent-findings query is issued when resolving_rule_ids is empty (FAILED/PERMISSION_DENIED) - Add test_scan_rule_outcomes_written to verify audit record is emitted - Add consecutive_success_count = 0 assertion to reopen test Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 4 +- api/services/lifecycle_service.py | 131 +++++--- api/services/pattern_service.py | 83 ++--- scanner/worker.py | 32 ++ tests/test_finding_lifecycle.py | 285 ++++++++---------- 5 files changed, 288 insertions(+), 247 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 1b1ccfa4..7d6d8d98 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -137,7 +137,9 @@ def upgrade() -> None: 'persistent_finding', 'cross_resource_recurrence', 'reopened_finding' - )) + )), + CONSTRAINT uq_patterns_type_lifecycle_scan + UNIQUE (pattern_type, lifecycle_id, scan_id) ) """ ) diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 9aff5e92..10a1ebbe 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -12,10 +12,6 @@ # Statuses that mean "we actively confirmed this rule was clean in the scan." _RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) -# Statuses that mean "we could not reliably evaluate the rule." -# Fail-closed: do NOT resolve findings when the scan could not see the resource. -_BLOCKING_STATUSES = frozenset({"PERMISSION_DENIED", "TIMEOUT", "FAILED"}) - def _normalize_resource_id(resource_id: str) -> str: """Return a stable, lowercased, stripped version of an ARM resource ID.""" @@ -72,7 +68,8 @@ def apply_scan( subscription_id: Azure subscription ID. tenant_id: Tenant identifier for isolation. rule_outcomes: List of dicts with at minimum keys 'rule_id' and - 'status' (one of the six allowed status values). + 'status' (one of the six allowed status values). Also used to + populate scan_rule_outcomes for durable audit history. findings: List of finding dicts from the scan, each with keys 'rule_id', 'resource_id', and optionally 'evidence_key'. Defaults to an empty list when omitted. @@ -89,11 +86,46 @@ def apply_scan( logger.info("Scan %s already applied; skipping lifecycle update", scan_id) return + # --- Write durable per-rule outcome records ------------------- + # Insert before lifecycle logic so the audit record exists even + # if lifecycle processing fails. The UNIQUE(scan_id, rule_id) + # constraint prevents duplicates on accidental double-writes. + for outcome in rule_outcomes: + rule_id_o = outcome.get("rule_id", "") + status_o = outcome.get("status", "FAILED") + if not rule_id_o: + continue + cur.execute( + """ + INSERT INTO scan_rule_outcomes ( + scan_id, rule_id, status, tenant_id, subscription_id, + inventory_boundary, started_at, completed_at + ) + VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) + ON CONFLICT (scan_id, rule_id) DO NOTHING + """, + ( + scan_id, + rule_id_o, + status_o, + tenant_id, + subscription_id, + outcome.get("started_at"), + ), + ) + # Build a lookup: rule_id -> outcome status outcome_by_rule: Dict[str, str] = { o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o } + # Collect rule IDs that actively confirmed a clean result. Only + # these can trigger resolution of absent findings (fail-closed). + resolving_rule_ids = [ + rule_id for rule_id, status in outcome_by_rule.items() + if status in _RESOLVING_STATUSES + ] + # Build the set of (rule_id, resource_id_normalized) pairs seen in # this scan. A fingerprint in this set was actively observed. seen_fingerprint_keys: set = set() @@ -131,6 +163,8 @@ def apply_scan( # --- Upsert fingerprints and lifecycles for seen findings ------- seen_fingerprint_ids: set = set() for fp in fingerprints_in_scan: + # ON CONFLICT touches fingerprint_hash to force RETURNING id + # even when the row already exists (DO NOTHING returns nothing). cur.execute( """ INSERT INTO finding_fingerprints ( @@ -208,7 +242,7 @@ def apply_scan( (scan_id, lifecycle_id), ) elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): - # Reappeared after resolution: reopen. + # Reappeared after resolution: reopen and reset success streak. cur.execute( """ UPDATE finding_lifecycles @@ -216,6 +250,7 @@ def apply_scan( last_seen_scan_id = %s, occurrence_count = occurrence_count + 1, reopen_count = reopen_count + 1, + consecutive_success_count = 0, updated_at = NOW(), row_version = row_version + 1 WHERE id = %s @@ -232,11 +267,13 @@ def apply_scan( ) # --- Resolve findings NOT seen in this scan --------------------- - # Only resolve if the rule's outcome actively confirmed the resource - # was clean (SUCCESS / EMPTY_SUCCESS). Fail closed for uncertain outcomes. - if seen_fingerprint_ids: - # Find fingerprints for this tenant/subscription that are currently - # OPEN or REOPENED but were NOT observed in this scan. + # Only lock and process rows for rules that had a resolving outcome. + # This avoids unnecessary contention on unrelated rules and is more + # efficient on subscriptions with many open findings. + if not resolving_rule_ids: + # No rule produced a clean outcome: nothing can be resolved. + pass + elif seen_fingerprint_ids: cur.execute( """ SELECT fl.id, fl.state, fl.row_version, ff.rule_id @@ -246,13 +283,14 @@ def apply_scan( AND ff.subscription_id = %s AND fl.state IN ('OPEN', 'REOPENED') AND fl.fingerprint_id NOT IN %s + AND ff.rule_id = ANY(%s) FOR UPDATE OF fl """, - (tenant_id, subscription_id, tuple(seen_fingerprint_ids)), + (tenant_id, subscription_id, tuple(seen_fingerprint_ids), resolving_rule_ids), ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) else: - # No findings seen at all: resolve all OPEN/REOPENED where rule - # had a resolving outcome. + # No findings seen at all: resolve for rules with clean outcomes. cur.execute( """ SELECT fl.id, fl.state, fl.row_version, ff.rule_id @@ -261,37 +299,12 @@ def apply_scan( WHERE ff.tenant_id = %s AND ff.subscription_id = %s AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) FOR UPDATE OF fl """, - (tenant_id, subscription_id), + (tenant_id, subscription_id, resolving_rule_ids), ) - - absent_rows = cur.fetchall() - for lc_id, state, row_version, rule_id in absent_rows: - outcome_status = outcome_by_rule.get(rule_id) - if outcome_status in _RESOLVING_STATUSES: - cur.execute( - """ - UPDATE finding_lifecycles - SET state = 'RESOLVED', - last_seen_scan_id = %s, - consecutive_success_count = consecutive_success_count + 1, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s - """, - (scan_id, lc_id), - ) - cur.execute( - """ - INSERT INTO finding_lifecycle_transitions - (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, %s, 'RESOLVED', %s, - 'Rule confirmed clean; finding absent from scan') - """, - (lc_id, state, scan_id), - ) - # else: PERMISSION_DENIED / TIMEOUT / FAILED / missing -> fail closed, no change. + _resolve_absent_rows(cur, scan_id, outcome_by_rule) # --- Idempotency sentinel (inserted last) ----------------------- cur.execute( @@ -304,3 +317,37 @@ def apply_scan( db_conn.commit() logger.info("Lifecycle application committed for scan %s", scan_id) + + +def _resolve_absent_rows( + cur: Any, + scan_id: str, + outcome_by_rule: Dict[str, str], +) -> None: + """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules.""" + absent_rows = cur.fetchall() + for lc_id, state, row_version, rule_id in absent_rows: + outcome_status = outcome_by_rule.get(rule_id) + if outcome_status in _RESOLVING_STATUSES: + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'RESOLVED', + last_seen_scan_id = %s, + consecutive_success_count = consecutive_success_count + 1, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lc_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'RESOLVED', %s, + 'Rule confirmed clean; finding absent from scan') + """, + (lc_id, state, scan_id), + ) + # else: outcome missing or blocking -> fail closed, no state change. diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 282bc2f9..6ebcd965 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -1,5 +1,6 @@ """PatternService: detects and publishes security patterns from lifecycle state.""" +import json import logging from typing import Any @@ -46,18 +47,7 @@ def detect_and_publish( """, (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), ) - for row in cur.fetchall(): - _upsert_pattern( - db_conn, - pattern_type="persistent_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_PERSISTENT_THRESHOLD, - ) - count += 1 + persistent_rows = cur.fetchall() # ---------------------------------------------------------------- # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED @@ -78,23 +68,7 @@ def detect_and_publish( """, (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), ) - for row in cur.fetchall(): - lifecycle_ids = row["lifecycle_ids"] - # Publish one pattern per lifecycle in the group so each is - # individually traceable; finding_ids carries the sibling IDs. - for lc_id in lifecycle_ids: - sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] - _upsert_pattern( - db_conn, - pattern_type="cross_resource_recurrence", - lifecycle_id=lc_id, - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=sibling_ids, - threshold=_CROSS_RESOURCE_THRESHOLD, - ) - count += 1 + cross_rows = cur.fetchall() # ---------------------------------------------------------------- # 3. reopened_finding: reopen_count >= 1 and state == REOPENED @@ -111,19 +85,54 @@ def detect_and_publish( """, (tenant_id, subscription_id, _REOPENED_THRESHOLD), ) - for row in cur.fetchall(): + reopened_rows = cur.fetchall() + + # All detection queries are finished; cursor is closed. Now upsert + # patterns using separate cursor calls to avoid open-cursor overlap. + for row in persistent_rows: + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + for row in cross_rows: + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle in the group so each is + # individually traceable; finding_ids carries the sibling IDs. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] _upsert_pattern( db_conn, - pattern_type="reopened_finding", - lifecycle_id=row["lifecycle_id"], + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, tenant_id=tenant_id, subscription_id=subscription_id, scan_id=scan_id, - finding_ids=[], - threshold=_REOPENED_THRESHOLD, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, ) count += 1 + for row in reopened_rows: + _upsert_pattern( + db_conn, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, + ) + count += 1 + db_conn.commit() logger.info( "Pattern detection for scan %s: %d pattern(s) upserted", @@ -143,9 +152,7 @@ def _upsert_pattern( finding_ids: list, threshold: int, ) -> None: - """Insert or update a single pattern record.""" - import json - + """Insert a pattern record if the (type, lifecycle, scan) combination is new.""" with db_conn.cursor() as cur: cur.execute( """ @@ -155,7 +162,7 @@ def _upsert_pattern( created_at, updated_at ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, NOW(), NOW()) - ON CONFLICT DO NOTHING + ON CONFLICT ON CONSTRAINT uq_patterns_type_lifecycle_scan DO NOTHING """, ( pattern_type, diff --git a/scanner/worker.py b/scanner/worker.py index 5c0d388a..dfd70095 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -25,6 +25,8 @@ configure_logging, init_sentry, ) +from api.services.lifecycle_service import LifecycleService +from api.services.pattern_service import PatternService from scanner.engine import ScanEngine from scanner.enrichment_worker import process_enrichment_job from scanner.graph.graph_populator import populate_graph @@ -231,6 +233,36 @@ def run_worker(): exc, ) + # Apply lifecycle tracking. Lifecycle failures are non-fatal: + # the scan is already persisted, so we log and continue rather + # than marking the scan as failed. + try: + tenant_id = os.environ.get("OPENSHIELD_TENANT_ID", subscription_id) + lc_svc = LifecycleService() + lc_svc.apply_scan( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + rule_outcomes=result.get("rule_outcomes", []), + findings=result.get("findings", []), + ) + pat_svc = PatternService() + pat_svc.detect_and_publish( + db_conn=db._get_conn(), + scan_id=scan_id, + subscription_id=subscription_id, + tenant_id=tenant_id, + ) + except Exception as lc_exc: + logger.error( + "Lifecycle/pattern update failed for scan %s (scan data intact): %s", + scan_id, + lc_exc, + extra={"scan_id": scan_id}, + ) + + SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 7e5a239e..3b31816c 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -40,24 +40,28 @@ def _make_outcome(rule_id: str, status: str) -> dict: # --------------------------------------------------------------------------- -# Simple DB simulation using a plain dict as in-memory state. -# We avoid mocking every cursor call individually by building a lightweight -# fake cursor that replays scripted return values in order. +# Fake cursor / connection. +# +# The cursor consumes from a shared deque so multiple cursor() calls on the +# same connection share one result stream. This mirrors how psycopg2 works +# (multiple cursors on one connection see the same transaction state) and +# avoids the "second cursor re-reads from the start" bug noted in code review. # --------------------------------------------------------------------------- +from collections import deque + class _FakeCursor: - """A fake psycopg2 cursor that works with pre-loaded fetchone/fetchall results.""" + """Fake psycopg2 cursor backed by a shared result deque.""" - def __init__(self, results: list): - # results is a list of return values; each execute() pops one. - self._results = list(results) + def __init__(self, results_deque: deque): + self._results = results_deque + self.executed: list = [] self._current = None - self.executed = [] def execute(self, sql, params=None): self.executed.append((sql.strip(), params)) - self._current = self._results.pop(0) if self._results else None + self._current = self._results.popleft() if self._results else None def fetchone(self): return self._current @@ -75,20 +79,24 @@ def __exit__(self, *args): class _FakeConn: - """Fake connection whose cursor() returns a _FakeCursor consuming a result list.""" + """Fake connection whose cursor() calls share one result deque.""" def __init__(self, results: list): - self._results = results + self._deque: deque = deque(results) self.committed = False - self._cursor_obj = None + self._cursor_obj: _FakeCursor | None = None def cursor(self, **_kwargs): - self._cursor_obj = _FakeCursor(self._results) + # Return a new cursor object but backed by the same shared deque. + self._cursor_obj = _FakeCursor(self._deque) return self._cursor_obj def commit(self): self.committed = True + def all_executed(self) -> list: + return self._cursor_obj.executed if self._cursor_obj else [] + # --------------------------------------------------------------------------- # Tests @@ -129,53 +137,41 @@ def test_strip_whitespace(self): class TestLifecycleServiceIdempotency: - """Applying the same scan_id twice must produce exactly one transition.""" + """Applying the same scan_id twice must commit exactly once.""" def test_second_apply_is_no_op(self): - # First call: idempotency row NOT present (fetchone -> None), then - # fingerprint upsert returns id=1, lifecycle select returns None - # (new finding), lifecycle insert returns id=10, commit. - # Second call: idempotency row IS present (fetchone -> (scan_id,)). - svc = LifecycleService() - - transitions = [] - - # We use a counting approach: track how many times commit() is called. committed_count = 0 class _TrackingConn: - def __init__(self, is_first_call): - self._is_first = is_first_call - self._results = self._build_results(is_first_call) - - def _build_results(self, first): - if first: - # idempotency check -> None (not applied) - # fingerprint upsert RETURNING id -> (1,) - # lifecycle FOR UPDATE -> None (new) - # lifecycle INSERT RETURNING id -> (10,) - # transition insert -> None - # absent-findings query -> [] - # idempotency insert -> None - return [None, (1,), None, (10,), None, [], None] + def __init__(self, already_applied: bool): + if already_applied: + # idempotency check returns a row -> return immediately + results = [("already-applied",)] else: - # idempotency check -> row found: stop immediately - return [("already-applied",)] + # idempotency check None, outcome insert None, + # fingerprint upsert -> (1,), lifecycle lock -> None (new), + # lifecycle insert -> (10,), transition insert None, + # absent-findings query -> [], idempotency insert None + results = [None, None, (1,), None, (10,), None, [], None] + self._deque: deque = deque(results) + self._cursor_obj = None def cursor(self, **_kwargs): - return _FakeCursor(self._results) + self._cursor_obj = _FakeCursor(self._deque) + return self._cursor_obj def commit(self): nonlocal committed_count committed_count += 1 + svc = LifecycleService() svc.apply_scan( - _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) svc.apply_scan( - _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) @@ -188,176 +184,133 @@ class TestLifecycleStateTransitions: """State machine correctness tests using scripted cursor results.""" def _run(self, results, findings, outcomes): - """Run apply_scan with scripted cursor results; return the fake conn.""" conn = _FakeConn(results) svc = LifecycleService() svc.apply_scan(conn, SCAN_ID_1, SUB_ID, TENANT_ID, outcomes, findings) return conn + def _all_sql(self, conn: _FakeConn) -> list[str]: + return [sql for sql, _ in conn.all_executed()] + def test_new_finding_creates_open_lifecycle(self): - # Scripted results in cursor execute order: + # Sequence (one shared deque, all execute() calls in order): # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> None (no existing row) - # 4. lifecycle INSERT RETURNING id -> (10,) - # 5. transition insert -> None - # 6. absent-findings query -> [] - # 7. idempotency insert -> None - results = [None, (1,), None, (10,), None, [], None] + # 2. scan_rule_outcomes insert (for RULE-001 outcome) -> None + # 3. fingerprint upsert RETURNING id -> (1,) + # 4. lifecycle FOR UPDATE -> None (new) + # 5. lifecycle INSERT RETURNING id -> (10,) + # 6. transition insert -> None + # 7. absent-findings query (no resolving ids after seen_fingerprint_ids) -> [] + # 8. idempotency insert -> None + results = [None, None, (1,), None, (10,), None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - - # Verify the lifecycle INSERT used 'OPEN' - executed = conn._cursor_obj.executed - insert_lc = next( - (sql for sql, _ in executed if "INSERT INTO finding_lifecycles" in sql), None + sqls = self._all_sql(conn) + assert any("INSERT INTO finding_lifecycles" in s for s in sqls) + # Transition to OPEN must be recorded. + assert any("finding_lifecycle_transitions" in s and "'OPEN'" in s for s in sqls) + + def test_scan_rule_outcomes_written(self): + """scan_rule_outcomes must be populated on every apply_scan call.""" + results = [None, None, (1,), None, (10,), None, [], None] + conn = self._run( + results, + [_make_finding("RULE-001", "/rg/foo")], + [_make_outcome("RULE-001", "SUCCESS")], ) - assert insert_lc is not None + sqls = self._all_sql(conn) + assert any("INSERT INTO scan_rule_outcomes" in s for s in sqls) def test_open_finding_not_seen_in_success_scan_is_resolved(self): - # No findings in this scan, outcome is SUCCESS -> existing OPEN should resolve. + # No findings; SUCCESS outcome -> existing OPEN resolved. # 1. idempotency check -> None - # 2. absent-findings query (no seen ids branch) -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] - # 3. UPDATE finding_lifecycles SET state='RESOLVED' -> None - # 4. transition insert -> None - # 5. idempotency insert -> None + # 2. scan_rule_outcomes insert -> None + # 3. absent-findings query (empty seen_ids branch) -> [(10,'OPEN',0,'RULE-001')] + # 4. UPDATE to RESOLVED -> None + # 5. transition insert (OPEN -> RESOLVED) -> None + # 6. idempotency insert -> None results = [ + None, None, [(10, "OPEN", 0, "RULE-001")], None, None, None, ] - conn = self._run( - results, - [], # no findings - [_make_outcome("RULE-001", "SUCCESS")], - ) + conn = self._run(results, [], [_make_outcome("RULE-001", "SUCCESS")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is not None + sqls = self._all_sql(conn) + assert any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) def test_open_finding_not_seen_in_failed_scan_stays_open(self): - # FAILED outcome -> fail-closed: finding should NOT be resolved. + # FAILED outcome -> fail-closed: no resolution. # 1. idempotency check -> None - # 2. absent-findings query -> [(lc_id=10, 'OPEN', 0, 'RULE-001')] + # 2. scan_rule_outcomes insert -> None + # (FAILED is not in resolving_rule_ids; no absent-findings query is issued) # 3. idempotency insert -> None - # (no UPDATE to RESOLVED because outcome is FAILED) - results = [ - None, - [(10, "OPEN", 0, "RULE-001")], - None, - ] - conn = self._run( - results, - [], - [_make_outcome("RULE-001", "FAILED")], - ) + results = [None, None, None] + conn = self._run(results, [], [_make_outcome("RULE-001", "FAILED")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is None + sqls = self._all_sql(conn) + assert not any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) - def test_open_finding_not_seen_in_permission_denied_stays_open(self): - results = [ - None, - [(10, "OPEN", 0, "RULE-001")], - None, - ] - conn = self._run( - results, - [], - [_make_outcome("RULE-001", "PERMISSION_DENIED")], - ) + def test_open_finding_not_seen_in_permission_denied_scan_stays_open(self): + results = [None, None, None] + conn = self._run(results, [], [_make_outcome("RULE-001", "PERMISSION_DENIED")]) assert conn.committed - executed = conn._cursor_obj.executed - resolve_sql = next( - (sql for sql, _ in executed if "RESOLVED" in sql and "UPDATE" in sql.upper()), None - ) - assert resolve_sql is None + sqls = self._all_sql(conn) + assert not any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) def test_resolved_finding_seen_again_becomes_reopened(self): - # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> should REOPEN. + # Fingerprint exists (id=1), lifecycle row has state=RESOLVED -> REOPEN. # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> (lc_id=10, 'RESOLVED', 1, 0, 2) - # (id, state, occurrence_count, reopen_count, row_version) - # 4. UPDATE to REOPENED -> None - # 5. transition insert (RESOLVED -> REOPENED) -> None - # 6. absent-findings query -> [] - # 7. idempotency insert -> None - results = [ - None, - (1,), - (10, "RESOLVED", 1, 0, 2), - None, - None, - [], - None, - ] + # 2. scan_rule_outcomes insert -> None + # 3. fingerprint upsert RETURNING id -> (1,) + # 4. lifecycle FOR UPDATE -> (10,'RESOLVED',1,0,2) + # 5. UPDATE to REOPENED (resets consecutive_success_count=0) -> None + # 6. transition insert -> None + # 7. absent-findings query -> [] + # 8. idempotency insert -> None + results = [None, None, (1,), (10, "RESOLVED", 1, 0, 2), None, None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - executed = conn._cursor_obj.executed - reopen_sql = next( - (sql for sql, _ in executed if "REOPENED" in sql and "UPDATE" in sql.upper()), None - ) - assert reopen_sql is not None + sqls = self._all_sql(conn) + assert any("REOPENED" in s and "UPDATE" in s.upper() for s in sqls) + # consecutive_success_count must be reset to 0 on reopen. + assert any("consecutive_success_count = 0" in s for s in sqls) def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): - # REOPENED + seen again: occurrence_count increments, state stays REOPENED. - # 1. idempotency check -> None - # 2. fingerprint upsert RETURNING id -> (1,) - # 3. lifecycle FOR UPDATE -> (10, 'REOPENED', 3, 1, 4) - # 4. UPDATE occurrence_count + 1 -> None (OPEN/REOPENED branch) - # 5. absent-findings query -> [] - # 6. idempotency insert -> None - results = [ - None, - (1,), - (10, "REOPENED", 3, 1, 4), - None, - [], - None, - ] + # REOPENED + seen in scan: occurrence_count increments, no new state transition. + # 1. idempotency -> None + # 2. scan_rule_outcomes insert -> None + # 3. fingerprint upsert -> (1,) + # 4. lifecycle FOR UPDATE -> (10,'REOPENED',3,1,4) + # 5. UPDATE occurrence_count + 1 -> None + # 6. absent-findings query -> [] + # 7. idempotency insert -> None + results = [None, None, (1,), (10, "REOPENED", 3, 1, 4), None, [], None] conn = self._run( results, [_make_finding("RULE-001", "/rg/foo")], [_make_outcome("RULE-001", "SUCCESS")], ) assert conn.committed - executed = conn._cursor_obj.executed - # Verify no transition to a NEW state was recorded for this finding - # (i.e. no INSERT INTO finding_lifecycle_transitions for OPEN/REOPENED). - reopened_transition = next( - ( - sql for sql, params in executed - if "INSERT INTO finding_lifecycle_transitions" in sql - and params is not None - and "REOPENED" in str(params) - ), - None, - ) - assert reopened_transition is None - - # Verify occurrence_count was incremented (UPDATE without state change). - occ_update = next( - ( - sql for sql, _ in executed - if "occurrence_count = occurrence_count + 1" in sql - ), - None, - ) - assert occ_update is not None + sqls = self._all_sql(conn) + + # No transition record should be emitted for REOPENED->REOPENED. + transition_to_reopened = [ + s for s in sqls + if "finding_lifecycle_transitions" in s and "REOPENED" in s + ] + assert not transition_to_reopened + + # occurrence_count should increment. + assert any("occurrence_count = occurrence_count + 1" in s for s in sqls) From 2d99c5a7bf19d855f8369c2698157f740a1017a6 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 11:20:16 +0100 Subject: [PATCH 30/67] fix(security): strip internal exception detail from all 500 error responses Five routes were echoing str(exc) directly to the client in error response bodies, exposing DB connection strings, filesystem paths, and stack trace fragments. The JWT middleware was also leaking the specific JWT validation failure reason via f-string interpolation. Changes: - api/app.py: InvalidTokenError now returns generic 'Invalid token' (no exc detail) - api/routes/compliance.py: FileNotFoundError logs path, returns opaque message; general handler drops 'detail' field - api/routes/findings.py: both endpoints drop 'detail: str(exc)' from 500 body - api/routes/scans.py: all four catch blocks drop 'detail: str(exc)' from 500 body Internal errors are still logged server-side at ERROR level. Signed-off-by: Tanvir Farhad --- api/routes/scans.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/api/routes/scans.py b/api/routes/scans.py index 3aed6dfe..42c90b9b 100644 --- a/api/routes/scans.py +++ b/api/routes/scans.py @@ -174,6 +174,35 @@ def trigger_scan(): } +def _run_enrichment_in_background(scan_id: str, findings: list, db_url: str) -> None: + """Run CVE enrichment off the request thread and persist the result. + + Runs outside the Flask request/app context (it's started via + threading.Thread), so it opens its own DatabaseManager rather than + reusing flask.g. + """ + db = DatabaseManager(db_url) + try: + enriched = enrich_findings(findings) + db.update_cve_fields(enriched) + db.update_scan_enrichment_status(scan_id, "COMPLETED") + logger.info("Background CVE enrichment complete for scan %s (%d findings)", scan_id, len(enriched)) + except Exception as exc: + logger.error("Background enrichment failed for scan %s: %s", scan_id, exc) + try: + # A failed write (e.g. in update_cve_fields) can leave db.conn in + # an aborted-transaction state. Roll back first, or this status + # update itself raises InFailedSqlTransaction and gets swallowed + # below, leaving the scan stuck at ENRICHING forever. + if db.conn is not None: + db.conn.rollback() + db.update_scan_enrichment_status(scan_id, "FAILED") + except Exception as status_exc: + logger.error("Failed to record FAILED status for scan %s: %s", scan_id, status_exc) + finally: + db.close() + + @scans_bp.post("/api/scans//enrich") def enrich_scan(scan_id): """Enqueue durable CVE enrichment; no request-owned thread is created.""" From 63abf31173f783f3deccbddc83c82f9f50f607a1 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 11:42:41 +0100 Subject: [PATCH 31/67] fix(lifecycle): address all reviewer-blocking issues before merge Critical: - lifecycle_service: wrap entire transaction in try/except with explicit rollback() to prevent connection wedge on mid-transaction failure - pattern_service: same rollback guard for detect_and_publish transaction - lifecycle_service: replace NOT IN %s + tuple() with != ALL(%s) + list() to fix single-element tuple syntax error (PostgreSQL rejects trailing comma) - patterns API: add tenant_id = %s to both SQL WHERE clauses to close cross-tenant pattern read in multi-tenant deployments Important: - patterns API: add _effective_tenant() helper using OPENSHIELD_TENANT_ID env - migration: add 4 missing indexes (fingerprints tenant+sub, lifecycles state partial, transitions lifecycle_id, patterns sub+created_at) - lifecycle_service: remove unused psycopg2.extras import - test_finding_lifecycle: add test_rule_a_success_does_not_resolve_rule_b_finding to verify cross-rule resolution isolation (the key correctness invariant) - test_finding_lifecycle: add rollback() to _FakeConn and _TrackingConn - test_patterns: assert subscription_id appears in SQL params for IDOR test and subscription scoping test (not just mock return value) Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 15 + api/routes/patterns.py | 25 +- api/services/lifecycle_service.py | 439 +++++++++--------- api/services/pattern_service.py | 196 ++++---- tests/test_finding_lifecycle.py | 51 ++ tests/test_patterns.py | 17 +- 6 files changed, 418 insertions(+), 325 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 7d6d8d98..d36e9444 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -145,6 +145,21 @@ def upgrade() -> None: ) + # Indexes for hot query paths + op.execute( + "CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)" + ) + op.execute( + "CREATE INDEX ix_finding_lifecycles_state ON finding_lifecycles (state) WHERE state IN ('OPEN', 'REOPENED')" + ) + op.execute( + "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" + ) + op.execute( + "CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)" + ) + + def downgrade() -> None: op.execute("DROP TABLE IF EXISTS patterns") op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") diff --git a/api/routes/patterns.py b/api/routes/patterns.py index be5d8f9e..abe7e805 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -42,6 +42,16 @@ def _validate_limit(raw: str) -> int: return value +def _effective_tenant(effective_sub: str) -> str: + """Return the tenant scope for this request. + + Uses OPENSHIELD_TENANT_ID env var when set (multi-tenant deployments). + Falls back to effective_sub for single-tenant deployments where + tenant_id == subscription_id by convention. + """ + return os.environ.get("OPENSHIELD_TENANT_ID", effective_sub) + + def _effective_subscription(subscription_id_param: str | None) -> str: """Return the subscription scope this request is authorized to see. @@ -116,6 +126,7 @@ def list_patterns(): limit = _validate_limit(request.args["limit"]) effective_sub = _effective_subscription(subscription_id_param) + tenant_id = _effective_tenant(effective_sub) db = _get_db() conn = db._get_conn() @@ -127,12 +138,13 @@ def list_patterns(): subscription_id, scan_id, finding_ids, threshold, algorithm_version, created_at, updated_at FROM patterns - WHERE subscription_id = %s + WHERE tenant_id = %s + AND subscription_id = %s AND (%s IS NULL OR pattern_type = %s) ORDER BY created_at DESC LIMIT %s """, - (effective_sub, pattern_type, pattern_type, limit), + (tenant_id, effective_sub, pattern_type, pattern_type, limit), ) rows = cur.fetchall() @@ -140,10 +152,11 @@ def list_patterns(): """ SELECT COUNT(*) AS count FROM patterns - WHERE subscription_id = %s + WHERE tenant_id = %s + AND subscription_id = %s AND (%s IS NULL OR pattern_type = %s) """, - (effective_sub, pattern_type, pattern_type), + (tenant_id, effective_sub, pattern_type, pattern_type), ) total_row = cur.fetchone() total = total_row["count"] if total_row else 0 @@ -176,6 +189,7 @@ def get_pattern(pattern_id: int): # Resolve scope first so an unscoped caller cannot enumerate IDs. effective_sub = _effective_subscription(None) + tenant_id = _effective_tenant(effective_sub) db = _get_db() conn = db._get_conn() @@ -188,9 +202,10 @@ def get_pattern(pattern_id: int): algorithm_version, created_at, updated_at FROM patterns WHERE id = %s + AND tenant_id = %s AND subscription_id = %s """, - (pattern_id, effective_sub), + (pattern_id, tenant_id, effective_sub), ) row = cur.fetchone() diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 10a1ebbe..67729f9e 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -5,8 +5,6 @@ import logging from typing import Any, Dict, List, Optional -import psycopg2.extras - logger = logging.getLogger(__name__) # Statuses that mean "we actively confirmed this rule was clean in the scan." @@ -48,7 +46,8 @@ class LifecycleService: All database work for a single apply_scan call executes inside one transaction. The idempotency sentinel (scan_lifecycle_applications) is inserted last, so a crash before commit means the operation never happened - and can safely be retried. + and can safely be retried. Any exception triggers an explicit rollback to + leave the connection in a clean state for the next operation. """ def apply_scan( @@ -67,256 +66,261 @@ def apply_scan( scan_id: UUID of the completed scan. subscription_id: Azure subscription ID. tenant_id: Tenant identifier for isolation. - rule_outcomes: List of dicts with at minimum keys 'rule_id' and - 'status' (one of the six allowed status values). Also used to - populate scan_rule_outcomes for durable audit history. - findings: List of finding dicts from the scan, each with keys - 'rule_id', 'resource_id', and optionally 'evidence_key'. - Defaults to an empty list when omitted. + rule_outcomes: List of dicts with keys 'rule_id' and 'status'. + findings: List of finding dicts with keys 'rule_id', 'resource_id', + and optionally 'evidence_key'. Defaults to empty list. """ findings = findings or [] - with db_conn.cursor() as cur: - # --- Idempotency check ---------------------------------------- - cur.execute( - "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", - (scan_id,), - ) - if cur.fetchone() is not None: - logger.info("Scan %s already applied; skipping lifecycle update", scan_id) - return - - # --- Write durable per-rule outcome records ------------------- - # Insert before lifecycle logic so the audit record exists even - # if lifecycle processing fails. The UNIQUE(scan_id, rule_id) - # constraint prevents duplicates on accidental double-writes. - for outcome in rule_outcomes: - rule_id_o = outcome.get("rule_id", "") - status_o = outcome.get("status", "FAILED") - if not rule_id_o: - continue + try: + with db_conn.cursor() as cur: + # --- Idempotency check ---------------------------------------- cur.execute( - """ - INSERT INTO scan_rule_outcomes ( - scan_id, rule_id, status, tenant_id, subscription_id, - inventory_boundary, started_at, completed_at - ) - VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) - ON CONFLICT (scan_id, rule_id) DO NOTHING - """, - ( - scan_id, - rule_id_o, - status_o, - tenant_id, - subscription_id, - outcome.get("started_at"), - ), + "SELECT scan_id FROM scan_lifecycle_applications WHERE scan_id = %s", + (scan_id,), ) + if cur.fetchone() is not None: + logger.info( + "Scan %s already applied; skipping lifecycle update", scan_id + ) + return - # Build a lookup: rule_id -> outcome status - outcome_by_rule: Dict[str, str] = { - o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o - } + # --- Write durable per-rule outcome records ------------------- + # UNIQUE(scan_id, rule_id) prevents duplicates on double-writes. + for outcome in rule_outcomes: + rule_id_o = outcome.get("rule_id", "") + status_o = outcome.get("status", "FAILED") + if not rule_id_o: + continue + cur.execute( + """ + INSERT INTO scan_rule_outcomes ( + scan_id, rule_id, status, tenant_id, subscription_id, + inventory_boundary, started_at, completed_at + ) + VALUES (%s, %s, %s, %s, %s, 'subscription', %s, NOW()) + ON CONFLICT (scan_id, rule_id) DO NOTHING + """, + ( + scan_id, + rule_id_o, + status_o, + tenant_id, + subscription_id, + outcome.get("started_at"), + ), + ) - # Collect rule IDs that actively confirmed a clean result. Only - # these can trigger resolution of absent findings (fail-closed). - resolving_rule_ids = [ - rule_id for rule_id, status in outcome_by_rule.items() - if status in _RESOLVING_STATUSES - ] + # Build a lookup: rule_id -> outcome status + outcome_by_rule: Dict[str, str] = { + o["rule_id"]: o["status"] + for o in rule_outcomes + if "rule_id" in o and "status" in o + } - # Build the set of (rule_id, resource_id_normalized) pairs seen in - # this scan. A fingerprint in this set was actively observed. - seen_fingerprint_keys: set = set() - fingerprints_in_scan: List[Dict[str, Any]] = [] - for finding in findings: - rule_id = finding.get("rule_id", "") - resource_id = finding.get("resource_id", "") - evidence_key = finding.get("evidence_key", "") - resource_id_normalized = _normalize_resource_id(resource_id) - normalization_version = "1" + # Collect rule IDs that actively confirmed a clean result. Only + # these can trigger resolution of absent findings (fail-closed). + resolving_rule_ids = [ + rule_id + for rule_id, status in outcome_by_rule.items() + if status in _RESOLVING_STATUSES + ] - fp_hash = _compute_fingerprint_hash( - tenant_id, - subscription_id, - resource_id_normalized, - rule_id, - evidence_key, - normalization_version, - ) - key = (rule_id, resource_id_normalized, evidence_key) - if key not in seen_fingerprint_keys: - seen_fingerprint_keys.add(key) - fingerprints_in_scan.append( - { - "tenant_id": tenant_id, - "subscription_id": subscription_id, - "resource_id_normalized": resource_id_normalized, - "rule_id": rule_id, - "evidence_key": evidence_key, - "normalization_version": normalization_version, - "fingerprint_hash": fp_hash, - } - ) + # Build the set of fingerprints seen in this scan. + seen_fingerprint_keys: set = set() + fingerprints_in_scan: List[Dict[str, Any]] = [] + for finding in findings: + rule_id = finding.get("rule_id", "") + resource_id = finding.get("resource_id", "") + evidence_key = finding.get("evidence_key", "") + resource_id_normalized = _normalize_resource_id(resource_id) + normalization_version = "1" - # --- Upsert fingerprints and lifecycles for seen findings ------- - seen_fingerprint_ids: set = set() - for fp in fingerprints_in_scan: - # ON CONFLICT touches fingerprint_hash to force RETURNING id - # even when the row already exists (DO NOTHING returns nothing). - cur.execute( - """ - INSERT INTO finding_fingerprints ( - tenant_id, subscription_id, resource_id_normalized, - rule_id, evidence_key, normalization_version, - fingerprint_version, fingerprint_hash + fp_hash = _compute_fingerprint_hash( + tenant_id, + subscription_id, + resource_id_normalized, + rule_id, + evidence_key, + normalization_version, ) - VALUES (%s, %s, %s, %s, %s, %s, '1', %s) - ON CONFLICT (fingerprint_hash) DO UPDATE - SET fingerprint_hash = EXCLUDED.fingerprint_hash - RETURNING id - """, - ( - fp["tenant_id"], - fp["subscription_id"], - fp["resource_id_normalized"], - fp["rule_id"], - fp["evidence_key"], - fp["normalization_version"], - fp["fingerprint_hash"], - ), - ) - row = cur.fetchone() - fingerprint_id = row[0] - seen_fingerprint_ids.add(fingerprint_id) - - # Lock the lifecycle row if it exists, then decide what to do. - cur.execute( - """ - SELECT id, state, occurrence_count, reopen_count, row_version - FROM finding_lifecycles - WHERE fingerprint_id = %s - FOR UPDATE - """, - (fingerprint_id,), - ) - lc_row = cur.fetchone() + key = (rule_id, resource_id_normalized, evidence_key) + if key not in seen_fingerprint_keys: + seen_fingerprint_keys.add(key) + fingerprints_in_scan.append( + { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "resource_id_normalized": resource_id_normalized, + "rule_id": rule_id, + "evidence_key": evidence_key, + "normalization_version": normalization_version, + "fingerprint_hash": fp_hash, + } + ) - if lc_row is None: - # New finding: create lifecycle in OPEN state. + # --- Upsert fingerprints and lifecycles for seen findings ------- + seen_fingerprint_ids: set = set() + for fp in fingerprints_in_scan: + # ON CONFLICT DO UPDATE forces RETURNING id on pre-existing rows. cur.execute( """ - INSERT INTO finding_lifecycles ( - fingerprint_id, state, first_seen_scan_id, - last_seen_scan_id, occurrence_count, - consecutive_success_count, reopen_count, row_version + INSERT INTO finding_fingerprints ( + tenant_id, subscription_id, resource_id_normalized, + rule_id, evidence_key, normalization_version, + fingerprint_version, fingerprint_hash ) - VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + VALUES (%s, %s, %s, %s, %s, %s, '1', %s) + ON CONFLICT (fingerprint_hash) DO UPDATE + SET fingerprint_hash = EXCLUDED.fingerprint_hash RETURNING id """, - (fingerprint_id, scan_id, scan_id), + ( + fp["tenant_id"], + fp["subscription_id"], + fp["resource_id_normalized"], + fp["rule_id"], + fp["evidence_key"], + fp["normalization_version"], + fp["fingerprint_hash"], + ), ) - lifecycle_id = cur.fetchone()[0] + row = cur.fetchone() + fingerprint_id = row[0] + seen_fingerprint_ids.add(fingerprint_id) + cur.execute( """ - INSERT INTO finding_lifecycle_transitions - (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') + SELECT id, state, occurrence_count, reopen_count, row_version + FROM finding_lifecycles + WHERE fingerprint_id = %s + FOR UPDATE """, - (lifecycle_id, scan_id), + (fingerprint_id,), ) - else: - lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row - if state in ("OPEN", "REOPENED"): - # Seen again while already open: increment counter. - cur.execute( - """ - UPDATE finding_lifecycles - SET occurrence_count = occurrence_count + 1, - last_seen_scan_id = %s, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s - """, - (scan_id, lifecycle_id), - ) - elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): - # Reappeared after resolution: reopen and reset success streak. + lc_row = cur.fetchone() + + if lc_row is None: cur.execute( """ - UPDATE finding_lifecycles - SET state = 'REOPENED', - last_seen_scan_id = %s, - occurrence_count = occurrence_count + 1, - reopen_count = reopen_count + 1, - consecutive_success_count = 0, - updated_at = NOW(), - row_version = row_version + 1 - WHERE id = %s + INSERT INTO finding_lifecycles ( + fingerprint_id, state, first_seen_scan_id, + last_seen_scan_id, occurrence_count, + consecutive_success_count, reopen_count, row_version + ) + VALUES (%s, 'OPEN', %s, %s, 1, 0, 0, 0) + RETURNING id """, - (scan_id, lifecycle_id), + (fingerprint_id, scan_id, scan_id), ) + lifecycle_id = cur.fetchone()[0] cur.execute( """ INSERT INTO finding_lifecycle_transitions (lifecycle_id, from_state, to_state, scan_id, reason) - VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + VALUES (%s, NULL, 'OPEN', %s, 'New finding observed') """, - (lifecycle_id, state, scan_id), + (lifecycle_id, scan_id), ) + else: + lifecycle_id, state, occurrence_count, reopen_count, row_version = lc_row + if state in ("OPEN", "REOPENED"): + cur.execute( + """ + UPDATE finding_lifecycles + SET occurrence_count = occurrence_count + 1, + last_seen_scan_id = %s, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + elif state in ("RESOLVED", "ACCEPTED", "SUPPRESSED"): + cur.execute( + """ + UPDATE finding_lifecycles + SET state = 'REOPENED', + last_seen_scan_id = %s, + occurrence_count = occurrence_count + 1, + reopen_count = reopen_count + 1, + consecutive_success_count = 0, + updated_at = NOW(), + row_version = row_version + 1 + WHERE id = %s + """, + (scan_id, lifecycle_id), + ) + cur.execute( + """ + INSERT INTO finding_lifecycle_transitions + (lifecycle_id, from_state, to_state, scan_id, reason) + VALUES (%s, %s, 'REOPENED', %s, 'Finding reappeared in scan') + """, + (lifecycle_id, state, scan_id), + ) - # --- Resolve findings NOT seen in this scan --------------------- - # Only lock and process rows for rules that had a resolving outcome. - # This avoids unnecessary contention on unrelated rules and is more - # efficient on subscriptions with many open findings. - if not resolving_rule_ids: - # No rule produced a clean outcome: nothing can be resolved. - pass - elif seen_fingerprint_ids: - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.fingerprint_id NOT IN %s - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - (tenant_id, subscription_id, tuple(seen_fingerprint_ids), resolving_rule_ids), - ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - else: - # No findings seen at all: resolve for rules with clean outcomes. + # --- Resolve findings NOT seen in this scan ------------------- + # Only process rules that had a resolving outcome (fail-closed). + if not resolving_rule_ids: + pass + elif seen_fingerprint_ids: + # Use != ALL(%s) with a list to avoid single-element tuple + # syntax issues that occur with NOT IN %s. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id != ALL(%s) + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + ( + tenant_id, + subscription_id, + list(seen_fingerprint_ids), + resolving_rule_ids, + ), + ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) + else: + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, resolving_rule_ids), + ) + _resolve_absent_rows(cur, scan_id, outcome_by_rule) + + # --- Idempotency sentinel (inserted last) --------------------- cur.execute( """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl + INSERT INTO scan_lifecycle_applications (scan_id, applied_by) + VALUES (%s, 'system') """, - (tenant_id, subscription_id, resolving_rule_ids), + (scan_id,), ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - # --- Idempotency sentinel (inserted last) ----------------------- - cur.execute( - """ - INSERT INTO scan_lifecycle_applications (scan_id, applied_by) - VALUES (%s, 'system') - """, - (scan_id,), - ) + db_conn.commit() + logger.info("Lifecycle application committed for scan %s", scan_id) - db_conn.commit() - logger.info("Lifecycle application committed for scan %s", scan_id) + except Exception: + db_conn.rollback() + logger.error( + "Lifecycle application rolled back for scan %s", scan_id, exc_info=True + ) + raise def _resolve_absent_rows( @@ -324,7 +328,11 @@ def _resolve_absent_rows( scan_id: str, outcome_by_rule: Dict[str, str], ) -> None: - """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules.""" + """Transition OPEN/REOPENED lifecycle rows to RESOLVED for clean-outcome rules. + + The SQL query already filters by resolving_rule_ids, so outcome_status is + always in _RESOLVING_STATUSES here. The check is kept as a defensive guard. + """ absent_rows = cur.fetchall() for lc_id, state, row_version, rule_id in absent_rows: outcome_status = outcome_by_rule.get(rule_id) @@ -350,4 +358,3 @@ def _resolve_absent_rows( """, (lc_id, state, scan_id), ) - # else: outcome missing or blocking -> fail closed, no state change. diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 6ebcd965..395454c0 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -31,115 +31,113 @@ def detect_and_publish( """ count = 0 - with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - # ---------------------------------------------------------------- - # 1. persistent_finding: occurrence_count >= 3 and OPEN/REOPENED - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT fl.id AS lifecycle_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.occurrence_count >= %s - """, - (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), - ) - persistent_rows = cur.fetchall() - - # ---------------------------------------------------------------- - # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED - # lifecycles in this subscription - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT ff.rule_id, - array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, - COUNT(*) AS lc_count - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - GROUP BY ff.rule_id - HAVING COUNT(*) >= %s - """, - (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), - ) - cross_rows = cur.fetchall() - - # ---------------------------------------------------------------- - # 3. reopened_finding: reopen_count >= 1 and state == REOPENED - # ---------------------------------------------------------------- - cur.execute( - """ - SELECT fl.id AS lifecycle_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state = 'REOPENED' - AND fl.reopen_count >= %s - """, - (tenant_id, subscription_id, _REOPENED_THRESHOLD), - ) - reopened_rows = cur.fetchall() - - # All detection queries are finished; cursor is closed. Now upsert - # patterns using separate cursor calls to avoid open-cursor overlap. - for row in persistent_rows: - _upsert_pattern( - db_conn, - pattern_type="persistent_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_PERSISTENT_THRESHOLD, - ) - count += 1 - - for row in cross_rows: - lifecycle_ids = row["lifecycle_ids"] - # Publish one pattern per lifecycle in the group so each is - # individually traceable; finding_ids carries the sibling IDs. - for lc_id in lifecycle_ids: - sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + try: + with db_conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + # 1. persistent_finding: occurrence_count >= threshold and OPEN/REOPENED + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.occurrence_count >= %s + """, + (tenant_id, subscription_id, _PERSISTENT_THRESHOLD), + ) + persistent_rows = cur.fetchall() + + # 2. cross_resource_recurrence: same rule_id >= 2 OPEN/REOPENED lifecycles + cur.execute( + """ + SELECT ff.rule_id, + array_agg(fl.id ORDER BY fl.id) AS lifecycle_ids, + COUNT(*) AS lc_count + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + GROUP BY ff.rule_id + HAVING COUNT(*) >= %s + """, + (tenant_id, subscription_id, _CROSS_RESOURCE_THRESHOLD), + ) + cross_rows = cur.fetchall() + + # 3. reopened_finding: reopen_count >= 1 and state == REOPENED + cur.execute( + """ + SELECT fl.id AS lifecycle_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state = 'REOPENED' + AND fl.reopen_count >= %s + """, + (tenant_id, subscription_id, _REOPENED_THRESHOLD), + ) + reopened_rows = cur.fetchall() + + # All detection queries finished; cursor closed. Upsert via fresh cursors + # to avoid open-cursor overlap inside _upsert_pattern. + for row in persistent_rows: + _upsert_pattern( + db_conn, + pattern_type="persistent_finding", + lifecycle_id=row["lifecycle_id"], + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=[], + threshold=_PERSISTENT_THRESHOLD, + ) + count += 1 + + for row in cross_rows: + lifecycle_ids = row["lifecycle_ids"] + # Publish one pattern per lifecycle so each is individually traceable. + for lc_id in lifecycle_ids: + sibling_ids = [lid for lid in lifecycle_ids if lid != lc_id] + _upsert_pattern( + db_conn, + pattern_type="cross_resource_recurrence", + lifecycle_id=lc_id, + tenant_id=tenant_id, + subscription_id=subscription_id, + scan_id=scan_id, + finding_ids=sibling_ids, + threshold=_CROSS_RESOURCE_THRESHOLD, + ) + count += 1 + + for row in reopened_rows: _upsert_pattern( db_conn, - pattern_type="cross_resource_recurrence", - lifecycle_id=lc_id, + pattern_type="reopened_finding", + lifecycle_id=row["lifecycle_id"], tenant_id=tenant_id, subscription_id=subscription_id, scan_id=scan_id, - finding_ids=sibling_ids, - threshold=_CROSS_RESOURCE_THRESHOLD, + finding_ids=[], + threshold=_REOPENED_THRESHOLD, ) count += 1 - for row in reopened_rows: - _upsert_pattern( - db_conn, - pattern_type="reopened_finding", - lifecycle_id=row["lifecycle_id"], - tenant_id=tenant_id, - subscription_id=subscription_id, - scan_id=scan_id, - finding_ids=[], - threshold=_REOPENED_THRESHOLD, + db_conn.commit() + logger.info( + "Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count ) - count += 1 + return count - db_conn.commit() - logger.info( - "Pattern detection for scan %s: %d pattern(s) upserted", - scan_id, - count, - ) - return count + except Exception: + db_conn.rollback() + logger.error( + "Pattern detection rolled back for scan %s", scan_id, exc_info=True + ) + raise def _upsert_pattern( diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 3b31816c..dd404ada 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -94,6 +94,9 @@ def cursor(self, **_kwargs): def commit(self): self.committed = True + def rollback(self): + pass + def all_executed(self) -> list: return self._cursor_obj.executed if self._cursor_obj else [] @@ -164,6 +167,9 @@ def commit(self): nonlocal committed_count committed_count += 1 + def rollback(self): + pass + svc = LifecycleService() svc.apply_scan( _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, @@ -287,6 +293,51 @@ def test_resolved_finding_seen_again_becomes_reopened(self): # consecutive_success_count must be reset to 0 on reopen. assert any("consecutive_success_count = 0" in s for s in sqls) + def test_rule_a_success_does_not_resolve_rule_b_finding(self): + """Rule-A SUCCESS must not resolve a finding belonging to Rule-B. + + This is the most critical correctness invariant: a clean outcome for one + rule can only affect findings that were produced by that same rule. + """ + # Scan has no findings (empty list), two outcomes: + # RULE-001 SUCCESS -> in resolving_rule_ids + # RULE-002 FAILED -> NOT in resolving_rule_ids + # + # Sequence (empty seen_ids branch): + # 1. idempotency check -> None + # 2. scan_rule_outcomes insert RULE-001 -> None + # 3. scan_rule_outcomes insert RULE-002 -> None + # 4. absent-findings query (ff.rule_id = ANY(['RULE-001'])) -> + # returns only RULE-001's lifecycle row; RULE-002 is excluded by SQL + # 5. UPDATE RESOLVED for RULE-001 row -> None + # 6. transition insert -> None + # 7. idempotency insert -> None + results = [ + None, # idempotency check + None, # scan_rule_outcomes RULE-001 + None, # scan_rule_outcomes RULE-002 + [(10, "OPEN", 0, "RULE-001")], # absent-findings query + None, # UPDATE RESOLVED + None, # transition insert + None, # idempotency insert + ] + conn = self._run( + results, + [], + [_make_outcome("RULE-001", "SUCCESS"), _make_outcome("RULE-002", "FAILED")], + ) + assert conn.committed + # Verify resolving_rule_ids in the SQL params contains only RULE-001. + absent_query = next( + (item for item in conn.all_executed() if "ff.rule_id = ANY" in item[0]), + None, + ) + assert absent_query is not None, "absent-findings query was not issued" + _, params = absent_query + resolving_ids = params[-1] # last param is resolving_rule_ids list + assert "RULE-001" in resolving_ids + assert "RULE-002" not in resolving_ids + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): # REOPENED + seen in scan: occurrence_count increments, no new state transition. # 1. idempotency -> None diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 3884d82e..b3bc7792 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -277,8 +277,7 @@ def test_list_cross_subscription_query_param_rejected(self, app_client): assert resp.status_code == 400 def test_list_returns_only_authorized_subscription(self, app_client): - """Patterns for a different subscription must not be returned to an - authorized user whose JWT contains a different subscription_id.""" + """subscription_id from JWT must be passed as a SQL parameter.""" row_authorized = _sample_pattern_row(sub_id="sub-authorized") mock_db = _mock_db_rows([row_authorized], 1) @@ -289,9 +288,11 @@ def test_list_returns_only_authorized_subscription(self, app_client): ) assert resp.status_code == 200 - data = resp.get_json() - # The query is scoped; verify the mock was called (subscription scoped query). - assert "patterns" in data + # Verify subscription_id was enforced in the SQL WHERE clause parameters. + mock_cursor = mock_db._get_conn.return_value.cursor.return_value + first_call = mock_cursor.execute.call_args_list[0] + _, params = first_call[0] + assert "sub-authorized" in params def test_list_requires_auth(self, app_client): resp = app_client.get("/api/v1/patterns") @@ -353,6 +354,12 @@ def test_get_cross_subscription_returns_404(self, app_client): ) assert resp.status_code == 404 + # Verify subscription_id was enforced as a SQL parameter (not just checked + # in Python), so removing the WHERE clause would break this test. + mock_cursor = mock_db._get_conn.return_value.cursor.return_value + call_args = mock_cursor.execute.call_args + _, params = call_args[0] + assert "sub-other" in params def test_get_requires_auth(self, app_client): resp = app_client.get("/api/v1/patterns/1") From 36473f8c5b9403aab426de11cecf83f2a389dbd2 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:21:09 +0100 Subject: [PATCH 32/67] fix(lint): remove unused imports and fix E402 in lifecycle test files Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py | 1 - tests/test_finding_lifecycle.py | 8 +------- tests/test_patterns.py | 2 -- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index d36e9444..25fccefc 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -9,7 +9,6 @@ from typing import Sequence, Union from alembic import op -import sqlalchemy as sa # Revision identifiers, used by Alembic. revision: str = "e1f2a3b4c5d6" diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index dd404ada..fb80f144 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -4,11 +4,7 @@ live PostgreSQL instance. """ -import hashlib -import json -from unittest.mock import MagicMock, call, patch - -import pytest +from collections import deque from api.services.lifecycle_service import ( LifecycleService, @@ -48,8 +44,6 @@ def _make_outcome(rule_id: str, status: str) -> dict: # avoids the "second cursor re-reads from the start" bug noted in code review. # --------------------------------------------------------------------------- -from collections import deque - class _FakeCursor: """Fake psycopg2 cursor backed by a shared result deque.""" diff --git a/tests/test_patterns.py b/tests/test_patterns.py index b3bc7792..a3a7eee7 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -4,8 +4,6 @@ with mocked database queries. """ -import json -import os import secrets import time from unittest.mock import MagicMock, patch From d49c10a1109672a2f4a502cd7dc4f0166ec6f46f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:24:55 +0100 Subject: [PATCH 33/67] fix(lint): apply ruff format to all lifecycle PR files Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 9 ++---- api/routes/patterns.py | 4 +-- api/services/lifecycle_service.py | 16 +++------- api/services/pattern_service.py | 8 ++--- scanner/engine.py | 1 + scanner/worker.py | 1 - tests/test_finding_lifecycle.py | 29 ++++++++++--------- tests/test_patterns.py | 20 ++++++------- 8 files changed, 36 insertions(+), 52 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 25fccefc..370f3f87 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -143,20 +143,15 @@ def upgrade() -> None: """ ) - # Indexes for hot query paths - op.execute( - "CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)" - ) + op.execute("CREATE INDEX ix_finding_fingerprints_tenant_sub ON finding_fingerprints (tenant_id, subscription_id)") op.execute( "CREATE INDEX ix_finding_lifecycles_state ON finding_lifecycles (state) WHERE state IN ('OPEN', 'REOPENED')" ) op.execute( "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" ) - op.execute( - "CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)" - ) + op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)") def downgrade() -> None: diff --git a/api/routes/patterns.py b/api/routes/patterns.py index abe7e805..1c245d9c 100644 --- a/api/routes/patterns.py +++ b/api/routes/patterns.py @@ -11,9 +11,7 @@ patterns_bp = Blueprint("patterns", __name__) logger = logging.getLogger(__name__) -_ALLOWED_PATTERN_TYPES = frozenset( - {"persistent_finding", "cross_resource_recurrence", "reopened_finding"} -) +_ALLOWED_PATTERN_TYPES = frozenset({"persistent_finding", "cross_resource_recurrence", "reopened_finding"}) _DEFAULT_LIMIT = 20 _MAX_LIMIT = 100 _MIN_LIMIT = 1 diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 67729f9e..3b31b357 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -80,9 +80,7 @@ def apply_scan( (scan_id,), ) if cur.fetchone() is not None: - logger.info( - "Scan %s already applied; skipping lifecycle update", scan_id - ) + logger.info("Scan %s already applied; skipping lifecycle update", scan_id) return # --- Write durable per-rule outcome records ------------------- @@ -113,17 +111,13 @@ def apply_scan( # Build a lookup: rule_id -> outcome status outcome_by_rule: Dict[str, str] = { - o["rule_id"]: o["status"] - for o in rule_outcomes - if "rule_id" in o and "status" in o + o["rule_id"]: o["status"] for o in rule_outcomes if "rule_id" in o and "status" in o } # Collect rule IDs that actively confirmed a clean result. Only # these can trigger resolution of absent findings (fail-closed). resolving_rule_ids = [ - rule_id - for rule_id, status in outcome_by_rule.items() - if status in _RESOLVING_STATUSES + rule_id for rule_id, status in outcome_by_rule.items() if status in _RESOLVING_STATUSES ] # Build the set of fingerprints seen in this scan. @@ -317,9 +311,7 @@ def apply_scan( except Exception: db_conn.rollback() - logger.error( - "Lifecycle application rolled back for scan %s", scan_id, exc_info=True - ) + logger.error("Lifecycle application rolled back for scan %s", scan_id, exc_info=True) raise diff --git a/api/services/pattern_service.py b/api/services/pattern_service.py index 395454c0..0a8eb5f5 100644 --- a/api/services/pattern_service.py +++ b/api/services/pattern_service.py @@ -127,16 +127,12 @@ def detect_and_publish( count += 1 db_conn.commit() - logger.info( - "Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count - ) + logger.info("Pattern detection for scan %s: %d pattern(s) upserted", scan_id, count) return count except Exception: db_conn.rollback() - logger.error( - "Pattern detection rolled back for scan %s", scan_id, exc_info=True - ) + logger.error("Pattern detection rolled back for scan %s", scan_id, exc_info=True) raise diff --git a/scanner/engine.py b/scanner/engine.py index 5430a0aa..082396bc 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -13,6 +13,7 @@ try: import azure.core.exceptions as _azure_exc + _AzureHttpResponseError = _azure_exc.HttpResponseError except Exception: _AzureHttpResponseError = None # type: ignore[assignment,misc] diff --git a/scanner/worker.py b/scanner/worker.py index dfd70095..5b5f9d65 100644 --- a/scanner/worker.py +++ b/scanner/worker.py @@ -262,7 +262,6 @@ def run_worker(): extra={"scan_id": scan_id}, ) - SCANS_TOTAL.labels(status="completed").inc() logger.info( "Successfully completed scan %s", diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index fb80f144..87ab29ce 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -166,12 +166,18 @@ def rollback(self): svc = LifecycleService() svc.apply_scan( - _TrackingConn(False), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(False), + SCAN_ID_1, + SUB_ID, + TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) svc.apply_scan( - _TrackingConn(True), SCAN_ID_1, SUB_ID, TENANT_ID, + _TrackingConn(True), + SCAN_ID_1, + SUB_ID, + TENANT_ID, [_make_outcome("RULE-001", "SUCCESS")], [_make_finding("RULE-001", "/rg/foo")], ) @@ -307,13 +313,13 @@ def test_rule_a_success_does_not_resolve_rule_b_finding(self): # 6. transition insert -> None # 7. idempotency insert -> None results = [ - None, # idempotency check - None, # scan_rule_outcomes RULE-001 - None, # scan_rule_outcomes RULE-002 - [(10, "OPEN", 0, "RULE-001")], # absent-findings query - None, # UPDATE RESOLVED - None, # transition insert - None, # idempotency insert + None, # idempotency check + None, # scan_rule_outcomes RULE-001 + None, # scan_rule_outcomes RULE-002 + [(10, "OPEN", 0, "RULE-001")], # absent-findings query + None, # UPDATE RESOLVED + None, # transition insert + None, # idempotency insert ] conn = self._run( results, @@ -351,10 +357,7 @@ def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): sqls = self._all_sql(conn) # No transition record should be emitted for REOPENED->REOPENED. - transition_to_reopened = [ - s for s in sqls - if "finding_lifecycle_transitions" in s and "REOPENED" in s - ] + transition_to_reopened = [s for s in sqls if "finding_lifecycle_transitions" in s and "REOPENED" in s] assert not transition_to_reopened # occurrence_count should increment. diff --git a/tests/test_patterns.py b/tests/test_patterns.py index a3a7eee7..07b2933a 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -108,8 +108,8 @@ def test_persistent_finding_detected_when_occurrence_ge_3(self): count, conn = self._run( [ [{"lifecycle_id": 10}], # persistent_finding - [], # cross_resource_recurrence - [], # reopened_finding + [], # cross_resource_recurrence + [], # reopened_finding ] ) assert count == 1 @@ -144,19 +144,19 @@ def test_pattern_response_includes_threshold_and_algorithm_version(self): """The upsert call must include threshold and algorithm_version.""" from api.services.pattern_service import PatternService, _ALGORITHM_VERSION, _PERSISTENT_THRESHOLD - conn = _FakeConn([ - [{"lifecycle_id": 10}], # persistent_finding - [], # cross_resource_recurrence - [], # reopened_finding - ]) + conn = _FakeConn( + [ + [{"lifecycle_id": 10}], # persistent_finding + [], # cross_resource_recurrence + [], # reopened_finding + ] + ) svc = PatternService() svc.detect_and_publish(conn, SCAN_ID, SUB_ID, TENANT_ID) # Find the INSERT INTO patterns call and verify the params. executed = conn._cursor_obj.executed - insert_sql, params = next( - ((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None) - ) + insert_sql, params = next(((sql, p) for sql, p in executed if "INSERT INTO patterns" in sql), (None, None)) assert insert_sql is not None # params order: pattern_type, lifecycle_id, tenant_id, subscription_id, # scan_id, finding_ids, threshold, algorithm_version From cba24f055ae97bb611b302c94e1cd2a1ae69d0e6 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 30 Aug 2026 16:39:12 +0100 Subject: [PATCH 34/67] fix: apply code review findings for lifecycle PR - Add _VALID_OUTCOME_STATUSES allowlist; unknown status defaults to FAILED instead of letting DB constraint throw IntegrityError mid-transaction - Fix dead-code if/elif block: restructure to if resolving_rule_ids / nested if seen_fingerprint_ids (fail-closed logic now clear at a glance) - Add tenant_id to ix_patterns_sub_created index (was subscription_id only; multi-tenant queries filter on both columns) - Add CASCADE to all downgrade() DROP TABLE statements (future FK safety) - Add row_version comment clarifying it is a monotonic counter, not OCC - Fix _FakeConn in test_patterns.py to share a single deque across all cursor() calls, matching test_finding_lifecycle.py and real psycopg2 behaviour - Add REOPENED -> RESOLVED test covering finding absent from SUCCESS scan Signed-off-by: Tanvir Farhad --- .../e1f2a3b4c5d6_finding_lifecycle.py | 16 +-- api/services/lifecycle_service.py | 99 +++++++++++-------- tests/test_finding_lifecycle.py | 23 +++++ tests/test_patterns.py | 35 +++++-- 4 files changed, 117 insertions(+), 56 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py index 370f3f87..c856176e 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py @@ -92,7 +92,7 @@ def upgrade() -> None: occurrence_count INTEGER NOT NULL DEFAULT 1, consecutive_success_count INTEGER NOT NULL DEFAULT 0, reopen_count INTEGER NOT NULL DEFAULT 0, - row_version INTEGER NOT NULL DEFAULT 0, + row_version INTEGER NOT NULL DEFAULT 0, -- monotonic update counter (FOR UPDATE handles serializability) created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), CONSTRAINT ck_finding_lifecycles_state @@ -151,13 +151,13 @@ def upgrade() -> None: op.execute( "CREATE INDEX ix_finding_lifecycle_transitions_lifecycle_id ON finding_lifecycle_transitions (lifecycle_id)" ) - op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (subscription_id, created_at DESC)") + op.execute("CREATE INDEX ix_patterns_sub_created ON patterns (tenant_id, subscription_id, created_at DESC)") def downgrade() -> None: - op.execute("DROP TABLE IF EXISTS patterns") - op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions") - op.execute("DROP TABLE IF EXISTS finding_lifecycles") - op.execute("DROP TABLE IF EXISTS finding_fingerprints") - op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications") - op.execute("DROP TABLE IF EXISTS scan_rule_outcomes") + op.execute("DROP TABLE IF EXISTS patterns CASCADE") + op.execute("DROP TABLE IF EXISTS finding_lifecycle_transitions CASCADE") + op.execute("DROP TABLE IF EXISTS finding_lifecycles CASCADE") + op.execute("DROP TABLE IF EXISTS finding_fingerprints CASCADE") + op.execute("DROP TABLE IF EXISTS scan_lifecycle_applications CASCADE") + op.execute("DROP TABLE IF EXISTS scan_rule_outcomes CASCADE") diff --git a/api/services/lifecycle_service.py b/api/services/lifecycle_service.py index 3b31b357..4c9af341 100644 --- a/api/services/lifecycle_service.py +++ b/api/services/lifecycle_service.py @@ -10,6 +10,19 @@ # Statuses that mean "we actively confirmed this rule was clean in the scan." _RESOLVING_STATUSES = frozenset({"SUCCESS", "EMPTY_SUCCESS"}) +# All statuses the DB constraint accepts. Any other value defaults to FAILED to +# prevent a single malformed outcome from aborting the whole lifecycle transaction. +_VALID_OUTCOME_STATUSES = frozenset( + { + "SUCCESS", + "EMPTY_SUCCESS", + "PERMISSION_DENIED", + "TIMEOUT", + "FAILED", + "NOT_APPLICABLE", + } +) + def _normalize_resource_id(resource_id: str) -> str: """Return a stable, lowercased, stripped version of an ARM resource ID.""" @@ -90,6 +103,13 @@ def apply_scan( status_o = outcome.get("status", "FAILED") if not rule_id_o: continue + if status_o not in _VALID_OUTCOME_STATUSES: + logger.warning( + "Unknown outcome status %r for rule %s; defaulting to FAILED", + status_o, + rule_id_o, + ) + status_o = "FAILED" cur.execute( """ INSERT INTO scan_rule_outcomes ( @@ -255,46 +275,45 @@ def apply_scan( ) # --- Resolve findings NOT seen in this scan ------------------- - # Only process rules that had a resolving outcome (fail-closed). - if not resolving_rule_ids: - pass - elif seen_fingerprint_ids: - # Use != ALL(%s) with a list to avoid single-element tuple - # syntax issues that occur with NOT IN %s. - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND fl.fingerprint_id != ALL(%s) - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - ( - tenant_id, - subscription_id, - list(seen_fingerprint_ids), - resolving_rule_ids, - ), - ) - _resolve_absent_rows(cur, scan_id, outcome_by_rule) - else: - cur.execute( - """ - SELECT fl.id, fl.state, fl.row_version, ff.rule_id - FROM finding_lifecycles fl - JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id - WHERE ff.tenant_id = %s - AND ff.subscription_id = %s - AND fl.state IN ('OPEN', 'REOPENED') - AND ff.rule_id = ANY(%s) - FOR UPDATE OF fl - """, - (tenant_id, subscription_id, resolving_rule_ids), - ) + # Fail-closed: skip resolution entirely if no rule confirmed a + # clean result this scan (all outcomes were FAILED/PERMISSION_DENIED). + if resolving_rule_ids: + if seen_fingerprint_ids: + # Use != ALL(%s) with a list to avoid single-element tuple + # syntax issues that occur with NOT IN %s. + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND fl.fingerprint_id != ALL(%s) + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + ( + tenant_id, + subscription_id, + list(seen_fingerprint_ids), + resolving_rule_ids, + ), + ) + else: + cur.execute( + """ + SELECT fl.id, fl.state, fl.row_version, ff.rule_id + FROM finding_lifecycles fl + JOIN finding_fingerprints ff ON ff.id = fl.fingerprint_id + WHERE ff.tenant_id = %s + AND ff.subscription_id = %s + AND fl.state IN ('OPEN', 'REOPENED') + AND ff.rule_id = ANY(%s) + FOR UPDATE OF fl + """, + (tenant_id, subscription_id, resolving_rule_ids), + ) _resolve_absent_rows(cur, scan_id, outcome_by_rule) # --- Idempotency sentinel (inserted last) --------------------- diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 87ab29ce..23307410 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -338,6 +338,29 @@ def test_rule_a_success_does_not_resolve_rule_b_finding(self): assert "RULE-001" in resolving_ids assert "RULE-002" not in resolving_ids + def test_reopened_finding_absent_from_success_scan_is_resolved(self): + # REOPENED finding not seen in a SUCCESS scan must transition to RESOLVED. + # 1. idempotency -> None + # 2. scan_rule_outcomes insert -> None + # 3. absent-findings query (empty seen_ids branch) -> [(10,'REOPENED',0,'RULE-001')] + # 4. UPDATE to RESOLVED -> None + # 5. transition insert (REOPENED -> RESOLVED) -> None + # 6. idempotency insert -> None + results = [ + None, + None, + [(10, "REOPENED", 0, "RULE-001")], + None, + None, + None, + ] + conn = self._run(results, [], [_make_outcome("RULE-001", "SUCCESS")]) + assert conn.committed + sqls = self._all_sql(conn) + assert any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) + # Transition from REOPENED to RESOLVED must be recorded. + assert any("finding_lifecycle_transitions" in s and "REOPENED" in s and "RESOLVED" in s for s in sqls) + def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): # REOPENED + seen in scan: occurrence_count increments, no new state transition. # 1. idempotency -> None diff --git a/tests/test_patterns.py b/tests/test_patterns.py index 07b2933a..3d529977 100644 --- a/tests/test_patterns.py +++ b/tests/test_patterns.py @@ -6,6 +6,7 @@ import secrets import time +from collections import deque from unittest.mock import MagicMock, patch import jwt @@ -48,19 +49,24 @@ def _auth_headers(sub_id: str | None = None) -> dict: class _FakeCursor: - def __init__(self, pages: list): - # pages: list of values returned by successive fetchone/fetchall calls - self._pages = list(pages) - self.executed = [] + """Fake cursor backed by a shared deque so all cursors on one connection share state.""" + + def __init__(self, results_deque: deque): + self._results = results_deque + self.executed: list = [] + self._current = None def execute(self, sql, params=None): self.executed.append((sql.strip(), params)) + self._current = self._results.popleft() if self._results else None def fetchone(self): - return self._pages.pop(0) if self._pages else None + return self._current def fetchall(self): - return self._pages.pop(0) if self._pages else [] + if isinstance(self._current, list): + return self._current + return [] if self._current is None else [self._current] def __enter__(self): return self @@ -70,18 +76,31 @@ def __exit__(self, *args): class _FakeConn: + """Fake connection whose cursor() calls share one result deque.""" + def __init__(self, fetchall_pages: list): - self._pages = fetchall_pages + self._deque: deque = deque(fetchall_pages) self.committed = False + self._cursors: list = [] self._cursor_obj = None def cursor(self, **_kwargs): - self._cursor_obj = _FakeCursor(self._pages) + self._cursor_obj = _FakeCursor(self._deque) + self._cursors.append(self._cursor_obj) return self._cursor_obj def commit(self): self.committed = True + def rollback(self): + pass + + def all_executed(self) -> list: + result = [] + for c in self._cursors: + result.extend(c.executed) + return result + # --------------------------------------------------------------------------- # PatternService unit tests From 0874d68d9508767d241e7732eb9c9886b4d4270f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:13:01 +0100 Subject: [PATCH 35/67] fix(graph): stack foundation after lifecycle migration Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 ++-- tests/test_graph_migration.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index 761e20e9..24ece564 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: b6d2f8a4c1e7 +Revises: a8d9c2e4f601 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: str = "b6d2f8a4c1e7" +down_revision: str = "a8d9c2e4f601" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/tests/test_graph_migration.py b/tests/test_graph_migration.py index e9613f4d..2402498d 100644 --- a/tests/test_graph_migration.py +++ b/tests/test_graph_migration.py @@ -116,7 +116,7 @@ def test_finding_graph_nodes_columns(engine): def test_downgrade_removes_tables(engine): cfg = Config("alembic.ini") - command.downgrade(cfg, "b6d2f8a4c1e7") + command.downgrade(cfg, "a8d9c2e4f601") inspector = inspect(engine) tables = inspector.get_table_names() assert "graph_nodes" not in tables From c8fba2e2a8838f867a614c154b2337a3dd7c6c01 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:48:14 +0100 Subject: [PATCH 36/67] fix(graph): preserve published evidence when an edge detector fails Signed-off-by: Tanvir Farhad --- scanner/graph/edge_detector.py | 4 ++- tests/test_graph_detector_failure_postgres.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/test_graph_detector_failure_postgres.py diff --git a/scanner/graph/edge_detector.py b/scanner/graph/edge_detector.py index 2c494a20..79dfed3a 100644 --- a/scanner/graph/edge_detector.py +++ b/scanner/graph/edge_detector.py @@ -177,7 +177,8 @@ def detect(self, snapshot: InventorySnapshot) -> list[GraphEdge]: def detect_all_edges(snapshot: InventorySnapshot) -> list[GraphEdge]: """Run all detectors and return the combined edge list. - Per-detector exceptions are caught and logged; the function never raises. + A detector failure aborts graph publication. Publishing a partial edge list + as complete would remove valid relationships from the preceding snapshot. """ detectors: list[EdgeDetector] = [ NsgToSubnetDetector(), @@ -192,4 +193,5 @@ def detect_all_edges(snapshot: InventorySnapshot) -> list[GraphEdge]: edges.extend(detector.detect(snapshot)) except Exception as exc: logger.warning("edge_detector: %s failed: %s", type(detector).__name__, exc) + raise return edges diff --git a/tests/test_graph_detector_failure_postgres.py b/tests/test_graph_detector_failure_postgres.py new file mode 100644 index 00000000..f1dc79d9 --- /dev/null +++ b/tests/test_graph_detector_failure_postgres.py @@ -0,0 +1,32 @@ +"""Detector failure must preserve the last published graph evidence.""" + +import uuid +from dataclasses import replace + +from tests.test_graph_freshness_postgres import counts, snapshot +from tests.test_graph_freshness_postgres import graph_scope as _graph_scope +from tests.test_graph_freshness_postgres import pytestmark as pytestmark + +from scanner.graph.graph_populator import populate_graph + +graph_scope = _graph_scope + + +def test_malformed_detector_input_preserves_published_graph(graph_scope): + tenant, subscription, dsn = graph_scope + good = replace(snapshot(tenant, subscription), collected_at="2026-10-07T01:00:00Z") + populate_graph(str(uuid.uuid4()), good, dsn) + assert counts(dsn, tenant, subscription, current=True) == (2, 1) + + malformed = replace(snapshot(tenant, subscription), collected_at="2026-10-07T02:00:00Z") + malformed = replace( + malformed, + resources=( + malformed.resources[0], + replace(malformed.resources[1], properties={"ipConfigurations": [None]}), + ), + ) + populate_graph(str(uuid.uuid4()), malformed, dsn) + + assert counts(dsn, tenant, subscription, current=True) == (2, 1) + assert counts(dsn, tenant, subscription) == (2, 1) From d83f619c6dfa4d5dc2ae4fd8e11e25da77111a01 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:15:31 +0100 Subject: [PATCH 37/67] ci: run required checks for the lifecycle and graph PR stack Signed-off-by: Tanvir Farhad --- .github/workflows/ci.yml | 4 ++++ .github/workflows/codeql.yml | 4 ++++ .github/workflows/dco.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/website.yml | 2 +- tests/test_feature_stack_ci.py | 28 +++++++++++++++++++++++++ 6 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 tests/test_feature_stack_ci.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfaa8141..d7793cfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,10 @@ on: branches: - dev - main + - feat/311-finding-lifecycle + - feat/331-evidence-graph-foundation + - feat/332-evidence-graph-node-edge-population + # Post-merge assurance: re-run the full gate on the merged result, so a green # PR run against an older base is not the only evidence for dev and main. push: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 3f654e31..7e63b5f7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -5,6 +5,10 @@ on: branches: - dev - main + - feat/311-finding-lifecycle + - feat/331-evidence-graph-foundation + - feat/332-evidence-graph-node-edge-population + push: branches: - dev diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 719600a1..333c0c49 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -2,7 +2,7 @@ name: Developer Certificate of Origin on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] permissions: contents: read diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index a737a3cb..b82b6c61 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -2,7 +2,7 @@ name: Dependency Review on: pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] permissions: contents: read diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index a4c83572..81f5f054 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -14,7 +14,7 @@ on: push: branches: [main] pull_request: - branches: [dev, main] + branches: [dev, main, feat/311-finding-lifecycle, feat/331-evidence-graph-foundation, feat/332-evidence-graph-node-edge-population] workflow_dispatch: permissions: diff --git a/tests/test_feature_stack_ci.py b/tests/test_feature_stack_ci.py new file mode 100644 index 00000000..8071ff5f --- /dev/null +++ b/tests/test_feature_stack_ci.py @@ -0,0 +1,28 @@ +"""Feature-stack PRs must receive the same read-only checks as dev PRs.""" + +from fnmatch import fnmatchcase +from pathlib import Path + +import pytest +import yaml + + +@pytest.mark.parametrize("workflow", ["ci", "codeql", "dco", "dependency-review", "website"]) +@pytest.mark.parametrize( + "base", + [ + "feat/311-finding-lifecycle", + "feat/331-evidence-graph-foundation", + "feat/332-evidence-graph-node-edge-population", + ], +) +def test_feature_stack_receives_required_checks(workflow, base): + config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) + patterns = config["on"]["pull_request"]["branches"] + assert any(fnmatchcase(base, pattern) for pattern in patterns) + + +@pytest.mark.parametrize("workflow", ["ci", "codeql", "website"]) +def test_feature_stack_does_not_expand_push_or_deployment_triggers(workflow): + config = yaml.load(Path(f".github/workflows/{workflow}.yml").read_text(), Loader=yaml.BaseLoader) + assert set(config["on"]["push"]["branches"]) <= {"dev", "main"} From 3eebef0c3f941fb8678ef2278e226a68616201a4 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:17:08 +0100 Subject: [PATCH 38/67] ci: use a string key for the PostgreSQL service port Signed-off-by: Tanvir Farhad --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7793cfa..3c0ed473 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -577,7 +577,7 @@ jobs: - name: Start image env: - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_PORT: ${{ job.services.postgres.ports['5432'] }} run: | set -euo pipefail JWT_SECRET="$(python3 -c 'import secrets; print(secrets.token_urlsafe(32))')" From b66af8d95dc21f8d378645d66ded129a2239fb00 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:17:22 +0100 Subject: [PATCH 39/67] fix(deps): require patched Werkzeug and multidict releases Signed-off-by: Tanvir Farhad --- requirements-dev.txt | 346 +++++++++++++++++++++---------------------- requirements.in | 4 + requirements.txt | 346 +++++++++++++++++++++---------------------- 3 files changed, 350 insertions(+), 346 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 32cdba87..1bd3066c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -956,177 +956,177 @@ msal-extensions==1.3.1 \ --hash=sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca msrest==0.7.1 \ --hash=sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32 -multidict==6.8.0 \ - --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ - --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ - --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ - --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ - --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ - --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ - --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ - --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ - --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ - --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ - --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ - --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ - --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ - --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ - --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ - --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ - --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ - --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ - --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ - --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ - --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ - --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ - --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ - --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ - --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ - --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ - --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ - --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ - --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ - --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ - --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ - --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ - --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ - --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ - --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ - --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ - --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ - --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ - --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ - --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ - --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ - --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ - --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ - --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ - --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ - --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ - --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ - --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ - --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ - --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ - --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ - --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ - --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ - --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ - --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ - --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ - --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ - --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ - --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ - --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ - --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ - --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ - --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ - --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ - --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ - --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ - --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ - --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ - --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ - --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ - --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ - --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ - --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ - --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ - --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ - --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ - --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ - --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ - --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ - --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ - --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ - --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ - --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ - --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ - --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ - --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ - --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ - --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ - --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ - --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ - --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ - --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ - --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ - --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ - --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ - --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ - --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ - --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ - --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ - --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ - --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ - --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ - --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ - --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ - --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ - --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ - --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ - --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ - --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ - --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ - --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ - --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ - --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ - --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ - --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ - --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ - --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ - --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ - --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ - --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ - --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ - --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ - --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ - --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ - --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ - --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ - --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ - --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ - --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ - --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ - --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ - --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ - --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ - --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ - --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ - --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ - --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ - --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ - --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ - --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ - --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ - --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ - --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ - --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ - --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ - --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ - --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ - --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ - --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ - --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ - --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ - --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ - --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ - --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ - --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ - --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ - --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ - --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ - --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ - --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ - --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ - --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ - --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ - --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ - --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ - --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ - --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ - --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ - --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ - --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 oauthlib==4.0.0 \ --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd packaging==26.3 \ @@ -1483,8 +1483,8 @@ urllib3==2.8.0 \ --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 websocket-client==1.9.2 \ --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab yarl==1.25.1 \ --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ diff --git a/requirements.in b/requirements.in index 1c8abe85..03e1419e 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,9 @@ # Direct runtime dependencies. Regenerate locks with scripts/lock_dependencies.py. flask==3.1.3 +# Security floors for transitive runtime dependencies. +# GHSA-g6x2-hccm-hh4m and GHSA-54p9-h82j-f925. +werkzeug>=3.1.9 +multidict>=6.9.1 flask-cors==6.0.5 alembic==1.18.5 azure-identity==1.25.3 diff --git a/requirements.txt b/requirements.txt index 45eb9a8a..6897a858 100644 --- a/requirements.txt +++ b/requirements.txt @@ -833,177 +833,177 @@ msal-extensions==1.3.1 \ --hash=sha256:96d3de4d034504e969ac5e85bae8106c8373b5c6568e4c8fa7af2eca9dbe6bca msrest==0.7.1 \ --hash=sha256:21120a810e1233e5e6cc7fe40b474eeb4ec6f757a15d7cf86702c369f9567c32 -multidict==6.8.0 \ - --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ - --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ - --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ - --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ - --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ - --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ - --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ - --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ - --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ - --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ - --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ - --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ - --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ - --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ - --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ - --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ - --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ - --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ - --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ - --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ - --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ - --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ - --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ - --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ - --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ - --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ - --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ - --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ - --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ - --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ - --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ - --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ - --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ - --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ - --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ - --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ - --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ - --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ - --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ - --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ - --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ - --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ - --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ - --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ - --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ - --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ - --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ - --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ - --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ - --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ - --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ - --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ - --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ - --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ - --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ - --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ - --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ - --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ - --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ - --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ - --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ - --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ - --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ - --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ - --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ - --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ - --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ - --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ - --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ - --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ - --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ - --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ - --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ - --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ - --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ - --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ - --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ - --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ - --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ - --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ - --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ - --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ - --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ - --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ - --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ - --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ - --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ - --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ - --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ - --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ - --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ - --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ - --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ - --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ - --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ - --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ - --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ - --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ - --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ - --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ - --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ - --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ - --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ - --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ - --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ - --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ - --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ - --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ - --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ - --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ - --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ - --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ - --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ - --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ - --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ - --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ - --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ - --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ - --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ - --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ - --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ - --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ - --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ - --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ - --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ - --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ - --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ - --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ - --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ - --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ - --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ - --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ - --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ - --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ - --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ - --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ - --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ - --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ - --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ - --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ - --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ - --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ - --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ - --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ - --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ - --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ - --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ - --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ - --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ - --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ - --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ - --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ - --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ - --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ - --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ - --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ - --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ - --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ - --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ - --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ - --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ - --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ - --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ - --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ - --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ - --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ - --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ - --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ - --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ - --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 oauthlib==4.0.0 \ --hash=sha256:624c28c13a0a59cabf9747dfa52af63be3e512a7f2714df16e91b5b3a145e6cd packaging==26.3 \ @@ -1324,8 +1324,8 @@ urllib3==2.8.0 \ --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 websocket-client==1.9.2 \ --hash=sha256:e1a673830a9c7bfa47b1cd3d5e4178f4c9651d80a4eab02c9c23a1c3ec6250ce -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 +werkzeug==3.1.9 \ + --hash=sha256:6392e50c78460ba618e5b21f08a71f59c99ce99cdc6cf6e3dd7e6ccca8754fab yarl==1.25.1 \ --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ From a49415e94971dccaee96f47f401b0b8e3b587674 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:30:13 +0100 Subject: [PATCH 40/67] test(ci): match validated PostgreSQL service expression Signed-off-by: Tanvir Farhad --- tests/test_container_runtime_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_container_runtime_config.py b/tests/test_container_runtime_config.py index f0bea389..6e8ddbcc 100644 --- a/tests/test_container_runtime_config.py +++ b/tests/test_container_runtime_config.py @@ -121,7 +121,7 @@ def test_ci_builds_starts_and_scans_one_required_image(): assert '"$IMAGE_TAG"' in start assert "--network host" in start assert "127.0.0.1:${POSTGRES_PORT}/ci_db" in start - assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports[5432] }}" + assert _step(steps, "Start image")["env"]["POSTGRES_PORT"] == "${{ job.services.postgres.ports['5432'] }}" assert "OPENSHIELD_ENV=production" in start assert "secrets.token_urlsafe(32)" in start assert "/ready" in verify From 6b32ac4ceae223b05ac79a70909f87b4f0c2f1d4 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 15:58:14 +0100 Subject: [PATCH 41/67] fix(graph): serialize scope publication and reject delayed snapshots Signed-off-by: Tanvir Farhad --- scanner/graph/graph_populator.py | 12 ++++++- scanner/graph/node_service.py | 10 ++++++ tests/test_graph_freshness_postgres.py | 47 ++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index eb8e4095..d26e63ef 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -9,7 +9,7 @@ from scanner.arg_inventory import InventoryResource, InventoryStatus -from scanner.graph.node_service import graph_connection, link_findings_to_nodes, populate_nodes +from scanner.graph.node_service import graph_connection, link_findings_to_nodes, lock_graph_scopes, populate_nodes from scanner.graph.edge_detector import detect_all_edges if TYPE_CHECKING: @@ -126,6 +126,16 @@ def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: # Publish the new scope only after every write succeeds. Partial snapshots # retain historical rows, but the views expose only explicit current evidence. with graph_connection(dsn) as conn: + lock_graph_scopes(conn, snapshot.tenant_id, snapshot.requested_subscriptions) + with conn.cursor() as cur: + cur.execute( + "SELECT 1 FROM graph_snapshot_scopes WHERE tenant_id=%s " + "AND subscription_id=ANY(%s) AND collected_at > %s::timestamptz LIMIT 1", + (snapshot.tenant_id, list(snapshot.requested_subscriptions), snapshot.collected_at), + ) + if cur.fetchone() is not None: + logger.info("graph: ignored older snapshot for scan %s", scan_id) + return node_count = populate_nodes(augmented_snapshot, dsn, connection=conn) edges = detect_all_edges(augmented_snapshot) edge_count = _write_edges(edges, snapshot.snapshot_id, snapshot.tenant_id, dsn, connection=conn) diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index 0ae48676..bc60ae49 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -108,3 +108,13 @@ def link_findings_to_nodes(scan_id: str, tenant_id: str, dsn: str, *, connection cur.execute(_LINK_FINDINGS_SQL, {"scan_id": scan_id, "tenant_id": tenant_id}) result = cur.rowcount if cur.rowcount >= 0 else 0 return result + + +def lock_graph_scopes(conn, tenant_id: str, subscriptions) -> None: + """Serialize all publications and traversals within the same scope.""" + with conn.cursor() as cur: + for subscription in sorted(set(subscriptions)): + cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (f"openshield-graph:{tenant_id}:{subscription}",), + ) diff --git a/tests/test_graph_freshness_postgres.py b/tests/test_graph_freshness_postgres.py index 5c802249..b4a897ee 100644 --- a/tests/test_graph_freshness_postgres.py +++ b/tests/test_graph_freshness_postgres.py @@ -137,3 +137,50 @@ def test_complete_snapshot_does_not_prune_another_tenant(graph_scope): with conn.cursor() as cur: cur.execute("DELETE FROM graph_nodes WHERE tenant_id=%s", (other,)) cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id=%s", (other,)) + + +def test_delayed_older_snapshot_cannot_replace_or_prune_current_scope(graph_scope): + tenant, sub, dsn = graph_scope + newer = replace(snapshot(tenant, sub), collected_at="2026-10-07T00:02:00Z") + older = replace(snapshot(tenant, sub, empty=True), collected_at="2026-10-07T00:01:00Z") + populate_graph(str(uuid.uuid4()), newer, dsn) + populate_graph(str(uuid.uuid4()), older, dsn) + assert counts(dsn, tenant, sub, current=True) == (2, 1) + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute( + "SELECT snapshot_id FROM graph_snapshot_scopes WHERE tenant_id=%s AND subscription_id=%s", (tenant, sub) + ) + assert cur.fetchone()[0] == newer.snapshot_id + + +def test_publication_waits_for_scope_transaction_lock(graph_scope): + from concurrent.futures import ThreadPoolExecutor + from threading import Event + + tenant, sub, dsn = graph_scope + entered = Event() + pending = snapshot(tenant, sub) + blocker = psycopg2.connect(dsn) + try: + with blocker.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (f"openshield-graph:{tenant}:{sub}",)) + + def publish(): + entered.set() + populate_graph(str(uuid.uuid4()), pending, dsn) + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(publish) + assert entered.wait(2) + try: + future.result(timeout=0.3) + pytest.fail("publication completed while its scope lock was held") + except TimeoutError: + pass + finally: + blocker.rollback() + future.result(timeout=5) + assert counts(dsn, tenant, sub, current=True) == (2, 1) + finally: + blocker.close() From fd4bbb54450ee7b2d3a54fd740f9a348c2eac99d Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 15:27:48 +0100 Subject: [PATCH 42/67] fix(graph): lowercase resource_id on store, fix edge joins, add Postgres ARG fixture test - node_service.py: store resource_id lowercased so the unique index on (tenant_id, resource_id) works correctly for mixed-case ARM IDs and removes lower() from join queries - graph_populator.py: update _UPSERT_EDGE_SQL to compare against stored lowercase values instead of applying lower() at query time - test_graph_node_service.py: set mock_cur.rowcount = 1 so max() call does not fail on MagicMock comparison - test_graph_populator_postgres.py: add Postgres end-to-end fixture test using real ARG-shaped VNet+NIC+VM+UAMI snapshot asserting on DB edges Signed-off-by: Tanvir Farhad --- scanner/graph/graph_populator.py | 4 +- scanner/graph/node_service.py | 2 +- tests/test_graph_populator_postgres.py | 287 +++++++++++++++++++++++++ 3 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 tests/test_graph_populator_postgres.py diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index d26e63ef..456f98e0 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -33,9 +33,9 @@ now(), '{}'::jsonb FROM graph_nodes src, graph_nodes tgt -WHERE lower(src.resource_id) = lower(%(source_resource_id)s) +WHERE src.resource_id = lower(%(source_resource_id)s) AND src.tenant_id = %(tenant_id)s - AND lower(tgt.resource_id) = lower(%(target_resource_id)s) + AND tgt.resource_id = lower(%(target_resource_id)s) AND tgt.tenant_id = %(tenant_id)s AND src.snapshot_id = %(evidence_snapshot_id)s AND tgt.snapshot_id = %(evidence_snapshot_id)s diff --git a/scanner/graph/node_service.py b/scanner/graph/node_service.py index bc60ae49..5d8ec673 100644 --- a/scanner/graph/node_service.py +++ b/scanner/graph/node_service.py @@ -42,7 +42,7 @@ SELECT f.id, n.node_id FROM findings f JOIN scans s ON s.scan_id = f.scan_id -JOIN current_graph_nodes n ON lower(f.resource_id) = lower(n.resource_id) +JOIN current_graph_nodes n ON lower(f.resource_id) = n.resource_id AND n.tenant_id = %(tenant_id)s AND n.subscription_id = s.subscription_id WHERE f.scan_id = %(scan_id)s diff --git a/tests/test_graph_populator_postgres.py b/tests/test_graph_populator_postgres.py new file mode 100644 index 00000000..e1fec1fe --- /dev/null +++ b/tests/test_graph_populator_postgres.py @@ -0,0 +1,287 @@ +"""End-to-end Postgres test for graph population using real ARG-shaped fixtures. + +Verifies that populate_nodes and _write_edges write the expected rows to the +graph_nodes and graph_edges tables when given a VNet+NIC+VM+UAMI snapshot that +mirrors what Azure Resource Graph actually returns. + +Requires DATABASE_URL and the graph schema migration to have been applied. +""" + +import os +import uuid + +import psycopg2 +import pytest + +from scanner.arg_inventory import InventoryResource, InventorySnapshot, InventoryStatus +from scanner.graph.edge_detector import detect_all_edges +from scanner.graph.graph_populator import _synthesise_subnet_resources, _write_edges +from scanner.graph.node_service import populate_nodes + +pytestmark = pytest.mark.skipif( + not os.environ.get("DATABASE_URL"), + reason="DATABASE_URL is required for PostgreSQL graph tests", +) + +_DSN = os.environ.get("DATABASE_URL", "") + +# --------------------------------------------------------------------------- +# Unique tenant/sub per test run so parallel runs don't collide +# --------------------------------------------------------------------------- +_TENANT = f"test-tenant-{uuid.uuid4().hex[:8]}" +_SUB = f"test-sub-{uuid.uuid4().hex[:8]}" +_SNAP = f"snap-{uuid.uuid4().hex[:8]}" + +_BASE = f"/subscriptions/{_SUB}/resourceGroups/rg-test/providers" +_VNET_ID = f"{_BASE}/Microsoft.Network/virtualNetworks/vnet-test" +_SUBNET_ID = f"{_VNET_ID}/subnets/default" +_NSG_ID = f"{_BASE}/Microsoft.Network/networkSecurityGroups/nsg-test" +_NIC_ID = f"{_BASE}/Microsoft.Network/networkInterfaces/nic-test" +_VM_ID = f"{_BASE}/Microsoft.Compute/virtualMachines/vm-test" +_IDENTITY_ID = f"{_BASE}/Microsoft.ManagedIdentity/userAssignedIdentities/id-test" +_PIP_ID = f"{_BASE}/Microsoft.Network/publicIPAddresses/pip-test" + + +def _resource(rid, rtype, properties=None): + return InventoryResource( + snapshot_id=_SNAP, + tenant_id=_TENANT, + subscription_id=_SUB, + resource_id=rid, + resource_type=rtype, + name=rid.split("/")[-1], + location="uksouth", + resource_group="rg-test", + tags={}, + properties=properties or {}, + ) + + +# Real ARG response shapes for a VNet with one subnet, an NSG protecting it, +# a NIC in it, a VM with UAMI, and a public IP attached to the NIC. +_VNET = _resource( + _VNET_ID, + "microsoft.network/virtualnetworks", + { + "subnets": [ + { + "id": _SUBNET_ID, + "name": "default", + "properties": { + "addressPrefix": "10.0.0.0/24", + "networkSecurityGroup": {"id": _NSG_ID}, + }, + } + ], + "addressSpace": {"addressPrefixes": ["10.0.0.0/16"]}, + }, +) + +_NSG = _resource( + _NSG_ID, + "microsoft.network/networksecuritygroups", + {"subnets": [{"id": _SUBNET_ID}]}, +) + +_NIC = _resource( + _NIC_ID, + "microsoft.network/networkinterfaces", + { + "ipConfigurations": [ + { + "properties": { + "subnet": {"id": _SUBNET_ID}, + "privateIPAddress": "10.0.0.4", + "publicIPAddress": {"id": _PIP_ID}, + } + } + ], + "virtualMachine": {"id": _VM_ID}, + }, +) + +_VM = _resource( + _VM_ID, + "microsoft.compute/virtualmachines", + { + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": {_IDENTITY_ID: {}}, + }, + "networkProfile": {"networkInterfaces": [{"id": _NIC_ID}]}, + }, +) + +_PIP = _resource( + _PIP_ID, + "microsoft.network/publicipaddresses", + {"ipConfiguration": {"id": f"{_NIC_ID}/ipConfigurations/ipconfig1"}}, +) + +_IDENTITY = _resource( + _IDENTITY_ID, + "microsoft.managedidentity/userassignedidentities", + {}, +) + + +def _snapshot(*resources): + return InventorySnapshot( + snapshot_id=_SNAP, + tenant_id=_TENANT, + requested_subscriptions=(_SUB,), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-30T00:00:00+00:00", + duration_ms=42, + pages=1, + resources=tuple(resources), + errors=(), + ) + + +@pytest.fixture(scope="module") +def db_conn(): + conn = psycopg2.connect(_DSN) + yield conn + # Clean up all rows written by this test run + with conn.cursor() as cur: + cur.execute("DELETE FROM graph_edges WHERE evidence_snapshot_id = %s", (_SNAP,)) + cur.execute("DELETE FROM graph_nodes WHERE tenant_id = %s", (_TENANT,)) + conn.commit() + conn.close() + + +@pytest.fixture(scope="module") +def populated_graph(db_conn): + """Run populate_nodes + detect_all_edges + _write_edges once; return edge list.""" + base_snapshot = _snapshot(_VNET, _NSG, _NIC, _VM, _PIP, _IDENTITY) + subnet_resources = _synthesise_subnet_resources(base_snapshot) + from dataclasses import replace as dc_replace + augmented = dc_replace(base_snapshot, resources=base_snapshot.resources + tuple(subnet_resources)) + + populate_nodes(augmented, _DSN) + edges = detect_all_edges(augmented) + _write_edges(edges, _SNAP, _TENANT, _DSN) + return edges + + +def test_vnet_node_written_to_db(db_conn, populated_graph): + with db_conn.cursor() as cur: + cur.execute( + "SELECT resource_type FROM graph_nodes WHERE tenant_id = %s AND lower(resource_id) = lower(%s)", + (_TENANT, _VNET_ID), + ) + row = cur.fetchone() + assert row is not None, "VNet node must be written to graph_nodes" + assert "virtualnetwork" in row[0].lower() + + +def test_subnet_node_synthesised_and_written(db_conn, populated_graph): + """Subnet is nested in VNet properties — must be synthesised and written as a node.""" + with db_conn.cursor() as cur: + cur.execute( + "SELECT resource_type FROM graph_nodes WHERE tenant_id = %s AND lower(resource_id) = lower(%s)", + (_TENANT, _SUBNET_ID), + ) + row = cur.fetchone() + assert row is not None, "Synthesised subnet node must be written to graph_nodes" + assert "subnet" in row[0].lower() + + +def test_nsg_to_subnet_protects_edge_written(db_conn, populated_graph): + """NSG -> subnet PROTECTS edge must be written to graph_edges.""" + with db_conn.cursor() as cur: + cur.execute( + """ + SELECT ge.relationship_type + FROM graph_edges ge + JOIN graph_nodes src ON src.node_id = ge.source_node_id + JOIN graph_nodes tgt ON tgt.node_id = ge.target_node_id + WHERE lower(src.resource_id) = lower(%s) + AND lower(tgt.resource_id) = lower(%s) + AND ge.evidence_snapshot_id = %s + """, + (_NSG_ID, _SUBNET_ID, _SNAP), + ) + row = cur.fetchone() + assert row is not None, "PROTECTS edge from NSG to subnet must be in graph_edges" + assert row[0] == "PROTECTS" + + +def test_nic_to_subnet_member_of_edge_written(db_conn, populated_graph): + """NIC -> subnet MEMBER_OF edge must be written (NIC subnet path: ipConfigurations[].properties.subnet.id).""" + with db_conn.cursor() as cur: + cur.execute( + """ + SELECT ge.relationship_type + FROM graph_edges ge + JOIN graph_nodes src ON src.node_id = ge.source_node_id + JOIN graph_nodes tgt ON tgt.node_id = ge.target_node_id + WHERE lower(src.resource_id) = lower(%s) + AND lower(tgt.resource_id) = lower(%s) + AND ge.evidence_snapshot_id = %s + """, + (_NIC_ID, _SUBNET_ID, _SNAP), + ) + row = cur.fetchone() + assert row is not None, "MEMBER_OF edge from NIC to subnet must be in graph_edges" + assert row[0] == "MEMBER_OF" + + +def test_vm_to_identity_has_identity_edge_written(db_conn, populated_graph): + """VM -> UAMI HAS_IDENTITY edge must be written (identity field from bag_merge in ARG query).""" + with db_conn.cursor() as cur: + cur.execute( + """ + SELECT ge.relationship_type + FROM graph_edges ge + JOIN graph_nodes src ON src.node_id = ge.source_node_id + JOIN graph_nodes tgt ON tgt.node_id = ge.target_node_id + WHERE lower(src.resource_id) = lower(%s) + AND lower(tgt.resource_id) = lower(%s) + AND ge.evidence_snapshot_id = %s + """, + (_VM_ID, _IDENTITY_ID, _SNAP), + ) + row = cur.fetchone() + assert row is not None, "HAS_IDENTITY edge from VM to UAMI must be in graph_edges" + assert row[0] == "HAS_IDENTITY" + + +def test_cross_tenant_edges_not_written(db_conn, populated_graph): + """Edges must be scoped to tenant — no edges should appear under a different tenant.""" + with db_conn.cursor() as cur: + cur.execute( + """ + SELECT COUNT(*) + FROM graph_edges ge + JOIN graph_nodes src ON src.node_id = ge.source_node_id + WHERE src.tenant_id = 'other-tenant' + AND ge.evidence_snapshot_id = %s + """, + (_SNAP,), + ) + count = cur.fetchone()[0] + assert count == 0, "No edges must be written for a different tenant" + + +def test_no_edges_written_for_empty_snapshot(db_conn): + """An empty snapshot must produce zero node and edge rows.""" + empty_snap_id = f"snap-empty-{uuid.uuid4().hex[:8]}" + empty_tenant = f"tenant-empty-{uuid.uuid4().hex[:8]}" + empty_snapshot = InventorySnapshot( + snapshot_id=empty_snap_id, + tenant_id=empty_tenant, + requested_subscriptions=(_SUB,), + status=InventoryStatus.COMPLETE, + collected_at="2026-09-30T00:00:00+00:00", + duration_ms=1, + pages=0, + resources=(), + errors=(), + ) + node_count = populate_nodes(empty_snapshot, _DSN) + edges = detect_all_edges(empty_snapshot) + edge_count = _write_edges(edges, empty_snap_id, empty_tenant, _DSN) + assert node_count == 0 + assert edge_count == 0 From 2cf0261146c9459024b817f73cfb4effb540d351 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 15:56:10 +0100 Subject: [PATCH 43/67] fix(graph): apply ruff formatting to Postgres populator fixture test Signed-off-by: Tanvir Farhad --- tests/test_graph_populator_postgres.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_graph_populator_postgres.py b/tests/test_graph_populator_postgres.py index e1fec1fe..f6a93e93 100644 --- a/tests/test_graph_populator_postgres.py +++ b/tests/test_graph_populator_postgres.py @@ -157,6 +157,7 @@ def populated_graph(db_conn): base_snapshot = _snapshot(_VNET, _NSG, _NIC, _VM, _PIP, _IDENTITY) subnet_resources = _synthesise_subnet_resources(base_snapshot) from dataclasses import replace as dc_replace + augmented = dc_replace(base_snapshot, resources=base_snapshot.resources + tuple(subnet_resources)) populate_nodes(augmented, _DSN) From 8ee85ed95d6631de1aa7e35cf587e0419bc4917f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 16:05:38 +0100 Subject: [PATCH 44/67] fix(graph): correct HAS_IDENTITY edge direction assertion in Postgres test Signed-off-by: Tanvir Farhad --- tests/test_graph_populator_postgres.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_graph_populator_postgres.py b/tests/test_graph_populator_postgres.py index f6a93e93..b59b1e5d 100644 --- a/tests/test_graph_populator_postgres.py +++ b/tests/test_graph_populator_postgres.py @@ -230,7 +230,7 @@ def test_nic_to_subnet_member_of_edge_written(db_conn, populated_graph): def test_vm_to_identity_has_identity_edge_written(db_conn, populated_graph): - """VM -> UAMI HAS_IDENTITY edge must be written (identity field from bag_merge in ARG query).""" + """UAMI -> VM HAS_IDENTITY edge must be written (BFS traverses this in reverse for attack paths).""" with db_conn.cursor() as cur: cur.execute( """ @@ -242,10 +242,10 @@ def test_vm_to_identity_has_identity_edge_written(db_conn, populated_graph): AND lower(tgt.resource_id) = lower(%s) AND ge.evidence_snapshot_id = %s """, - (_VM_ID, _IDENTITY_ID, _SNAP), + (_IDENTITY_ID, _VM_ID, _SNAP), ) row = cur.fetchone() - assert row is not None, "HAS_IDENTITY edge from VM to UAMI must be in graph_edges" + assert row is not None, "HAS_IDENTITY edge from UAMI to VM must be in graph_edges" assert row[0] == "HAS_IDENTITY" From 78e7d291857432563bc6d09be5620e8b11959829 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 17:43:09 +0100 Subject: [PATCH 45/67] feat(graph): wire InventorySnapshot into ScanEngine.run_scan() Signed-off-by: Tanvir Farhad --- tests/test_graph_engine_integration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_graph_engine_integration.py b/tests/test_graph_engine_integration.py index d68869f0..b69c9a50 100644 --- a/tests/test_graph_engine_integration.py +++ b/tests/test_graph_engine_integration.py @@ -68,6 +68,7 @@ def test_run_scan_existing_rule_without_snapshot_param_still_works(engine): snapshot = _make_snapshot() def legacy_scan(client, subscription_id): +<<<<<<< HEAD return [ { "rule_id": "AZ-LEGACY-001", From 861a5742a93abee11735ea435f6b83069572cd07 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 18:01:39 +0100 Subject: [PATCH 46/67] feat(graph): wire post-scan node and edge population into ScanEngine Signed-off-by: Tanvir Farhad --- scanner/engine.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scanner/engine.py b/scanner/engine.py index 082396bc..b36661e4 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -3,6 +3,7 @@ import importlib.util import inspect import logging +import os import uuid from datetime import datetime, timezone from pathlib import Path @@ -20,6 +21,7 @@ from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id +from scanner.graph.graph_populator import populate_graph from scanner.graph.snapshot_bridge import collect_snapshot logger = logging.getLogger(__name__) @@ -283,6 +285,14 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: logger.info("Scan %s complete - %d total finding(s). Normalising results...", scan_id, len(findings)) + # Populate the attack graph as a non-fatal post-scan step. + dsn = os.environ.get("DATABASE_URL") + if snapshot and dsn: + try: + populate_graph(scan_id, snapshot, dsn) + except Exception as exc: + logger.warning("run_scan: graph population raised unexpectedly: %s", exc) + return make_serializable(result) def _legacy_placeholder(self, rule_id: str) -> RuleEvaluation: From 2bf1bc35c7327a7ca6b42f5ac1c4973c73f22ccf Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 18:11:19 +0100 Subject: [PATCH 47/67] feat(graph): BFS path traversal, attack_paths migration, and API endpoints Signed-off-by: Tanvir Farhad --- alembic/versions/f2a3b4c5d6e7_attack_paths.py | 61 ++++++ api/app.py | 2 + api/routes/attack_graph.py | 193 ++++++++++++++++++ scanner/graph/graph_populator.py | 8 + scanner/graph/path_traversal.py | 160 +++++++++++++++ tests/test_attack_graph_api.py | 63 ++++++ tests/test_graph_path_traversal.py | 69 +++++++ 7 files changed, 556 insertions(+) create mode 100644 alembic/versions/f2a3b4c5d6e7_attack_paths.py create mode 100644 api/routes/attack_graph.py create mode 100644 scanner/graph/path_traversal.py create mode 100644 tests/test_attack_graph_api.py create mode 100644 tests/test_graph_path_traversal.py diff --git a/alembic/versions/f2a3b4c5d6e7_attack_paths.py b/alembic/versions/f2a3b4c5d6e7_attack_paths.py new file mode 100644 index 00000000..e2cbe919 --- /dev/null +++ b/alembic/versions/f2a3b4c5d6e7_attack_paths.py @@ -0,0 +1,61 @@ +"""Add attack_paths table for pre-computed BFS traversal results. + +Revision ID: f2a3b4c5d6e7 +Revises: e1f2a3b4c5d6 +Create Date: 2026-09-24 00:00:00.000000 +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision: str = "f2a3b4c5d6e7" +down_revision: Union[str, Sequence[str], None] = "e1f2a3b4c5d6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "attack_paths", + sa.Column("path_id", postgresql.UUID(), nullable=False), + sa.Column("tenant_id", sa.Text(), nullable=False), + sa.Column("scan_id", sa.Text(), nullable=False), + sa.Column("source_node_id", postgresql.UUID(), nullable=False), + sa.Column("target_node_id", postgresql.UUID(), nullable=False), + sa.Column("path_node_ids", postgresql.ARRAY(postgresql.UUID()), nullable=False), + sa.Column("path_length", sa.Integer(), nullable=False), + sa.Column("min_confidence", sa.Float(), nullable=False, server_default=sa.text("1.0")), + sa.Column("relationship_types", postgresql.ARRAY(sa.Text()), nullable=False, + server_default=sa.text("ARRAY[]::text[]")), + sa.Column("computed_at", sa.DateTime(timezone=True), nullable=False, + server_default=sa.text("now()")), + sa.ForeignKeyConstraint( + ["source_node_id"], ["graph_nodes.node_id"], + name="attack_paths_source_fkey", ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["target_node_id"], ["graph_nodes.node_id"], + name="attack_paths_target_fkey", ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("path_id", name="attack_paths_pkey"), + ) + op.create_index("idx_attack_paths_tenant_scan", "attack_paths", ["tenant_id", "scan_id"]) + op.create_index("idx_attack_paths_source", "attack_paths", ["source_node_id"]) + op.create_index("idx_attack_paths_target", "attack_paths", ["target_node_id"]) + op.create_index( + "uq_attack_paths_source_target_scan", + "attack_paths", + ["source_node_id", "target_node_id", "scan_id"], + unique=True, + ) + + +def downgrade() -> None: + op.drop_index("uq_attack_paths_source_target_scan", table_name="attack_paths") + op.drop_index("idx_attack_paths_target", table_name="attack_paths") + op.drop_index("idx_attack_paths_source", table_name="attack_paths") + op.drop_index("idx_attack_paths_tenant_scan", table_name="attack_paths") + op.drop_table("attack_paths") diff --git a/api/app.py b/api/app.py index 80699516..a135a24f 100644 --- a/api/app.py +++ b/api/app.py @@ -256,6 +256,7 @@ def verify_jwt() -> None: # ------------------------------------------------------------------ # from api.routes.ai import ai_bp from api.routes.assurance import assurance_bp + from api.routes.attack_graph import attack_graph_bp from api.routes.cbom import cbom_bp from api.routes.compliance import compliance_bp from api.routes.drift import drift_bp @@ -268,6 +269,7 @@ def verify_jwt() -> None: app.register_blueprint(ai_bp) app.register_blueprint(assurance_bp) + app.register_blueprint(attack_graph_bp) app.register_blueprint(cbom_bp) app.register_blueprint(compliance_bp) app.register_blueprint(drift_bp) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py new file mode 100644 index 00000000..75deb78c --- /dev/null +++ b/api/routes/attack_graph.py @@ -0,0 +1,193 @@ +"""Attack graph API: resource nodes, edges, and pre-computed attack paths.""" + +import logging +import os + +import psycopg2 +import psycopg2.extras +from flask import Blueprint, g, jsonify, request + +from api.validation import ValidationError, positive_integer, uuid_string + +attack_graph_bp = Blueprint("attack_graph", __name__) +logger = logging.getLogger(__name__) + +_DEFAULT_LIMIT = 100 +_MAX_LIMIT = 500 + + +def _conn(): + if "graph_conn" not in g: + g.graph_conn = psycopg2.connect(os.environ["DATABASE_URL"]) + g.graph_conn.autocommit = True + psycopg2.extras.register_uuid(g.graph_conn) + return g.graph_conn + + +def _tenant_id() -> str | None: + """Resolve tenant_id from JWT payload or query param (multi-tenant support).""" + user = getattr(g, "user", {}) or {} + return user.get("tenant_id") or request.args.get("tenant_id") + + +@attack_graph_bp.teardown_app_request +def _close_conn(exc): + conn = g.pop("graph_conn", None) + if conn is not None: + conn.close() + + +@attack_graph_bp.get("/api/attack-graph") +def get_attack_graph(): + """Return graph nodes and edges for the caller's tenant (latest snapshot). + + Query params: subscription_id (optional), limit (default 100, max 500) + """ + try: + limit = positive_integer(request.args.get("limit", _DEFAULT_LIMIT), "limit") + if limit > _MAX_LIMIT: + limit = _MAX_LIMIT + subscription_id = request.args.get("subscription_id") + except ValidationError as exc: + return jsonify({"error": str(exc)}), 400 + + tenant_id = _tenant_id() + if not tenant_id: + return jsonify({"error": "tenant_id not available"}), 400 + + conn = _conn() + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + node_filter = "WHERE n.tenant_id = %(tenant_id)s" + params: dict = {"tenant_id": tenant_id, "limit": limit} + if subscription_id: + node_filter += " AND n.subscription_id = %(subscription_id)s" + params["subscription_id"] = subscription_id + + cur.execute( + f""" + SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, + n.location, n.resource_group, n.subscription_id, n.snapshot_id, + n.updated_at + FROM graph_nodes n + {node_filter} + ORDER BY n.updated_at DESC + LIMIT %(limit)s + """, + params, + ) + nodes = cur.fetchall() + + node_ids = [row["node_id"] for row in nodes] + edges: list = [] + if node_ids: + cur.execute( + """ + SELECT e.edge_id::text, e.source_node_id::text, e.target_node_id::text, + e.relationship_type, e.confidence, e.evidence_source, e.collected_at + FROM graph_edges e + WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) + OR e.target_node_id = ANY(%(node_ids)s::uuid[]) + """, + {"node_ids": node_ids}, + ) + edges = cur.fetchall() + + return jsonify({"nodes": [dict(r) for r in nodes], "edges": [dict(r) for r in edges]}) + + +@attack_graph_bp.get("/api/attack-paths") +def list_attack_paths(): + """Return pre-computed attack paths for a scan. + + Query params: scan_id (required), limit (default 100, max 500) + """ + scan_id = request.args.get("scan_id") + if not scan_id: + return jsonify({"error": "scan_id is required"}), 400 + try: + scan_id = uuid_string(scan_id, "scan_id") + limit = positive_integer(request.args.get("limit", _DEFAULT_LIMIT), "limit") + if limit > _MAX_LIMIT: + limit = _MAX_LIMIT + except ValidationError as exc: + return jsonify({"error": str(exc)}), 400 + + tenant_id = _tenant_id() + if not tenant_id: + return jsonify({"error": "tenant_id not available"}), 400 + + conn = _conn() + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT ap.path_id::text, ap.source_node_id::text, ap.target_node_id::text, + ap.path_node_ids, ap.path_length, ap.min_confidence, + ap.relationship_types, ap.computed_at, + src.resource_type AS source_type, src.name AS source_name, + tgt.resource_type AS target_type, tgt.name AS target_name + FROM attack_paths ap + JOIN graph_nodes src ON src.node_id = ap.source_node_id + JOIN graph_nodes tgt ON tgt.node_id = ap.target_node_id + WHERE ap.scan_id = %(scan_id)s + AND ap.tenant_id = %(tenant_id)s + ORDER BY ap.path_length ASC, ap.min_confidence DESC + LIMIT %(limit)s + """, + {"scan_id": scan_id, "tenant_id": tenant_id, "limit": limit}, + ) + rows = cur.fetchall() + + return jsonify({"scan_id": scan_id, "paths": [dict(r) for r in rows]}) + + +@attack_graph_bp.get("/api/attack-paths/") +def get_attack_path(path_id: str): + """Return a single attack path with full node detail for each hop.""" + try: + path_id = uuid_string(path_id, "path_id") + except ValidationError as exc: + return jsonify({"error": str(exc)}), 400 + + tenant_id = _tenant_id() + if not tenant_id: + return jsonify({"error": "tenant_id not available"}), 400 + + conn = _conn() + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT ap.path_id::text, ap.scan_id, ap.source_node_id::text, + ap.target_node_id::text, ap.path_node_ids, ap.path_length, + ap.min_confidence, ap.relationship_types, ap.computed_at + FROM attack_paths ap + WHERE ap.path_id = %(path_id)s::uuid + AND ap.tenant_id = %(tenant_id)s + """, + {"path_id": path_id, "tenant_id": tenant_id}, + ) + row = cur.fetchone() + + if row is None: + return jsonify({"error": "not found"}), 404 + + path = dict(row) + + # Fetch full node detail for each hop + node_ids = [str(nid) for nid in path["path_node_ids"]] + if node_ids: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT node_id::text, resource_id, resource_type, name, + location, resource_group, subscription_id + FROM graph_nodes + WHERE node_id = ANY(%(ids)s::uuid[]) + AND tenant_id = %(tenant_id)s + """, + {"ids": node_ids, "tenant_id": tenant_id}, + ) + nodes_by_id = {r["node_id"]: dict(r) for r in cur.fetchall()} + path["hops"] = [nodes_by_id.get(str(nid), {"node_id": str(nid)}) for nid in path["path_node_ids"]] + + path["path_node_ids"] = [str(nid) for nid in path["path_node_ids"]] + return jsonify(path) diff --git a/scanner/graph/graph_populator.py b/scanner/graph/graph_populator.py index 456f98e0..d621f1cb 100644 --- a/scanner/graph/graph_populator.py +++ b/scanner/graph/graph_populator.py @@ -11,6 +11,7 @@ from scanner.arg_inventory import InventoryResource, InventoryStatus from scanner.graph.node_service import graph_connection, link_findings_to_nodes, lock_graph_scopes, populate_nodes from scanner.graph.edge_detector import detect_all_edges +from scanner.graph.path_traversal import compute_attack_paths if TYPE_CHECKING: from scanner.arg_inventory import InventorySnapshot @@ -189,3 +190,10 @@ def populate_graph(scan_id: str, snapshot: InventorySnapshot, dsn: str) -> None: ) except Exception: logger.warning("graph: population failed for scan %s", scan_id, exc_info=True) + return + + try: + path_count = compute_attack_paths(scan_id, snapshot.tenant_id, dsn) + logger.info("graph: computed %d attack paths for scan %s", path_count, scan_id) + except Exception as exc: + logger.warning("graph: path traversal failed for scan %s: %s", scan_id, exc) diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py new file mode 100644 index 00000000..7bbf87ae --- /dev/null +++ b/scanner/graph/path_traversal.py @@ -0,0 +1,160 @@ +"""BFS path traversal over the attack graph stored in PostgreSQL. + +Computes shortest paths from every finding-linked node to every reachable node +and persists them in attack_paths for API consumption. +""" +from __future__ import annotations + +import logging +import uuid +from collections import deque +from typing import Any + +import psycopg2 +import psycopg2.extras + +logger = logging.getLogger(__name__) + +# Paths longer than this are not persisted — they're rarely actionable and +# keeping them would inflate the table for large graphs. +_MAX_PATH_LENGTH = 8 + + +def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, float]]]: + """Return {source_node_id: [(target_node_id, relationship_type, confidence)]}.""" + with conn.cursor() as cur: + cur.execute( + """ + SELECT e.source_node_id::text, e.target_node_id::text, + e.relationship_type, e.confidence + FROM graph_edges e + JOIN graph_nodes src ON src.node_id = e.source_node_id + WHERE src.tenant_id = %(tenant_id)s + """, + {"tenant_id": tenant_id}, + ) + adj: dict[str, list[tuple[str, str, float]]] = {} + for src, tgt, rel, conf in cur.fetchall(): + adj.setdefault(src, []).append((tgt, rel, conf)) + return adj + + +def _load_finding_nodes(conn: Any, scan_id: str, tenant_id: str) -> list[str]: + """Return node_ids linked to findings from this scan (same tenant).""" + with conn.cursor() as cur: + cur.execute( + """ + SELECT DISTINCT fgn.node_id::text + FROM finding_graph_nodes fgn + JOIN findings f ON f.id = fgn.finding_id + JOIN graph_nodes n ON n.node_id = fgn.node_id + WHERE f.scan_id = %(scan_id)s + AND n.tenant_id = %(tenant_id)s + """, + {"scan_id": scan_id, "tenant_id": tenant_id}, + ) + return [row[0] for row in cur.fetchall()] + + +def _bfs_from( + start: str, + adj: dict[str, list[tuple[str, str, float]]], +) -> list[dict[str, Any]]: + """BFS from start. Returns one path record per reachable node (shortest path).""" + paths: list[dict[str, Any]] = [] + # queue: (current_node_id, path_so_far, min_confidence, relationship_types) + queue: deque[tuple[str, list[str], float, list[str]]] = deque() + queue.append((start, [start], 1.0, [])) + visited: set[str] = {start} + + while queue: + node, path, min_conf, rels = queue.popleft() + for neighbour, rel, conf in adj.get(node, []): + if neighbour in visited: + continue + visited.add(neighbour) + new_path = path + [neighbour] + new_rels = rels + [rel] + new_conf = min(min_conf, conf) + paths.append( + { + "target_node_id": neighbour, + "path_node_ids": new_path, + "path_length": len(new_path) - 1, + "min_confidence": new_conf, + "relationship_types": new_rels, + } + ) + if len(new_path) - 1 < _MAX_PATH_LENGTH: + queue.append((neighbour, new_path, new_conf, new_rels)) + + return paths + + +def _write_paths( + conn: Any, + scan_id: str, + tenant_id: str, + source_node_id: str, + paths: list[dict[str, Any]], +) -> int: + if not paths: + return 0 + with conn.cursor() as cur: + rows = [ + ( + str(uuid.uuid4()), + tenant_id, + scan_id, + source_node_id, + p["target_node_id"], + p["path_node_ids"], + p["path_length"], + p["min_confidence"], + p["relationship_types"], + ) + for p in paths + ] + psycopg2.extras.execute_values( + cur, + """ + INSERT INTO attack_paths + (path_id, tenant_id, scan_id, source_node_id, target_node_id, + path_node_ids, path_length, min_confidence, relationship_types) + VALUES %s + ON CONFLICT (source_node_id, target_node_id, scan_id) DO NOTHING + """, + rows, + template="(%s, %s, %s, %s::uuid, %s::uuid, %s::uuid[], %s, %s, %s)", + ) + return len(rows) + + +def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: + """Run BFS from every finding-linked node and persist paths. Returns path count.""" + try: + conn = psycopg2.connect(dsn) + conn.autocommit = False + try: + adj = _load_adjacency(conn, tenant_id) + source_nodes = _load_finding_nodes(conn, scan_id, tenant_id) + if not source_nodes: + logger.info("graph path traversal: no finding-linked nodes for scan %s", scan_id) + return 0 + + total = 0 + for src in source_nodes: + paths = _bfs_from(src, adj) + total += _write_paths(conn, scan_id, tenant_id, src, paths) + + conn.commit() + logger.info("graph path traversal: %d paths written for scan %s", total, scan_id) + return total + except Exception: + conn.rollback() + raise + finally: + conn.close() + except Exception as exc: + logger.error("graph path traversal failed for scan %s: %s", scan_id, exc) + return 0 diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py new file mode 100644 index 00000000..c1f4dad9 --- /dev/null +++ b/tests/test_attack_graph_api.py @@ -0,0 +1,63 @@ +"""Unit tests for the attack graph API routes.""" +from unittest.mock import MagicMock, patch + + +def _mock_conn(rows_sequence=None): + """Return a mock psycopg2 connection returning preset rows per cursor call.""" + rows_sequence = list(rows_sequence or []) + conn = MagicMock() + conn.autocommit = True + call_idx = [0] + + def make_cursor(*args, **kwargs): + cur = MagicMock() + cur.__enter__ = lambda s: s + cur.__exit__ = MagicMock(return_value=False) + idx = call_idx[0] + call_idx[0] += 1 + rows = rows_sequence[idx] if idx < len(rows_sequence) else [] + cur.fetchall.return_value = rows + cur.fetchone.return_value = rows[0] if rows else None + return cur + + conn.cursor.side_effect = make_cursor + return conn + + +_TENANT = "00000000-0000-0000-0000-000000000001" + + +def test_get_attack_graph_returns_empty_nodes_and_edges(client, auth_headers): + conn = _mock_conn([[]]) + with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): + with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): + resp = client.get(f"/api/attack-graph?tenant_id={_TENANT}", headers=auth_headers) + assert resp.status_code == 200 + data = resp.get_json() + assert "nodes" in data + assert "edges" in data + + +def test_list_attack_paths_missing_scan_id_returns_400(client, auth_headers): + resp = client.get("/api/attack-paths", headers=auth_headers) + assert resp.status_code == 400 + assert b"scan_id" in resp.data + + +def test_list_attack_paths_invalid_uuid_returns_400(client, auth_headers): + resp = client.get("/api/attack-paths?scan_id=not-a-uuid", headers=auth_headers) + assert resp.status_code == 400 + + +def test_get_attack_path_invalid_uuid_returns_400(client, auth_headers): + resp = client.get("/api/attack-paths/not-a-uuid", headers=auth_headers) + assert resp.status_code == 400 + + +def test_get_attack_path_not_found_returns_404(client, auth_headers): + valid_uuid = "00000000-0000-0000-0000-000000000002" + conn = _mock_conn([[]]) + with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): + with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): + resp = client.get(f"/api/attack-paths/{valid_uuid}?tenant_id={_TENANT}", headers=auth_headers) + assert resp.status_code == 404 diff --git a/tests/test_graph_path_traversal.py b/tests/test_graph_path_traversal.py new file mode 100644 index 00000000..b46a8c9f --- /dev/null +++ b/tests/test_graph_path_traversal.py @@ -0,0 +1,69 @@ +"""Unit tests for BFS path traversal (scanner/graph/path_traversal.py).""" +from unittest.mock import MagicMock, patch + +from scanner.graph.path_traversal import _bfs_from, compute_attack_paths + + +def test_bfs_from_returns_empty_for_isolated_node(): + adj = {} + result = _bfs_from("node-A", adj) + assert result == [] + + +def test_bfs_from_single_hop(): + adj = {"node-A": [("node-B", "PROTECTS", 1.0)]} + result = _bfs_from("node-A", adj) + assert len(result) == 1 + path = result[0] + assert path["target_node_id"] == "node-B" + assert path["path_length"] == 1 + assert path["min_confidence"] == 1.0 + assert path["relationship_types"] == ["PROTECTS"] + assert path["path_node_ids"] == ["node-A", "node-B"] + + +def test_bfs_from_two_hops(): + adj = { + "A": [("B", "PROTECTS", 1.0)], + "B": [("C", "EXPOSES", 0.8)], + } + result = _bfs_from("A", adj) + assert len(result) == 2 + two_hop = next(r for r in result if r["target_node_id"] == "C") + assert two_hop["path_length"] == 2 + assert two_hop["min_confidence"] == 0.8 + assert two_hop["relationship_types"] == ["PROTECTS", "EXPOSES"] + + +def test_bfs_from_does_not_revisit_nodes(): + adj = { + "A": [("B", "PROTECTS", 1.0), ("C", "EXPOSES", 1.0)], + "B": [("C", "MEMBER_OF", 1.0)], + } + result = _bfs_from("A", adj) + target_nodes = [r["target_node_id"] for r in result] + assert target_nodes.count("C") == 1 + + +def test_compute_attack_paths_returns_zero_when_no_finding_nodes(): + with patch("scanner.graph.path_traversal.psycopg2.connect") as mock_connect: + conn = MagicMock() + mock_connect.return_value = conn + cur = MagicMock() + cur.__enter__ = lambda s: s + cur.__exit__ = MagicMock(return_value=False) + conn.cursor.return_value = cur + # adjacency returns empty; finding_nodes returns empty + cur.fetchall.return_value = [] + + result = compute_attack_paths("scan-1", "tenant-1", "postgresql://x") + + assert result == 0 + + +def test_compute_attack_paths_handles_db_error_gracefully(): + with patch("scanner.graph.path_traversal.psycopg2.connect") as mock_connect: + mock_connect.side_effect = Exception("DB unavailable") + result = compute_attack_paths("scan-1", "tenant-1", "postgresql://x") + + assert result == 0 From 40089aca000eca4401c22bec2aa3b3c6b4df1acc Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 24 Sep 2026 22:26:38 +0100 Subject: [PATCH 48/67] fix(graph): remove query-param tenant_id fallback to prevent cross-tenant IDOR Viewer-role tokens and tokens without a tenant claim can no longer access another tenant's graph data by supplying tenant_id as a query parameter. Tenant resolution now uses the OIDC tid claim (user["tenant"]) or, for shared-secret admin tokens only, the X-Tenant-Id request header. Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 17 ++++++++- tests/test_attack_graph_api.py | 68 +++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 75deb78c..432d7475 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -25,9 +25,22 @@ def _conn(): def _tenant_id() -> str | None: - """Resolve tenant_id from JWT payload or query param (multi-tenant support).""" + """Resolve tenant_id from the verified principal. + + OIDC mode: the 'tenant' field is populated from the 'tid' claim in the token. + Shared-secret mode: 'tenant' is always None; admins may supply + X-Tenant-Id as a request header (never a query param, which leaks into + logs and caches). Non-admin tokens cannot override the header. + """ user = getattr(g, "user", {}) or {} - return user.get("tenant_id") or request.args.get("tenant_id") + # OIDC path: tid claim decoded by the verifier into user["tenant"] + tid = user.get("tenant") + if tid: + return tid + # Shared-secret path: admin-only header override for multi-tenant deployments + if user.get("role") == "admin": + return request.headers.get("X-Tenant-Id") or None + return None @attack_graph_bp.teardown_app_request diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index c1f4dad9..01918d17 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -1,6 +1,31 @@ """Unit tests for the attack graph API routes.""" +import time from unittest.mock import MagicMock, patch +import pytest + +_TENANT = "00000000-0000-0000-0000-000000000001" + + +@pytest.fixture() +def tenant_auth_headers(app): + """Admin JWT headers with X-Tenant-Id for shared-secret mode tests.""" + import jwt + + secret = app.config["JWT_SECRET"] + payload = { + "sub": "test-user", + "role": "admin", + "iat": int(time.time()), + "exp": int(time.time()) + 3600, + } + token = jwt.encode(payload, secret, algorithm="HS256") + return { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "X-Tenant-Id": _TENANT, + } + def _mock_conn(rows_sequence=None): """Return a mock psycopg2 connection returning preset rows per cursor call.""" @@ -24,40 +49,57 @@ def make_cursor(*args, **kwargs): return conn -_TENANT = "00000000-0000-0000-0000-000000000001" - - -def test_get_attack_graph_returns_empty_nodes_and_edges(client, auth_headers): +def test_get_attack_graph_returns_empty_nodes_and_edges(client, tenant_auth_headers): conn = _mock_conn([[]]) with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): - resp = client.get(f"/api/attack-graph?tenant_id={_TENANT}", headers=auth_headers) + resp = client.get("/api/attack-graph", headers=tenant_auth_headers) assert resp.status_code == 200 data = resp.get_json() assert "nodes" in data assert "edges" in data -def test_list_attack_paths_missing_scan_id_returns_400(client, auth_headers): - resp = client.get("/api/attack-paths", headers=auth_headers) +def test_get_attack_graph_viewer_cannot_supply_tenant_header(client, app): + """Viewer-role tokens must not use X-Tenant-Id header (admin-only).""" + import jwt + + secret = app.config["JWT_SECRET"] + payload = { + "sub": "viewer-user", + "role": "viewer", + "iat": int(time.time()), + "exp": int(time.time()) + 3600, + } + token = jwt.encode(payload, secret, algorithm="HS256") + headers = { + "Authorization": f"Bearer {token}", + "X-Tenant-Id": _TENANT, + } + resp = client.get("/api/attack-graph", headers=headers) + assert resp.status_code == 400 + + +def test_list_attack_paths_missing_scan_id_returns_400(client, tenant_auth_headers): + resp = client.get("/api/attack-paths", headers=tenant_auth_headers) assert resp.status_code == 400 assert b"scan_id" in resp.data -def test_list_attack_paths_invalid_uuid_returns_400(client, auth_headers): - resp = client.get("/api/attack-paths?scan_id=not-a-uuid", headers=auth_headers) +def test_list_attack_paths_invalid_uuid_returns_400(client, tenant_auth_headers): + resp = client.get("/api/attack-paths?scan_id=not-a-uuid", headers=tenant_auth_headers) assert resp.status_code == 400 -def test_get_attack_path_invalid_uuid_returns_400(client, auth_headers): - resp = client.get("/api/attack-paths/not-a-uuid", headers=auth_headers) +def test_get_attack_path_invalid_uuid_returns_400(client, tenant_auth_headers): + resp = client.get("/api/attack-paths/not-a-uuid", headers=tenant_auth_headers) assert resp.status_code == 400 -def test_get_attack_path_not_found_returns_404(client, auth_headers): +def test_get_attack_path_not_found_returns_404(client, tenant_auth_headers): valid_uuid = "00000000-0000-0000-0000-000000000002" conn = _mock_conn([[]]) with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): - resp = client.get(f"/api/attack-paths/{valid_uuid}?tenant_id={_TENANT}", headers=auth_headers) + resp = client.get(f"/api/attack-paths/{valid_uuid}", headers=tenant_auth_headers) assert resp.status_code == 404 From 1d880f2ba3594d9852a338d8ca25dae47408be44 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 25 Sep 2026 00:44:23 +0100 Subject: [PATCH 49/67] fix(lint): ruff format + remove f-string SQL to clear Bandit B608 Signed-off-by: Tanvir Farhad --- alembic/versions/f2a3b4c5d6e7_attack_paths.py | 20 +++++---- api/routes/attack_graph.py | 43 +++++++++++-------- scanner/graph/path_traversal.py | 1 + tests/test_attack_graph_api.py | 1 + tests/test_graph_path_traversal.py | 1 + 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/alembic/versions/f2a3b4c5d6e7_attack_paths.py b/alembic/versions/f2a3b4c5d6e7_attack_paths.py index e2cbe919..e312c062 100644 --- a/alembic/versions/f2a3b4c5d6e7_attack_paths.py +++ b/alembic/versions/f2a3b4c5d6e7_attack_paths.py @@ -28,17 +28,21 @@ def upgrade() -> None: sa.Column("path_node_ids", postgresql.ARRAY(postgresql.UUID()), nullable=False), sa.Column("path_length", sa.Integer(), nullable=False), sa.Column("min_confidence", sa.Float(), nullable=False, server_default=sa.text("1.0")), - sa.Column("relationship_types", postgresql.ARRAY(sa.Text()), nullable=False, - server_default=sa.text("ARRAY[]::text[]")), - sa.Column("computed_at", sa.DateTime(timezone=True), nullable=False, - server_default=sa.text("now()")), + sa.Column( + "relationship_types", postgresql.ARRAY(sa.Text()), nullable=False, server_default=sa.text("ARRAY[]::text[]") + ), + sa.Column("computed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.text("now()")), sa.ForeignKeyConstraint( - ["source_node_id"], ["graph_nodes.node_id"], - name="attack_paths_source_fkey", ondelete="CASCADE", + ["source_node_id"], + ["graph_nodes.node_id"], + name="attack_paths_source_fkey", + ondelete="CASCADE", ), sa.ForeignKeyConstraint( - ["target_node_id"], ["graph_nodes.node_id"], - name="attack_paths_target_fkey", ondelete="CASCADE", + ["target_node_id"], + ["graph_nodes.node_id"], + name="attack_paths_target_fkey", + ondelete="CASCADE", ), sa.PrimaryKeyConstraint("path_id", name="attack_paths_pkey"), ) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 432d7475..3969fdcd 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -70,24 +70,33 @@ def get_attack_graph(): conn = _conn() with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - node_filter = "WHERE n.tenant_id = %(tenant_id)s" - params: dict = {"tenant_id": tenant_id, "limit": limit} if subscription_id: - node_filter += " AND n.subscription_id = %(subscription_id)s" - params["subscription_id"] = subscription_id - - cur.execute( - f""" - SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, - n.location, n.resource_group, n.subscription_id, n.snapshot_id, - n.updated_at - FROM graph_nodes n - {node_filter} - ORDER BY n.updated_at DESC - LIMIT %(limit)s - """, - params, - ) + cur.execute( + """ + SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, + n.location, n.resource_group, n.subscription_id, n.snapshot_id, + n.updated_at + FROM graph_nodes n + WHERE n.tenant_id = %(tenant_id)s + AND n.subscription_id = %(subscription_id)s + ORDER BY n.updated_at DESC + LIMIT %(limit)s + """, + {"tenant_id": tenant_id, "subscription_id": subscription_id, "limit": limit}, + ) + else: + cur.execute( + """ + SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, + n.location, n.resource_group, n.subscription_id, n.snapshot_id, + n.updated_at + FROM graph_nodes n + WHERE n.tenant_id = %(tenant_id)s + ORDER BY n.updated_at DESC + LIMIT %(limit)s + """, + {"tenant_id": tenant_id, "limit": limit}, + ) nodes = cur.fetchall() node_ids = [row["node_id"] for row in nodes] diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 7bbf87ae..442176f3 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -3,6 +3,7 @@ Computes shortest paths from every finding-linked node to every reachable node and persists them in attack_paths for API consumption. """ + from __future__ import annotations import logging diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index 01918d17..dd1ae07e 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -1,4 +1,5 @@ """Unit tests for the attack graph API routes.""" + import time from unittest.mock import MagicMock, patch diff --git a/tests/test_graph_path_traversal.py b/tests/test_graph_path_traversal.py index b46a8c9f..7f3b8e06 100644 --- a/tests/test_graph_path_traversal.py +++ b/tests/test_graph_path_traversal.py @@ -1,4 +1,5 @@ """Unit tests for BFS path traversal (scanner/graph/path_traversal.py).""" + from unittest.mock import MagicMock, patch from scanner.graph.path_traversal import _bfs_from, compute_attack_paths From 252dab6d6db5c0c5c9b2d5744f4771f5b9422ada Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 11:37:42 +0100 Subject: [PATCH 50/67] fix(graph): bidirectional BFS traversal and AND filter for attack-graph edge query Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 2 +- scanner/graph/path_traversal.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 3969fdcd..4289c0e7 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -108,7 +108,7 @@ def get_attack_graph(): e.relationship_type, e.confidence, e.evidence_source, e.collected_at FROM graph_edges e WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) - OR e.target_node_id = ANY(%(node_ids)s::uuid[]) + AND e.target_node_id = ANY(%(node_ids)s::uuid[]) """, {"node_ids": node_ids}, ) diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 442176f3..f2f298d4 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -37,6 +37,10 @@ def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, adj: dict[str, list[tuple[str, str, float]]] = {} for src, tgt, rel, conf in cur.fetchall(): adj.setdefault(src, []).append((tgt, rel, conf)) + # Add reverse direction so BFS from a finding can reach nodes that + # point TO it (e.g. PublicIP -EXPOSES-> VM: starting from the flagged + # VM can now reach the PublicIP that exposes it). + adj.setdefault(tgt, []).append((src, rel + "_REV", conf)) return adj From 1acef400674bb02f03cd996f5efec040ae867fc2 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 27 Sep 2026 23:11:47 +0100 Subject: [PATCH 51/67] fix(graph): remove double populate_graph, scope BFS reversal, fix path count, use DatabaseManager - Remove populate_graph call from ScanEngine.run_scan(); worker.py is the authoritative post-save call site. Running it inside run_scan() fires before findings are persisted and duplicates the population on every scan. - Restrict _REV adjacency edges to EXPOSES and HAS_IDENTITY only. Reversing MEMBER_OF would connect any two VMs sharing a subnet through the subnet node, producing spurious lateral-movement paths with no real attack vector. - _write_paths: return cur.rowcount instead of len(rows). ON CONFLICT DO NOTHING silently drops duplicate insertions; len(rows) overcounts whereas rowcount reflects actual rows written. - Replace raw psycopg2.connect in attack_graph routes with DatabaseManager to use the shared connection pool and match the pattern used by all other routes. - Remove unused MagicMock import (ruff F401). Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 27 +++++++++++++-------------- scanner/engine.py | 9 --------- scanner/graph/path_traversal.py | 13 ++++++++----- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 4289c0e7..ec97b200 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -3,10 +3,10 @@ import logging import os -import psycopg2 import psycopg2.extras from flask import Blueprint, g, jsonify, request +from api.models.finding import DatabaseManager from api.validation import ValidationError, positive_integer, uuid_string attack_graph_bp = Blueprint("attack_graph", __name__) @@ -16,12 +16,11 @@ _MAX_LIMIT = 500 -def _conn(): - if "graph_conn" not in g: - g.graph_conn = psycopg2.connect(os.environ["DATABASE_URL"]) - g.graph_conn.autocommit = True - psycopg2.extras.register_uuid(g.graph_conn) - return g.graph_conn +def _get_db() -> DatabaseManager: + if "db" not in g: + g.db = DatabaseManager(os.environ["DATABASE_URL"]) + g.db.connect() + return g.db def _tenant_id() -> str | None: @@ -44,10 +43,10 @@ def _tenant_id() -> str | None: @attack_graph_bp.teardown_app_request -def _close_conn(exc): - conn = g.pop("graph_conn", None) - if conn is not None: - conn.close() +def _close_db(exc): + db = g.pop("db", None) + if db is not None: + db.close() @attack_graph_bp.get("/api/attack-graph") @@ -68,7 +67,7 @@ def get_attack_graph(): if not tenant_id: return jsonify({"error": "tenant_id not available"}), 400 - conn = _conn() + conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: if subscription_id: cur.execute( @@ -138,7 +137,7 @@ def list_attack_paths(): if not tenant_id: return jsonify({"error": "tenant_id not available"}), 400 - conn = _conn() + conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( """ @@ -174,7 +173,7 @@ def get_attack_path(path_id: str): if not tenant_id: return jsonify({"error": "tenant_id not available"}), 400 - conn = _conn() + conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( """ diff --git a/scanner/engine.py b/scanner/engine.py index b36661e4..c94fa01d 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -21,7 +21,6 @@ from scanner.azure_client import AzureClient from scanner.evaluation import EvaluationStatus, RuleEvaluation, subscription_scope_id -from scanner.graph.graph_populator import populate_graph from scanner.graph.snapshot_bridge import collect_snapshot logger = logging.getLogger(__name__) @@ -285,14 +284,6 @@ def run_scan(self, scan_id: Optional[str] = None) -> Dict[str, Any]: logger.info("Scan %s complete - %d total finding(s). Normalising results...", scan_id, len(findings)) - # Populate the attack graph as a non-fatal post-scan step. - dsn = os.environ.get("DATABASE_URL") - if snapshot and dsn: - try: - populate_graph(scan_id, snapshot, dsn) - except Exception as exc: - logger.warning("run_scan: graph population raised unexpectedly: %s", exc) - return make_serializable(result) def _legacy_placeholder(self, rule_id: str) -> RuleEvaluation: diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index f2f298d4..9355ae1e 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -34,13 +34,16 @@ def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, """, {"tenant_id": tenant_id}, ) + # Only traverse in reverse for relationships where the reverse direction + # is semantically meaningful. MEMBER_OF and PROTECTS are not reversed: + # reversing MEMBER_OF would let any two VMs in the same subnet reach + # each other via the subnet node, producing spurious lateral-movement paths. + _REVERSE_RELS = {"EXPOSES", "HAS_IDENTITY"} adj: dict[str, list[tuple[str, str, float]]] = {} for src, tgt, rel, conf in cur.fetchall(): adj.setdefault(src, []).append((tgt, rel, conf)) - # Add reverse direction so BFS from a finding can reach nodes that - # point TO it (e.g. PublicIP -EXPOSES-> VM: starting from the flagged - # VM can now reach the PublicIP that exposes it). - adj.setdefault(tgt, []).append((src, rel + "_REV", conf)) + if rel in _REVERSE_RELS: + adj.setdefault(tgt, []).append((src, rel + "_REV", conf)) return adj @@ -132,7 +135,7 @@ def _write_paths( rows, template="(%s, %s, %s, %s::uuid, %s::uuid, %s::uuid[], %s, %s, %s)", ) - return len(rows) + return cur.rowcount def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: From 1f01cf3fe72f1daa767406e1f069330b91e7466f Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Mon, 28 Sep 2026 00:09:35 +0100 Subject: [PATCH 52/67] fix(graph): add subnet-bridging test, return 403 for shared-secret callers, document retention - tests/test_graph_path_traversal.py: add test asserting MEMBER_OF edges are not reversed (two VMs in one subnet must not be reachable from each other through the subnet node). Add test for _load_adjacency verifying EXPOSES gets a _REV edge but MEMBER_OF does not. - api/routes/attack_graph.py: return 403 instead of 400 when tenant_id is absent in shared-secret mode. The request is well-formed; the auth method is insufficient. Error message now explains OIDC is required. - docs/api-reference.md: document the OIDC-only requirement for attack graph endpoints and note the attack_paths retention follow-up (issue #333). Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 15 +++++++++--- docs/api-reference.md | 14 +++++++++++ tests/test_graph_path_traversal.py | 39 +++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index ec97b200..1d653c27 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -65,7 +65,10 @@ def get_attack_graph(): tenant_id = _tenant_id() if not tenant_id: - return jsonify({"error": "tenant_id not available"}), 400 + # Shared-secret callers have no tenant claim; OIDC is required for + # tenant-scoped graph endpoints. Return 403 (not 400) since the + # request is well-formed but the auth method is insufficient. + return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: @@ -135,7 +138,10 @@ def list_attack_paths(): tenant_id = _tenant_id() if not tenant_id: - return jsonify({"error": "tenant_id not available"}), 400 + # Shared-secret callers have no tenant claim; OIDC is required for + # tenant-scoped graph endpoints. Return 403 (not 400) since the + # request is well-formed but the auth method is insufficient. + return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: @@ -171,7 +177,10 @@ def get_attack_path(path_id: str): tenant_id = _tenant_id() if not tenant_id: - return jsonify({"error": "tenant_id not available"}), 400 + # Shared-secret callers have no tenant claim; OIDC is required for + # tenant-scoped graph endpoints. Return 403 (not 400) since the + # request is well-formed but the auth method is insufficient. + return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: diff --git a/docs/api-reference.md b/docs/api-reference.md index 7192e140..c60526e3 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -697,3 +697,17 @@ The following endpoints are called by the frontend but have no backend implement | Endpoint | Used by | Status | |---|---|---| | `GET /api/monitoring` | Monitoring page — score trend chart, category distribution | Deferred. Score and findings data come from `GET /api/score` and `GET /api/findings` instead. | + +--- + +## Attack graph endpoints + +`GET /api/attack-graph`, `GET /api/attack-paths`, and `GET /api/attack-paths/` expose the attack graph computed from the most recent scan for the caller's tenant. + +### Authentication requirement + +These endpoints are **OIDC-only**. In `shared_secret` mode no tenant claim is present in the token, so the endpoints return `403 {"error": "tenant_id not available; OIDC authentication required"}`. This is intentional: the graph is strictly scoped per tenant and there is no safe way to infer a tenant from a shared-secret token. + +### Attack-path retention + +Attack paths are written per scan and are not automatically pruned. In long-running deployments, old scan paths accumulate in `attack_paths`. Retention policy (e.g. keep only the N most recent scans per tenant) is tracked in issue #333 and will be addressed in a follow-up. diff --git a/tests/test_graph_path_traversal.py b/tests/test_graph_path_traversal.py index 7f3b8e06..969cf899 100644 --- a/tests/test_graph_path_traversal.py +++ b/tests/test_graph_path_traversal.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock, patch -from scanner.graph.path_traversal import _bfs_from, compute_attack_paths +from scanner.graph.path_traversal import _bfs_from, _load_adjacency, compute_attack_paths def test_bfs_from_returns_empty_for_isolated_node(): @@ -46,6 +46,43 @@ def test_bfs_from_does_not_revisit_nodes(): assert target_nodes.count("C") == 1 +def test_member_of_not_reversed_prevents_subnet_bridging(): + # Two VMs sharing a subnet. MEMBER_OF is forward-only (VM -> subnet), + # so BFS from vm-a must not reach vm-b through the subnet. + adj = { + "vm-a": [("subnet-1", "MEMBER_OF", 0.9)], + "vm-b": [("subnet-1", "MEMBER_OF", 0.9)], + # subnet-1 has no outgoing edges: MEMBER_OF is not reversed + } + result = _bfs_from("vm-a", adj) + target_ids = {r["target_node_id"] for r in result} + assert "vm-b" not in target_ids + assert "subnet-1" in target_ids # forward hop is still present + + +def test_load_adjacency_reverses_exposes_not_member_of(): + conn = MagicMock() + cur = MagicMock() + cur.__enter__ = lambda s: s + cur.__exit__ = MagicMock(return_value=False) + conn.cursor.return_value = cur + # One EXPOSES edge and one MEMBER_OF edge + cur.fetchall.return_value = [ + ("pub-ip", "vm-1", "EXPOSES", 1.0), + ("vm-1", "subnet-1", "MEMBER_OF", 0.9), + ] + + adj = _load_adjacency(conn, "tenant-x") + + # EXPOSES should have a reverse entry (vm-1 -> pub-ip via EXPOSES_REV) + rev_exposes = [n for n, rel, _ in adj.get("vm-1", []) if rel == "EXPOSES_REV"] + assert rev_exposes, "EXPOSES_REV edge missing from vm-1" + + # MEMBER_OF should NOT have a reverse entry (subnet-1 should not point to vm-1) + rev_member = [n for n, rel, _ in adj.get("subnet-1", []) if rel == "MEMBER_OF_REV"] + assert not rev_member, "MEMBER_OF_REV edge should not exist (spurious bridging)" + + def test_compute_attack_paths_returns_zero_when_no_finding_nodes(): with patch("scanner.graph.path_traversal.psycopg2.connect") as mock_connect: conn = MagicMock() From 4fc7b6688d1e67f029d6352ae44f2610efaeb321 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Mon, 28 Sep 2026 01:18:26 +0100 Subject: [PATCH 53/67] fix(lint+test): remove unused os import, update 400->403 assertion, fix stacked admission test - scanner/engine.py: remove 'import os' now unused after populate_graph was removed from run_scan(). - tests/test_attack_graph_api.py: update viewer-token test to expect 403 (auth method insufficient) instead of 400 (bad request) to match the updated response for shared-secret callers without a tenant claim. - tests/test_scan_admission_migration_postgres.py: resolve _HEAD dynamically via ScriptDirectory.get_current_head() so stacked migration tests do not pin a hardcoded revision. Signed-off-by: Tanvir Farhad --- scanner/engine.py | 1 - tests/test_attack_graph_api.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/scanner/engine.py b/scanner/engine.py index c94fa01d..082396bc 100644 --- a/scanner/engine.py +++ b/scanner/engine.py @@ -3,7 +3,6 @@ import importlib.util import inspect import logging -import os import uuid from datetime import datetime, timezone from pathlib import Path diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index dd1ae07e..bff52966 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -78,7 +78,7 @@ def test_get_attack_graph_viewer_cannot_supply_tenant_header(client, app): "X-Tenant-Id": _TENANT, } resp = client.get("/api/attack-graph", headers=headers) - assert resp.status_code == 400 + assert resp.status_code == 403 def test_list_attack_paths_missing_scan_id_returns_400(client, tenant_auth_headers): From 0214b0a0b360fb3ea915241f4f1eb6e66a3df3f3 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 1 Oct 2026 00:39:58 +0100 Subject: [PATCH 54/67] fix(graph): return attempted path count instead of rowcount from execute_values psycopg2 execute_values returns -1 for rowcount when ON CONFLICT DO NOTHING is used, making compute_attack_paths always report 0 paths written to logs. Return len(rows) (paths attempted) instead so the log line is meaningful. Signed-off-by: Tanvir Farhad --- scanner/graph/path_traversal.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 9355ae1e..9501e634 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -135,7 +135,9 @@ def _write_paths( rows, template="(%s, %s, %s, %s::uuid, %s::uuid, %s::uuid[], %s, %s, %s)", ) - return cur.rowcount + # execute_values returns -1 for rowcount with ON CONFLICT DO NOTHING; + # return the number of rows attempted instead. + return len(rows) def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: From 4cdf560a4cb5cef8cdb112d344172c841ec887ce Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 02:44:22 +0100 Subject: [PATCH 55/67] fix(graph): fix rowcount from execute_values, add path retention, wrap DB exceptions in routes Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 204 +++++++++++++++++--------------- scanner/graph/path_traversal.py | 33 +++++- 2 files changed, 138 insertions(+), 99 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 1d653c27..d7774b2c 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -70,51 +70,55 @@ def get_attack_graph(): # request is well-formed but the auth method is insufficient. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 - conn = _get_db().conn - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - if subscription_id: - cur.execute( - """ - SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, - n.location, n.resource_group, n.subscription_id, n.snapshot_id, - n.updated_at - FROM graph_nodes n - WHERE n.tenant_id = %(tenant_id)s - AND n.subscription_id = %(subscription_id)s - ORDER BY n.updated_at DESC - LIMIT %(limit)s - """, - {"tenant_id": tenant_id, "subscription_id": subscription_id, "limit": limit}, - ) - else: - cur.execute( - """ - SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, - n.location, n.resource_group, n.subscription_id, n.snapshot_id, - n.updated_at - FROM graph_nodes n - WHERE n.tenant_id = %(tenant_id)s - ORDER BY n.updated_at DESC - LIMIT %(limit)s - """, - {"tenant_id": tenant_id, "limit": limit}, - ) - nodes = cur.fetchall() - - node_ids = [row["node_id"] for row in nodes] - edges: list = [] - if node_ids: - cur.execute( - """ - SELECT e.edge_id::text, e.source_node_id::text, e.target_node_id::text, - e.relationship_type, e.confidence, e.evidence_source, e.collected_at - FROM graph_edges e - WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) - AND e.target_node_id = ANY(%(node_ids)s::uuid[]) - """, - {"node_ids": node_ids}, - ) - edges = cur.fetchall() + try: + conn = _get_db().conn + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + if subscription_id: + cur.execute( + """ + SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, + n.location, n.resource_group, n.subscription_id, n.snapshot_id, + n.updated_at + FROM graph_nodes n + WHERE n.tenant_id = %(tenant_id)s + AND n.subscription_id = %(subscription_id)s + ORDER BY n.updated_at DESC + LIMIT %(limit)s + """, + {"tenant_id": tenant_id, "subscription_id": subscription_id, "limit": limit}, + ) + else: + cur.execute( + """ + SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, + n.location, n.resource_group, n.subscription_id, n.snapshot_id, + n.updated_at + FROM graph_nodes n + WHERE n.tenant_id = %(tenant_id)s + ORDER BY n.updated_at DESC + LIMIT %(limit)s + """, + {"tenant_id": tenant_id, "limit": limit}, + ) + nodes = cur.fetchall() + + node_ids = [row["node_id"] for row in nodes] + edges: list = [] + if node_ids: + cur.execute( + """ + SELECT e.edge_id::text, e.source_node_id::text, e.target_node_id::text, + e.relationship_type, e.confidence, e.evidence_source, e.collected_at + FROM graph_edges e + WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) + AND e.target_node_id = ANY(%(node_ids)s::uuid[]) + """, + {"node_ids": node_ids}, + ) + edges = cur.fetchall() + except Exception: + logger.exception("get_attack_graph failed for tenant %s", tenant_id) + return jsonify({"error": "internal server error"}), 500 return jsonify({"nodes": [dict(r) for r in nodes], "edges": [dict(r) for r in edges]}) @@ -143,26 +147,30 @@ def list_attack_paths(): # request is well-formed but the auth method is insufficient. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 - conn = _get_db().conn - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute( - """ - SELECT ap.path_id::text, ap.source_node_id::text, ap.target_node_id::text, - ap.path_node_ids, ap.path_length, ap.min_confidence, - ap.relationship_types, ap.computed_at, - src.resource_type AS source_type, src.name AS source_name, - tgt.resource_type AS target_type, tgt.name AS target_name - FROM attack_paths ap - JOIN graph_nodes src ON src.node_id = ap.source_node_id - JOIN graph_nodes tgt ON tgt.node_id = ap.target_node_id - WHERE ap.scan_id = %(scan_id)s - AND ap.tenant_id = %(tenant_id)s - ORDER BY ap.path_length ASC, ap.min_confidence DESC - LIMIT %(limit)s - """, - {"scan_id": scan_id, "tenant_id": tenant_id, "limit": limit}, - ) - rows = cur.fetchall() + try: + conn = _get_db().conn + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT ap.path_id::text, ap.source_node_id::text, ap.target_node_id::text, + ap.path_node_ids, ap.path_length, ap.min_confidence, + ap.relationship_types, ap.computed_at, + src.resource_type AS source_type, src.name AS source_name, + tgt.resource_type AS target_type, tgt.name AS target_name + FROM attack_paths ap + JOIN graph_nodes src ON src.node_id = ap.source_node_id + JOIN graph_nodes tgt ON tgt.node_id = ap.target_node_id + WHERE ap.scan_id = %(scan_id)s + AND ap.tenant_id = %(tenant_id)s + ORDER BY ap.path_length ASC, ap.min_confidence DESC + LIMIT %(limit)s + """, + {"scan_id": scan_id, "tenant_id": tenant_id, "limit": limit}, + ) + rows = cur.fetchall() + except Exception: + logger.exception("list_attack_paths failed for scan %s", scan_id) + return jsonify({"error": "internal server error"}), 500 return jsonify({"scan_id": scan_id, "paths": [dict(r) for r in rows]}) @@ -182,42 +190,46 @@ def get_attack_path(path_id: str): # request is well-formed but the auth method is insufficient. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 - conn = _get_db().conn - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute( - """ - SELECT ap.path_id::text, ap.scan_id, ap.source_node_id::text, - ap.target_node_id::text, ap.path_node_ids, ap.path_length, - ap.min_confidence, ap.relationship_types, ap.computed_at - FROM attack_paths ap - WHERE ap.path_id = %(path_id)s::uuid - AND ap.tenant_id = %(tenant_id)s - """, - {"path_id": path_id, "tenant_id": tenant_id}, - ) - row = cur.fetchone() - - if row is None: - return jsonify({"error": "not found"}), 404 - - path = dict(row) - - # Fetch full node detail for each hop - node_ids = [str(nid) for nid in path["path_node_ids"]] - if node_ids: + try: + conn = _get_db().conn with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: cur.execute( """ - SELECT node_id::text, resource_id, resource_type, name, - location, resource_group, subscription_id - FROM graph_nodes - WHERE node_id = ANY(%(ids)s::uuid[]) - AND tenant_id = %(tenant_id)s + SELECT ap.path_id::text, ap.scan_id, ap.source_node_id::text, + ap.target_node_id::text, ap.path_node_ids, ap.path_length, + ap.min_confidence, ap.relationship_types, ap.computed_at + FROM attack_paths ap + WHERE ap.path_id = %(path_id)s::uuid + AND ap.tenant_id = %(tenant_id)s """, - {"ids": node_ids, "tenant_id": tenant_id}, + {"path_id": path_id, "tenant_id": tenant_id}, ) - nodes_by_id = {r["node_id"]: dict(r) for r in cur.fetchall()} - path["hops"] = [nodes_by_id.get(str(nid), {"node_id": str(nid)}) for nid in path["path_node_ids"]] + row = cur.fetchone() + + if row is None: + return jsonify({"error": "not found"}), 404 + + path = dict(row) + + # Fetch full node detail for each hop + node_ids = [str(nid) for nid in path["path_node_ids"]] + if node_ids: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: + cur.execute( + """ + SELECT node_id::text, resource_id, resource_type, name, + location, resource_group, subscription_id + FROM graph_nodes + WHERE node_id = ANY(%(ids)s::uuid[]) + AND tenant_id = %(tenant_id)s + """, + {"ids": node_ids, "tenant_id": tenant_id}, + ) + nodes_by_id = {r["node_id"]: dict(r) for r in cur.fetchall()} + path["hops"] = [nodes_by_id.get(str(nid), {"node_id": str(nid)}) for nid in path["path_node_ids"]] + except Exception: + logger.exception("get_attack_path failed for path_id %s", path_id) + return jsonify({"error": "internal server error"}), 500 path["path_node_ids"] = [str(nid) for nid in path["path_node_ids"]] return jsonify(path) diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 9501e634..8660241d 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -99,6 +99,30 @@ def _bfs_from( return paths +def _delete_stale_paths(conn: Any, scan_id: str, tenant_id: str) -> None: + """Delete attack paths from previous scans for the same subscription, keeping only the current scan.""" + with conn.cursor() as cur: + cur.execute( + """ + DELETE FROM attack_paths + WHERE tenant_id = %(tenant_id)s + AND scan_id != %(scan_id)s + AND scan_id IN ( + SELECT DISTINCT ap2.scan_id + FROM attack_paths ap2 + JOIN findings f ON f.scan_id = ap2.scan_id + WHERE ap2.tenant_id = %(tenant_id)s + AND f.subscription_id = ( + SELECT subscription_id FROM findings + WHERE scan_id = %(scan_id)s + LIMIT 1 + ) + ) + """, + {"tenant_id": tenant_id, "scan_id": scan_id}, + ) + + def _write_paths( conn: Any, scan_id: str, @@ -135,9 +159,10 @@ def _write_paths( rows, template="(%s, %s, %s, %s::uuid, %s::uuid, %s::uuid[], %s, %s, %s)", ) - # execute_values returns -1 for rowcount with ON CONFLICT DO NOTHING; - # return the number of rows attempted instead. - return len(rows) + inserted = cur.rowcount + # cur.rowcount reflects actual inserts after ON CONFLICT DO NOTHING; + # fall back to attempted count if the driver reports -1. + return inserted if inserted >= 0 else len(rows) def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: @@ -146,10 +171,12 @@ def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: conn = psycopg2.connect(dsn) conn.autocommit = False try: + _delete_stale_paths(conn, scan_id, tenant_id) adj = _load_adjacency(conn, tenant_id) source_nodes = _load_finding_nodes(conn, scan_id, tenant_id) if not source_nodes: logger.info("graph path traversal: no finding-linked nodes for scan %s", scan_id) + conn.commit() return 0 total = 0 From 27088069cc7177581daf1c23c5647b4ef99ac644 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 11:55:45 +0100 Subject: [PATCH 56/67] fix(tests): remove leftover merge-conflict marker in integration test Signed-off-by: Tanvir Farhad --- tests/test_graph_engine_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_graph_engine_integration.py b/tests/test_graph_engine_integration.py index b69c9a50..d68869f0 100644 --- a/tests/test_graph_engine_integration.py +++ b/tests/test_graph_engine_integration.py @@ -68,7 +68,6 @@ def test_run_scan_existing_rule_without_snapshot_param_still_works(engine): snapshot = _make_snapshot() def legacy_scan(client, subscription_id): -<<<<<<< HEAD return [ { "rule_id": "AZ-LEGACY-001", From 11d42f9423c0759045ab77281487e674fcf34777 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 4 Oct 2026 14:26:37 +0100 Subject: [PATCH 57/67] fix: correct teardown scope, int coerce limit param, module-level _REVERSE_RELS, safe logger.error Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 6 +++--- scanner/graph/path_traversal.py | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index d7774b2c..76f3db0c 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -42,7 +42,7 @@ def _tenant_id() -> str | None: return None -@attack_graph_bp.teardown_app_request +@attack_graph_bp.teardown_request def _close_db(exc): db = g.pop("db", None) if db is not None: @@ -56,7 +56,7 @@ def get_attack_graph(): Query params: subscription_id (optional), limit (default 100, max 500) """ try: - limit = positive_integer(request.args.get("limit", _DEFAULT_LIMIT), "limit") + limit = positive_integer(int(request.args.get("limit", _DEFAULT_LIMIT)), "limit") if limit > _MAX_LIMIT: limit = _MAX_LIMIT subscription_id = request.args.get("subscription_id") @@ -134,7 +134,7 @@ def list_attack_paths(): return jsonify({"error": "scan_id is required"}), 400 try: scan_id = uuid_string(scan_id, "scan_id") - limit = positive_integer(request.args.get("limit", _DEFAULT_LIMIT), "limit") + limit = positive_integer(int(request.args.get("limit", _DEFAULT_LIMIT)), "limit") if limit > _MAX_LIMIT: limit = _MAX_LIMIT except ValidationError as exc: diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 8660241d..0e8c1e4f 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -20,6 +20,11 @@ # keeping them would inflate the table for large graphs. _MAX_PATH_LENGTH = 8 +# Relationships whose edges are also traversed in reverse during BFS. +# MEMBER_OF and PROTECTS are excluded: reversing MEMBER_OF would let any two +# VMs in the same subnet reach each other via the subnet node. +_REVERSE_RELS: frozenset[str] = frozenset({"EXPOSES", "HAS_IDENTITY"}) + def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, float]]]: """Return {source_node_id: [(target_node_id, relationship_type, confidence)]}.""" @@ -38,7 +43,6 @@ def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, # is semantically meaningful. MEMBER_OF and PROTECTS are not reversed: # reversing MEMBER_OF would let any two VMs in the same subnet reach # each other via the subnet node, producing spurious lateral-movement paths. - _REVERSE_RELS = {"EXPOSES", "HAS_IDENTITY"} adj: dict[str, list[tuple[str, str, float]]] = {} for src, tgt, rel, conf in cur.fetchall(): adj.setdefault(src, []).append((tgt, rel, conf)) @@ -192,6 +196,6 @@ def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: raise finally: conn.close() - except Exception as exc: - logger.error("graph path traversal failed for scan %s: %s", scan_id, exc) + except Exception: + logger.error("graph path traversal failed for scan %s", scan_id, exc_info=True) return 0 From 922a903a09ec00ed45a733240392626d6bdc5bbd Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:10:27 +0100 Subject: [PATCH 58/67] fix(graph-api): reject malformed limits and describe admin tenant fallback Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 6 ++--- docs/api-reference.md | 48 +++++++++++++++++----------------- tests/test_attack_graph_api.py | 24 +++++++++++++++++ 3 files changed, 51 insertions(+), 27 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 76f3db0c..66a3f266 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -60,7 +60,7 @@ def get_attack_graph(): if limit > _MAX_LIMIT: limit = _MAX_LIMIT subscription_id = request.args.get("subscription_id") - except ValidationError as exc: + except (ValidationError, ValueError) as exc: return jsonify({"error": str(exc)}), 400 tenant_id = _tenant_id() @@ -137,7 +137,7 @@ def list_attack_paths(): limit = positive_integer(int(request.args.get("limit", _DEFAULT_LIMIT)), "limit") if limit > _MAX_LIMIT: limit = _MAX_LIMIT - except ValidationError as exc: + except (ValidationError, ValueError) as exc: return jsonify({"error": str(exc)}), 400 tenant_id = _tenant_id() @@ -180,7 +180,7 @@ def get_attack_path(path_id: str): """Return a single attack path with full node detail for each hop.""" try: path_id = uuid_string(path_id, "path_id") - except ValidationError as exc: + except (ValidationError, ValueError) as exc: return jsonify({"error": str(exc)}), 400 tenant_id = _tenant_id() diff --git a/docs/api-reference.md b/docs/api-reference.md index c60526e3..9a7b2464 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -7,7 +7,7 @@ routes become public only when the deliberate demo-mode setting is enabled. ## Authentication -`/`, `/health`, `/ready`, and `/metrics` are always public. All other routes — including all `/api/*` GET endpoints — require an `Authorization: Bearer ` header, verified by `api/auth.py` according to `OPENSHIELD_AUTH_MODE`: +`/`, `/health`, `/ready`, and `/metrics` are always public. All other routes — including all `/api/*` GET endpoints — require an `Authorization: Bearer ` header, verified by `api/auth.py` according to `OPENSHIELD_AUTH_MODE`: | Mode | Verification | Role source | Intended for | |---|---|---|---| @@ -27,7 +27,7 @@ The dashboard never embeds a token (see [authentication and containment](securit ### Subscription authorization -`POST /api/scans/trigger` also checks `subscription_id` against `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS`, a comma-separated allowlist. A valid `operator`/`admin` token can otherwise trigger a scan against *any* subscription_id — role alone doesn't say which subscription a caller is entitled to. Left unset, every subscription_id is accepted (matches historical behavior); the API logs a loud startup warning when it's unset. This is a single-tenant containment boundary, not a substitute for real per-tenant authorization — see issue #294 for the remaining tenant-ownership scope this is a stopgap for. +`POST /api/scans/trigger` also checks `subscription_id` against `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS`, a comma-separated allowlist. A valid `operator`/`admin` token can otherwise trigger a scan against *any* subscription_id — role alone doesn't say which subscription a caller is entitled to. Left unset, every subscription_id is accepted (matches historical behavior); the API logs a loud startup warning when it's unset. This is a single-tenant containment boundary, not a substitute for real per-tenant authorization — see issue #294 for the remaining tenant-ownership scope this is a stopgap for. ## Input limits @@ -185,7 +185,7 @@ Example response: Returns the details and current status of a specific scan. -Path parameters: `scan_id` — UUID of the scan. +Path parameters: `scan_id` — UUID of the scan. Example response: @@ -208,7 +208,7 @@ Example response: Admits an asynchronous scan against the configured subscription. Execution happens in a background worker process; the response returns as soon as the scan is durably recorded. -Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): +Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): ```json { @@ -221,7 +221,7 @@ Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): Admission is serialized per subscription and enforced by the database, so concurrent and replayed triggers converge on one logical scan rather than creating duplicates: - **At most one active scan per subscription.** While a `pending` or `running` scan exists, a further trigger returns that existing scan instead of queueing another. -- **`Idempotency-Key` (optional request header, 1–200 characters).** A repeat of the same key for the same subscription returns the original scan. The key is scoped to the subscription; the same key under a different subscription is a different request. A trigger carries no request input other than `subscription_id`, so a key that resolves to an existing scan is always a replay of the same logical request and there is no changed-payload conflict to report. +- **`Idempotency-Key` (optional request header, 1–200 characters).** A repeat of the same key for the same subscription returns the original scan. The key is scoped to the subscription; the same key under a different subscription is a different request. A trigger carries no request input other than `subscription_id`, so a key that resolves to an existing scan is always a replay of the same logical request and there is no changed-payload conflict to report. - **`OPENSHIELD_MAX_SCANS_PER_SUBSCRIPTION_PER_HOUR`** adds an optional hourly admission quota. Unset or `0` (the default) applies no time-window limit; the one-active-scan rule still applies. ### Responses @@ -229,8 +229,8 @@ Admission is serialized per subscription and enforced by the database, so concur | Status | When | Body | | --- | --- | --- | | `202 Accepted` | A new scan was admitted and queued. | `scan_id`, `status: "pending"`, `message` | -| `200 OK` | The request resolved to an existing logical scan — an `Idempotency-Key` replay of the same request, or a trigger while a scan is already active for the subscription. | `scan_id`, `status` (the existing scan's `pending`/`running`), `message: "Existing logical scan returned."` | -| `400 Bad Request` | Malformed body, invalid `subscription_id`, missing subscription, or an `Idempotency-Key` outside 1–200 characters. | `error` | +| `200 OK` | The request resolved to an existing logical scan — an `Idempotency-Key` replay of the same request, or a trigger while a scan is already active for the subscription. | `scan_id`, `status` (the existing scan's `pending`/`running`), `message: "Existing logical scan returned."` | +| `400 Bad Request` | Malformed body, invalid `subscription_id`, missing subscription, or an `Idempotency-Key` outside 1–200 characters. | `error` | | `403 Forbidden` | `subscription_id` is not on the `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS` allowlist. | `error` | | `429 Too Many Requests` | The configured hourly quota for this subscription is exhausted. | `error: "Scan quota exceeded for this subscription."` | @@ -266,7 +266,7 @@ Missing subscription response (`400`): ## POST /api/scans/<scan_id>/enrich -Queues durable CVE enrichment for a completed scan's findings. Enrichment runs as a database-backed job claimed by the background worker — the request never owns a thread, so the work survives an API restart. +Queues durable CVE enrichment for a completed scan's findings. Enrichment runs as a database-backed job claimed by the background worker — the request never owns a thread, so the work survives an API restart. There is **never more than one enrichment job per scan**. Repeat calls are safe: they report the state of the single job rather than creating another. @@ -280,11 +280,11 @@ Every response carries `scan_id`, `job_id`, `status` (the job row's state) and a | `202 Accepted` | `requeued` | A previously **failed** job was reset to `pending` and will be retried. | | `202 Accepted` | `active` | A `pending` or `running` job already exists and was returned unchanged. A live claim is never interrupted. | | `200 OK` | `completed` | Enrichment already finished; nothing was restarted. | -| `404 Not Found` | — | Unknown `scan_id`, or the scan has no findings to enrich and has not already been enriched. | +| `404 Not Found` | — | Unknown `scan_id`, or the scan has no findings to enrich and has not already been enriched. | An already-enriched scan always reports `completed`, including a clean scan that had no findings to enrich in the first place. -A job that exhausts its retry budget becomes `failed`. Re-POSTing this endpoint is the supported operator recovery: it atomically returns the job to `pending` with a fresh retry budget, clears the lease, and keeps the last `error_message` and the `checkpoint` so the retry resumes rather than re-enriching findings that already succeeded. Concurrent re-POSTs converge — exactly one reports `requeued` and the rest report `active`. +A job that exhausts its retry budget becomes `failed`. Re-POSTing this endpoint is the supported operator recovery: it atomically returns the job to `pending` with a fresh retry budget, clears the lease, and keeps the last `error_message` and the `checkpoint` so the retry resumes rather than re-enriching findings that already succeeded. Concurrent re-POSTs converge — exactly one reports `requeued` and the rest report `active`. Newly queued (`202`): @@ -316,7 +316,7 @@ Poll `GET /api/scans/` for `cve_enrichment_status` (`PENDING`, `ENRICHI ## GET /api/score -Returns the overall security posture score from 0 to 100. Under [severity contract v1](severity-contract.md), the score starts at 100 and deducts 20 per CRITICAL finding, 10 per HIGH finding, 5 per MEDIUM finding, and 2 per LOW finding; INFO findings deduct zero. Scoped to the most recent **completed** scan — if no completed scan exists yet, this returns `status: "NO_SCAN_DATA"` with `score: null` rather than a misleading 100 (a scan with no findings and no evidence at all would otherwise be indistinguishable). +Returns the overall security posture score from 0 to 100. Under [severity contract v1](severity-contract.md), the score starts at 100 and deducts 20 per CRITICAL finding, 10 per HIGH finding, 5 per MEDIUM finding, and 2 per LOW finding; INFO findings deduct zero. Scoped to the most recent **completed** scan — if no completed scan exists yet, this returns `status: "NO_SCAN_DATA"` with `score: null` rather than a misleading 100 (a scan with no findings and no evidence at all would otherwise be indistinguishable). Query parameters: @@ -345,13 +345,13 @@ Example response (no completed scan exists yet): } ``` -Consumers must check `status` and treat a `null` `score` as "not assessed" — never coerce it to `0`, which would misrepresent absence of evidence as a confirmed worst-case score. +Consumers must check `status` and treat a `null` `score` as "not assessed" — never coerce it to `0`, which would misrepresent absence of evidence as a confirmed worst-case score. --- ## GET /api/compliance/<framework> -Returns technical-evidence coverage against a compliance framework mapping pack, scoped to the most recent **completed** scan. This is coverage, not a certification or a claim of full framework compliance — see `docs/compliance-mapping-pack.md` for the full mapping-pack schema and `evaluation_basis` semantics. +Returns technical-evidence coverage against a compliance framework mapping pack, scoped to the most recent **completed** scan. This is coverage, not a certification or a claim of full framework compliance — see `docs/compliance-mapping-pack.md` for the full mapping-pack schema and `evaluation_basis` semantics. Supported frameworks: @@ -371,10 +371,10 @@ Query parameters: | `subscription_id` | UUID string | No | Scopes the "most recent completed scan" lookup to one Azure subscription. Defaults to the deployment's `AZURE_SUBSCRIPTION_ID`; if neither is set the latest completed scan from *any* subscription is used, which is only correct for a single-tenant database. A malformed value is a `400`, never a silently unscoped result. | `status` is one of: -- `OK` — a completed scan exists and at least one mapped control is in scope; `score_percent` is a real evaluated percentage. -- `NO_SCAN_DATA` — no completed scan exists yet, so there is no evidence to report; `score_percent` is `null`. -- `NO_REVIEWED_CONTROLS` — a completed scan exists, but every mapping awaits review; `score_percent` is `null` and no direct-evidence score exists. -- `NO_IN_SCOPE_CONTROLS` — a completed scan exists, but every reviewed mapped control for this framework is `not_applicable`/`organizational` and excluded from the denominator; `score_percent` is `null`. +- `OK` — a completed scan exists and at least one mapped control is in scope; `score_percent` is a real evaluated percentage. +- `NO_SCAN_DATA` — no completed scan exists yet, so there is no evidence to report; `score_percent` is `null`. +- `NO_REVIEWED_CONTROLS` — a completed scan exists, but every mapping awaits review; `score_percent` is `null` and no direct-evidence score exists. +- `NO_IN_SCOPE_CONTROLS` — a completed scan exists, but every reviewed mapped control for this framework is `not_applicable`/`organizational` and excluded from the denominator; `score_percent` is `null`. Per-control `status` is evaluation-derived (issue #263) only after its mapping has been reviewed: `PASS`/`FAIL`/`UNKNOWN`/`ERROR` is the rolled-up status of @@ -391,7 +391,7 @@ Any control with `review_status` other than `reviewed` has control status `UNREVIEWED_MAPPING`, regardless of its `mapping_type`. It is excluded from the denominator alongside `not_applicable` and `organizational` controls, and does not contribute to `passed` or `failed`. Consumers must check `status` and -never treat a `null` `score_percent` as `0` — a missing/excluded score is a +never treat a `null` `score_percent` as `0` — a missing/excluded score is a different fact from a real, evaluated 0%. `reviewed_controls`, `unreviewed_controls`, `not_applicable`, `organizational`, and `excluded_controls` make the reason explicit. @@ -440,7 +440,7 @@ Example response (`OK`): } ``` -Example response (`NO_SCAN_DATA`, HTTP 200 — never 500): +Example response (`NO_SCAN_DATA`, HTTP 200 — never 500): ```json { @@ -471,7 +471,7 @@ Unknown framework response (HTTP 400): ## GET /api/resources -Returns unique Azure resources derived from the most recent scan that has findings. Resources are aggregated from findings — one entry per distinct `resource_id`. Risk level is computed from the maximum severity finding on each resource. +Returns unique Azure resources derived from the most recent scan that has findings. Resources are aggregated from findings — one entry per distinct `resource_id`. Risk level is computed from the maximum severity finding on each resource. Query parameters: none @@ -513,7 +513,7 @@ No findings response (no scan with findings exists): ## GET /api/prioritization -Returns findings from the most recent scan grouped and ranked by risk score (`severity_weight × affected_resource_count`). Produces a matrix view, a ranked list, and recommended action items. +Returns findings from the most recent scan grouped and ranked by risk score (`severity_weight × affected_resource_count`). Produces a matrix view, a ranked list, and recommended action items. Query parameters: none @@ -613,7 +613,7 @@ No drift response (fewer than two scans): Returns the structured remediation playbook for a specific finding. Loads the matching `playbooks/cli/fix_.sh` script and wraps the finding's remediation text as a portal step. Appends NVD links from any CVE references on the finding. -Path parameters: `id` — integer finding ID from `GET /api/findings`. +Path parameters: `id` — integer finding ID from `GET /api/findings`. Example response: @@ -696,7 +696,7 @@ The following endpoints are called by the frontend but have no backend implement | Endpoint | Used by | Status | |---|---|---| -| `GET /api/monitoring` | Monitoring page — score trend chart, category distribution | Deferred. Score and findings data come from `GET /api/score` and `GET /api/findings` instead. | +| `GET /api/monitoring` | Monitoring page — score trend chart, category distribution | Deferred. Score and findings data come from `GET /api/score` and `GET /api/findings` instead. | --- @@ -706,7 +706,7 @@ The following endpoints are called by the frontend but have no backend implement ### Authentication requirement -These endpoints are **OIDC-only**. In `shared_secret` mode no tenant claim is present in the token, so the endpoints return `403 {"error": "tenant_id not available; OIDC authentication required"}`. This is intentional: the graph is strictly scoped per tenant and there is no safe way to infer a tenant from a shared-secret token. +OIDC callers use the verified tenant claim. A supplied `X-Tenant-Id` header cannot override that claim. In `shared_secret` mode, an authenticated admin may supply `X-Tenant-Id` explicitly; viewer and operator tokens cannot select a tenant. Requests without a usable tenant return `403 {"error": "tenant_id not available; OIDC authentication required"}`. Tenant query parameters never determine graph scope. ### Attack-path retention diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index bff52966..f021a6a5 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -104,3 +104,27 @@ def test_get_attack_path_not_found_returns_404(client, tenant_auth_headers): with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): resp = client.get(f"/api/attack-paths/{valid_uuid}", headers=tenant_auth_headers) assert resp.status_code == 404 + + +@pytest.mark.parametrize("limit", ["abc", "", "1.5", "0", "-1"]) +@pytest.mark.parametrize( + "endpoint", ["/api/attack-graph", "/api/attack-paths?scan_id=00000000-0000-0000-0000-000000000002"] +) +def test_malformed_limit_returns_400(client, tenant_auth_headers, endpoint, limit): + separator = "&" if "?" in endpoint else "?" + response = client.get(f"{endpoint}{separator}limit={limit}", headers=tenant_auth_headers) + assert response.status_code == 400 + + +def test_verified_viewer_token_cannot_select_tenant_in_query(client, app): + import jwt + + token = jwt.encode( + {"sub": "viewer", "role": "viewer", "tid": _TENANT, "exp": int(time.time()) + 60}, + app.config["JWT_SECRET"], + algorithm="HS256", + ) + response = client.get( + f"/api/attack-graph?tenant_id={_TENANT}", headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": _TENANT} + ) + assert response.status_code == 403 From 66c7cc902bcc2b46250c2124bd8cc2f867c40b06 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 14:15:02 +0100 Subject: [PATCH 59/67] fix(graph): traverse current evidence and prune clean scan paths by scan scope Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 6 +- scanner/graph/path_traversal.py | 30 +++--- tests/test_graph_paths_postgres.py | 151 +++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+), 21 deletions(-) create mode 100644 tests/test_graph_paths_postgres.py diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 66a3f266..815b642a 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -79,7 +79,7 @@ def get_attack_graph(): SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, n.location, n.resource_group, n.subscription_id, n.snapshot_id, n.updated_at - FROM graph_nodes n + FROM current_graph_nodes n WHERE n.tenant_id = %(tenant_id)s AND n.subscription_id = %(subscription_id)s ORDER BY n.updated_at DESC @@ -93,7 +93,7 @@ def get_attack_graph(): SELECT n.node_id::text, n.resource_id, n.resource_type, n.name, n.location, n.resource_group, n.subscription_id, n.snapshot_id, n.updated_at - FROM graph_nodes n + FROM current_graph_nodes n WHERE n.tenant_id = %(tenant_id)s ORDER BY n.updated_at DESC LIMIT %(limit)s @@ -109,7 +109,7 @@ def get_attack_graph(): """ SELECT e.edge_id::text, e.source_node_id::text, e.target_node_id::text, e.relationship_type, e.confidence, e.evidence_source, e.collected_at - FROM graph_edges e + FROM current_graph_edges e WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) AND e.target_node_id = ANY(%(node_ids)s::uuid[]) """, diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 0e8c1e4f..bf4f1b94 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -16,7 +16,7 @@ logger = logging.getLogger(__name__) -# Paths longer than this are not persisted — they're rarely actionable and +# Paths longer than this are not persisted — they're rarely actionable and # keeping them would inflate the table for large graphs. _MAX_PATH_LENGTH = 8 @@ -33,8 +33,8 @@ def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, """ SELECT e.source_node_id::text, e.target_node_id::text, e.relationship_type, e.confidence - FROM graph_edges e - JOIN graph_nodes src ON src.node_id = e.source_node_id + FROM current_graph_edges e + JOIN current_graph_nodes src ON src.node_id = e.source_node_id WHERE src.tenant_id = %(tenant_id)s """, {"tenant_id": tenant_id}, @@ -59,7 +59,7 @@ def _load_finding_nodes(conn: Any, scan_id: str, tenant_id: str) -> list[str]: SELECT DISTINCT fgn.node_id::text FROM finding_graph_nodes fgn JOIN findings f ON f.id = fgn.finding_id - JOIN graph_nodes n ON n.node_id = fgn.node_id + JOIN current_graph_nodes n ON n.node_id = fgn.node_id WHERE f.scan_id = %(scan_id)s AND n.tenant_id = %(tenant_id)s """, @@ -108,20 +108,14 @@ def _delete_stale_paths(conn: Any, scan_id: str, tenant_id: str) -> None: with conn.cursor() as cur: cur.execute( """ - DELETE FROM attack_paths - WHERE tenant_id = %(tenant_id)s - AND scan_id != %(scan_id)s - AND scan_id IN ( - SELECT DISTINCT ap2.scan_id - FROM attack_paths ap2 - JOIN findings f ON f.scan_id = ap2.scan_id - WHERE ap2.tenant_id = %(tenant_id)s - AND f.subscription_id = ( - SELECT subscription_id FROM findings - WHERE scan_id = %(scan_id)s - LIMIT 1 - ) - ) + DELETE FROM attack_paths ap + USING scans previous, scans current + WHERE ap.tenant_id = %(tenant_id)s + AND ap.scan_id <> %(scan_id)s + AND previous.scan_id::text = ap.scan_id + AND current.scan_id = %(scan_id)s::uuid + AND current.status = 'completed' + AND previous.subscription_id = current.subscription_id """, {"tenant_id": tenant_id, "scan_id": scan_id}, ) diff --git a/tests/test_graph_paths_postgres.py b/tests/test_graph_paths_postgres.py new file mode 100644 index 00000000..7f167396 --- /dev/null +++ b/tests/test_graph_paths_postgres.py @@ -0,0 +1,151 @@ +"""Database regressions for current traversal and successful clean-scan retention.""" + +import os +import uuid +from dataclasses import replace + +import psycopg2 +import pytest + +from scanner.graph.graph_populator import populate_graph +from scanner.graph.path_traversal import _delete_stale_paths, _load_adjacency, compute_attack_paths +from tests.test_graph_freshness_postgres import snapshot + +pytestmark = pytest.mark.skipif(not os.environ.get("DATABASE_URL"), reason="requires PostgreSQL") + + +@pytest.fixture +def path_scope(): + tenant, sub = str(uuid.uuid4()), str(uuid.uuid4()) + dsn = os.environ["DATABASE_URL"] + scans = [] + + def scan(subscription=sub): + sid = str(uuid.uuid4()) + scans.append(sid) + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO scans (scan_id, subscription_id, started_at, status) VALUES (%s,%s,now(),'completed')", + (sid, subscription), + ) + return sid + + yield tenant, sub, dsn, scan + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM attack_paths WHERE tenant_id=%s", (tenant,)) + cur.execute("DELETE FROM graph_nodes WHERE tenant_id=%s", (tenant,)) + cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id=%s", (tenant,)) + cur.execute("DELETE FROM findings WHERE scan_id=ANY(%s::uuid[])", (scans,)) + cur.execute("DELETE FROM scans WHERE scan_id=ANY(%s::uuid[])", (scans,)) + + +def risky_scan(tenant, sub, dsn, sid): + snap = snapshot(tenant, sub) + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO findings (scan_id, rule_id, rule_name, severity, resource_id, detected_at, finding_key) " + "VALUES (%s,'AZ-TEST-001','test','HIGH',%s,now(),%s)", + (sid, snap.resources[1].resource_id, uuid.uuid4().hex), + ) + populate_graph(sid, snap, dsn) + return snap + + +def path_count(dsn, tenant, sid): + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("SELECT count(*) FROM attack_paths WHERE tenant_id=%s AND scan_id=%s", (tenant, sid)) + return cur.fetchone()[0] + + +def test_successful_clean_scan_clears_previous_paths_from_authoritative_scope(path_scope): + tenant, sub, dsn, scan = path_scope + risky = scan() + risky_scan(tenant, sub, dsn, risky) + # Seed one prior path so the cleanup regression is independent of traversal. + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("SELECT node_id::text FROM graph_nodes WHERE tenant_id=%s ORDER BY resource_id", (tenant,)) + nodes = [row[0] for row in cur.fetchall()] + cur.execute( + "INSERT INTO attack_paths " + "(path_id,tenant_id,scan_id,source_node_id,target_node_id,path_node_ids,path_length,relationship_types) " + "VALUES (%s,%s,%s,%s,%s,%s::uuid[],1,ARRAY['MEMBER_OF']) ON CONFLICT DO NOTHING", + (str(uuid.uuid4()), tenant, risky, nodes[0], nodes[1], nodes), + ) + clean = scan() + assert path_count(dsn, tenant, risky) == 1 + with psycopg2.connect(dsn) as conn: + _delete_stale_paths(conn, clean, tenant) + assert path_count(dsn, tenant, risky) == 0 + + +def test_partial_snapshot_does_not_traverse_retained_old_edge(path_scope): + from scanner.arg_inventory import InventoryStatus + + tenant, sub, dsn, scan = path_scope + first = scan() + risky_scan(tenant, sub, dsn, first) + with psycopg2.connect(dsn) as conn: + assert _load_adjacency(conn, tenant) + partial = snapshot(tenant, sub, status=InventoryStatus.PARTIAL, linked=False) + populate_graph(scan(), replace(partial, resources=partial.resources[:1]), dsn) + with psycopg2.connect(dsn) as conn: + assert _load_adjacency(conn, tenant) == {} + + +def test_risky_then_clean_scan_computes_zero_and_removes_old_paths(path_scope): + tenant, sub, dsn, scan = path_scope + first = scan() + risky_scan(tenant, sub, dsn, first) + assert path_count(dsn, tenant, first) == 1 + clean = scan() + populate_graph(clean, snapshot(tenant, sub, linked=False), dsn) + assert compute_attack_paths(clean, tenant, dsn) == 0 + assert path_count(dsn, tenant, first) == 0 + + +def test_clean_scan_retention_preserves_other_subscription_paths(path_scope): + tenant, sub, dsn, scan = path_scope + first = scan() + risky_scan(tenant, sub, dsn, first) + other_sub = str(uuid.uuid4()) + other = scan(other_sub) + risky_scan(tenant, other_sub, dsn, other) + clean = scan() + assert compute_attack_paths(clean, tenant, dsn) == 0 + assert path_count(dsn, tenant, first) == 0 + assert path_count(dsn, tenant, other) == 1 + + +def test_unsuccessful_scan_cannot_clear_previous_paths(path_scope): + tenant, sub, dsn, scan = path_scope + first = scan() + risky_scan(tenant, sub, dsn, first) + failed = scan() + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("UPDATE scans SET status='failed' WHERE scan_id=%s", (failed,)) + _delete_stale_paths(conn, failed, tenant) + assert path_count(dsn, tenant, first) == 1 + + +def test_cleanup_keeps_other_tenants_paths(path_scope): + tenant, sub, dsn, scan = path_scope + other_tenant = str(uuid.uuid4()) + first, other = scan(), scan() + try: + risky_scan(tenant, sub, dsn, first) + risky_scan(other_tenant, sub, dsn, other) + clean = scan() + compute_attack_paths(clean, tenant, dsn) + assert path_count(dsn, tenant, first) == 0 + assert path_count(dsn, other_tenant, other) == 1 + finally: + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("DELETE FROM graph_nodes WHERE tenant_id=%s", (other_tenant,)) + cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id=%s", (other_tenant,)) From 3546ed957e2530a23300dd2b7e1b0f9c21a87556 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 16:00:17 +0100 Subject: [PATCH 60/67] fix(graph): guard path replay and sanitize graph API validation Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 31 ++++++------ docs/api-reference.md | 6 +-- scanner/graph/path_traversal.py | 29 ++++++++++-- tests/test_attack_graph_api.py | 19 ++++++-- tests/test_graph_paths_postgres.py | 76 +++++++++++++++++++++++++++++- 5 files changed, 132 insertions(+), 29 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 815b642a..443d4887 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -26,8 +26,8 @@ def _get_db() -> DatabaseManager: def _tenant_id() -> str | None: """Resolve tenant_id from the verified principal. - OIDC mode: the 'tenant' field is populated from the 'tid' claim in the token. - Shared-secret mode: 'tenant' is always None; admins may supply + The 'tenant' field comes from the verified token's tenant claim. + For tokens without a tenant claim, admins may supply X-Tenant-Id as a request header (never a query param, which leaks into logs and caches). Non-admin tokens cannot override the header. """ @@ -60,14 +60,13 @@ def get_attack_graph(): if limit > _MAX_LIMIT: limit = _MAX_LIMIT subscription_id = request.args.get("subscription_id") - except (ValidationError, ValueError) as exc: - return jsonify({"error": str(exc)}), 400 + except (ValidationError, ValueError): + return jsonify({"error": "Invalid request parameters"}), 400 tenant_id = _tenant_id() if not tenant_id: - # Shared-secret callers have no tenant claim; OIDC is required for - # tenant-scoped graph endpoints. Return 403 (not 400) since the - # request is well-formed but the auth method is insufficient. + # A caller without a verified tenant or admin fallback cannot access + # tenant-scoped graph evidence. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 try: @@ -137,14 +136,13 @@ def list_attack_paths(): limit = positive_integer(int(request.args.get("limit", _DEFAULT_LIMIT)), "limit") if limit > _MAX_LIMIT: limit = _MAX_LIMIT - except (ValidationError, ValueError) as exc: - return jsonify({"error": str(exc)}), 400 + except (ValidationError, ValueError): + return jsonify({"error": "Invalid request parameters"}), 400 tenant_id = _tenant_id() if not tenant_id: - # Shared-secret callers have no tenant claim; OIDC is required for - # tenant-scoped graph endpoints. Return 403 (not 400) since the - # request is well-formed but the auth method is insufficient. + # A caller without a verified tenant or admin fallback cannot access + # tenant-scoped graph evidence. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 try: @@ -180,14 +178,13 @@ def get_attack_path(path_id: str): """Return a single attack path with full node detail for each hop.""" try: path_id = uuid_string(path_id, "path_id") - except (ValidationError, ValueError) as exc: - return jsonify({"error": str(exc)}), 400 + except (ValidationError, ValueError): + return jsonify({"error": "Invalid request parameters"}), 400 tenant_id = _tenant_id() if not tenant_id: - # Shared-secret callers have no tenant claim; OIDC is required for - # tenant-scoped graph endpoints. Return 403 (not 400) since the - # request is well-formed but the auth method is insufficient. + # A caller without a verified tenant or admin fallback cannot access + # tenant-scoped graph evidence. return jsonify({"error": "tenant_id not available; OIDC authentication required"}), 403 try: diff --git a/docs/api-reference.md b/docs/api-reference.md index 9a7b2464..f9b3979d 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -702,12 +702,12 @@ The following endpoints are called by the frontend but have no backend implement ## Attack graph endpoints -`GET /api/attack-graph`, `GET /api/attack-paths`, and `GET /api/attack-paths/` expose the attack graph computed from the most recent scan for the caller's tenant. +`GET /api/attack-graph` returns explicitly observed nodes and relationships from the latest published inventory snapshot for each subscription in the caller's tenant. Complete snapshots expire absent resources and relationships within their tenant/subscription scope. Partial snapshots retain historical evidence in storage, while current graph queries and traversal exclude unobserved resources and relationships. Failed collection keeps the previous published snapshot. `GET /api/attack-paths` and `GET /api/attack-paths/` expose paths computed for a requested scan. Successful subsequent scans replace prior paths for the same tenant/subscription, including scans with no findings. ### Authentication requirement -OIDC callers use the verified tenant claim. A supplied `X-Tenant-Id` header cannot override that claim. In `shared_secret` mode, an authenticated admin may supply `X-Tenant-Id` explicitly; viewer and operator tokens cannot select a tenant. Requests without a usable tenant return `403 {"error": "tenant_id not available; OIDC authentication required"}`. Tenant query parameters never determine graph scope. +Callers use the tenant claim preserved by the token verifier in either authentication mode. A supplied `X-Tenant-Id` header cannot override that claim. An authenticated admin without a tenant claim may supply `X-Tenant-Id` explicitly; viewer and operator tokens cannot select a tenant. Requests without a usable tenant return `403 {"error": "tenant_id not available; OIDC authentication required"}`. Tenant query parameters never determine graph scope. ### Attack-path retention -Attack paths are written per scan and are not automatically pruned. In long-running deployments, old scan paths accumulate in `attack_paths`. Retention policy (e.g. keep only the N most recent scans per tenant) is tracked in issue #333 and will be addressed in a follow-up. +Attack paths are replaced by a newer successful scan within the same tenant/subscription. Clean scans remove prior paths even when there are no findings. Failed scans and delayed older scans retain the newer published evidence. Scope publication and traversal serialize on the same database lock. diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index bf4f1b94..19e5ae7a 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -14,9 +14,11 @@ import psycopg2 import psycopg2.extras +from scanner.graph.node_service import lock_graph_scopes + logger = logging.getLogger(__name__) -# Paths longer than this are not persisted — they're rarely actionable and +# Paths longer than this are not persisted because they are rarely actionable and # keeping them would inflate the table for large graphs. _MAX_PATH_LENGTH = 8 @@ -103,9 +105,25 @@ def _bfs_from( return paths -def _delete_stale_paths(conn: Any, scan_id: str, tenant_id: str) -> None: +def _delete_stale_paths(conn: Any, scan_id: str, tenant_id: str) -> bool: """Delete attack paths from previous scans for the same subscription, keeping only the current scan.""" with conn.cursor() as cur: + cur.execute("SELECT subscription_id FROM scans WHERE scan_id=%s::uuid", (scan_id,)) + scope = cur.fetchone() + if scope is None: + return False + lock_graph_scopes(conn, tenant_id, [scope[0]]) + with conn.cursor() as cur: + cur.execute( + "SELECT current.status='completed' AND NOT EXISTS (" + "SELECT 1 FROM scans newer WHERE newer.subscription_id=current.subscription_id " + "AND newer.status='completed' AND newer.started_at > current.started_at) " + "FROM scans current WHERE current.scan_id=%s::uuid", + (scan_id,), + ) + row = cur.fetchone() + if row is None or not row[0]: + return False cur.execute( """ DELETE FROM attack_paths ap @@ -116,10 +134,13 @@ def _delete_stale_paths(conn: Any, scan_id: str, tenant_id: str) -> None: AND current.scan_id = %(scan_id)s::uuid AND current.status = 'completed' AND previous.subscription_id = current.subscription_id + AND previous.started_at <= current.started_at """, {"tenant_id": tenant_id, "scan_id": scan_id}, ) + return True + def _write_paths( conn: Any, @@ -169,7 +190,9 @@ def compute_attack_paths(scan_id: str, tenant_id: str, dsn: str) -> int: conn = psycopg2.connect(dsn) conn.autocommit = False try: - _delete_stale_paths(conn, scan_id, tenant_id) + if not _delete_stale_paths(conn, scan_id, tenant_id): + conn.commit() + return 0 adj = _load_adjacency(conn, tenant_id) source_nodes = _load_finding_nodes(conn, scan_id, tenant_id) if not source_nodes: diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index f021a6a5..20e7f1ee 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -116,7 +116,7 @@ def test_malformed_limit_returns_400(client, tenant_auth_headers, endpoint, limi assert response.status_code == 400 -def test_verified_viewer_token_cannot_select_tenant_in_query(client, app): +def test_verified_viewer_tenant_cannot_be_overridden_by_header_or_query(client, app): import jwt token = jwt.encode( @@ -124,7 +124,16 @@ def test_verified_viewer_token_cannot_select_tenant_in_query(client, app): app.config["JWT_SECRET"], algorithm="HS256", ) - response = client.get( - f"/api/attack-graph?tenant_id={_TENANT}", headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": _TENANT} - ) - assert response.status_code == 403 + foreign = "00000000-0000-0000-0000-000000000099" + conn = MagicMock() + cursor = conn.cursor.return_value.__enter__.return_value + cursor.fetchall.return_value = [] + db = MagicMock() + db.conn = conn + with patch("api.routes.attack_graph._get_db", return_value=db): + response = client.get( + f"/api/attack-graph?tenant_id={foreign}", + headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": foreign}, + ) + assert response.status_code == 200 + assert cursor.execute.call_args[0][1]["tenant_id"] == _TENANT diff --git a/tests/test_graph_paths_postgres.py b/tests/test_graph_paths_postgres.py index 7f167396..fca30aa2 100644 --- a/tests/test_graph_paths_postgres.py +++ b/tests/test_graph_paths_postgres.py @@ -72,7 +72,8 @@ def test_successful_clean_scan_clears_previous_paths_from_authoritative_scope(pa nodes = [row[0] for row in cur.fetchall()] cur.execute( "INSERT INTO attack_paths " - "(path_id,tenant_id,scan_id,source_node_id,target_node_id,path_node_ids,path_length,relationship_types) " + "(path_id,tenant_id,scan_id,source_node_id,target_node_id," + "path_node_ids,path_length,relationship_types) " "VALUES (%s,%s,%s,%s,%s,%s::uuid[],1,ARRAY['MEMBER_OF']) ON CONFLICT DO NOTHING", (str(uuid.uuid4()), tenant, risky, nodes[0], nodes[1], nodes), ) @@ -149,3 +150,76 @@ def test_cleanup_keeps_other_tenants_paths(path_scope): with conn.cursor() as cur: cur.execute("DELETE FROM graph_nodes WHERE tenant_id=%s", (other_tenant,)) cur.execute("DELETE FROM graph_snapshot_scopes WHERE tenant_id=%s", (other_tenant,)) + + +def test_authenticated_graph_api_selects_only_current_verified_tenant_evidence(path_scope, client, app): + import time + import jwt + from scanner.arg_inventory import InventoryStatus + + tenant, sub, dsn, scan = path_scope + first = scan() + risky_scan(tenant, sub, dsn, first) + partial = snapshot(tenant, sub, status=InventoryStatus.PARTIAL, linked=False) + populate_graph(scan(), replace(partial, resources=partial.resources[:1]), dsn) + token = jwt.encode( + {"sub": "graph-viewer", "role": "viewer", "tid": tenant, "exp": int(time.time()) + 60}, + app.config["JWT_SECRET"], + algorithm="HS256", + ) + foreign = str(uuid.uuid4()) + response = client.get( + f"/api/attack-graph?subscription_id={sub}&tenant_id={foreign}", + headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": foreign}, + ) + assert response.status_code == 200 + result = response.get_json() + assert len(result["nodes"]) == 1 + assert result["nodes"][0]["resource_id"] == partial.resources[0].resource_id.lower() + assert result["edges"] == [] + + +def test_delayed_older_traversal_cannot_delete_newer_paths(path_scope): + tenant, sub, dsn, scan = path_scope + older, newer = scan(), scan() + with psycopg2.connect(dsn) as conn: + with conn.cursor() as cur: + cur.execute("UPDATE scans SET started_at=now()-interval '2 minutes' WHERE scan_id=%s", (older,)) + cur.execute("UPDATE scans SET started_at=now()-interval '1 minute' WHERE scan_id=%s", (newer,)) + risky_scan(tenant, sub, dsn, newer) + assert path_count(dsn, tenant, newer) == 1 + assert compute_attack_paths(older, tenant, dsn) == 0 + assert path_count(dsn, tenant, newer) == 1 + + +def test_traversal_waits_for_scope_publication_lock(path_scope): + from concurrent.futures import ThreadPoolExecutor + from threading import Event + + tenant, sub, dsn, scan = path_scope + sid = scan() + risky_scan(tenant, sub, dsn, sid) + entered = Event() + blocker = psycopg2.connect(dsn) + try: + with blocker.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (f"openshield-graph:{tenant}:{sub}",)) + + def traverse(): + entered.set() + return compute_attack_paths(sid, tenant, dsn) + + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(traverse) + assert entered.wait(2) + try: + future.result(timeout=0.3) + pytest.fail("traversal completed while its scope lock was held") + except TimeoutError: + pass + finally: + blocker.rollback() + assert future.result(timeout=5) == 0 + assert path_count(dsn, tenant, sid) == 1 + finally: + blocker.close() From 68ec94373ab5d772f7dee02c4142214450eecd16 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Thu, 8 Oct 2026 16:00:57 +0100 Subject: [PATCH 61/67] docs(graph): preserve original API reference encoding Signed-off-by: Tanvir Farhad --- docs/api-reference.md | 46 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/api-reference.md b/docs/api-reference.md index f9b3979d..a1771df0 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -7,7 +7,7 @@ routes become public only when the deliberate demo-mode setting is enabled. ## Authentication -`/`, `/health`, `/ready`, and `/metrics` are always public. All other routes — including all `/api/*` GET endpoints — require an `Authorization: Bearer ` header, verified by `api/auth.py` according to `OPENSHIELD_AUTH_MODE`: +`/`, `/health`, `/ready`, and `/metrics` are always public. All other routes — including all `/api/*` GET endpoints — require an `Authorization: Bearer ` header, verified by `api/auth.py` according to `OPENSHIELD_AUTH_MODE`: | Mode | Verification | Role source | Intended for | |---|---|---|---| @@ -27,7 +27,7 @@ The dashboard never embeds a token (see [authentication and containment](securit ### Subscription authorization -`POST /api/scans/trigger` also checks `subscription_id` against `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS`, a comma-separated allowlist. A valid `operator`/`admin` token can otherwise trigger a scan against *any* subscription_id — role alone doesn't say which subscription a caller is entitled to. Left unset, every subscription_id is accepted (matches historical behavior); the API logs a loud startup warning when it's unset. This is a single-tenant containment boundary, not a substitute for real per-tenant authorization — see issue #294 for the remaining tenant-ownership scope this is a stopgap for. +`POST /api/scans/trigger` also checks `subscription_id` against `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS`, a comma-separated allowlist. A valid `operator`/`admin` token can otherwise trigger a scan against *any* subscription_id — role alone doesn't say which subscription a caller is entitled to. Left unset, every subscription_id is accepted (matches historical behavior); the API logs a loud startup warning when it's unset. This is a single-tenant containment boundary, not a substitute for real per-tenant authorization — see issue #294 for the remaining tenant-ownership scope this is a stopgap for. ## Input limits @@ -185,7 +185,7 @@ Example response: Returns the details and current status of a specific scan. -Path parameters: `scan_id` — UUID of the scan. +Path parameters: `scan_id` — UUID of the scan. Example response: @@ -208,7 +208,7 @@ Example response: Admits an asynchronous scan against the configured subscription. Execution happens in a background worker process; the response returns as soon as the scan is durably recorded. -Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): +Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): ```json { @@ -221,7 +221,7 @@ Request body (optional — falls back to `AZURE_SUBSCRIPTION_ID`): Admission is serialized per subscription and enforced by the database, so concurrent and replayed triggers converge on one logical scan rather than creating duplicates: - **At most one active scan per subscription.** While a `pending` or `running` scan exists, a further trigger returns that existing scan instead of queueing another. -- **`Idempotency-Key` (optional request header, 1–200 characters).** A repeat of the same key for the same subscription returns the original scan. The key is scoped to the subscription; the same key under a different subscription is a different request. A trigger carries no request input other than `subscription_id`, so a key that resolves to an existing scan is always a replay of the same logical request and there is no changed-payload conflict to report. +- **`Idempotency-Key` (optional request header, 1–200 characters).** A repeat of the same key for the same subscription returns the original scan. The key is scoped to the subscription; the same key under a different subscription is a different request. A trigger carries no request input other than `subscription_id`, so a key that resolves to an existing scan is always a replay of the same logical request and there is no changed-payload conflict to report. - **`OPENSHIELD_MAX_SCANS_PER_SUBSCRIPTION_PER_HOUR`** adds an optional hourly admission quota. Unset or `0` (the default) applies no time-window limit; the one-active-scan rule still applies. ### Responses @@ -229,8 +229,8 @@ Admission is serialized per subscription and enforced by the database, so concur | Status | When | Body | | --- | --- | --- | | `202 Accepted` | A new scan was admitted and queued. | `scan_id`, `status: "pending"`, `message` | -| `200 OK` | The request resolved to an existing logical scan — an `Idempotency-Key` replay of the same request, or a trigger while a scan is already active for the subscription. | `scan_id`, `status` (the existing scan's `pending`/`running`), `message: "Existing logical scan returned."` | -| `400 Bad Request` | Malformed body, invalid `subscription_id`, missing subscription, or an `Idempotency-Key` outside 1–200 characters. | `error` | +| `200 OK` | The request resolved to an existing logical scan — an `Idempotency-Key` replay of the same request, or a trigger while a scan is already active for the subscription. | `scan_id`, `status` (the existing scan's `pending`/`running`), `message: "Existing logical scan returned."` | +| `400 Bad Request` | Malformed body, invalid `subscription_id`, missing subscription, or an `Idempotency-Key` outside 1–200 characters. | `error` | | `403 Forbidden` | `subscription_id` is not on the `OPENSHIELD_AUTHORIZED_SUBSCRIPTIONS` allowlist. | `error` | | `429 Too Many Requests` | The configured hourly quota for this subscription is exhausted. | `error: "Scan quota exceeded for this subscription."` | @@ -266,7 +266,7 @@ Missing subscription response (`400`): ## POST /api/scans/<scan_id>/enrich -Queues durable CVE enrichment for a completed scan's findings. Enrichment runs as a database-backed job claimed by the background worker — the request never owns a thread, so the work survives an API restart. +Queues durable CVE enrichment for a completed scan's findings. Enrichment runs as a database-backed job claimed by the background worker — the request never owns a thread, so the work survives an API restart. There is **never more than one enrichment job per scan**. Repeat calls are safe: they report the state of the single job rather than creating another. @@ -280,11 +280,11 @@ Every response carries `scan_id`, `job_id`, `status` (the job row's state) and a | `202 Accepted` | `requeued` | A previously **failed** job was reset to `pending` and will be retried. | | `202 Accepted` | `active` | A `pending` or `running` job already exists and was returned unchanged. A live claim is never interrupted. | | `200 OK` | `completed` | Enrichment already finished; nothing was restarted. | -| `404 Not Found` | — | Unknown `scan_id`, or the scan has no findings to enrich and has not already been enriched. | +| `404 Not Found` | — | Unknown `scan_id`, or the scan has no findings to enrich and has not already been enriched. | An already-enriched scan always reports `completed`, including a clean scan that had no findings to enrich in the first place. -A job that exhausts its retry budget becomes `failed`. Re-POSTing this endpoint is the supported operator recovery: it atomically returns the job to `pending` with a fresh retry budget, clears the lease, and keeps the last `error_message` and the `checkpoint` so the retry resumes rather than re-enriching findings that already succeeded. Concurrent re-POSTs converge — exactly one reports `requeued` and the rest report `active`. +A job that exhausts its retry budget becomes `failed`. Re-POSTing this endpoint is the supported operator recovery: it atomically returns the job to `pending` with a fresh retry budget, clears the lease, and keeps the last `error_message` and the `checkpoint` so the retry resumes rather than re-enriching findings that already succeeded. Concurrent re-POSTs converge — exactly one reports `requeued` and the rest report `active`. Newly queued (`202`): @@ -316,7 +316,7 @@ Poll `GET /api/scans/` for `cve_enrichment_status` (`PENDING`, `ENRICHI ## GET /api/score -Returns the overall security posture score from 0 to 100. Under [severity contract v1](severity-contract.md), the score starts at 100 and deducts 20 per CRITICAL finding, 10 per HIGH finding, 5 per MEDIUM finding, and 2 per LOW finding; INFO findings deduct zero. Scoped to the most recent **completed** scan — if no completed scan exists yet, this returns `status: "NO_SCAN_DATA"` with `score: null` rather than a misleading 100 (a scan with no findings and no evidence at all would otherwise be indistinguishable). +Returns the overall security posture score from 0 to 100. Under [severity contract v1](severity-contract.md), the score starts at 100 and deducts 20 per CRITICAL finding, 10 per HIGH finding, 5 per MEDIUM finding, and 2 per LOW finding; INFO findings deduct zero. Scoped to the most recent **completed** scan — if no completed scan exists yet, this returns `status: "NO_SCAN_DATA"` with `score: null` rather than a misleading 100 (a scan with no findings and no evidence at all would otherwise be indistinguishable). Query parameters: @@ -345,13 +345,13 @@ Example response (no completed scan exists yet): } ``` -Consumers must check `status` and treat a `null` `score` as "not assessed" — never coerce it to `0`, which would misrepresent absence of evidence as a confirmed worst-case score. +Consumers must check `status` and treat a `null` `score` as "not assessed" — never coerce it to `0`, which would misrepresent absence of evidence as a confirmed worst-case score. --- ## GET /api/compliance/<framework> -Returns technical-evidence coverage against a compliance framework mapping pack, scoped to the most recent **completed** scan. This is coverage, not a certification or a claim of full framework compliance — see `docs/compliance-mapping-pack.md` for the full mapping-pack schema and `evaluation_basis` semantics. +Returns technical-evidence coverage against a compliance framework mapping pack, scoped to the most recent **completed** scan. This is coverage, not a certification or a claim of full framework compliance — see `docs/compliance-mapping-pack.md` for the full mapping-pack schema and `evaluation_basis` semantics. Supported frameworks: @@ -371,10 +371,10 @@ Query parameters: | `subscription_id` | UUID string | No | Scopes the "most recent completed scan" lookup to one Azure subscription. Defaults to the deployment's `AZURE_SUBSCRIPTION_ID`; if neither is set the latest completed scan from *any* subscription is used, which is only correct for a single-tenant database. A malformed value is a `400`, never a silently unscoped result. | `status` is one of: -- `OK` — a completed scan exists and at least one mapped control is in scope; `score_percent` is a real evaluated percentage. -- `NO_SCAN_DATA` — no completed scan exists yet, so there is no evidence to report; `score_percent` is `null`. -- `NO_REVIEWED_CONTROLS` — a completed scan exists, but every mapping awaits review; `score_percent` is `null` and no direct-evidence score exists. -- `NO_IN_SCOPE_CONTROLS` — a completed scan exists, but every reviewed mapped control for this framework is `not_applicable`/`organizational` and excluded from the denominator; `score_percent` is `null`. +- `OK` — a completed scan exists and at least one mapped control is in scope; `score_percent` is a real evaluated percentage. +- `NO_SCAN_DATA` — no completed scan exists yet, so there is no evidence to report; `score_percent` is `null`. +- `NO_REVIEWED_CONTROLS` — a completed scan exists, but every mapping awaits review; `score_percent` is `null` and no direct-evidence score exists. +- `NO_IN_SCOPE_CONTROLS` — a completed scan exists, but every reviewed mapped control for this framework is `not_applicable`/`organizational` and excluded from the denominator; `score_percent` is `null`. Per-control `status` is evaluation-derived (issue #263) only after its mapping has been reviewed: `PASS`/`FAIL`/`UNKNOWN`/`ERROR` is the rolled-up status of @@ -391,7 +391,7 @@ Any control with `review_status` other than `reviewed` has control status `UNREVIEWED_MAPPING`, regardless of its `mapping_type`. It is excluded from the denominator alongside `not_applicable` and `organizational` controls, and does not contribute to `passed` or `failed`. Consumers must check `status` and -never treat a `null` `score_percent` as `0` — a missing/excluded score is a +never treat a `null` `score_percent` as `0` — a missing/excluded score is a different fact from a real, evaluated 0%. `reviewed_controls`, `unreviewed_controls`, `not_applicable`, `organizational`, and `excluded_controls` make the reason explicit. @@ -440,7 +440,7 @@ Example response (`OK`): } ``` -Example response (`NO_SCAN_DATA`, HTTP 200 — never 500): +Example response (`NO_SCAN_DATA`, HTTP 200 — never 500): ```json { @@ -471,7 +471,7 @@ Unknown framework response (HTTP 400): ## GET /api/resources -Returns unique Azure resources derived from the most recent scan that has findings. Resources are aggregated from findings — one entry per distinct `resource_id`. Risk level is computed from the maximum severity finding on each resource. +Returns unique Azure resources derived from the most recent scan that has findings. Resources are aggregated from findings — one entry per distinct `resource_id`. Risk level is computed from the maximum severity finding on each resource. Query parameters: none @@ -513,7 +513,7 @@ No findings response (no scan with findings exists): ## GET /api/prioritization -Returns findings from the most recent scan grouped and ranked by risk score (`severity_weight × affected_resource_count`). Produces a matrix view, a ranked list, and recommended action items. +Returns findings from the most recent scan grouped and ranked by risk score (`severity_weight × affected_resource_count`). Produces a matrix view, a ranked list, and recommended action items. Query parameters: none @@ -613,7 +613,7 @@ No drift response (fewer than two scans): Returns the structured remediation playbook for a specific finding. Loads the matching `playbooks/cli/fix_.sh` script and wraps the finding's remediation text as a portal step. Appends NVD links from any CVE references on the finding. -Path parameters: `id` — integer finding ID from `GET /api/findings`. +Path parameters: `id` — integer finding ID from `GET /api/findings`. Example response: @@ -696,7 +696,7 @@ The following endpoints are called by the frontend but have no backend implement | Endpoint | Used by | Status | |---|---|---| -| `GET /api/monitoring` | Monitoring page — score trend chart, category distribution | Deferred. Score and findings data come from `GET /api/score` and `GET /api/findings` instead. | +| `GET /api/monitoring` | Monitoring page — score trend chart, category distribution | Deferred. Score and findings data come from `GET /api/score` and `GET /api/findings` instead. | --- From 571a4799e063d16f12ab65d79152c369adfc4878 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 15:28:05 +0100 Subject: [PATCH 62/67] fix(graph): add /v1/ prefix to all attack-graph and attack-paths API routes Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 6 +++--- tests/test_attack_graph_api.py | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 443d4887..748234cd 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -49,7 +49,7 @@ def _close_db(exc): db.close() -@attack_graph_bp.get("/api/attack-graph") +@attack_graph_bp.get("/api/v1/attack-graph") def get_attack_graph(): """Return graph nodes and edges for the caller's tenant (latest snapshot). @@ -122,7 +122,7 @@ def get_attack_graph(): return jsonify({"nodes": [dict(r) for r in nodes], "edges": [dict(r) for r in edges]}) -@attack_graph_bp.get("/api/attack-paths") +@attack_graph_bp.get("/api/v1/attack-paths") def list_attack_paths(): """Return pre-computed attack paths for a scan. @@ -173,7 +173,7 @@ def list_attack_paths(): return jsonify({"scan_id": scan_id, "paths": [dict(r) for r in rows]}) -@attack_graph_bp.get("/api/attack-paths/") +@attack_graph_bp.get("/api/v1/attack-paths/") def get_attack_path(path_id: str): """Return a single attack path with full node detail for each hop.""" try: diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index 20e7f1ee..6617fa63 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -54,7 +54,7 @@ def test_get_attack_graph_returns_empty_nodes_and_edges(client, tenant_auth_head conn = _mock_conn([[]]) with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): - resp = client.get("/api/attack-graph", headers=tenant_auth_headers) + resp = client.get("/api/v1/attack-graph", headers=tenant_auth_headers) assert resp.status_code == 200 data = resp.get_json() assert "nodes" in data @@ -77,23 +77,23 @@ def test_get_attack_graph_viewer_cannot_supply_tenant_header(client, app): "Authorization": f"Bearer {token}", "X-Tenant-Id": _TENANT, } - resp = client.get("/api/attack-graph", headers=headers) + resp = client.get("/api/v1/attack-graph", headers=headers) assert resp.status_code == 403 def test_list_attack_paths_missing_scan_id_returns_400(client, tenant_auth_headers): - resp = client.get("/api/attack-paths", headers=tenant_auth_headers) + resp = client.get("/api/v1/attack-paths", headers=tenant_auth_headers) assert resp.status_code == 400 assert b"scan_id" in resp.data def test_list_attack_paths_invalid_uuid_returns_400(client, tenant_auth_headers): - resp = client.get("/api/attack-paths?scan_id=not-a-uuid", headers=tenant_auth_headers) + resp = client.get("/api/v1/attack-paths?scan_id=not-a-uuid", headers=tenant_auth_headers) assert resp.status_code == 400 def test_get_attack_path_invalid_uuid_returns_400(client, tenant_auth_headers): - resp = client.get("/api/attack-paths/not-a-uuid", headers=tenant_auth_headers) + resp = client.get("/api/v1/attack-paths/not-a-uuid", headers=tenant_auth_headers) assert resp.status_code == 400 @@ -102,7 +102,7 @@ def test_get_attack_path_not_found_returns_404(client, tenant_auth_headers): conn = _mock_conn([[]]) with patch("api.routes.attack_graph.psycopg2.connect", return_value=conn): with patch.dict("os.environ", {"DATABASE_URL": "postgresql://fake/db"}): - resp = client.get(f"/api/attack-paths/{valid_uuid}", headers=tenant_auth_headers) + resp = client.get(f"/api/v1/attack-paths/{valid_uuid}", headers=tenant_auth_headers) assert resp.status_code == 404 From f38e3c38b9635187a4216b3bd70857bd2e614922 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 15:32:13 +0100 Subject: [PATCH 63/67] fix(graph): fix remaining old-path references in attack-graph test parametrize and tenant test Signed-off-by: Tanvir Farhad --- tests/test_attack_graph_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_attack_graph_api.py b/tests/test_attack_graph_api.py index 6617fa63..453b08a2 100644 --- a/tests/test_attack_graph_api.py +++ b/tests/test_attack_graph_api.py @@ -108,7 +108,7 @@ def test_get_attack_path_not_found_returns_404(client, tenant_auth_headers): @pytest.mark.parametrize("limit", ["abc", "", "1.5", "0", "-1"]) @pytest.mark.parametrize( - "endpoint", ["/api/attack-graph", "/api/attack-paths?scan_id=00000000-0000-0000-0000-000000000002"] + "endpoint", ["/api/v1/attack-graph", "/api/v1/attack-paths?scan_id=00000000-0000-0000-0000-000000000002"] ) def test_malformed_limit_returns_400(client, tenant_auth_headers, endpoint, limit): separator = "&" if "?" in endpoint else "?" @@ -132,7 +132,7 @@ def test_verified_viewer_tenant_cannot_be_overridden_by_header_or_query(client, db.conn = conn with patch("api.routes.attack_graph._get_db", return_value=db): response = client.get( - f"/api/attack-graph?tenant_id={foreign}", + f"/api/v1/attack-graph?tenant_id={foreign}", headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": foreign}, ) assert response.status_code == 200 From a40be234fa36aa72079e9997c96356772520755d Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Fri, 9 Oct 2026 15:56:18 +0100 Subject: [PATCH 64/67] fix(graph): update /api/attack-graph path to /api/v1/attack-graph in postgres integration test Signed-off-by: Tanvir Farhad --- tests/test_graph_paths_postgres.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_graph_paths_postgres.py b/tests/test_graph_paths_postgres.py index fca30aa2..7b3f288e 100644 --- a/tests/test_graph_paths_postgres.py +++ b/tests/test_graph_paths_postgres.py @@ -169,7 +169,7 @@ def test_authenticated_graph_api_selects_only_current_verified_tenant_evidence(p ) foreign = str(uuid.uuid4()) response = client.get( - f"/api/attack-graph?subscription_id={sub}&tenant_id={foreign}", + f"/api/v1/attack-graph?subscription_id={sub}&tenant_id={foreign}", headers={"Authorization": f"Bearer {token}", "X-Tenant-Id": foreign}, ) assert response.status_code == 200 From e0a38cd03083dfcede8c7f30a12aa065d7148794 Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sun, 11 Oct 2026 13:56:54 +0100 Subject: [PATCH 65/67] fix(graph): rechain attack_paths migration onto graph_current_scope to restore single alembic head Signed-off-by: Tanvir Farhad --- alembic/versions/f2a3b4c5d6e7_attack_paths.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alembic/versions/f2a3b4c5d6e7_attack_paths.py b/alembic/versions/f2a3b4c5d6e7_attack_paths.py index e312c062..f60b5cdc 100644 --- a/alembic/versions/f2a3b4c5d6e7_attack_paths.py +++ b/alembic/versions/f2a3b4c5d6e7_attack_paths.py @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "f2a3b4c5d6e7" -down_revision: Union[str, Sequence[str], None] = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "e2c4a6b8d013" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None From 373d5994cc8bf242b83b79186cf2bf099d2db6de Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Sat, 17 Oct 2026 15:53:36 +0100 Subject: [PATCH 66/67] fix(graph): cap unbounded edges query and adjacency load in BFS traversal api/routes/attack_graph.py: added LIMIT to the edges query (limit * 10) to match the node cap and prevent unbounded response sizes on dense graphs. scanner/graph/path_traversal.py: added _MAX_ADJACENCY_EDGES = 50_000 cap to _load_adjacency so large tenants do not OOM the worker process during BFS. BFS still runs correctly on the capped subset. tests/test_graph_paths_postgres.py: fixed TimeoutError catch to use concurrent.futures.TimeoutError explicitly for Python 3.10 compatibility. Signed-off-by: Tanvir Farhad --- api/routes/attack_graph.py | 3 ++- scanner/graph/path_traversal.py | 12 ++++++++++-- tests/test_graph_paths_postgres.py | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/api/routes/attack_graph.py b/api/routes/attack_graph.py index 748234cd..ef57febc 100644 --- a/api/routes/attack_graph.py +++ b/api/routes/attack_graph.py @@ -111,8 +111,9 @@ def get_attack_graph(): FROM current_graph_edges e WHERE e.source_node_id = ANY(%(node_ids)s::uuid[]) AND e.target_node_id = ANY(%(node_ids)s::uuid[]) + LIMIT %(edges_limit)s """, - {"node_ids": node_ids}, + {"node_ids": node_ids, "edges_limit": limit * 10}, ) edges = cur.fetchall() except Exception: diff --git a/scanner/graph/path_traversal.py b/scanner/graph/path_traversal.py index 19e5ae7a..9e3c5fb9 100644 --- a/scanner/graph/path_traversal.py +++ b/scanner/graph/path_traversal.py @@ -28,8 +28,15 @@ _REVERSE_RELS: frozenset[str] = frozenset({"EXPOSES", "HAS_IDENTITY"}) +_MAX_ADJACENCY_EDGES = 50_000 + + def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, float]]]: - """Return {source_node_id: [(target_node_id, relationship_type, confidence)]}.""" + """Return {source_node_id: [(target_node_id, relationship_type, confidence)]}. + + Capped at _MAX_ADJACENCY_EDGES rows. Tenants that exceed this threshold get + a partial graph; BFS still runs correctly on the subset. + """ with conn.cursor() as cur: cur.execute( """ @@ -38,8 +45,9 @@ def _load_adjacency(conn: Any, tenant_id: str) -> dict[str, list[tuple[str, str, FROM current_graph_edges e JOIN current_graph_nodes src ON src.node_id = e.source_node_id WHERE src.tenant_id = %(tenant_id)s + LIMIT %(max_edges)s """, - {"tenant_id": tenant_id}, + {"tenant_id": tenant_id, "max_edges": _MAX_ADJACENCY_EDGES}, ) # Only traverse in reverse for relationships where the reverse direction # is semantically meaningful. MEMBER_OF and PROTECTS are not reversed: diff --git a/tests/test_graph_paths_postgres.py b/tests/test_graph_paths_postgres.py index 7b3f288e..d10fa68f 100644 --- a/tests/test_graph_paths_postgres.py +++ b/tests/test_graph_paths_postgres.py @@ -193,7 +193,7 @@ def test_delayed_older_traversal_cannot_delete_newer_paths(path_scope): def test_traversal_waits_for_scope_publication_lock(path_scope): - from concurrent.futures import ThreadPoolExecutor + from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError from threading import Event tenant, sub, dsn, scan = path_scope @@ -215,7 +215,7 @@ def traverse(): try: future.result(timeout=0.3) pytest.fail("traversal completed while its scope lock was held") - except TimeoutError: + except FuturesTimeoutError: pass finally: blocker.rollback() From b1c636a01e0a69f1072c8d79f021908b49ed6c5a Mon Sep 17 00:00:00 2001 From: Tanvir Farhad Date: Tue, 20 Oct 2026 01:29:56 +0100 Subject: [PATCH 67/67] fix(graph): rebase onto dev, fix lint in graph populator and path traversal - Rebase feat/333-evidence-graph-path-api onto origin/dev (66 commits) - Resolve merge conflicts in scanner/engine.py and scanner/worker.py preserving both the evaluate() path with failed_rule_ids and the lifecycle rule_outcomes tracking - Fix duplicate alembic revision e1f2a3b4c5d6: rename finding_lifecycle migration to f3a5b7c9d1e2 and rechain graph_schema after it - Fix scans.py missing enrich_findings import introduced by conflict merge - Add subscription_id and tid claim passthrough in shared-secret JWT verifier so patterns route and attack-graph tenant override tests pass - Fix test_finding_lifecycle assertion to check SQL params for from_state Signed-off-by: Tanvir Farhad --- alembic/versions/e1f2a3b4c5d6_graph_schema.py | 4 ++-- ...ing_lifecycle.py => f3a5b7c9d1e2_finding_lifecycle.py} | 8 ++++---- api/auth.py | 4 +++- api/routes/scans.py | 1 + tests/test_finding_lifecycle.py | 8 +++++++- 5 files changed, 17 insertions(+), 8 deletions(-) rename alembic/versions/{e1f2a3b4c5d6_finding_lifecycle.py => f3a5b7c9d1e2_finding_lifecycle.py} (97%) diff --git a/alembic/versions/e1f2a3b4c5d6_graph_schema.py b/alembic/versions/e1f2a3b4c5d6_graph_schema.py index 24ece564..fec4c270 100644 --- a/alembic/versions/e1f2a3b4c5d6_graph_schema.py +++ b/alembic/versions/e1f2a3b4c5d6_graph_schema.py @@ -1,7 +1,7 @@ """Add graph_nodes, graph_edges, and finding_graph_nodes for attack graph. Revision ID: e1f2a3b4c5d6 -Revises: a8d9c2e4f601 +Revises: f3a5b7c9d1e2 Create Date: 2026-09-24 00:00:00.000000 """ @@ -12,7 +12,7 @@ from sqlalchemy.dialects import postgresql revision: str = "e1f2a3b4c5d6" -down_revision: str = "a8d9c2e4f601" +down_revision: str = "f3a5b7c9d1e2" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py b/alembic/versions/f3a5b7c9d1e2_finding_lifecycle.py similarity index 97% rename from alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py rename to alembic/versions/f3a5b7c9d1e2_finding_lifecycle.py index c856176e..86917e7f 100644 --- a/alembic/versions/e1f2a3b4c5d6_finding_lifecycle.py +++ b/alembic/versions/f3a5b7c9d1e2_finding_lifecycle.py @@ -1,8 +1,8 @@ """Add finding lifecycle tables: scan_rule_outcomes, scan_lifecycle_applications, finding_fingerprints, finding_lifecycles, finding_lifecycle_transitions, patterns. -Revision ID: e1f2a3b4c5d6 -Revises: d8e4f6a1b2c3 +Revision ID: f3a5b7c9d1e2 +Revises: b6d2f8a4c1e7 Create Date: 2026-08-30 00:00:00.000000 """ @@ -11,8 +11,8 @@ from alembic import op # Revision identifiers, used by Alembic. -revision: str = "e1f2a3b4c5d6" -down_revision: Union[str, Sequence[str], None] = "d8e4f6a1b2c3" +revision: str = "f3a5b7c9d1e2" +down_revision: Union[str, Sequence[str], None] = "b6d2f8a4c1e7" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/api/auth.py b/api/auth.py index 97c2d91a..a2b66a6f 100644 --- a/api/auth.py +++ b/api/auth.py @@ -197,12 +197,14 @@ def _verify_shared_secret(self, token: str) -> Dict[str, Any]: if role not in KNOWN_ROLES: logger.warning("JWT rejected: missing or unrecognized role %r", role) raise TokenRejected("Invalid token") + raw_tid = claims.get("tid") return { "sub": claims["sub"], "role": role, - "tenant": None, + "tenant": str(raw_tid).lower() if raw_tid else None, "issuer": claims.get("iss"), "auth_mode": SHARED_SECRET_MODE, + "subscription_id": claims.get("subscription_id"), } def _get_jwks_client(self) -> Any: diff --git a/api/routes/scans.py b/api/routes/scans.py index 42c90b9b..83b07f29 100644 --- a/api/routes/scans.py +++ b/api/routes/scans.py @@ -6,6 +6,7 @@ from flask import Blueprint, g, jsonify, request from api.models.finding import DatabaseManager, ScanQuotaExceeded +from scanner.cve_correlator import enrich_findings from api.validation import ( VALIDATION_ERROR_MESSAGE, ValidationError, diff --git a/tests/test_finding_lifecycle.py b/tests/test_finding_lifecycle.py index 23307410..ab45fa4b 100644 --- a/tests/test_finding_lifecycle.py +++ b/tests/test_finding_lifecycle.py @@ -359,7 +359,13 @@ def test_reopened_finding_absent_from_success_scan_is_resolved(self): sqls = self._all_sql(conn) assert any("RESOLVED" in s and "UPDATE" in s.upper() for s in sqls) # Transition from REOPENED to RESOLVED must be recorded. - assert any("finding_lifecycle_transitions" in s and "REOPENED" in s and "RESOLVED" in s for s in sqls) + # from_state is passed as a parameter (%s), so check both SQL and params. + assert any( + "finding_lifecycle_transitions" in sql + and "RESOLVED" in sql + and "REOPENED" in str(params) + for sql, params in conn.all_executed() + ) def test_reopened_finding_seen_again_increments_occurrence_stays_reopened(self): # REOPENED + seen in scan: occurrence_count increments, no new state transition.