diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1fcdbc..b182a02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,21 @@ jobs: - name: Run tests run: python -m pytest tests/ -v --tb=short + + - name: Generate SBOM (CycloneDX) + # Never fails the job; the dedicated scan below is the vuln gate. + continue-on-error: true + run: pip-audit -r requirements.txt --format cyclonedx-json --output sbom.json + + - name: Upload SBOM artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: sbom-cyclonedx + path: sbom.json + if-no-files-found: warn + + - name: Vulnerability scan (pip-audit) + # Fails CI on known-vuln pinned deps. Document any unavoidable, no-fix + # advisory with `--ignore-vuln ` and a reason here — never blanket-ignore. + run: pip-audit -r requirements.txt diff --git a/core/procore/review_store.py b/core/procore/review_store.py index 3fe641d..81e66ef 100644 --- a/core/procore/review_store.py +++ b/core/procore/review_store.py @@ -9,6 +9,7 @@ import json import logging +import os import threading from pathlib import Path @@ -19,8 +20,17 @@ _LOCK = threading.Lock() +def _local_jsonl_enabled() -> bool: + """Local JSONL persistence is off by default (production-safe); the durable + record is the Supabase audit trail. Enable with PROCORE_LOCAL_JSONL_ENABLED=true.""" + return os.getenv("PROCORE_LOCAL_JSONL_ENABLED", "false").strip().lower() == "true" + + def store_artifact(artifact: dict) -> None: """Append a full review artifact to the store.""" + if not _local_jsonl_enabled(): + log.debug("Local JSONL disabled; skipping artifact store write") + return _DATA_DIR.mkdir(parents=True, exist_ok=True) with _LOCK: with open(_ARTIFACT_STORE, "a", encoding="utf-8") as f: diff --git a/core/procore/webhook_handler.py b/core/procore/webhook_handler.py index befabb1..8d95b98 100644 --- a/core/procore/webhook_handler.py +++ b/core/procore/webhook_handler.py @@ -203,8 +203,21 @@ def extract_submittal_attachments(event: WebhookEvent) -> list[SubmittalAttachme return result +def _local_jsonl_enabled() -> bool: + """Whether local JSONL persistence under src/data is enabled. + + Default off (production-safe): the durable record is the Supabase audit + trail (migration 008). Local JSONL is dev convenience only; enable with + PROCORE_LOCAL_JSONL_ENABLED=true. + """ + return os.getenv("PROCORE_LOCAL_JSONL_ENABLED", "false").strip().lower() == "true" + + def log_payload(event: WebhookEvent) -> None: """Append the raw payload to the payload log for replay/debugging.""" + if not _local_jsonl_enabled(): + log.debug("Local JSONL disabled; skipping payload log write") + return _DATA_DIR.mkdir(parents=True, exist_ok=True) record = { "logged_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), @@ -220,6 +233,9 @@ def log_payload(event: WebhookEvent) -> None: def log_review(review_artifact: dict, event: WebhookEvent) -> None: """Append the review artifact to the review log.""" + if not _local_jsonl_enabled(): + log.debug("Local JSONL disabled; skipping review log write") + return _DATA_DIR.mkdir(parents=True, exist_ok=True) record = { "logged_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), diff --git a/requirements-dev.txt b/requirements-dev.txt index 558e846..6f99bcd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,4 @@ pytest-xdist==3.8.0 pytest-asyncio flake8 pre-commit +pip-audit[cyclonedx] diff --git a/requirements.txt b/requirements.txt index 4c5acc2..d11a531 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ anthropic==0.84.0 -cryptography==46.0.7 +cryptography==48.0.1 ecdsa==0.19.2 fastapi==0.135.1 httpx==0.28.1 @@ -10,11 +10,11 @@ Pillow==12.2.0 pyasn1==0.6.3 pydantic[email]==2.12.5 pymupdf==1.27.1 -pypdf==6.9.2 +pypdf==6.13.0 python-docx==1.2.0 -python-dotenv==1.1.0 +python-dotenv==1.2.2 python-jose[cryptography]==3.5.0 -python-multipart==0.0.22 +python-multipart==0.0.31 rapidfuzz==3.14.3 requests==2.33.0 sentry-sdk[fastapi]==2.54.0 diff --git a/supabase/migrations/008_procore_audit.sql b/supabase/migrations/008_procore_audit.sql new file mode 100644 index 0000000..58511c4 --- /dev/null +++ b/supabase/migrations/008_procore_audit.sql @@ -0,0 +1,161 @@ +-- 008_procore_audit.sql +-- Minimal, append-only audit trail for Procore SWMS reviews (T6). +-- +-- Stores ONLY audit metadata (hashes, rule-pack versions, finding counts, +-- status, write-back metadata, human-override trail) — never raw SWMS text or +-- Procore document content. Supports the "Procore remains system of record / +-- minimal audit metadata retained" data-handling statement. +-- +-- Hardening (per docs/procore/STAGE3_CERTIFICATION_PLAN_V2.md §5 + pre-flight): +-- * table lives in a PRIVATE (non-PostgREST-exposed) schema with RLS; +-- * append-only: UPDATE always blocked; DELETE only via the controlled +-- retention/deletion functions (SECURITY DEFINER, run as owner); +-- * writes/purges/deletions go through SECURITY DEFINER functions in public +-- whose EXECUTE is granted to service_role only. + +create schema if not exists private; + +create table if not exists private.procore_audit ( + id bigint generated always as identity primary key, + record_type text not null default 'review', -- 'review' | 'override' + review_run_id text, + delivery_key text, + correlation_id text, + company_id bigint, + project_id bigint, + document_hash text, -- sha256 of the document; never raw SWMS text + rule_pack_version text, + rule_library_version text, + project_review_status text, + status_recommendation text, + workflow_state text, + review_confidence text, + finding_count integer, + hard_fail_count integer, + writeback jsonb not null default '{}'::jsonb, + reviewer_override jsonb, -- present on record_type = 'override' + retention_days integer not null default 365, + created_at timestamptz not null default now() +); + +alter table private.procore_audit enable row level security; +-- No policies on purpose: only the SECURITY DEFINER functions (owner) and +-- service_role (which bypasses RLS) may touch this table. + +create index if not exists idx_procore_audit_company_created + on private.procore_audit (company_id, created_at desc); +create index if not exists idx_procore_audit_created + on private.procore_audit (created_at); + +-- Append-only: block UPDATE always; block DELETE for non-admin/owner roles. +-- The retention/deletion functions below are SECURITY DEFINER and run as the +-- function owner, so they pass the DELETE guard while ad-hoc deletes do not. +create or replace function private.prevent_procore_audit_mutation() +returns trigger +language plpgsql +security definer +as $$ +begin + if tg_op = 'UPDATE' then + raise exception 'private.procore_audit is append-only; UPDATE not permitted.'; + end if; + if current_user not in ('audit_admin', 'postgres') then + raise exception 'private.procore_audit is append-only; DELETE not permitted.'; + end if; + return old; +end; +$$; + +drop trigger if exists no_update_procore_audit on private.procore_audit; +create trigger no_update_procore_audit + before update on private.procore_audit + for each row execute function private.prevent_procore_audit_mutation(); + +drop trigger if exists no_delete_procore_audit on private.procore_audit; +create trigger no_delete_procore_audit + before delete on private.procore_audit + for each row execute function private.prevent_procore_audit_mutation(); + +-- Write one audit record (review or override). SECURITY DEFINER so PostgREST +-- callers reach the private table; EXECUTE restricted to service_role. +create or replace function public.record_procore_audit(p_record jsonb) +returns bigint +language plpgsql +security definer +set search_path = private, public +as $$ +declare + v_id bigint; +begin + insert into private.procore_audit ( + record_type, review_run_id, delivery_key, correlation_id, + company_id, project_id, document_hash, rule_pack_version, + rule_library_version, project_review_status, status_recommendation, + workflow_state, review_confidence, finding_count, hard_fail_count, + writeback, reviewer_override, retention_days + ) + values ( + coalesce(p_record->>'record_type', 'review'), + p_record->>'review_run_id', + p_record->>'delivery_key', + p_record->>'correlation_id', + (p_record->>'company_id')::bigint, + (p_record->>'project_id')::bigint, + p_record->>'document_hash', + p_record->>'rule_pack_version', + p_record->>'rule_library_version', + p_record->>'project_review_status', + p_record->>'status_recommendation', + p_record->>'workflow_state', + p_record->>'review_confidence', + (p_record->>'finding_count')::integer, + (p_record->>'hard_fail_count')::integer, + coalesce(p_record->'writeback', '{}'::jsonb), + p_record->'reviewer_override', + coalesce((p_record->>'retention_days')::integer, 365) + ) + returning id into v_id; + return v_id; +end; +$$; + +-- Retention purge: delete rows past their per-row retention window. +create or replace function public.purge_procore_audit() +returns integer +language plpgsql +security definer +set search_path = private, public +as $$ +declare + v_deleted integer; +begin + delete from private.procore_audit + where created_at < now() - (retention_days || ' days')::interval; + get diagnostics v_deleted = row_count; + return v_deleted; +end; +$$; + +-- Customer deletion request: remove all audit rows for a company. +create or replace function public.delete_procore_audit_for_company(p_company_id bigint) +returns integer +language plpgsql +security definer +set search_path = private, public +as $$ +declare + v_deleted integer; +begin + delete from private.procore_audit where company_id = p_company_id; + get diagnostics v_deleted = row_count; + return v_deleted; +end; +$$; + +-- Least privilege: only the service role may call these RPCs. +revoke all on function public.record_procore_audit(jsonb) from public, anon, authenticated; +revoke all on function public.purge_procore_audit() from public, anon, authenticated; +revoke all on function public.delete_procore_audit_for_company(bigint) from public, anon, authenticated; +grant execute on function public.record_procore_audit(jsonb) to service_role; +grant execute on function public.purge_procore_audit() to service_role; +grant execute on function public.delete_procore_audit_for_company(bigint) to service_role; diff --git a/tests/test_procore_audit_migration.py b/tests/test_procore_audit_migration.py new file mode 100644 index 0000000..e38bb22 --- /dev/null +++ b/tests/test_procore_audit_migration.py @@ -0,0 +1,57 @@ +"""Contract test for migration 008 — Procore audit trail (T6). + +The migration cannot be applied in CI, so this guards the hardening invariants +against silent weakening: private schema + RLS, append-only triggers, retention +and customer-deletion functions, service-role-only EXECUTE, and no raw text. +""" +from pathlib import Path + +MIGRATION = ( + Path(__file__).resolve().parent.parent + / "supabase" / "migrations" / "008_procore_audit.sql" +) + + +def _sql() -> str: + return MIGRATION.read_text(encoding="utf-8").lower() + + +def test_migration_file_exists(): + assert MIGRATION.exists() + + +def test_table_is_private_with_rls(): + sql = _sql() + assert "create table if not exists private.procore_audit" in sql + assert "alter table private.procore_audit enable row level security" in sql + + +def test_append_only_triggers(): + sql = _sql() + assert "before update on private.procore_audit" in sql + assert "before delete on private.procore_audit" in sql + assert "append-only" in sql + + +def test_retention_and_deletion_functions(): + sql = _sql() + assert "function public.purge_procore_audit" in sql + assert "function public.delete_procore_audit_for_company" in sql + assert "retention_days" in sql + + +def test_service_role_only_execute(): + sql = _sql() + for fn in ( + "public.record_procore_audit(jsonb)", + "public.purge_procore_audit()", + "public.delete_procore_audit_for_company(bigint)", + ): + assert f"grant execute on function {fn} to service_role" in sql + assert f"revoke all on function {fn} from public, anon, authenticated" in sql + + +def test_stores_hash_not_raw_text(): + sql = _sql() + assert "document_hash" in sql + assert "never raw swms text" in sql diff --git a/tests/test_procore_webhook.py b/tests/test_procore_webhook.py index e6b7f92..11b7b8f 100644 --- a/tests/test_procore_webhook.py +++ b/tests/test_procore_webhook.py @@ -328,6 +328,7 @@ def test_no_approval_in_statuses(self): class TestPayloadLogging: def test_log_creates_file(self, tmp_path, monkeypatch): import core.procore.webhook_handler as wh + monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true") monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl") monkeypatch.setattr(wh, "_DATA_DIR", tmp_path) event = parse_event(_load_fixture("submittal_created")) @@ -335,6 +336,52 @@ def test_log_creates_file(self, tmp_path, monkeypatch): assert (tmp_path / "payloads.jsonl").exists() +# ── Local JSONL gate (T5) ──────────────────────────────────────────────────── + +class TestLocalJsonlGate: + """T5: local JSONL writes are gated off by default (production-safe).""" + + def test_log_payload_disabled_writes_nothing(self, tmp_path, monkeypatch): + import core.procore.webhook_handler as wh + monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False) + monkeypatch.setattr(wh, "_DATA_DIR", tmp_path) + monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl") + wh.log_payload(parse_event(_load_fixture("submittal_created"))) + assert not (tmp_path / "payloads.jsonl").exists() + + def test_log_review_disabled_writes_nothing(self, tmp_path, monkeypatch): + import core.procore.webhook_handler as wh + monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False) + monkeypatch.setattr(wh, "_DATA_DIR", tmp_path) + monkeypatch.setattr(wh, "_REVIEW_LOG", tmp_path / "reviews.jsonl") + wh.log_review({"status_recommendation": "Escalate"}, + parse_event(_load_fixture("submittal_created"))) + assert not (tmp_path / "reviews.jsonl").exists() + + def test_enabled_writes_payload_and_review(self, tmp_path, monkeypatch): + import core.procore.webhook_handler as wh + monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true") + monkeypatch.setattr(wh, "_DATA_DIR", tmp_path) + monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl") + monkeypatch.setattr(wh, "_REVIEW_LOG", tmp_path / "reviews.jsonl") + event = parse_event(_load_fixture("submittal_created")) + wh.log_payload(event) + wh.log_review({"status_recommendation": "Escalate"}, event) + assert (tmp_path / "payloads.jsonl").exists() + assert (tmp_path / "reviews.jsonl").exists() + + def test_store_artifact_gated(self, tmp_path, monkeypatch): + import core.procore.review_store as rs + monkeypatch.setattr(rs, "_DATA_DIR", tmp_path) + monkeypatch.setattr(rs, "_ARTIFACT_STORE", tmp_path / "artifacts.jsonl") + monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False) + rs.store_artifact({"review_run_id": "r1"}) + assert not (tmp_path / "artifacts.jsonl").exists() + monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true") + rs.store_artifact({"review_run_id": "r1"}) + assert (tmp_path / "artifacts.jsonl").exists() + + # ── API endpoint ──────────────────────────────────────────────────────────── class TestWebhookEndpoint: diff --git a/tests/test_resubmission_comparison.py b/tests/test_resubmission_comparison.py index 0543530..c3c1640 100644 --- a/tests/test_resubmission_comparison.py +++ b/tests/test_resubmission_comparison.py @@ -247,6 +247,12 @@ def test_status_in_allowed(self): # ── Review store ──────────────────────────────────────────────────────────── class TestReviewStore: + @pytest.fixture(autouse=True) + def _enable_local_jsonl(self, monkeypatch): + # These tests exercise the local JSONL store/find roundtrip, which is + # gated off by default (T5); enable it for this class. + monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true") + def setup_method(self): clear_store()