feat(procore): Track A batch 2 — audit retention, JSONL gate, SBOM/vuln scan (T5,T6,T9)

.github/workflows/ci.yml

@@ -39,3 +39,21 @@ jobs:
 
       - name: Run tests
         run: python -m pytest tests/ -v --tb=short
+
+      - name: Generate SBOM (CycloneDX)
+        # Never fails the job; the dedicated scan below is the vuln gate.
+        continue-on-error: true
+        run: pip-audit -r requirements.txt --format cyclonedx-json --output sbom.json
+
+      - name: Upload SBOM artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sbom-cyclonedx
+          path: sbom.json
+          if-no-files-found: warn
+
+      - name: Vulnerability scan (pip-audit)
+        # Fails CI on known-vuln pinned deps. Document any unavoidable, no-fix
+        # advisory with `--ignore-vuln <id>` and a reason here — never blanket-ignore.
+        run: pip-audit -r requirements.txt

core/procore/review_store.py

@@ -9,6 +9,7 @@
 
 import json
 import logging
+import os
 import threading
 from pathlib import Path
 
@@ -19,8 +20,17 @@
 _LOCK = threading.Lock()
 
 
+def _local_jsonl_enabled() -> bool:
+    """Local JSONL persistence is off by default (production-safe); the durable
+    record is the Supabase audit trail. Enable with PROCORE_LOCAL_JSONL_ENABLED=true."""
+    return os.getenv("PROCORE_LOCAL_JSONL_ENABLED", "false").strip().lower() == "true"
+
+
 def store_artifact(artifact: dict) -> None:
     """Append a full review artifact to the store."""
+    if not _local_jsonl_enabled():
+        log.debug("Local JSONL disabled; skipping artifact store write")
+        return
     _DATA_DIR.mkdir(parents=True, exist_ok=True)
     with _LOCK:
         with open(_ARTIFACT_STORE, "a", encoding="utf-8") as f:

core/procore/webhook_handler.py

@@ -203,8 +203,21 @@ def extract_submittal_attachments(event: WebhookEvent) -> list[SubmittalAttachme
     return result
 
 
+def _local_jsonl_enabled() -> bool:
+    """Whether local JSONL persistence under src/data is enabled.
+
+    Default off (production-safe): the durable record is the Supabase audit
+    trail (migration 008). Local JSONL is dev convenience only; enable with
+    PROCORE_LOCAL_JSONL_ENABLED=true.
+    """
+    return os.getenv("PROCORE_LOCAL_JSONL_ENABLED", "false").strip().lower() == "true"
+
+
 def log_payload(event: WebhookEvent) -> None:
     """Append the raw payload to the payload log for replay/debugging."""
+    if not _local_jsonl_enabled():
+        log.debug("Local JSONL disabled; skipping payload log write")
+        return
     _DATA_DIR.mkdir(parents=True, exist_ok=True)
     record = {
         "logged_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
@@ -220,6 +233,9 @@ def log_payload(event: WebhookEvent) -> None:
 
 def log_review(review_artifact: dict, event: WebhookEvent) -> None:
     """Append the review artifact to the review log."""
+    if not _local_jsonl_enabled():
+        log.debug("Local JSONL disabled; skipping review log write")
+        return
     _DATA_DIR.mkdir(parents=True, exist_ok=True)
     record = {
         "logged_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),

requirements-dev.txt

@@ -4,3 +4,4 @@ pytest-xdist==3.8.0
 pytest-asyncio
 flake8
 pre-commit
+pip-audit[cyclonedx]

requirements.txt

@@ -1,5 +1,5 @@
 anthropic==0.84.0
-cryptography==46.0.7
+cryptography==48.0.1
 ecdsa==0.19.2
 fastapi==0.135.1
 httpx==0.28.1
@@ -10,11 +10,11 @@ Pillow==12.2.0
 pyasn1==0.6.3
 pydantic[email]==2.12.5
 pymupdf==1.27.1
-pypdf==6.9.2
+pypdf==6.13.0
 python-docx==1.2.0
-python-dotenv==1.1.0
+python-dotenv==1.2.2
 python-jose[cryptography]==3.5.0
-python-multipart==0.0.22
+python-multipart==0.0.31
 rapidfuzz==3.14.3
 requests==2.33.0
 sentry-sdk[fastapi]==2.54.0

supabase/migrations/008_procore_audit.sql

@@ -0,0 +1,161 @@
+-- 008_procore_audit.sql
+-- Minimal, append-only audit trail for Procore SWMS reviews (T6).
+--
+-- Stores ONLY audit metadata (hashes, rule-pack versions, finding counts,
+-- status, write-back metadata, human-override trail) — never raw SWMS text or
+-- Procore document content. Supports the "Procore remains system of record /
+-- minimal audit metadata retained" data-handling statement.
+--
+-- Hardening (per docs/procore/STAGE3_CERTIFICATION_PLAN_V2.md §5 + pre-flight):
+--   * table lives in a PRIVATE (non-PostgREST-exposed) schema with RLS;
+--   * append-only: UPDATE always blocked; DELETE only via the controlled
+--     retention/deletion functions (SECURITY DEFINER, run as owner);
+--   * writes/purges/deletions go through SECURITY DEFINER functions in public
+--     whose EXECUTE is granted to service_role only.
+
+create schema if not exists private;
+
+create table if not exists private.procore_audit (
+    id                   bigint generated always as identity primary key,
+    record_type          text not null default 'review',  -- 'review' | 'override'
+    review_run_id        text,
+    delivery_key         text,
+    correlation_id       text,
+    company_id           bigint,
+    project_id           bigint,
+    document_hash        text,        -- sha256 of the document; never raw SWMS text
+    rule_pack_version    text,
+    rule_library_version text,
+    project_review_status text,
+    status_recommendation text,
+    workflow_state       text,
+    review_confidence    text,
+    finding_count        integer,
+    hard_fail_count      integer,
+    writeback            jsonb not null default '{}'::jsonb,
+    reviewer_override    jsonb,        -- present on record_type = 'override'
+    retention_days       integer not null default 365,
+    created_at           timestamptz not null default now()
+);
+
+alter table private.procore_audit enable row level security;
+-- No policies on purpose: only the SECURITY DEFINER functions (owner) and
+-- service_role (which bypasses RLS) may touch this table.
+
+create index if not exists idx_procore_audit_company_created
+    on private.procore_audit (company_id, created_at desc);
+create index if not exists idx_procore_audit_created
+    on private.procore_audit (created_at);
+
+-- Append-only: block UPDATE always; block DELETE for non-admin/owner roles.
+-- The retention/deletion functions below are SECURITY DEFINER and run as the
+-- function owner, so they pass the DELETE guard while ad-hoc deletes do not.
+create or replace function private.prevent_procore_audit_mutation()
+returns trigger
+language plpgsql
+security definer
+as $$
+begin
+    if tg_op = 'UPDATE' then
+        raise exception 'private.procore_audit is append-only; UPDATE not permitted.';
+    end if;
+    if current_user not in ('audit_admin', 'postgres') then
+        raise exception 'private.procore_audit is append-only; DELETE not permitted.';
+    end if;
+    return old;
+end;
+$$;
+
+drop trigger if exists no_update_procore_audit on private.procore_audit;
+create trigger no_update_procore_audit
+    before update on private.procore_audit
+    for each row execute function private.prevent_procore_audit_mutation();
+
+drop trigger if exists no_delete_procore_audit on private.procore_audit;
+create trigger no_delete_procore_audit
+    before delete on private.procore_audit
+    for each row execute function private.prevent_procore_audit_mutation();
+
+-- Write one audit record (review or override). SECURITY DEFINER so PostgREST
+-- callers reach the private table; EXECUTE restricted to service_role.
+create or replace function public.record_procore_audit(p_record jsonb)
+returns bigint
+language plpgsql
+security definer
+set search_path = private, public
+as $$
+declare
+    v_id bigint;
+begin
+    insert into private.procore_audit (
+        record_type, review_run_id, delivery_key, correlation_id,
+        company_id, project_id, document_hash, rule_pack_version,
+        rule_library_version, project_review_status, status_recommendation,
+        workflow_state, review_confidence, finding_count, hard_fail_count,
+        writeback, reviewer_override, retention_days
+    )
+    values (
+        coalesce(p_record->>'record_type', 'review'),
+        p_record->>'review_run_id',
+        p_record->>'delivery_key',
+        p_record->>'correlation_id',
+        (p_record->>'company_id')::bigint,
+        (p_record->>'project_id')::bigint,
+        p_record->>'document_hash',
+        p_record->>'rule_pack_version',
+        p_record->>'rule_library_version',
+        p_record->>'project_review_status',
+        p_record->>'status_recommendation',
+        p_record->>'workflow_state',
+        p_record->>'review_confidence',
+        (p_record->>'finding_count')::integer,
+        (p_record->>'hard_fail_count')::integer,
+        coalesce(p_record->'writeback', '{}'::jsonb),
+        p_record->'reviewer_override',
+        coalesce((p_record->>'retention_days')::integer, 365)
+    )
+    returning id into v_id;
+    return v_id;
+end;
+$$;
+
+-- Retention purge: delete rows past their per-row retention window.
+create or replace function public.purge_procore_audit()
+returns integer
+language plpgsql
+security definer
+set search_path = private, public
+as $$
+declare
+    v_deleted integer;
+begin
+    delete from private.procore_audit
+    where created_at < now() - (retention_days || ' days')::interval;
+    get diagnostics v_deleted = row_count;
+    return v_deleted;
+end;
+$$;
+
+-- Customer deletion request: remove all audit rows for a company.
+create or replace function public.delete_procore_audit_for_company(p_company_id bigint)
+returns integer
+language plpgsql
+security definer
+set search_path = private, public
+as $$
+declare
+    v_deleted integer;
+begin
+    delete from private.procore_audit where company_id = p_company_id;
+    get diagnostics v_deleted = row_count;
+    return v_deleted;
+end;
+$$;
+
+-- Least privilege: only the service role may call these RPCs.
+revoke all on function public.record_procore_audit(jsonb) from public, anon, authenticated;
+revoke all on function public.purge_procore_audit() from public, anon, authenticated;
+revoke all on function public.delete_procore_audit_for_company(bigint) from public, anon, authenticated;
+grant execute on function public.record_procore_audit(jsonb) to service_role;
+grant execute on function public.purge_procore_audit() to service_role;
+grant execute on function public.delete_procore_audit_for_company(bigint) to service_role;

tests/test_procore_audit_migration.py

@@ -0,0 +1,57 @@
+"""Contract test for migration 008 — Procore audit trail (T6).
+
+The migration cannot be applied in CI, so this guards the hardening invariants
+against silent weakening: private schema + RLS, append-only triggers, retention
+and customer-deletion functions, service-role-only EXECUTE, and no raw text.
+"""
+from pathlib import Path
+
+MIGRATION = (
+    Path(__file__).resolve().parent.parent
+    / "supabase" / "migrations" / "008_procore_audit.sql"
+)
+
+
+def _sql() -> str:
+    return MIGRATION.read_text(encoding="utf-8").lower()
+
+
+def test_migration_file_exists():
+    assert MIGRATION.exists()
+
+
+def test_table_is_private_with_rls():
+    sql = _sql()
+    assert "create table if not exists private.procore_audit" in sql
+    assert "alter table private.procore_audit enable row level security" in sql
+
+
+def test_append_only_triggers():
+    sql = _sql()
+    assert "before update on private.procore_audit" in sql
+    assert "before delete on private.procore_audit" in sql
+    assert "append-only" in sql
+
+
+def test_retention_and_deletion_functions():
+    sql = _sql()
+    assert "function public.purge_procore_audit" in sql
+    assert "function public.delete_procore_audit_for_company" in sql
+    assert "retention_days" in sql
+
+
+def test_service_role_only_execute():
+    sql = _sql()
+    for fn in (
+        "public.record_procore_audit(jsonb)",
+        "public.purge_procore_audit()",
+        "public.delete_procore_audit_for_company(bigint)",
+    ):
+        assert f"grant execute on function {fn} to service_role" in sql
+        assert f"revoke all on function {fn} from public, anon, authenticated" in sql
+
+
+def test_stores_hash_not_raw_text():
+    sql = _sql()
+    assert "document_hash" in sql
+    assert "never raw swms text" in sql

tests/test_procore_webhook.py

@@ -328,13 +328,60 @@ def test_no_approval_in_statuses(self):
 class TestPayloadLogging:
     def test_log_creates_file(self, tmp_path, monkeypatch):
         import core.procore.webhook_handler as wh
+        monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true")
         monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl")
         monkeypatch.setattr(wh, "_DATA_DIR", tmp_path)
         event = parse_event(_load_fixture("submittal_created"))
         log_payload(event)
         assert (tmp_path / "payloads.jsonl").exists()
 
 
+# ── Local JSONL gate (T5) ────────────────────────────────────────────────────
+
+class TestLocalJsonlGate:
+    """T5: local JSONL writes are gated off by default (production-safe)."""
+
+    def test_log_payload_disabled_writes_nothing(self, tmp_path, monkeypatch):
+        import core.procore.webhook_handler as wh
+        monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False)
+        monkeypatch.setattr(wh, "_DATA_DIR", tmp_path)
+        monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl")
+        wh.log_payload(parse_event(_load_fixture("submittal_created")))
+        assert not (tmp_path / "payloads.jsonl").exists()
+
+    def test_log_review_disabled_writes_nothing(self, tmp_path, monkeypatch):
+        import core.procore.webhook_handler as wh
+        monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False)
+        monkeypatch.setattr(wh, "_DATA_DIR", tmp_path)
+        monkeypatch.setattr(wh, "_REVIEW_LOG", tmp_path / "reviews.jsonl")
+        wh.log_review({"status_recommendation": "Escalate"},
+                      parse_event(_load_fixture("submittal_created")))
+        assert not (tmp_path / "reviews.jsonl").exists()
+
+    def test_enabled_writes_payload_and_review(self, tmp_path, monkeypatch):
+        import core.procore.webhook_handler as wh
+        monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true")
+        monkeypatch.setattr(wh, "_DATA_DIR", tmp_path)
+        monkeypatch.setattr(wh, "_PAYLOAD_LOG", tmp_path / "payloads.jsonl")
+        monkeypatch.setattr(wh, "_REVIEW_LOG", tmp_path / "reviews.jsonl")
+        event = parse_event(_load_fixture("submittal_created"))
+        wh.log_payload(event)
+        wh.log_review({"status_recommendation": "Escalate"}, event)
+        assert (tmp_path / "payloads.jsonl").exists()
+        assert (tmp_path / "reviews.jsonl").exists()
+
+    def test_store_artifact_gated(self, tmp_path, monkeypatch):
+        import core.procore.review_store as rs
+        monkeypatch.setattr(rs, "_DATA_DIR", tmp_path)
+        monkeypatch.setattr(rs, "_ARTIFACT_STORE", tmp_path / "artifacts.jsonl")
+        monkeypatch.delenv("PROCORE_LOCAL_JSONL_ENABLED", raising=False)
+        rs.store_artifact({"review_run_id": "r1"})
+        assert not (tmp_path / "artifacts.jsonl").exists()
+        monkeypatch.setenv("PROCORE_LOCAL_JSONL_ENABLED", "true")
+        rs.store_artifact({"review_run_id": "r1"})
+        assert (tmp_path / "artifacts.jsonl").exists()
+
+
 # ── API endpoint ────────────────────────────────────────────────────────────
 
 class TestWebhookEndpoint: