diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 98886de71..2af36b78b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -39,8 +39,8 @@ updates: # its blocker below clears, then let CI judge the upgrade on its own. # eslint 10 — eslint-config-next 16.2.12 calls the pre-10 scope-manager # API (scopeManager.addGlobals); `eslint .` crashes outright. - # jsdom 30 — requires Node >=22.22.2; CI runs Node 20. - # jest-dom 7 — requires Node >=22 and a new @testing-library/dom peer. + # (jsdom 30 and jest-dom 7 were ignored while copy-gate ran Node 20; + # every workflow now reads .nvmrc (Node 22), so CI judges those majors.) # typescript 7 — typescript-eslint 8.x (via eslint-config-next) peers # typescript <6.1.0; `npm run lint` crashes in # typescript-estree ("reading 'Cjs'"). Backed out in @@ -50,10 +50,6 @@ updates: # package.json's range and `npm ci` fails (#3138, #3157). - dependency-name: "eslint" update-types: ["version-update:semver-major"] - - dependency-name: "jsdom" - update-types: ["version-update:semver-major"] - - dependency-name: "@testing-library/jest-dom" - update-types: ["version-update:semver-major"] - dependency-name: "typescript" update-types: ["version-update:semver-major"] - dependency-name: "@vitest/coverage-v8" diff --git a/.github/workflows/backend-ci.yml b/.github/workflows/backend-ci.yml index aa40ff90e..3cf85c12e 100644 --- a/.github/workflows/backend-ci.yml +++ b/.github/workflows/backend-ci.yml @@ -21,6 +21,10 @@ on: - 'frontend/src/lib/public-openapi.ts' - 'frontend/package-lock.json' - 'alembic/**' + # The two non-Alembic schema channels, guarded by + # scripts/check_schema_channels.py in alembic-revision-check. + - 'supabase/migrations/**' + - 'sql/prod-apply/**' - 'server/**' - 'services/**' - 'shared/**' @@ -43,6 +47,10 @@ on: - 'frontend/src/lib/public-openapi.ts' - 'frontend/package-lock.json' - 'alembic/**' + # The two non-Alembic schema channels, guarded by + # scripts/check_schema_channels.py in alembic-revision-check. + - 'supabase/migrations/**' + - 'sql/prod-apply/**' - 'server/**' - 'services/**' - 'shared/**' @@ -355,6 +363,10 @@ jobs: --extend-exclude __pycache__,.venv,venv,build,dist,services \ --statistics + # Every top-level entry in scripts/ has a row in scripts/README.md (WI-08). + - name: Guard the scripts/ registry + run: python scripts/check_scripts_registered.py + # ── RLS fallback-UUID guard (hard gate) ────────────────── # Scans migrations for the fail-open RLS pattern that #1091 fixed. # Any new migration that reintroduces `COALESCE(..., )` @@ -425,6 +437,16 @@ jobs: - name: Guard against new orphan migrations run: python scripts/check_orphan_migrations.py + # ── Schema channel guard (hard gate) ────────────────── + # supabase/migrations/ and sql/prod-apply/ may hold rendered copies of + # Alembic revisions and nothing else. A file there must start with + # ``-- alembic-revision: `` and ``-- rendered-from: ...`` headers + # naming an existing revision. Pre-decision files are grandfathered + # by name inside the script; the list only shrinks. See + # docs/decisions/2026-10-30-schema-change-channel.md. + - name: Guard the non-Alembic schema channels + run: python scripts/check_schema_channels.py + # ── Dep version drift guard (hard gate) ───────────────── # Scans services/*/requirements*.txt to ensure no per-service file # lowers the floor for shared packages below pyproject.toml. Prevents diff --git a/.github/workflows/copy-gate.yml b/.github/workflows/copy-gate.yml index 0f0b05ebe..5b66be3a1 100644 --- a/.github/workflows/copy-gate.yml +++ b/.github/workflows/copy-gate.yml @@ -34,7 +34,8 @@ jobs: persist-credentials: false - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v6 with: - node-version: '20' + # One Node version for every workflow: .nvmrc (WI-13, 2026-10-30). + node-version-file: .nvmrc - run: node --test scripts/copy-lint/check-copy.test.mjs - name: Read base copy baseline env: diff --git a/ASYNC_PROCESSES.md b/ASYNC_PROCESSES.md index 5e3abeaca..b70d397ab 100644 --- a/ASYNC_PROCESSES.md +++ b/ASYNC_PROCESSES.md @@ -21,10 +21,10 @@ trigger was removed in migration v059. |-----------|---------|--------------------|----------------------| | `nlp_extraction` | `_handle_nlp_extraction` → `services.nlp.app.extractor.extract_entities` | 300s | topic `documents.ingested` | | `review_item` | `_handle_review_item` → admin HallucinationTracker | 30s | topic `nlp.needs_review` | -| `graph_update` | **No handler registered** — a timeout (120s) is configured, but `TASK_HANDLERS` registers only `nlp_extraction` and `review_item`; an enqueued `graph_update` row sits unclaimed | 120s | topic `graph.update` | -(Extraction-path hardening for the pg pipeline, including the `review_item` handler, is -in flight on a sibling branch — treat the handler details above as the pre-fix baseline.) +`graph_update` was declared as a task type with a 120 s timeout but nothing enqueued it and +no handler existed; it was removed on 2026-10-30 (tech-debt WI-15). High-confidence +extractions are logged as `fsma_high_confidence_no_graph_sink` until a graph sink exists. Mechanics: diff --git a/docs/MIGRATION_CONSOLIDATION.md b/docs/MIGRATION_CONSOLIDATION.md index b4ee95c8b..638640a73 100644 --- a/docs/MIGRATION_CONSOLIDATION.md +++ b/docs/MIGRATION_CONSOLIDATION.md @@ -1,88 +1,10 @@ # Migration Consolidation Plan -## Current State (3 parallel systems) - -| System | Location | Count | Format | -|--------|----------|-------|--------| -| Alembic | `alembic/versions/` | 10 | Python, revision chain | -| Raw SQL (main) | `migrations/` | 28 | Flyway V-numbered | -| Raw SQL (admin) | `services/admin/migrations/` | 46 | Flyway V-numbered | - -**Total: 84 migration files**, many with overlapping operations. - -## Why Alembic Wins - -- Alembic's baseline (`20260324_baseline_consolidate_v002_through_v042.py`) already consolidates the first 41 Flyway migrations -- Alembic has revision-chain integrity (down_revision links) -- Alembic supports both online (live DB) and offline (SQL script) modes -- Python migrations allow conditional logic (IF NOT EXISTS, data backfill) -- All new migrations since March 2026 are already Alembic - -## What's Already Covered by Alembic - -V002-V042 (baseline), V043-V050 (control plane + task queue), V051-V055 (RLS + tenant tables + indexes). - -## What Needs Porting (raw SQL not yet in Alembic) - -### High Priority — unique schema changes not in any Alembic migration - -| File | Operation | Priority | -|------|-----------|----------| -| `V049__transformation_links_adjacency.sql` | Lot-to-lot ingredient traceability | HIGH | -| `V053__add_growing_cte_type.sql` | Add 'growing' to event_type CHECK | HIGH | -| `V057__ftl_cottage_cheese_exemption.sql` | FTL update | MEDIUM | -| `SQL_V048__rls_sysadmin_defense_in_depth.sql` | RLS hardening | HIGH | -| `SQL_V050__rls_reference_tables.sql` | RLS on reference tables | HIGH | -| `V054__rls_variable_standardization.sql` | Fix RLS variable names | CRITICAL | -| `V056__rls_obligations_and_ftl.sql` | Close last RLS gaps | HIGH | - -### Already duplicated in Alembic (can skip) - -| File | Covered by | -|------|-----------| -| `V043__canonical_traceability_events.sql` | Alembic 20260325 | -| `V044__versioned_rules_engine.sql` | Alembic 20260325 | -| `V045__exception_remediation_queue.sql` | Alembic 20260325 | -| `V046__request_response_workflow.sql` | Alembic 20260325 | -| `V047__identity_resolution.sql` | Alembic 20260325 | -| `V052__ensure_tenant_tables.sql` | Alembic 20260411 V052 | -| `V052__cte_events_composite_idempotency_key.sql` | Alembic 20260411 V052 | - -### Admin migrations — disposition - -- **V1-V35**: Core admin schema. These run against `ADMIN_DATABASE_URL` (separate DB). Keep as-is until admin DB is merged into main DB during monolith consolidation. -- **V36-V40**: Funnel events, password reset, tool leads, supplier portal, PCOS removal. Same — admin-DB-scoped. - -## Execution Plan - -### Phase 1: Port remaining raw SQL to Alembic (1 day) -Create 3 new Alembic migrations: -1. `20260415_rls_hardening_v056.py` — combines SQL_V048, SQL_V050, V054, V056 -2. `20260415_transformation_links_v057.py` — ports V049 -3. `20260415_schema_updates_v058.py` — ports V053, V057 - -### Phase 2: Validate against real database (half day) -```bash -# Against a fresh database -alembic upgrade head - -# Against an existing database (stamp first) -alembic stamp head # if migrations were already applied via Flyway -``` - -### Phase 3: Remove raw SQL directory (after validation) -- Delete `migrations/` directory -- Update `Dockerfile` (already done — only copies `alembic/`) -- Update `scripts/run-migrations.sh` to use `alembic upgrade head` only - -### Phase 4: Admin DB merge (deferred to monolith consolidation) -- Port admin V1-V40 into main Alembic chain -- Requires merging `ADMIN_DATABASE_URL` into `DATABASE_URL` -- Depends on monolith consolidation completing first - -## Status - -- [x] Phase 1: Port remaining raw SQL to Alembic (3 migrations created, April 15 2026) -- [x] Phase 2: Validate against real database (Supabase, alembic upgrade head passes) -- [x] Phase 3: Remove raw SQL directory (`migrations/` deleted) -- [ ] Phase 4: Admin DB merge (deferred to monolith consolidation) +Superseded on 2026-10-30. This file described the March 2026 state (10 +Alembic revisions, a Flyway-style `migrations/` directory that no longer +exists) and was not kept current. + +The current decision and inventory of every schema channel is +`docs/decisions/2026-10-30-schema-change-channel.md`. The procedure for +reconciling the production version row is +`docs/runbooks/ALEMBIC_VERSION_RECONCILE.md`. diff --git a/docs/audits/TECH_DEBT_AUDIT_2026-10-28.md b/docs/audits/TECH_DEBT_AUDIT_2026-10-28.md new file mode 100644 index 000000000..197ebf0aa --- /dev/null +++ b/docs/audits/TECH_DEBT_AUDIT_2026-10-28.md @@ -0,0 +1,580 @@ +# Technical Debt Audit — 2026-10-28 + +Scope: the production repository at commit `af5beee` (branch +`claude/fervent-clarke-0aw2am`). Every count below was measured against the +working tree on this date with `wc`, `grep`, `find`, or by reading the file +named. Where an existing document already tracks an item, that document is +linked rather than restated. + +Method: each item is scored on Impact (how much it slows work, 1–5), Risk +(what happens if it stays, 1–5) and Effort (1 = under a day, 5 = multi-week). +Priority = (Impact + Risk) × (6 − Effort). Effort is for one engineer working +with AI assistance alongside feature work. + +Second opinion: TypeSafe Jev was asked which item to fix first and how likely +three of them are to cause an incident. Its answers are reported next to mine +in §2 and §3. + +Open GitHub backlog at audit time: 3 issues, all feature follow-ups. Nothing +below is tracked there yet. + +--- + +## 1. Summary + +| # | Item | Category | Impact | Risk | Effort | Priority | Status | +|--:|---|---|:-:|:-:|:-:|--:|---| +| 1 | Three schema-change channels and a stale production `alembic_version` row | Infrastructure | 4 | 5 | 3 | **27** | delivered in PR #3699 (ADR, inventory, precheck, runbook, CI guard); Appendix B decisions open | +| 2 | Six `app` packages collide; bare `from app.` imports; 347-line `sys.path` conftest | Architecture | 4 | 4 | 3 | **24** | open | +| 3 | Lint and type gates are near-empty (ruff selects 4 rule groups, mypy not a gate) | Code | 3 | 3 | 2 | **24** | open | +| 4 | 33 CI workflows, four of them over 850 lines, plus a workflow that exists to re-run flaky gates | Infrastructure | 4 | 3 | 3 | **21** | open | +| 5 | Legacy Kafka backbone still in tree and in the production image | Architecture | 2 | 3 | 2 | **20** | open | +| 6 | Production image carries test and optional AI dependencies | Infrastructure | 2 | 3 | 2 | **20** | open | +| 7 | Orientation docs disagree with the tree; duplicate doc directories; one-off reports at repo root | Documentation | 3 | 2 | 2 | **20** | open | +| 8 | 31 of 87 top-level scripts are referenced by nothing | Code | 2 | 2 | 1 | **20** | delivered in PR #3699 (registry `scripts/README.md`, guard `scripts/check_scripts_registered.py` in backend-ci); 16 archive candidates await Christopher's approval by name | +| 9 | Retired six-service topology still has entrypoints, k8s manifests, a PR-time CI job, and legacy runbooks | Infrastructure | 2 | 2 | 1 | **20** | open | +| 10 | DB-dependent tests skip vacuously without a migrated Postgres | Test | 3 | 3 | 3 | **18** | open | +| 11 | Frontend API access split three ways; 8 hand-rolled proxy routes | Code | 3 | 2 | 3 | **15** | open | +| 12 | Legacy `cte_events` dual-write still live, with 12 non-legacy readers | Architecture | 3 | 4 | 4 | **14** | open | +| 13 | Blocked major upgrades and Node version drift across CI | Dependency | 2 | 2 | 3 | **12** | delivered in PR #3699 (copy-gate reads `.nvmrc`; jsdom/jest-dom Dependabot ignores lifted) | +| 14 | 25 source modules over 1,000 lines (20 under `services/` and `server/`) | Code | 3 | 2 | 4 | **10** | open | +| 15 | `graph_update` task type declared, never enqueued, never handled | Code | 1 | 1 | 1 | **10** | delivered in PR #3699 | +| 16 | Frontend type looseness: 192 `any`, 14 eslint suppressions | Code | 1 | 1 | 2 | **8** | open | + +What is **not** debt here, and should stay as is: + +- Backend dependency management. `requirements.in` → hash-pinned + `requirements.lock`, CVE floors documented inline, Dependabot with a + 7-day cooldown, `supply-chain.yml` on lock changes. This is better than + most funded teams. +- Router gating. Production fails closed on an explicit allowlist + (`server/router_policy.py`), and `ENABLE_EXPERIMENTAL_ROUTERS=true` is + rejected in production. +- Frontend coverage gate. Floors were set 6 points under measured values in + August and are enforced in CI. + +--- + +## 2. Items in priority order + +### 1. Three schema-change channels and a stale production version row — Priority 27 + +**Evidence** + +- `alembic/versions/` holds 105 revisions; the newest is v137 + (`20261024_a3f7ce361b0e_v137_epcis_document_receipts.py`). +- `supabase/migrations/` holds 45 SQL files, the newest dated 2026-10-26. + No document in `docs/`, `ARCHITECTURE.md`, or + `docs/engineering/SCHEMA_CHANGE_POLICY.md` mentions this directory. +- `sql/prod-apply/` holds 8 hand-rendered apply, precheck and rollback + scripts for v128, v131 and SEL-314. +- `DEVOPS_NOTES.md` (2026-10-20): "`public.alembic_version` on the Supabase + project still reads `267d8053ced5` (v079) although v128 policies are + present, so prod-apply SQL has been landing without a version bump. + `alembic upgrade head` against this database would re-run v080–v133; do + not run it until the version row is reconciled." +- `docs/MIGRATION_CONSOLIDATION.md` says Alembic has 10 migrations and + describes porting Flyway files from a `migrations/` directory that no + longer exists. +- The Docker `CMD` runs `scripts/run-migrations.sh` before Gunicorn starts. + +**Why it matters** + +The product promise is tamper-evident evidence. The schema that holds it has +three write paths and the recorded version is about 50 revisions behind what +is applied. Any new deploy that runs migrations is one mis-set variable from +replaying v080 to v137 on production. Jev: 0.77 confidence this is the item +to fix first; incident likelihood within six months scored 2.72 on a 0–3 +scale (critical, confidence 0.73). I agree with both. + +**Fix (3 days)** + +1. Write a one-page ADR naming Alembic as the only channel and + `supabase/migrations/` and `sql/prod-apply/` as rendered outputs of it, + never hand-edited. Delete `docs/MIGRATION_CONSOLIDATION.md` or replace + it with the ADR. +2. Reconcile the production `alembic_version` row to the revision whose + effects are actually present, using the PRECHECK pattern already in + `sql/prod-apply/`. Record the result in `DEVOPS_NOTES.md`. +3. Add a CI guard: fail if a file under `supabase/migrations/` or + `sql/prod-apply/` changes without a matching Alembic revision in the + same diff. + +### 2. `app` package collision and bare imports — Priority 24 + +**Evidence** + +- `services/{admin,compliance,graph,ingestion,nlp,scheduler}/app/` are six + packages with the same import name. +- Root `conftest.py` (347 lines) rewrites `sys.path` before each service's + tests collect so `from app.X` resolves to the right one. +- 20 non-test files still use bare `from app.` imports; 27 use + `from services..app`. Seven of the bare ones are live ingestion + modules (`services/ingestion/app/sandbox/__init__.py`, + `services/ingestion/app/stripe_billing/rate_limiting.py`, + `services/ingestion/app/webhook_compat.py`, + `services/ingestion/app/labels/gs1_128.py`, + `services/ingestion/app/audit_export_log.py`, + `services/ingestion/regengine_ingestion/parsers/edi_parser.py`, + `services/ingestion/regengine_ingestion/utils/fetch_utils.py`); one is + `services/admin/app/logging_config.py`. +- `Dockerfile` sets `PYTHONPATH=/app:/app/services:/app/services/ingestion:/app/services/admin:...`. + In the container `app` always resolves to `services/ingestion/app`, so a + bare import from another service either raises `ImportError` (no module + of that name under ingestion) or, when the name collides, loads + ingestion's module. The admin case is a compatibility shim reached only + from the legacy `services/admin/main.py`. +- `pyproject.toml` comments explain the hazard rather than remove it + ("Listing multiple `services/` here would cause `app` to permanently + resolve against whichever service sorted first"). +- `pytest.ini_options.testpaths` lists `services/test_data/tests`, which + does not exist. + +**Why it matters** + +Every new test file and every new module inherits a trap that already +required two CI guards (`test-suite-check.yml`, the testpaths comment for +#2923 and #3247). Jev scored wrong-module import likelihood in production +1.97 on 0–3 (high, confidence 0.60). My own read is lower for production, +because the live bare imports are in ingestion, which sorts first, but the +test-side cost is paid on every PR. + +**Fix (3 days, mechanical)** + +1. Rewrite the 20 bare imports to `services..app` form. +2. Delete the seven legacy entrypoints (`services/*/main.py`, + `services/graph/app/main.py`, 1,818 lines) or move them to + scripts/legacy/ outside the import path. `ARCHITECTURE.md` already + says they are not the deploy target. +3. Reduce `Dockerfile` `PYTHONPATH` to `/app:/app/services`. +4. Shrink `conftest.py` to the fixtures only; remove the `sys.path` + switching once nothing depends on it. +5. Remove the dead `services/test_data/tests` entry from `testpaths`. + +### 3. Lint and type gates are near-empty — Priority 24 + +**Evidence** + +- `backend-ci.yml` runs `ruff check --select E9,F63,F7,F82` (syntax errors, + invalid prints and comparisons, undefined names). `pyproject.toml` + selects only `I` (import sorting). +- `[tool.mypy]` exists with the note "not yet a CI gate". No workflow runs + mypy. +- 766 `noqa` markers and 223 `type: ignore` markers in non-test backend + code. +- `BREAK-REPORT.md` records that a full `mypy .` still stops on a duplicate + module (`tests/audit/verify_chain.py` vs + `frontend/public/sdk/verify_chain.py`). + +**Why it matters** + +Two of the five commits before this audit are security fixes found by +external code scanning (ReDoS, header sanitisation, shell exec, URL host checks). +`ruff`'s `B`, `S` and `PL` families catch several of those classes before +review. With one engineer, static gates are the cheapest reviewer available. + +**Fix (2 days to start, then incremental)** + +1. Add `F`, `E`, `B`, `S102`, `S105`–`S108`, `S113`, `S301`, `S324`, + `S602`–`S607` to the ruff select for `services/shared/` only. Fix or + explicitly `noqa` with a reason. Expand one service per week. +2. Add `mypy services/shared --strict-optional` as a non-blocking job, + then make it blocking once clean. Resolve the `verify_chain.py` + duplicate by excluding `frontend/public/sdk` from mypy. +3. Ratchet: a CI step that fails if the count of `noqa` without a reason + rises above the committed baseline. + +### 4. CI sprawl — Priority 21 + +**Evidence** + +- 33 workflow files. `backend-ci.yml` 1,530 lines, `pr-guards.yml` 1,401, + `pr-gate.yml` 1,123, `security.yml` 875. +- `pr-gate-frontend-rerun.yml` exists to re-run the frontend gate after a + flaky failure. 29 lines across the three main workflows mention flake, + retry or re-run. The `test:e2e:authenticated-core` script in `frontend/package.json` + hard-codes `--retries=2`. +- `infra-k8s-ci.yml` runs on PRs for manifests `docs/CODEBASE_MAP.md` + calls retired. +- `docs/CODEBASE_MAP.md` §2 says 23 workflows. + +**Why it matters** + +Each PR waits on the union of these. The ingestion test leg alone has a +25-minute timeout. A founder paying per-PR wall-clock time cannot afford +duplicated jobs, and a rerun workflow institutionalises flakiness instead +of removing it. + +**Fix (4 days)** + +1. Measure: pull the last 30 PR runs and list job durations and failure + causes. Decide from data which jobs are redundant with + `pr-gate.yml`. +2. Move shared steps (checkout, Python setup, lock install) into one + reusable workflow called by the rest. +3. Delete `pr-gate-frontend-rerun.yml` after fixing the top flaky spec it + was created for. Drop `infra-k8s-ci.yml` with item 9. +4. Set `concurrency` groups so a new push cancels the previous run on the + same PR, if not already universal. + +### 5. Legacy Kafka backbone — Priority 20 + +**Evidence** + +- 13 non-test modules import `confluent_kafka` or read + `KAFKA_BOOTSTRAP_SERVERS`. The main ones total 3,505 lines: + `services/shared/kafka_auth.py` (786), `services/shared/kafka_compat.py`, + `services/shared/kafka_consumer_base.py`, `services/shared/observability/dlq_producer.py`, + `services/admin/app/review_consumer.py`, `services/nlp/app/consumer/__init__.py` (1,118), + `services/ingestion/app/kafka_utils.py`. +- `services/shared/event_backbone.py` defaults `EVENT_BACKBONE` to `pg`. + `ASYNC_PROCESSES.md` states the backbone is the Postgres task queue "not + Kafka". +- `confluent-kafka[schema-registry,avro]` and `fastavro` are in + `requirements.in` and the production image. + +**Fix (2 days)** + +Delete the Kafka path and its dependencies. Keep `services/shared/observability/kafka_propagation.py` +only if the tracing header helpers are used by the PG path (check callers +first). `ARCHITECTURE.md` already calls this "remediation drift, not the +primary deploy contract". + +### 6. Production image carries test and optional dependencies — Priority 20 + +**Evidence** + +- `requirements.in` includes pytest, pytest-asyncio, pytest-cov, + pytest-mock, pytest-xdist, hypothesis, aiosqlite and testcontainers with + the comment "move to dev-requirements.txt if image size matters". + `docker` (a testcontainers transitive) is in the lock. +- The `Dockerfile` compensates with `RUN find /app -name "test_*.py" -delete`. +- The image also ships langchain, langchain-community, langgraph, + langsmith, openai, llama-cloud, pandas, numpy, pytesseract, pdf2image and + pyzbar. The routers that use most of these (`nlp`, `agent_org`, + `label_vision`, `qr_decoder`) are in `EXPERIMENTAL_ROUTERS` and never + mount in production. + +**Why it matters** + +Attack surface and Trivy noise in `security.yml`, slower Railway builds, +and `--require-hashes` resolution work for packages production never +imports. + +**Fix (2 days)** + +1. Split `requirements.in` into `requirements.in` and + requirements-dev.in; compile two locks; CI installs both, Docker + installs one. +2. Move the experimental-only packages behind an extras file + (requirements-experimental.in) that the image omits until a router + graduates to the production allowlist. + +### 7. Documentation drift and clutter — Priority 20 + +**Evidence** + +- `docs/CODEBASE_MAP.md` claims every line was checked against the tree, + and says: 23 workflows (there are 33), 81 `include_router` calls in + `server/main.py` (there are 2; registration moved to + `server/router_policy.py`), and 14 experimental routers (the frozenset + has 15; `agent_org` is missing from the doc). +- `ARCHITECTURE.md` large-module table is dated 2026-07-31 and lists 13 + files; there are now 25 non-test modules over 1,000 lines, 20 of them under + `services/` and `server/`. +- `docs/MIGRATION_CONSOLIDATION.md`: see item 1. +- 369 Markdown files under `docs/`; 34 carry an obsolete, superseded or + historical banner; 53 sit in `docs/archive/` plus 2 in `docs/_archive/`. +- Duplicate directories: `docs/archive` and `docs/_archive`; `docs/schema` + and `docs/schemas`; `nfr/` and `docs/nfr`; `runbooks/` and + `docs/runbooks`. +- 20 Markdown files at the repo root, including one-off reports + (`BREAK-REPORT.md`, `FIX-REPORT.md`) and `GEMINI.md`. + +**Why it matters** + +Agent sessions are told to read `ARCHITECTURE.md` and `CODEBASE_MAP.md` +first. Wrong counts there cost every session a verification pass, which is +exactly why `scripts/check_docs_links.py` was written for paths. Counts +need the same treatment. + +**Fix (2 days)** + +1. Extend `scripts/check_docs_links.py` to assert the three numeric claims + above, or remove the numbers from the docs. +2. Merge the four duplicate directory pairs; move `BREAK-REPORT.md` and + `FIX-REPORT.md` under `docs/engineering/` with dates in the filename. +3. Re-stamp `ARCHITECTURE.md` after items 2, 5 and 9 land, since they + change its content. + +### 8. Orphaned scripts — Priority 20 + +**Evidence** + +31 of 87 top-level files in `scripts/` are not referenced by any workflow, +the `Dockerfile`, `railway.toml`, either runbooks directory, another +script, or the README. The largest: `scripts/verify_canonical_parity_live.py` +(1,702 lines), `scripts/backfill_canonical_from_legacy.py` (1,423), +`scripts/seed_demo_data.py` (680), `scripts/audit-issues.py` (665), +`scripts/e2e_brutal_scenario.py` (654), `scripts/jev_header_preview.py` (602). + +**Fix (1 day)** + +For each: delete, or add a one-line entry to a scripts/README.md naming +the runbook or workflow that uses it. The two canonical-parity scripts +belong to item 12 and should be deleted with it. + +### 9. Retired topology still in tree — Priority 20 + +**Evidence** + +- Seven legacy entrypoints (item 2), 1,818 lines. +- `infra/k8s/` with `base/` and `overlays/`, plus `infra-k8s-ci.yml` on + every PR that touches it. +- `runbooks/` at the root: five files, each with a `LEGACY:` banner per + `docs/CODEBASE_MAP.md`. +- `regengine/swarm/` (about 2,500 lines) gated by + `REGENGINE_ENABLE_LEGACY_SWARM`. +- `infra/gateway/nginx.conf` is "reference-only" with its own test suite in + the pytest `testpaths`. + +**Fix (1 day)** + +Move `infra/k8s`, root `runbooks/`, `regengine/swarm` and the legacy +entrypoints to docs/archive/retired-topology/ or delete them. Git history +keeps them. Remove `infra-k8s-ci.yml` and the gateway testpath. + +### 10. DB-dependent tests skip vacuously — Priority 18 + +**Evidence** + +- 170 `pytest.skip` / `skipif` sites. The most common reasons: "migrated + Postgres with fsma schema not available" (8 sites), "Explicit disposable + PostgreSQL required", "DATABASE_URL not set — live DB required". +- Skip sites by directory: `services/ingestion` 20 files, + `tests/migrations` 16, `tests/shared` 14. +- CI does run `live-db-suite` against `postgres:17` and a + `seeded-migration-rehearsal`, so production is covered. Locally, a bare + `pytest` passes while exercising none of the RLS, migration or + persistence paths. +- Backend coverage floors per service: compliance 80, admin 70, + scheduler 65, graph 60, ingestion 48, shared 35 (ratcheted 2026-04-20; + the comment in `pyproject.toml` explains the shared floor was set on an + inflated denominator). + +**Fix (3 days)** + +1. Add a `pytest --require-db` option that turns the skip into a failure, + and use it in the live-db CI job so a lost `DATABASE_URL` cannot turn + the suite green. +2. Document the one-command local path: `docker compose -f + docker-compose.dev.yml up -d` then `alembic upgrade head`, in + `CONTRIBUTING.md`, and make the skip message print it. +3. Re-measure the `shared` and `ingestion` floors and ratchet them. + +### 11. Frontend API access is split three ways — Priority 15 + +**Evidence** + +- `frontend/src/lib/api-client.ts` is 1,163 lines and is imported by 15 + files under `app/`, `components/` and `hooks/`. 33 non-test files in + those directories call `fetch(` directly. +- 30 route handlers under `frontend/src/app/api/`; 22 use `frontend/src/lib/proxy-factory.ts` (706 + lines); 8 are hand-rolled (`client-errors`, `inflow-lab/[...path]`, + `openapi.json`, `session`, `health`, `contact`, `tools/check-access`, + `tools/verify`). +- Largest page components: `frontend/src/app/tools/ftl-checker/components/FTLCheckerClient.tsx` + 1,299 lines, `frontend/src/app/ingest/page.tsx` 1,118, `frontend/src/app/dashboard/suppliers/page.tsx` + 1,097, `frontend/src/app/compliance/snapshots/page.tsx` 1,058. + +**Fix (3 days, incremental)** + +Route every backend call through `api-client.ts` or a typed hook. Convert +the 8 hand-rolled routes to `proxy-factory` where they proxy the backend +(`tools/*`, `inflow-lab`); leave `health`, `session`, `contact` and +`client-errors` as they are. Add an ESLint `no-restricted-syntax` rule +against bare `fetch(` in `src/app` and `src/components`. + +### 12. Legacy `cte_events` dual-write — Priority 14 + +**Evidence** + +- `services/shared/canonical_persistence/writer.py` calls + `legacy_dual_write.dual_write_legacy` on both single and batch writes; + since #1277 a failure raises and rolls back the canonical write. +- `services/shared/canonical_persistence/legacy_dual_write.py` (532 lines) is headed "TEMPORARY". + `CANONICAL_OWNERSHIP.md` calls `cte_events` "a legacy derived copy + (dual-write) that will be removed". +- 12 non-legacy modules still read `cte_events`, including + `services/admin/app/supplier_onboarding_routes.py`, `services/admin/app/supplier_facilities_routes.py`, + `services/admin/app/data_export_routes.py`, `services/admin/app/supplier_funnel_routes.py`, + `services/shared/workflows/export_output.py` and `services/shared/supplier_data_quality.py`. + +**Why it matters** + +Every evidence write pays for two tables, and a legacy-table failure can +reject a valid canonical event. Jev scored six-month incident likelihood +1.98 on 0–3 (high, confidence 0.74). Effort is the limiter: the 12 readers +must be re-pointed and verified with parity checks first. + +**Fix (2 weeks, in three PRs)** + +1. Re-point the 12 readers to `fsma.traceability_events` one route at a + time, each with a parity test against a seeded database. +2. Flip the dual-write to fail-open with a metric, run two weeks of + parity via the existing `scripts/verify_canonical_parity_live.py`. +3. Remove `services/shared/canonical_persistence/legacy_dual_write.py`, the two parity/backfill scripts, and + add a migration that drops or freezes `cte_events` and `cte_kdes`. + +### 13. Blocked upgrades and Node drift — Priority 12 + +**Evidence** + +- `dependabot.yml` ignores majors for eslint 10, typescript 7, jsdom 30, + jest-dom 7, vitest coverage 5, next, react, react-dom, vite and vitest, + each with a documented blocker. React stays at 18.3. +- Node: `.nvmrc` says 22; `copy-gate.yml` pins 20; one `frontend-ci.yml` + job pins 24; `package.json` `engines` says 22 or 24. The jsdom 30 + blocker note says "CI runs Node 20". +- Python 3.11 base image; security support ends 2027-10. + +**Fix (3 days)** + +Pin every workflow to `node-version-file: .nvmrc`. That alone unblocks +jsdom 30 and jest-dom 7. Schedule the eslint 10 and typescript 7 pair for +when `eslint-config-next` catches up; check monthly. Plan a Python 3.12 +image bump in Q1 2027. + +### 14. Modules over 1,000 lines — Priority 10 + +25 non-test modules (20 under `services/` and `server/`), up from 13 in the July table in `ARCHITECTURE.md`. +Top: `services/scheduler/service.py` 1,544, `services/shared/api_key_store.py` +1,488, `services/shared/canonical_event.py` 1,466, `services/ingestion/app/edi_ingestion/routes.py` +1,464, `services/shared/canonical_persistence/writer.py` 1,340, +`services/ingestion/app/fda_export/router.py` 1,323, `services/ingestion/app/inflow_workbench.py` +1,267, `server/main.py` 1,256, `services/graph/app/fsma_audit.py` 1,249, +`services/ingestion/app/compliance_score.py` 1,209. + +Split only when a feature already touches the file, following the "likely +split direction" column already in `ARCHITECTURE.md`. `server/main.py` is +the exception worth doing on its own: the lifespan, startup checks and +worker boot are the first thing every reader meets. + +### 15. `graph_update` dead task type — Priority 10 + +Declared in `services/shared/task_types.py` and given a 120-second +visibility timeout in `server/workers/task_processor.py`, but no production +code enqueues it and no handler is registered. `ASYNC_PROCESSES.md` +describes rows that "sit unclaimed"; today none are produced. Remove the +type and the timeout entry, or register the handler the day graph sync +moves to the queue. One hour. + +### 16. Frontend type looseness — Priority 8 + +192 `any` usages, all in test files (top file +`frontend/src/__tests__/pages/dashboard.test.tsx` with 18). 8 `react-hooks/set-state-in-effect` and 4 +`exhaustive-deps` suppressions in product code. `tsconfig.json` has +`strict: true`. Leave the tests; fix the 12 hook suppressions when the +page is next touched. + +--- + +## 3. Jev second opinion + +Asked one question set with the nine largest items as state (no customer +data, no secrets). + +| Question | Jev answer | Confidence | My view | +|---|---|---|---| +| Which item first? | schema_channels (0.81), test_gaps (0.16), legacy_dual_write (0.03) | 0.77 | Agree | +| Schema channel split incident likelihood, 6 months | 2.72 / 3 ≈ critical | 0.73 | Agree | +| Dual-write incident likelihood, 6 months | 1.98 / 3 ≈ high | 0.74 | Agree on risk; effort keeps it at rank 12 | +| `app` collision wrong-import in production | 1.97 / 3 ≈ high | 0.60 | Lower for production (ingestion sorts first); high for test cost | + +--- + +## 4. Phased plan + +Each phase fits beside feature work. Days are for one engineer. + +### Phase 0 — this week (4 days) + +| Item | Action | Days | +|--:|---|:-:| +| 1 | ADR: Alembic is the only channel. Reconcile production `alembic_version`. CI guard on `supabase/migrations/` and `sql/prod-apply/`. | 3 | +| 8 | Delete or register the 31 orphan scripts. | 0.5 | +| 15 | Remove `graph_update` type and timeout. | 0.1 | +| 13 | Point every workflow at `.nvmrc`. | 0.4 | + +Exit: one schema channel documented and enforced; production version row +matches applied state. + +### Phase 1 — next four weeks (9 days) + +| Item | Action | Days | +|--:|---|:-:| +| 2 | Rewrite bare imports, delete legacy entrypoints, trim `PYTHONPATH`, shrink `conftest.py`. | 3 | +| 9 | Archive `infra/k8s`, root `runbooks/`, `regengine/swarm`; drop `infra-k8s-ci.yml`. | 1 | +| 3 | Ruff `F E B S*` on `services/shared`; mypy non-blocking on `shared`; `noqa` ratchet. | 2 | +| 7 | Fix the three numeric claims in `CODEBASE_MAP.md`; merge duplicate doc dirs; move root reports. | 2 | +| 6 | Split dev requirements out of the production lock. | 1 | + +Exit: `pytest` from the root needs no path tricks; the image installs only +runtime packages; orientation docs pass a count check. + +### Phase 2 — weeks five to eight (9 days) + +| Item | Action | Days | +|--:|---|:-:| +| 4 | Measure PR CI durations; extract a reusable workflow; delete the rerun workflow after fixing its flake. | 4 | +| 5 | Delete the Kafka path and its two packages. | 2 | +| 10 | `--require-db` flag; local DB path in `CONTRIBUTING.md`; re-ratchet `shared` and `ingestion` floors. | 3 | + +Exit: PR wall-clock measured and reduced; a lost `DATABASE_URL` fails CI +instead of passing it. + +### Phase 3 — quarter (14 days) + +| Item | Action | Days | +|--:|---|:-:| +| 12 | Re-point 12 legacy readers, fail-open dual-write with parity metric, drop `cte_events`. | 10 | +| 11 | Bare-`fetch` lint rule; convert 4 proxy routes to `proxy-factory`. | 3 | +| 14 | Split `server/main.py` into lifespan, startup checks and worker boot. | 1 | + +Exit: one evidence table; one frontend API path. + +Items 3 (per-service ruff expansion), 14 (other large modules) and 16 are +continuous: do them inside feature PRs that already touch the file. + +--- + +## 5. Re-measure commands + +Run these before the next audit so the numbers above can be compared. + +```bash +# modules over 1,000 lines (non-test) +find services server regengine packages scripts -name '*.py' -not -path '*/tests/*' -not -name 'test_*' | xargs wc -l | awk '$1>1000' | sort -rn +``` + +```bash +# bare `from app.` imports in production code +grep -rlE '^\s*(from|import) app\b' --include='*.py' services | grep -v tests +``` + +```bash +# skip sites, noqa and type-ignore counts +grep -rlE 'pytest\.(mark\.)?skip|pytest\.skip\(' --include='*.py' tests services infra | wc -l +grep -rn 'noqa' --include='*.py' services server regengine packages | wc -l +grep -rn 'type: ignore' --include='*.py' services server regengine packages | wc -l +``` + +```bash +# workflow count and the four largest +ls .github/workflows | wc -l; wc -l .github/workflows/*.yml | sort -rn | head -5 +``` + +```bash +# Kafka surface +find services -name '*.py' -not -path '*/tests/*' | xargs grep -lE 'confluent_kafka|KAFKA_BOOTSTRAP' | wc -l +``` diff --git a/docs/decisions/2026-10-30-schema-change-channel.md b/docs/decisions/2026-10-30-schema-change-channel.md new file mode 100644 index 000000000..776b2317e --- /dev/null +++ b/docs/decisions/2026-10-30-schema-change-channel.md @@ -0,0 +1,211 @@ +# Schema change channel — Alembic is the only authority + +**Decision date:** 2026-10-30 +**Owner:** Christopher Sellers (solo technical founder) +**Status:** Proposed (WI-01 of `docs/plans/TECH_DEBT_REMEDIATION_PRD_2026-10-28.md`); becomes Active when Christopher merges it + +## Decision + +1. The Alembic chain under `alembic/versions/` is the only authoritative + description of the production database schema. Its head is the schema + the code expects. +2. `supabase/migrations/` is the Supabase CLI project history. It is frozen + as of this decision. No new file is added there unless it is a render of + an Alembic revision and carries the header in §Header. +3. `sql/prod-apply/` holds rendered outputs of Alembic revisions only: a + `PRECHECK_`, `apply_` and `rollback_` triple per revision, each carrying + the header in §Header. Nothing in that directory is hand-authored DDL. +4. Applying schema change to production by any path other than + `scripts/run-migrations.sh` (which runs `alembic upgrade head`) is a + break-glass action. Break-glass changes are allowed for security fixes + only, and the same PR must add an idempotent Alembic revision whose + `upgrade()` converges on the hand-applied state, so the chain stays true. +5. The production database is the Supabase project named in + `docs/operations/MIGRATION_V079_TO_V120_2026_09_05.md`, and + `docs/operations/DATABASE_ROLE_CUTOVER.md` is the blocking procedure for + running the chain against it. Railway hosts the application container + only. Documents that say the tenant database is Railway Postgres are + stale (see §Consequences). +6. `public.alembic_version` is never edited by hand except through + `docs/runbooks/ALEMBIC_VERSION_RECONCILE.md`, and that runbook permits an + edit only when the catalog proves every observable effect through the + target revision is present. + +## Header + +Every file under `supabase/migrations/` or `sql/prod-apply/` added after +this date starts with: + +```sql +-- alembic-revision: <12-hex revision id> +-- rendered-from: alembic/versions/.py () +``` + +A CI guard (`scripts/check_schema_channels.py`, run by the backend CI +"Alembic revision ID guard" job) fails a PR that adds or changes a file in those directories without +the header or with a revision id that does not exist. + +## Context + +Measured against the tree at `f87ed2a` on 2026-10-30. + +- Alembic: 105 revision files, one head `a3f7ce361b0e` (v137). The + revision DAG is clean (`scripts/check_alembic_revisions.py`). +- Production was last observed at `267d8053ced5` (v079), on 2026-09-05 + (`docs/operations/MIGRATION_V079_TO_V120_2026_09_05.md`) and again on + 2026-10-20 (`DEVOPS_NOTES.md`). 71 revisions sit between that row and + head. +- The controlled execution of the chain has been "prepared for review; + execution blocked" since 2026-09-05. Its prerequisites are role + provisioning, object-ownership reconciliation and a verified backup, all + in `docs/operations/DATABASE_ROLE_CUTOVER.md`. None is recorded as done. +- Meanwhile schema changes kept reaching production by two other channels: + 45 files in `supabase/migrations/` (17 from March–April 2026, 28 from + July–October 2026) and 12 files in `sql/prod-apply/`. Ten of the 45 render + an Alembic revision; 35 have no Alembic counterpart (Appendix A). +- `docs/MIGRATION_CONSOLIDATION.md` described a state from March 2026 (10 + Alembic revisions, a `migrations/` directory that no longer exists). It + now points here. +- Two documents disagree on the database's identity: + `docs/CODEBASE_MAP.md` §1 says Supabase; `frontend/COPY_REMEDIATION_PLAN.md` + (2026-08-02) says Railway Postgres. The operations documents and the + observed `alembic_version` row settle it: Supabase. + +Why this matters: the product promise is tamper-evident evidence, and the +schema that holds it has three write paths and a version row 71 revisions +behind the applied state. `scripts/run-migrations.sh` runs +`alembic upgrade head` whenever the version table is populated, so a deploy +whose `MIGRATIONS_DATABASE_URL` points at production would replay v080 to +v137. The 2026-09-05 rehearsal proved the chain from v079 on a synthetic +restore, but production has since received hand-applied changes that the +rehearsal did not have (Appendix A, "no counterpart"). + +## Consequences + +- No more hand-applied SQL without a paired Alembic revision (rule 4). + The eleven hand-applied changes since 2026-10-16 that have no counterpart + (Appendix A) need back-porting decisions before the chain runs; otherwise + `alembic upgrade head` will either fail on an object that exists or + silently diverge from what production has. +- The reconcile runbook, not a stamp, resolves the version row. Stamping + `head` would record 60 revisions whose effects the catalog cannot show. +- `frontend/COPY_REMEDIATION_PLAN.md` row "Tenant DB" is corrected in WI-07. +- `docs/engineering/SCHEMA_CHANGE_POLICY.md` §3.1 still describes Flyway + style `V{N}__description.sql` files and a `schema_migrations` table. That + policy predates Alembic ownership and is updated in WI-07 to name Alembic. + +## Alternatives considered + +- **Stamp production to head and forward-port the hand-applied changes.** + Rejected: the catalog shows most v080–v127 effects absent (passwordless + auth tables, receipts, dead-letter, evidence snapshots). Stamping would + lie about them and the application would fail on first use. +- **Make `supabase/migrations/` the authority and retire Alembic.** Rejected: + it has no revision graph, no downgrade path, and CI's migration tests, + parity audit and seeded rehearsals are all Alembic-based. +- **Keep three channels and document them.** Rejected: that is the current + state, and it produced a version row that cannot be trusted. + +Second opinion: TypeSafe Jev, given the catalog pattern above, chose "run +the chain, leave the row" at 0.89 confidence, called stamping to head unsafe +at p=0.95, and rated replay without back-porting the hand-applied changes a +high risk (2.22 of 3, low confidence 0.23). Request +`req_01a18b886d7f76d6b797d4ea7f5065ca`, 2026-10-30. + +## Appendix A — Channel inventory (2026-10-30) + +Counterpart = the Alembic revision whose `upgrade()` produces the same +objects. "Partial" means some of the file's statements have a counterpart. +"Auxiliary" means the objects exist only for Supabase-side features +(anonymous lead capture, developer portal RPCs, Supabase roles) and the +decision in Appendix B applies. + +### `supabase/migrations/` + +| File | Counterpart | Disposition | +|---|---|---| +| `20260314053557_create_waitlist_leads.sql` | none | Auxiliary. Decision B1. | +| `20260314073855_create_developer_portal_tables.sql` | v058 (`20260415_v058_docs_and_data.py`) | Rendered history. | +| `20260314074042_create_increment_invite_usage_rpc.sql` | none | Auxiliary. Decision B1. | +| `20260314093510_create_assessment_submissions.sql` | v058 | Rendered history. | +| `20260317074952_create_fsma_organizations.sql` | baseline (`20260324_97588ba8edf3_baseline_consolidate_v002_through_v042.py`) | Rendered history. | +| `20260319014551_create_search_queries_table.sql` | none | Auxiliary. Decision B1. | +| `20260320220954_extend_organizations_schema.sql` | baseline / v084 | Rendered history. | +| `20260321014327_seed_fsma_204_obligations.sql` | seeds in code (`services/shared/rules/seeds.py`, v116/v119) | Superseded by code-owned seeds. | +| `20260321014418_seed_obligation_cte_rules.sql` | same | Superseded. | +| `20260330002451_enable_rls_missing_tables.sql` | v051 / v056 | Rendered history. | +| `20260330031620_developer_portal_schema_doc.sql` | none (COMMENT ON only) | No schema effect. Freeze. | +| `20260403234813_create_traceability_events_and_ingestion_runs.sql` | v043–v047 (`20260325_compliance_control_plane_v043_v047.py`) | Rendered history. | +| `20260403235145_drop_hash_chain_fk_to_cte_events.sql` | v090 (`4c5d6e7f8a9b`) | Rendered history. | +| `20260403235708_create_rules_engine_tables.sql` | v043–v047 | Rendered history. | +| `20260404000051_fix_cte_events_idempotency_constraint.sql` | v072 (`71e79e111c5d`) | Rendered history. | +| `20260404011602_create_transformation_links_adjacency.sql` | v057 / v068 | Rendered history. | +| `20260413044744_add_last_login_at_to_users.sql` | none found | Decision B2: confirm whether `public.users.last_login_at` is in the ORM; if yes, add an idempotent revision. | +| `20260729012459_v092_lockdown_rls_and_revokes_only.sql` | v092 (`b3c8e1f4a627`) | Rendered history. | +| `20260730025741_lockdown_anon_lead_capture_and_definer_rpcs.sql` | none | Auxiliary. Decision B1. | +| `20260730025751_revoke_pgaudit_event_trigger_fn_execute.sql` | none | Auxiliary (provider extension). Decision B1. | +| `20260730033519_drop_transitional_anon_insert_on_assessment_submissions.sql` | none | Auxiliary. Decision B1. | +| `20261001054030_security_fsma_rls_and_function_search_path_20260930.sql` | partial: v111 (`b71d9e4c2a60`), v128 | Decision B3: diff against v111/v128 and back-port the remainder. | +| `20261016072839_search_path_extensions.sql` | none | Decision B4 (advisor batch). | +| `20261016073047_a_move_extensions_to_extensions_schema.sql` | none | Decision B4. Conflicts with the v111 extension-ownership note in `MIGRATION_V079_TO_V120_2026_09_05.md`. | +| `20261016073057_b_drop_duplicate_indexes.sql` | none | Decision B4. Drops `fsma.ux_cte_events_tenant_idempotency_key`, which v072 creates; the chain will recreate it unless back-ported. | +| `20261016073107_c_rls_initplan_select_wrap.sql` | none | Decision B4. Rewrites 73 policy expressions that v128/v134 later replace; order matters. | +| `20261016073122_d_index_unindexed_foreign_keys.sql` | none | Decision B4. | +| `20261016234347_v128_fsma_rls_policy_repair.sql` | v128 (`a5e7c9b2d341`) | Render. Header added 2026-10-30. | +| `20261016234352_v128_consolidate_permissive_policies.sql` | v128 (`93eb33f262a1`) | Render. Header added 2026-10-30. | +| `20261017021229_e_wrap_get_tenant_context.sql` | none | Decision B4. 86 policies rewritten; v134 assumes the v096 `CASE` form. | +| `20261017021244_f_tenant_id_indexes.sql` | partial: v055 | Decision B4. | +| `20261020072328_assessment_lead_abuse_guard.sql` | none | Auxiliary. Decision B1. | +| `20261020085506_sel314_ftl_read_only_drop_sysadmin_overlap.sql` | v134 (`1faec7148e78`) | Render. Header added 2026-10-30. | +| `20261020094817_sel314_drop_fallback_tenant_policies.sql` | v134 | Render. Header added 2026-10-30. | +| `20261022101703_lock_down_submit_assessment_lead.sql` | none | Auxiliary. Decision B1. | +| `20261022103548_enable_rls_internal_tables.sql` | partial: v056 / v111 | Decision B3. | +| `20261023050348_sel401_notify_new_lead_secret.sql` | none | Auxiliary. Decision B1. | +| `20261025033803_data_gap_time_to_fix.sql` | v135 (`41f96588d87c`) | Render. Header added 2026-10-30. | +| `20261025090000_supplier_agreed_sending_hours.sql` | v136 (`b005ddbf1dba`) | Render. Header added 2026-10-30. | +| `20261025090100_epcis_document_receipts.sql` | v137 (`a3f7ce361b0e`) | Render. Header added 2026-10-30. | +| `20261025100000_fix_obligation_1455_regulation_refs.sql` | none | Decision B5: data correction; needs a data revision or a seed update. | +| `20261025100100_waitlist_leads_drop_authenticated_read.sql` | none | Auxiliary. Decision B1. | +| `20261025100200_task_queue_fail_closed_tenant_policy.sql` | none; **conflicts** with v090 `a128f7f95db8` and v096 `c3a51e2f7b90`, which define `tenant_isolation_tasks` differently | Decision B6: back-port as v138 so the chain's last word matches production. | +| `20261025100300_data_gap_close_receipt_not_null.sql` | none; post-v135 fix | Decision B6: back-port as a revision after v135. | +| `20261026132000_sel399_rls_split_api_keys_sessions_users.sql` | none; file says "PREP ONLY. Do not apply without C.Lee's yes" | Decision B7: record whether it was applied; if yes, back-port; if no, move to a `proposed/` subdirectory. | + +### `sql/prod-apply/` + +| File | Counterpart | Disposition | +|---|---|---| +| `PRECHECK_v128_fsma_rls.sql`, `apply_v128_fsma_rls.sql`, `rollback_v128_fsma_rls.sql` | v128 (`a5e7c9b2d341`) | Render. Header added 2026-10-30. | +| `PRECHECK_v128_policy_merges.sql`, `apply_v128_policy_merges.sql`, `rollback_v128_policy_merges.sql` | v128 (`93eb33f262a1`) | Render. Header added 2026-10-30. | +| `PRECHECK_v131_tool_leads_utm.sql`, `apply_v131_tool_leads_utm.sql`, `rollback_v131_tool_leads_utm.sql` | v131 (`0a329588b824`) | Render. Header added 2026-10-30. | +| `apply_sel314_rls_dedupe.sql`, `rollback_sel314_rls_dedupe.sql` | v134 (`1faec7148e78`) | Render. Header added 2026-10-30. | +| `rollback_sel399_rls_split_api_keys_sessions_users.sql` | none (pairs with the Supabase file above) | Decision B7. | +| `PRECHECK_alembic_version_reconcile.sql` | all revisions after v079 | Read-only; owned by the reconcile runbook. | + +### Hand-applied revisions and their idempotency + +These revisions already have their effects on production. The chain must +pass over them without error. Counts are `IF NOT EXISTS` / +`DROP ... IF EXISTS` / `to_regclass` guards in each `upgrade()`: + +| Revision | Guards | Verdict | +|---|---|---| +| v128 `a5e7c9b2d341` | 2 / 2 / 2, plus catalog fingerprint checks | Idempotent by design (its header says so). | +| v128 `93eb33f262a1` | 1 / 0 / 0 | Accepts "already merged" pairs per its docstring; verify on staging. | +| v131 `0a329588b824` | 1 / 0 / 0 | `ADD COLUMN IF NOT EXISTS`; idempotent. | +| v134 `1faec7148e78` | 0 / 2 / 3 | Documented idempotent. | +| v135 `41f96588d87c` | 5 / 6 / 0 | Documented idempotent. | +| v136 `b005ddbf1dba` | 2 / 1 / 0 | Documented idempotent. | +| v137 `a3f7ce361b0e` | 3 / 4 / 0 | Documented idempotent. | + +## Appendix B — Decisions needed from Christopher + +| # | Decision | Default if no answer | +|---|---|---| +| B1 | Are the Supabase auxiliary objects (`waitlist_leads`, `search_queries`, `increment_invite_usage`, `submit_assessment_lead`, `notify_new_lead`, anon/service_role policies) in scope for Alembic? | Out of scope. Record them in a docs/schema/supabase-auxiliary-objects.md list and leave `supabase/migrations/` as their only history. | +| B2 | Is `public.users.last_login_at` used by the ORM? | Check `services/admin/app/sqlalchemy_models.py`; if present, add an idempotent revision. | +| B3 | Back-port the parts of the 2026-09-30 and 2026-10-22 RLS files that v111/v128 do not cover? | Yes, as one revision, after a diff. | +| B4 | Back-port the six advisor-driven files (extensions move, duplicate indexes, initplan wrap, FK indexes, tenant_id indexes, get_tenant_context wrap)? | Yes for indexes; no for the policy rewrites, which v128/v134 supersede. Extension move needs the provider-ownership review in `DATABASE_ROLE_CUTOVER.md`. | +| B5 | Obligation 1455 citation fix: data revision or seed update? | Seed update in `services/shared/rules/seeds.py` plus a data revision that applies it. | +| B6 | Back-port `task_queue_fail_closed_tenant_policy` and `data_gap_close_receipt_not_null` as v138/v139? | Yes; both are small and idempotent. | +| B7 | Was SEL-399 applied to production? | Treat as not applied until the precheck shows `api_keys_backend` on `public.api_keys`. | +| B8 | Who runs the reconcile runbook and when? | Christopher, after B1–B7, on the staging restore the 2026-09-05 document requires. | diff --git a/docs/plans/TECH_DEBT_REMEDIATION_PRD_2026-10-28.md b/docs/plans/TECH_DEBT_REMEDIATION_PRD_2026-10-28.md new file mode 100644 index 000000000..0ae118d5f --- /dev/null +++ b/docs/plans/TECH_DEBT_REMEDIATION_PRD_2026-10-28.md @@ -0,0 +1,1026 @@ +# PRD: Technical Debt Remediation, RegEngine — 2026-10-28 + +Audience: an AI implementation agent working inside the RegEngine repository +with no memory of the audit that produced this document. Everything the +agent needs is in this file or in the files it names. + +Source analysis: `docs/audits/TECH_DEBT_AUDIT_2026-10-28.md` (the audit). +This PRD turns the audit's sixteen findings into executable work items with +acceptance criteria. Where the two disagree, this PRD wins; it incorporates +the owner's standing rules that the audit did not. + +Owner and decision maker: Christopher (GitHub `PetrefiedThunder`). The agent +executes; it does not decide product direction, delete files, or change +production. + +--- + +## 0. Read this first + +### 0.1 What RegEngine is + +A pre-production FSMA 204 food-traceability evidence platform. One FastAPI +monolith (`server/main.py`) deployed on Railway from the root `Dockerfile`, +one Next.js app (`frontend/`) on Vercel, PostgreSQL on Supabase with +row-level security, Redis, optional Neo4j. The production spine is: + +```text +ingest -> canonicalize -> validate -> persist evidence -> export +``` + +Every change in this PRD must protect that spine or stay out of its way. +Read, in this order, before touching code: + +1. `CLAUDE.md` (agent rules, verification commands) +2. `ARCHITECTURE.md` +3. `docs/CODEBASE_MAP.md` +4. `CANONICAL_OWNERSHIP.md` +5. `docs/audits/TECH_DEBT_AUDIT_2026-10-28.md` + +### 0.2 Standing rules from the owner (override anything below that conflicts) + +- **Never delete files.** Where the audit says "delete", this PRD says + "move to an archive directory and record it". Deletion of any file + happens only after Christopher approves it by name in a PR review or a + message. Git history is not a substitute for this rule. +- **Keep diffs small and in one lane.** One work item per pull request. + Do not bundle. +- **No product decisions.** If a requirement needs a choice the PRD does + not make, stop and put the question in the PR description under + "Decision needed", then continue with the parts that do not depend on + it. +- **Secrets:** never ask for, print, log or commit a key. Verify a secret by + its name or effect only (see `CLAUDE.md` → Secrets). +- **Use `python3` and `npm`.** There is no `Makefile`. +- **Tests that need PostgreSQL, Redis or Neo4j:** if the local environment + lacks them, say so in the PR; do not fake a pass. + +### 0.3 Repository mechanics the agent will hit + +| Mechanic | What to know | +|---|---| +| Branch prefix | The `Engine Review Pass` CI gate derives the author engine from the branch prefix. Use `/`, where engine is one of `claude`, `codex`, `gemini`, `grok`, `muse`, `kimi`, `pi`, `deepseek`, `hermes`, `antigravity`. Example: `codex/wi-02-app-namespace`. `CONTRIBUTING.md` lists `fix/`, `feat/`, `chore/`, `docs/`; the gate rejects those, so use the engine form. | +| Cross-engine review | A PR merges only after a *different* engine posts a comment containing, alone on one line, `REVIEW: PASS `. The author engine's own PASS is ignored. After every push the SHA changes and the marker must be re-posted. Do not post a marker under an engine name that is not yours. | +| PR template | `.github/pull_request_template.md`. Fill every section. | +| Draft PRs | CodeRabbit skips drafts. Open PRs as ready for review unless the work is incomplete. | +| `*AUDIT*.md` is gitignored | `.gitignore` line 153. `docs/audits/*.md` is re-allowed. Put audit-type reports there. | +| Paths in this PRD | A backticked path that does not exist in the tree is a file this PRD tells you to create or a destination to move to. Appendix A lists every path that must already exist. | +| Docs link gate | `scripts/check_docs_links.py` fails CI when a backticked path in a scoped document does not exist. Scope is `runbooks/`, `docs/CODEBASE_MAP.md`, `docs/compliance/`. Run it on any document you edit anyway. A path named deliberately because it was retired goes in `.docs-link-allowlist` with a reason. | +| Workflow inventory gate | `scripts/check_workflow_inventory.py` fails CI when the set of workflows drifts from its checked-in inventory or when a required branch-protection context is emitted by nothing. Adding, renaming or archiving a workflow means updating that inventory in the same PR. | +| Legacy dead-code gate | `scripts/check_legacy_dead_code.py` blocks re-introduction of code paths already removed. Extend it when you retire something, so it cannot come back. | +| Visual report | `visual-report (non-required)` fails on every PR today (Playwright 1.64 rejects extensionless screenshot names). Fix is open as PR #3666. Ignore this check; do not try to fix it inside another work item. | +| Test discovery | `pyproject.toml` `[tool.pytest.ini_options].testpaths` lists every test root. A test directory not listed there never runs. `test-suite-check.yml` guards import failures. | +| Pre-commit | `.pre-commit-config.yaml` exists; install it outside the hash-pinned requirements. | + +### 0.4 Verification commands (from `CLAUDE.md`) + +```bash +python3 -m pytest tests/swarm -q +python3 -m pytest services/ingestion/tests -q --no-cov +python3 -m pytest services/admin/tests -q --no-cov +python3 -m pytest services/shared/tests -q --no-cov +python3 -m pytest tests/scripts -q --no-cov +``` + +```bash +cd frontend && npm run lint +cd frontend && npm run test:run +cd frontend && npm run build +``` + +Backend lint as CI runs it: + +```bash +ruff check . --select E9,F63,F7,F82 --line-length 120 +``` + +--- + +## 1. Goals and non-goals + +### Goals + +1. One schema-change channel, with production's recorded migration version + matching what is applied (WI-01). +2. A test and import layout with no hidden path tricks (WI-02, WI-10). +3. Static gates that catch the bug classes recent security fixes were for + (WI-03). +4. A production image that installs only what production imports (WI-05, + WI-06). +5. Orientation documents whose numbers are checked by CI, not by readers + (WI-07). +6. Lower PR wall-clock time with no loss of coverage (WI-04). +7. A single evidence table and a single frontend API path (WI-12, WI-11). + +### Non-goals + +- New product features or router graduations. +- Changing tenant isolation, RLS policy, auth, or the audit hash chain. +- Any change to production configuration, Railway variables, Vercel + settings or the Supabase project. The agent prepares; Christopher + applies. +- Rewriting modules only because they are long (WI-14 is opportunistic). +- Fixing `visual-report (non-required)` (owned by PR #3666). + +--- + +## 2. Scoring and sequencing + +Priority = (Impact + Risk) × (6 − Effort), from the audit. Days are for one +agent with review turnaround excluded. + +| WI | Title | Priority | Phase | Days | Depends on | +|--:|---|--:|:-:|:-:|---| +| 01 | Single schema channel and version-row reconciliation kit | 27 | 0 | 3 | — | +| 08 | Register or archive orphan scripts | 20 | 0 | 0.5 | — | +| 15 | Retire the dead `graph_update` task type | 10 | 0 | 0.1 | — | +| 13 | One Node version across CI | 12 | 0 | 0.4 | — | +| 02 | Remove the `app` package collision | 24 | 1 | 3 | 09 | +| 09 | Archive the retired six-service topology | 20 | 1 | 1 | — | +| 03 | Widen ruff; stage mypy | 24 | 1 | 2 | 02 | +| 07 | Make orientation docs count-checked | 20 | 1 | 2 | 02, 09 | +| 06 | Split dev and experimental requirements from the production lock | 20 | 1 | 1 | — | +| 04 | CI consolidation | 21 | 2 | 4 | 09, 13 | +| 05 | Retire the Kafka backbone | 20 | 2 | 2 | 06 | +| 10 | DB-dependent tests fail loudly when the DB is missing | 18 | 2 | 3 | 02 | +| 12 | Retire the `cte_events` dual-write | 14 | 3 | 10 | 01, 10 | +| 11 | One frontend API path | 15 | 3 | 3 | — | +| 14 | Split `server/main.py` | 10 | 3 | 1 | 02 | +| 16 | Frontend hook-rule suppressions | 8 | ongoing | — | — | + +Execute in phase order. Inside a phase, the order shown. + +--- + +## 3. Work items + +Each item has: Problem (with the evidence the audit measured), Scope, +Requirements (numbered, testable), Acceptance criteria, Verification, +Out of scope, Rollback. "Baseline" numbers come from the audit and must be +re-measured with the commands in Appendix A before the work starts, so the +PR can show before and after. + +### WI-01 — Single schema channel and version-row reconciliation kit + +**Status 2026-10-30:** requirements 1, 2, 3, 4 and 6 delivered in PR #3699 +(`docs/decisions/2026-10-30-schema-change-channel.md`, +`sql/prod-apply/PRECHECK_alembic_version_reconcile.sql`, +`docs/runbooks/ALEMBIC_VERSION_RECONCILE.md`, `docs/MIGRATION_CONSOLIDATION.md`). +Requirement 5 (`scripts/check_schema_channels.py`, its tests, and the step in +`backend-ci.yml`) landed in the same PR on 2026-10-30, so WI-01 is complete +except for the decisions in Appendix B of the ADR. + +**Problem.** Schema changes reach production three ways: the Alembic chain +(`alembic/versions/`, 105 revisions, head v137), `supabase/migrations/` +(45 SQL files, newest 2026-10-26, mentioned by no document), and +hand-rendered scripts in `sql/prod-apply/`. `DEVOPS_NOTES.md` (2026-10-20) +records that production's `public.alembic_version` still reads v079 while +v128 policies are applied, so `alembic upgrade head` there would replay +v080 to v137. `docs/MIGRATION_CONSOLIDATION.md` describes a state from +March (10 Alembic revisions, a `migrations/` directory that no longer +exists). The Docker `CMD` runs `scripts/run-migrations.sh` before the app +starts. + +**Scope.** Documentation, a read-only SQL kit, one CI guard with tests. No +database is touched by the agent. + +**Requirements.** + +1. Write `docs/decisions/2026-10-30-schema-change-channel.md` (use the + date the PR opens; the directory's existing files are named + `YYYY-MM-DD-topic.md`, follow their format). It must + state: Alembic is the only authoritative channel; `supabase/migrations/` + and `sql/prod-apply/` are *rendered outputs* of Alembic revisions, never + hand-edited; every rendered file names the Alembic revision it renders in + a header comment; and the rendering command used. +2. Inventory `supabase/migrations/` and `sql/prod-apply/`: for every file, + record in the ADR appendix which Alembic revision it corresponds to, or + "no Alembic counterpart". Files with no counterpart are a "Decision + needed" list for Christopher; do not write migrations for them. +3. Produce `sql/prod-apply/PRECHECK_alembic_version_reconcile.sql` + containing only `SELECT` statements that report: the current + `alembic_version` row; for each Alembic revision from v080 to head, + whether its observable effect is present (table, column, policy or + function existence, derived from each revision's `upgrade()`); and a + final computed "highest revision whose effects are all present". +4. Produce `docs/runbooks/ALEMBIC_VERSION_RECONCILE.md`: steps for + Christopher to run the precheck, read the result, set the version row + with a single `UPDATE public.alembic_version SET version_num = ''` + inside a transaction, and re-run the precheck. State explicitly that + the agent did not run it. +5. Add `scripts/check_schema_channels.py` with tests in + `tests/scripts/test_check_schema_channels.py`: fails when a PR diff adds + or changes a file under `supabase/migrations/` or `sql/prod-apply/` that + lacks the `-- alembic-revision: ` header, or whose named revision + does not exist under `alembic/versions/`. Wire it into + `.github/workflows/pr-guards.yml` next to the other guards. +6. Replace `docs/MIGRATION_CONSOLIDATION.md` body with a two-line pointer to + the ADR (do not delete the file; other docs may link it). + +**Acceptance criteria.** + +- `python3 -m pytest tests/scripts/test_check_schema_channels.py -q` passes, + including a case for a file without the header and a file naming a + non-existent revision. +- `python3 scripts/check_docs_links.py docs/decisions/2026-10-30-schema-change-channel.md docs/runbooks/ALEMBIC_VERSION_RECONCILE.md` passes. +- The PRECHECK file contains no statement other than `BEGIN READ ONLY`, + `SET LOCAL`, `WITH`, `SELECT`, `ROLLBACK` and comments (add a test + asserting this). +- The "Decision needed" list is in the PR description. + +**Out of scope.** Running anything against a database. Writing new Alembic +revisions. Changing `scripts/run-migrations.sh`. + +**Rollback.** Revert the PR. Nothing outside the repo changes. + +### WI-02 — Remove the `app` package collision + +**Problem.** `services/{admin,compliance,graph,ingestion,nlp,scheduler}/app/` +are six packages with one import name. Root `conftest.py` (347 lines) +rewrites `sys.path` per service so tests resolve `from app.X`. 20 +non-test files still use bare `from app.`/`import app`; 27 use +`services..app`. `Dockerfile` sets `PYTHONPATH` to all six service +directories, so in the container `app` always means +`services/ingestion/app`. Baseline: the exact 20 files are listed by +Appendix A command A2. + +**Scope.** Imports, legacy entrypoints, `Dockerfile` `PYTHONPATH`, +`conftest.py`, `pyproject.toml`. + +**Requirements.** + +1. Rewrite every bare `from app.` / `import app` in non-test code to the + `services..app` form. Keep behaviour identical; this is a rename. +2. Legacy entrypoints `services/admin/main.py`, `services/compliance/main.py`, + `services/graph/main.py`, `services/graph/app/main.py`, + `services/ingestion/main.py`, `services/nlp/main.py`, + `services/scheduler/main.py` (1,818 lines): move to + `docs/archive/retired-topology/entrypoints/_main.py` with a header + comment giving the original path and the date (WI-09 creates the + directory; if WI-09 has not landed, create it here). Do not delete. + If any of these is imported by live code or a workflow, that is a + "Decision needed"; leave it in place and report. +3. `services/admin/app/logging_config.py` is a compatibility shim for the + bare form. Keep its public function; remove the bare-import fallback once + no caller uses it. +4. Reduce `Dockerfile` `PYTHONPATH` to `/app:/app/services`. Keep the + build-time path check that follows it. +5. Reduce `conftest.py` to fixtures. Remove the `sys.path` switching once + every test imports through `services..app`. Tests that still use + `from app.` must be rewritten in the same PR (count them first with + Appendix A command A3; if above 150 files, split this into two PRs: + production imports first, tests second, and keep the conftest shim until + the second lands). +6. Remove the non-existent `services/test_data/tests` entry from + `pyproject.toml` `testpaths`. +7. Extend `scripts/check_legacy_dead_code.py` so a new bare `from app.` + import in `services/` fails CI, with a test. + +**Acceptance criteria.** + +- Appendix A command A2 returns zero files. +- `python3 -m pytest --collect-only -q` from the repo root reports no + collection errors. +- The five pytest commands in §0.4 pass with the same pass counts as before + the change (record both in the PR). +- `docker build .` succeeds locally if Docker is available; if not, say so + and rely on the `docker-build` CI job. +- `conftest.py` no longer contains `sys.path` manipulation. + +**Out of scope.** Renaming the `app` directories themselves. Changing what +any module does. + +**Rollback.** Revert the PR. + +### WI-03 — Widen ruff; stage mypy + +**Problem.** CI runs `ruff --select E9,F63,F7,F82`; `pyproject.toml` +selects only `I`. mypy is configured but not a gate. 766 `noqa` and 223 +`type: ignore` markers in non-test backend code. Two of the five commits +before the audit were security fixes found by external scanning (ReDoS, +header sanitisation, shell exec, URL host checks). + +**Scope.** `pyproject.toml`, `.github/workflows/backend-ci.yml`, the code +under `services/shared/` for the first pass. + +**Requirements.** + +1. In `pyproject.toml` `[tool.ruff.lint]`, add a per-directory override for + `services/shared/**` selecting `F`, `E`, `B`, `S102`, `S105`, `S106`, + `S107`, `S108`, `S113`, `S301`, `S324`, `S602`, `S603`, `S604`, `S605`, + `S606`, `S607`. Fix every finding or add `# noqa: -- ` + with a reason. A `noqa` without a reason is not acceptable in this PR. +2. Make `backend-ci.yml`'s `lint-shared` job run `ruff check services/shared` + with the project config (drop the explicit narrow `--select` for that + job only). +3. Add a non-blocking `mypy` job for `services/shared` using the existing + `[tool.mypy]` config, uploading its report as an artifact. Exclude + `frontend/public/sdk` from mypy so the `verify_chain.py` duplicate + (noted in `BREAK-REPORT.md`) does not stop it. +4. Add `scripts/check_noqa_ratchet.py` with tests: counts `noqa` markers + lacking a `--` reason under `services/` and fails if the count exceeds + the number committed in `scripts/noqa_baseline.txt`. Commit the baseline + measured after step 1. Wire into `pr-guards.yml`. +5. Document in `CONTRIBUTING.md` → Code Guidelines: the rule set, the + `noqa` reason convention, and that one more service directory joins the + override each time a PR touches it substantially. + +**Acceptance criteria.** + +- `ruff check services/shared` exits 0 with the new config. +- `tests/scripts/test_check_noqa_ratchet.py` passes. +- The mypy job runs in CI and is marked non-blocking in `ci-status`. +- Count of `noqa` without reason under `services/shared` is zero. + +**Out of scope.** Enabling the rules repo-wide. Making mypy blocking. + +**Rollback.** Revert the PR; no runtime change. + +### WI-04 — CI consolidation + +**Problem.** 33 workflows. `backend-ci.yml` 1,530 lines, `pr-guards.yml` +1,401, `pr-gate.yml` 1,123, `security.yml` 875. +`pr-gate-frontend-rerun.yml` exists to re-run flaky frontend gates. The +ingestion test leg has a 25-minute timeout. `infra-k8s-ci.yml` runs on PRs +for a retired topology. + +**Scope.** `.github/workflows/`, the workflow inventory consumed by +`scripts/check_workflow_inventory.py`, `docs/CODEBASE_MAP.md` §7. + +**Requirements.** + +1. Measure first. Using the GitHub API (the `gh` CLI if available, else the + REST API with the token the environment provides), pull the last 30 + completed `pull_request` runs of `pr-gate.yml`, `pr-guards.yml`, + `backend-ci.yml`, `frontend-ci.yml`, `security.yml`. Produce + `docs/engineering/CI_TIMING_2026-11.md` with, per job: median duration, + p90, failure count, and the top failure cause from logs. Commit this + before changing any workflow. +2. Extract the shared setup steps (checkout, Python setup with + `requirements.lock`, Node setup from `.nvmrc`, caches) into one reusable + workflow (`workflow_call`) and call it from the five workflows above. +3. Identify the flaky spec behind `pr-gate-frontend-rerun.yml` from the + timing document. Fix that spec's root cause (waits, selectors, test + isolation), prove ten consecutive green local runs, then archive the + rerun workflow to `docs/archive/retired-topology/workflows/` and update + the inventory. Do not delete. +4. Archive `infra-k8s-ci.yml` the same way (depends on WI-09). +5. Ensure every PR-triggered workflow has a `concurrency` group keyed on the + PR number with `cancel-in-progress: true`, except jobs the repo marks as + must-not-cancel (the `engine-review-pass` job already says so). +6. Update `docs/CODEBASE_MAP.md` §7 with the real workflow count (WI-07 + adds the CI check for it). + +**Acceptance criteria.** + +- `python3 scripts/check_workflow_inventory.py` passes. +- `python3 -m pytest tests/scripts/test_check_workflow_inventory.py -q` passes. +- A follow-up timing measurement after the change, appended to the timing + document, shows median PR wall-clock reduced. If it is not reduced, say + so; do not remove jobs to force the number. +- No required branch-protection context is left without an emitter. + +**Out of scope.** Changing what any test checks. Removing coverage gates. +Lowering thresholds. + +**Rollback.** Revert the PR; restore inventory. + +### WI-05 — Retire the Kafka backbone + +**Problem.** 13 non-test modules import `confluent_kafka` or read +`KAFKA_BOOTSTRAP_SERVERS`; the main ones total 3,505 lines +(`services/shared/kafka_auth.py` 786, `services/nlp/app/consumer/__init__.py` +1,118, `services/shared/kafka_compat.py`, `services/shared/kafka_consumer_base.py`, +`services/shared/observability/dlq_producer.py`, +`services/admin/app/review_consumer.py`, `services/ingestion/app/kafka_utils.py`). +`services/shared/event_backbone.py` defaults `EVENT_BACKBONE` to `pg`; +`ASYNC_PROCESSES.md` says the backbone is the Postgres task queue. +`confluent-kafka[schema-registry,avro]` and `fastavro` ship in the +production image. + +**Scope.** The modules above, `requirements.in`/`requirements.lock`, +`server/main.py` gating, docs. + +**Requirements.** + +1. Before removing anything, produce a call graph: for each Kafka module, + list every importer outside the Kafka set. Any importer on the `pg` path + (for example tracing helpers in + `services/shared/observability/kafka_propagation.py` used by + `server/main.py`) must keep its function; move the needed function to a + non-Kafka module first and update the importer. +2. Move the Kafka-only modules to `docs/archive/retired-topology/kafka/` + preserving relative paths. Do not delete. Remove their imports and the + `EVENT_BACKBONE=kafka` branch in `services/nlp/main.py` (which WI-02 has + already archived) and in `services/shared/event_backbone.py`. +3. Remove `confluent-kafka` and `fastavro` from `requirements.in`; + recompile `requirements.lock` with the repo's pip-tools command + (`pip-compile --generate-hashes`; check `ARCHITECTURE.md` → Dependency + Management and `.github/workflows/dependabot-lock-sync.yml` for the + exact invocation). Confirm no other package depended on them. +4. Remove Kafka-related environment variables from `.env.example` and any + `validate_env.py` checks; list them in the PR. +5. Extend `scripts/check_legacy_dead_code.py` so `import confluent_kafka` + in `services/` fails CI. +6. Update `ARCHITECTURE.md` → Infrastructure and `ASYNC_PROCESSES.md` to + drop the "legacy Kafka paths still exist" language. + +**Acceptance criteria.** + +- Appendix A command A5 returns zero. +- `pip install --require-hashes -r requirements.lock` succeeds in a clean + virtualenv. +- All five §0.4 pytest commands pass; tests that only exercised Kafka move + with the modules to the archive directory and are listed in the PR. +- `python3 scripts/check_docs_links.py ARCHITECTURE.md ASYNC_PROCESSES.md` passes. + +**Out of scope.** Any change to `services/shared/task_queue.py` or +`server/workers/task_processor.py`. + +**Rollback.** Revert the PR, which restores the lock. + +### WI-06 — Split dev and experimental requirements from the production lock + +**Problem.** `requirements.in` includes pytest, pytest-asyncio, pytest-cov, +pytest-mock, pytest-xdist, hypothesis, aiosqlite and testcontainers; +`docker` is a transitive. The `Dockerfile` compensates with +`find /app -name "test_*.py" -delete`. The image also ships langchain, +langchain-community, langgraph, langsmith, openai, llama-cloud, +pytesseract, pdf2image, pyzbar, whose routers (`nlp`, `agent_org`, +`label_vision`, `qr_decoder`) are in `EXPERIMENTAL_ROUTERS` +(`server/router_policy.py`) and never mount in production. + +**Scope.** Requirements files, `Dockerfile`, `backend-ci.yml` install +steps, `dependabot.yml`, `supply-chain.yml`. + +**Requirements.** + +1. Create `requirements-dev.in` holding the testing block from + `requirements.in`; compile `requirements-dev.lock` with hashes, + constrained by `requirements.lock` (`-c requirements.lock`) so versions + never diverge. +2. Create `requirements-experimental.in` for packages imported only by + modules behind `EXPERIMENTAL_ROUTERS`. Determine the set by import + tracing, not by guessing; list the trace in the PR. Packages imported by + any always-mounted or production-allowlisted router stay in + `requirements.in`. +3. `Dockerfile` installs `requirements.lock` only. Remove the + `find ... -delete` line only if no `test_*.py` file is copied by the + remaining `COPY` lines (check `.dockerignore`); otherwise leave it and + say why. +4. CI jobs install `requirements.lock` plus `requirements-dev.lock`, and + `requirements-experimental.lock` only in jobs that run experimental + tests (identify them by their test paths). +5. Add the new files to `dependabot.yml` and to the `supply-chain.yml` + `paths` trigger. +6. Add a startup guard test: with only `requirements.lock` installed in a + clean virtualenv, `python3 -c "import server.main"` succeeds with + `ENABLE_EXPERIMENTAL_ROUTERS` unset. Automate this as a CI job + (`image-import-check`) that builds the venv from the production lock + alone. + +**Acceptance criteria.** + +- `image-import-check` passes in CI. +- Count of packages in `requirements.lock` is lower than before; report + before and after. +- `docker-build` CI job passes. +- Dependabot opens no "lock out of sync" PR for the new files (verify the + `dependabot-lock-sync.yml` logic covers them). + +**Out of scope.** Removing any experimental router or its code. + +**Rollback.** Revert the PR. + +### WI-07 — Make orientation docs count-checked + +**Problem.** `docs/CODEBASE_MAP.md` claims every line was checked against +the tree, and says 23 workflows (there are 33), 81 `include_router` calls +in `server/main.py` (there are 2; registration moved to +`server/router_policy.py`), and lists 14 experimental routers (the +frozenset has 15; `agent_org` is missing). `ARCHITECTURE.md`'s +large-module table is dated 2026-07-31. Four duplicate doc directory +pairs: `docs/archive` and `docs/_archive`; `docs/schema` and +`docs/schemas`; `nfr/` and `docs/nfr`; `runbooks/` and `docs/runbooks`. +20 Markdown files at the repo root including one-off reports +`BREAK-REPORT.md`, `FIX-REPORT.md`. + +**Scope.** Documentation, `scripts/check_docs_links.py` or a sibling +script, `.docs-link-allowlist`. + +**Requirements.** + +1. Add `scripts/check_docs_counts.py` with tests: a small table of + (document, regex capturing a number, shell-free Python function that + computes the true value). Seed it with: workflow count in + `docs/CODEBASE_MAP.md` §2 and §7; `include_router` count in §4; the + experimental router list in §4 versus `server/router_policy.py` + `EXPERIMENTAL_ROUTERS`; the number of files in the `ARCHITECTURE.md` + large-module table versus the live count. Fails when they differ. Wire + into `pr-guards.yml`. +2. Correct the documents so the check passes. For the experimental router + list, render it from the frozenset rather than hand-typing. +3. Merge directory pairs by moving files (never deleting): `docs/_archive/*` + → `docs/archive/`; `docs/schemas/*` → `docs/schema/`; `nfr/*` → + `docs/nfr/`; root `runbooks/*` → `docs/runbooks/legacy/` (they carry a + `LEGACY:` banner already). Update every link; run the link checker on + every document that referenced a moved file. Update + `scripts/check_docs_links.py` `DEFAULT_SCOPE` so it still covers the + moved runbooks. +4. Move `BREAK-REPORT.md` → `docs/engineering/cli-break-report-2026-10.md` + and `FIX-REPORT.md` → `docs/engineering/about-copy-fix-report-2026-10.md` + (confirm dates from their content). Leave `GEMINI.md` in place; it is an + agent instruction file and its removal is a decision for Christopher. +5. Re-stamp `ARCHITECTURE.md` "Last verified" after re-checking each + section against the tree; record what changed in the PR. + +**Acceptance criteria.** + +- `python3 scripts/check_docs_counts.py` and its tests pass. +- `python3 scripts/check_docs_links.py` with default scope passes. +- `ls docs/_archive docs/schemas nfr runbooks` each report "No such file" + or an empty directory with only a `README.md` pointer. + +**Out of scope.** Rewriting document prose beyond the counts and moved +paths. + +**Rollback.** Revert the PR. + +### WI-08 — Register or archive orphan scripts + +**Status 2026-10-30:** delivered in PR #3699 with two deviations. +`scripts/README.md` registers all 99 top-level files and 11 subdirectories +(60 used, 4 data, 21 manual/one-time/retire-with-WI-12, 14 archive +candidates). `scripts/check_scripts_registered.py` (10 tests) runs in +`backend-ci.yml` job `lint-repo-roots` instead of `pr-guards.yml`, because +backend-ci already triggers on `scripts/**` and has Python set up. The 14 +archive candidates are listed in the README but not moved: the owner's +"never delete files" rule makes any move a decision for Christopher by +name, so the move is a one-commit follow-up once approved. + + +**Problem.** 31 of 87 top-level files in `scripts/` are referenced by no +workflow, `Dockerfile`, `railway.toml`, runbook, other script, README, +CONTRIBUTING or DEVOPS_NOTES. Largest: +`scripts/verify_canonical_parity_live.py` (1,702 lines), +`scripts/backfill_canonical_from_legacy.py` (1,423), +`scripts/seed_demo_data.py` (680), `scripts/audit-issues.py` (665), +`scripts/e2e_brutal_scenario.py` (654), `scripts/jev_header_preview.py` (602). + +**Scope.** `scripts/`, a new `scripts/README.md`. + +**Requirements.** + +1. Re-run Appendix A command A8 to get the current orphan list. +2. For each orphan, classify: (a) used by a documented manual procedure + (register it), (b) tied to WI-12 (`verify_canonical_parity_live.py`, + `backfill_canonical_from_legacy.py`: register under "retire with + WI-12"), (c) no known use (archive). +3. Create `scripts/README.md`: one line per script in `scripts/` and its + subdirectories: purpose, the workflow or runbook that invokes it, or + "manual: ". Scripts in class (c) move to `scripts/_archive/` with + the same name and a line in the README saying so and when. Do not + delete. +4. Any orphan with tests under `tests/scripts/` stays registered (the test + is a use). +5. Add a check to `scripts/check_legacy_dead_code.py` or a new + `scripts/check_scripts_registered.py` (with tests): every file in + `scripts/` top level must have a README line. Wire into `pr-guards.yml`. + +**Acceptance criteria.** + +- The registration check passes. +- `tests/scripts` suite passes. +- The PR lists every moved script with its classification reason. + +**Out of scope.** Editing any script's behaviour. + +**Rollback.** Revert the PR. + +### WI-09 — Archive the retired six-service topology + +**Problem.** `infra/k8s/` (`base/`, `overlays/`) plus `infra-k8s-ci.yml`; +five root `runbooks/` with `LEGACY:` banners; `regengine/swarm/` (about +2,600 lines) behind `REGENGINE_ENABLE_LEGACY_SWARM`; `infra/gateway/` +"reference-only" nginx config with its own tests in `testpaths`; the seven +legacy entrypoints (WI-02). + +**Scope.** Those paths, `pyproject.toml` `testpaths`, workflow inventory, +`docs/CODEBASE_MAP.md`. + +**Requirements.** + +1. Create `docs/archive/retired-topology/README.md` stating what the + six-service topology was, when it was retired, and that nothing under + this directory is loaded by any build, test or deploy. +2. Move (do not delete) `infra/k8s/` → `docs/archive/retired-topology/k8s/`; + `infra/gateway/` → `docs/archive/retired-topology/gateway/` **only if** + `ARCHITECTURE.md`'s claim that it is reference-only still holds (check + `Dockerfile`, `railway.toml`, workflows for any use); `regengine/swarm/` + → `docs/archive/retired-topology/swarm/`. +3. Remove `infra/gateway/tests` from `testpaths`; move those tests with the + config. Remove `tests/swarm` from `CLAUDE.md`'s verification list and + from `testpaths` after confirming `tests/swarm` tests only the swarm + runtime (if any test there covers live code, keep that test and say so). +4. Archive `infra-k8s-ci.yml` to `docs/archive/retired-topology/workflows/` + and update the workflow inventory. +5. Update `docs/CODEBASE_MAP.md` §1, §2 and `CLAUDE.md` (the + `regengine.swarm` sentence and the verification list) to match. +6. `regengine/` is also the published SDK. Confirm `regengine/setup.py` (or + `pyproject`) does not package `regengine.swarm`; if it does, exclude it + and say so. + +**Acceptance criteria.** + +- `python3 -m pytest --collect-only -q` has no errors. +- `python3 scripts/check_workflow_inventory.py` passes. +- `python3 scripts/check_docs_links.py` default scope passes. +- `python3 -m pytest tests/sdk -q` passes. + +**Out of scope.** Anything under `services/`. + +**Rollback.** Revert the PR. + +### WI-10 — DB-dependent tests fail loudly when the DB is missing + +**Problem.** 170 skip sites; the most common reason is "migrated Postgres +with fsma schema not available". A bare `pytest` locally passes while +exercising none of the RLS, migration or persistence paths. CI does run +`live-db-suite` (postgres:17) and `seeded-migration-rehearsal`, so +production is covered; the risk is a lost `DATABASE_URL` turning the suite +green. Per-service coverage floors: compliance 80, admin 70, scheduler 65, +graph 60, ingestion 48, shared 35. + +**Scope.** `conftest.py` fixtures, `backend-ci.yml`, `CONTRIBUTING.md`, +`docker-compose.dev.yml` docs. + +**Requirements.** + +1. Add a pytest option `--require-db` (in root `conftest.py`): when set, + every skip whose reason matches the DB-unavailable patterns (collect the + exact strings with Appendix A command A10) becomes a failure. Implement + by routing those skips through one helper `skip_or_fail_without_db()` + and replacing the 170 call sites' inline conditions with the helper + where the reason is DB availability. +2. Pass `--require-db` in `live-db-suite` and `seeded-migration-rehearsal`. +3. The helper's skip message must print the exact local setup commands: + `docker compose -f docker-compose.dev.yml up -d`, the `alembic upgrade + head` invocation with the dev `DATABASE_URL`, and the environment + variables needed. Put the same text in `CONTRIBUTING.md` → Development + Setup under a new "Running database tests locally" heading. +4. Re-measure `shared` and `ingestion` coverage on CI after WI-02 and WI-05 + land; if measured value minus 5 exceeds the floor, raise the floor to + that value in `backend-ci.yml` with a dated comment, following the + existing comment style there. + +**Acceptance criteria.** + +- With `DATABASE_URL` unset: `python3 -m pytest services/shared/tests -q --no-cov --require-db` fails with the helper's message, and without the flag skips as before. +- `live-db-suite` passes in CI with the flag. +- Skip-site count (Appendix A command A10) drops; report before and after. + +**Out of scope.** Writing new tests. Changing what the DB tests assert. + +**Rollback.** Revert the PR. + +### WI-11 — One frontend API path + +**Problem.** `frontend/src/lib/api-client.ts` (1,163 lines) is imported by +15 files under `app/`, `components/`, `hooks/`; 33 non-test files in those +directories call `fetch(` directly. 30 route handlers under +`frontend/src/app/api/`; 22 use `frontend/src/lib/proxy-factory.ts`, 8 are +hand-rolled: `client-errors`, `inflow-lab/[...path]`, `openapi.json`, +`session`, `health`, `contact`, `tools/check-access`, `tools/verify`. + +**Scope.** `frontend/src/app`, `frontend/src/components`, +`frontend/src/hooks`, `frontend/src/lib`, `frontend/eslint.config.mjs`. + +**Requirements.** + +1. Add an ESLint `no-restricted-syntax` rule forbidding bare `fetch(` under + `src/app/**`, `src/components/**`, `src/hooks/**`, excluding + `src/app/api/**` (route handlers legitimately fetch the backend) and + test files. Message must name `api-client.ts` as the replacement. +2. Convert the 33 call sites to `api-client.ts` functions or typed hooks. + Where no function exists, add one next to its neighbours in + `api-client.ts`, typed with the shapes in `frontend/src/types/api.ts`. + Preserve request headers, credentials mode, tenant header handling and + error mapping exactly; diff the network request in a test for at least + five converted sites using the existing MSW or mock pattern in + `frontend/src/test/`. +3. Convert `tools/check-access`, `tools/verify` and `inflow-lab/[...path]` + to `proxy-factory.ts` if they proxy the backend with no local logic; + leave `health`, `session`, `contact`, `client-errors`, `openapi.json` as + they are and say why in a comment at the top of each. +4. Do not change any page's rendered output. Run the existing page tests. + +**Acceptance criteria.** + +- `cd frontend && npm run lint` passes with the new rule enabled. +- `cd frontend && npm run test:run` passes with coverage not below the + thresholds in `frontend/vitest.config.mts`. +- `cd frontend && npm run build` passes. +- Appendix A command A11 returns zero outside `src/app/api`. + +**Out of scope.** Splitting the large page components. Changing API +shapes. + +**Rollback.** Revert the PR. + +### WI-12 — Retire the `cte_events` dual-write + +**Problem.** `services/shared/canonical_persistence/writer.py` calls +`legacy_dual_write.dual_write_legacy` on single and batch writes; since +#1277 a failure raises and rolls back the canonical write. +`services/shared/canonical_persistence/legacy_dual_write.py` (532 lines) is +headed "TEMPORARY". `CANONICAL_OWNERSHIP.md` calls `cte_events` a legacy +derived copy to be removed. 12 non-legacy modules still read `cte_events`, +including `services/admin/app/supplier_onboarding_routes.py`, +`services/admin/app/supplier_facilities_routes.py`, +`services/admin/app/data_export_routes.py`, +`services/admin/app/supplier_funnel_routes.py`, +`services/shared/workflows/export_output.py`, +`services/shared/supplier_data_quality.py`. Jev scored six-month incident +likelihood high (1.98 of 3). + +**Scope.** The 12 readers, the writer, one feature flag, one Alembic +revision at the end. Three PRs minimum. + +**Requirements.** + +PR A (readers): + +1. For each of the 12 readers (Appendix A command A12 lists them), re-point + the query to `fsma.traceability_events`, mapping KDE columns from the + `kdes` JSONB as `CANONICAL_OWNERSHIP.md` documents. One commit per + reader. +2. For each, add a parity test that seeds the same event through the + canonical writer and asserts the route's response is byte-identical + before and after the change (use the live-db fixtures). +3. Readers that are export surfaces (`export_output.py`, + `data_export_routes.py`) additionally run the existing FDA export + fixtures under `tests/` and `services/ingestion/tests` unchanged. + +PR B (flag): + +4. Add `CANONICAL_LEGACY_DUAL_WRITE` setting (default `on`) in + `services/shared/base_config.py`; when `off`, `writer.py` skips the + legacy write and increments a Prometheus counter + `legacy_dual_write_skipped_total`. When `on`, behaviour is unchanged. +5. Document in `DEVOPS_NOTES.md` the plan: set `off` on staging for two + weeks, run `scripts/verify_canonical_parity_live.py` daily, then + production. Christopher flips the flag; the agent does not. + +PR C (removal, only after Christopher confirms the two-week window passed): + +6. Remove the dual-write call and move `legacy_dual_write.py`, + `scripts/verify_canonical_parity_live.py` and + `scripts/backfill_canonical_from_legacy.py` to + `docs/archive/retired-topology/legacy-cte/`. Do not delete. +7. Add an Alembic revision that renames `fsma.cte_events` and + `fsma.cte_kdes` to `fsma.cte_events_retired_` (rename, not drop; + dropping is a later decision) and removes their RLS policies per + `docs/engineering/SCHEMA_CHANGE_POLICY.md`. Render the SQL per WI-01's + ADR. +8. Update `CANONICAL_OWNERSHIP.md` → Canonical Event → Persistence. + +**Acceptance criteria.** + +- PR A: all parity tests pass in `live-db-suite`; no route's response + changes. +- PR B: with the flag `off` in a test, the writer commits without touching + `cte_events` (assert row count unchanged) and the counter increments. +- PR C: `grep -rn cte_events services server` matches only the migration + and the archive; `alembic-revision-check` job passes. + +**Out of scope.** Dropping tables. Neo4j graph sync (`publish_graph_sync` +in the same module): keep it, move it to a non-legacy module in PR C. + +**Rollback.** PR A and B revert cleanly. PR C's migration has a +`downgrade()` that renames back. + +### WI-13 — One Node version across CI + +**Status 2026-10-30:** delivered in PR #3699. `copy-gate.yml` reads +`.nvmrc`; the Node 24 pin in `frontend-ci.yml` stays for the job that needs +`node:sqlite` and says so in a comment. The jsdom 30 and jest-dom 7 +Dependabot ignores were removed; CI now judges those majors. + + +**Problem.** `.nvmrc` says 22. `copy-gate.yml` pins Node 20; one +`frontend-ci.yml` job pins 24; `package.json` `engines` says 22 or 24. +`dependabot.yml` ignores jsdom 30 and jest-dom 7 with the note "CI runs +Node 20". + +**Scope.** Workflows, `dependabot.yml`. + +**Requirements.** + +1. Replace every `node-version: ''` in `.github/workflows/` with + `node-version-file: .nvmrc`. List each change in the PR. +2. If any job needed 24 for a reason (read its comments), keep 24 there, + and change `.nvmrc` to 24 only if `package.json` `engines` allows and + Vercel's Node setting (ask Christopher; do not change Vercel) matches. + Otherwise leave `.nvmrc` at 22. +3. Remove the jsdom and jest-dom major ignores from `dependabot.yml`, + leaving a comment that the Node blocker is gone and CI will judge the + upgrade. + +**Acceptance criteria.** + +- `grep -rn "node-version:" .github/workflows` returns nothing. +- `frontend-ci.yml` passes on the PR. + +**Out of scope.** Performing the jsdom or jest-dom upgrade. + +**Rollback.** Revert. + +### WI-14 — Split `server/main.py` + +**Problem.** `server/main.py` is 1,256 lines holding the lifespan, startup +checks, worker boot and router policy wiring. 25 non-test modules exceed +1,000 lines (20 under `services/` and `server/`); the others are split +only when a feature touches them. + +**Scope.** `server/`. + +**Requirements.** + +1. Extract into `server/startup_checks.py` (environment, RLS role posture, + secret presence checks), `server/workers_boot.py` (task processor and + embedded scheduler start/stop), and keep `server/main.py` as app + construction, middleware order and lifespan that calls the two. Keep + every log line and its order; the deploy log is parsed by humans + (`jev_configured`, `document_extract_configured` lines in `CLAUDE.md`). +2. No behaviour change. Add a test that imports `server.main` with + production-like env (`ENVIRONMENT=production`, no experimental routers) + and asserts the same set of mounted route paths as before the split + (snapshot the list in the test). + +**Acceptance criteria.** + +- `server/main.py` under 500 lines. +- The route-set snapshot test passes. +- `tests/security` suite passes. + +**Out of scope.** The other 24 large modules. + +**Rollback.** Revert. + +### WI-15 — Retire the dead `graph_update` task type + +**Status 2026-10-30:** delivered in PR #3699 (`services/shared/task_types.py`, +`server/workers/task_processor.py`, two test files, `ASYNC_PROCESSES.md`). + + +**Problem.** `graph_update` is declared in `services/shared/task_types.py` +and given a 120-second timeout in `server/workers/task_processor.py`, but +nothing enqueues it and `TASK_HANDLERS` has no handler. +`ASYNC_PROCESSES.md` describes unclaimed rows; none are produced today. + +**Requirements.** + +1. Remove the type and the timeout entry. If a CHECK constraint or enum in + the database lists task types (search `alembic/versions` for + `graph_update`), leave the database alone and say so; only code changes. +2. Update `ASYNC_PROCESSES.md`'s table. + +**Acceptance criteria.** `grep -rn graph_update services server` returns +nothing outside migrations; `tests/shared` passes. + +### WI-16 — Frontend hook-rule suppressions (ongoing) + +**Problem.** 14 `eslint-disable` comments in product code: 8 +`react-hooks/set-state-in-effect`, 4 `react-hooks/exhaustive-deps`, 2 +`no-console`. All 192 `any` usages are in test files and are not in +scope. + +**Requirements.** No standalone PR. When a PR touches one of these files +(`frontend/src/components/lead-gate/LeadGate.tsx`, +`frontend/src/lib/tenant-context.tsx`, `frontend/src/components/ui/tutorial.tsx`, +`frontend/src/components/layout/theme-toggle.tsx`, +`frontend/src/components/accessibility/AccessibilityWidget.tsx`, +`frontend/src/app/status/StatusBoard.tsx`, +`frontend/src/app/settings/profile/page.tsx`, +`frontend/src/app/tools/knowledge-graph/components/KnowledgeGraph.tsx`, +`frontend/src/app/tools/label-scanner/page.tsx`, +`frontend/src/app/dashboard/settings/users/page.tsx`), remove the +suppression by restructuring the effect, and keep the page's tests green. + +--- + +## 4. Definition of done, per PR + +A work-item PR is done when all of the following hold: + +1. Branch named `/wi--`. +2. PR body follows `.github/pull_request_template.md`, with three extra + sections: **Baseline vs after** (the Appendix A numbers for this item), + **Decision needed** (empty or a list), **Files moved to archive** + (empty or a list with original paths). +3. Every acceptance criterion above is quoted in the PR body with the + command run and its last line of output. +4. Required CI is green on the head commit. `visual-report (non-required)` + is ignored. `Engine Review Pass` needs a non-author engine's marker; the + agent asks Christopher to arrange that and does not post one itself. +5. `docs/audits/TECH_DEBT_AUDIT_2026-10-28.md` §1 table gets a new + "Status" column entry for the item (`PR #nnnn open` / `merged + `), in the same PR. +6. No file deleted. `git diff --diff-filter=D --name-only main...HEAD` is + empty, or every listed path is a rename (`--diff-filter=R` shows it) + or was approved by name. + +--- + +## 5. Decisions needed from Christopher (collected) + +The agent proceeds on everything else and records these in each PR: + +| Item | Decision | +|---|---| +| WI-01 | Which `supabase/migrations/` and `sql/prod-apply/` files without an Alembic counterpart should become Alembic revisions, and who runs the version-row reconcile. | +| WI-02 | Whether any legacy entrypoint is still used for local debugging and should stay as a thin shim. | +| WI-05 | Confirmation that no customer or partner integration is on the Kafka path (the code default is `pg`). | +| WI-06 | Whether any experimental router will graduate in the next quarter (its packages would then stay in the production lock). | +| WI-07 | Whether `GEMINI.md` stays at the root. | +| WI-08 | Approval, by name, for any script the agent proposes to archive. | +| WI-12 | The go-ahead for PR C after the two-week flag window, and whether to rename or drop the legacy tables. | +| WI-13 | Vercel's Node version, so `.nvmrc` can match it. | + +--- + +## Appendix A — Baseline measurement commands + +Run from the repo root before and after each item; paste both into the PR. + +```bash +# A2: non-test files with bare `app` imports (WI-02 target: 0) +grep -rlE '^\s*(from|import) app\b' --include='*.py' services | grep -v tests +``` + +```bash +# A3: test files with bare `app` imports (WI-02 sizing) +grep -rlE '^\s*(from|import) app\b' --include='*.py' services tests | grep tests | wc -l +``` + +```bash +# A5: production modules on the Kafka path (WI-05 target: 0) +find services -name '*.py' -not -path '*/tests/*' | xargs grep -lE 'confluent_kafka|KAFKA_BOOTSTRAP' | wc -l +``` + +```bash +# A7: workflow count and the four largest (WI-04, WI-07) +ls .github/workflows | wc -l; wc -l .github/workflows/*.yml | sort -rn | head -5 +``` + +```bash +# A8: top-level scripts referenced by nothing (WI-08 target: 0) +for s in scripts/*.py scripts/*.sh; do b=$(basename "$s"); grep -rq "$b" .github/workflows Dockerfile railway.toml docs/runbooks runbooks scripts/*.sh README.md CONTRIBUTING.md DEVOPS_NOTES.md scripts/README.md 2>/dev/null || echo "$s"; done +``` + +```bash +# A10: pytest skip sites and DB-unavailable reasons (WI-10) +grep -rlE 'pytest\.(mark\.)?skip|pytest\.skip\(' --include='*.py' tests services infra | wc -l +grep -rhoE 'skip(if)?\([^)]*reason\s*=\s*f?"[^"]{0,70}' --include='*.py' tests services infra | sed -E 's/.*reason\s*=\s*f?"//' | sort | uniq -c | sort -rn +``` + +```bash +# A11: bare fetch( in frontend product code (WI-11 target: 0 outside src/app/api) +grep -rlE '\bfetch\(' --include='*.ts' --include='*.tsx' frontend/src/app frontend/src/components frontend/src/hooks | grep -v __tests__ | grep -v 'src/app/api' +``` + +```bash +# A12: non-legacy readers of cte_events (WI-12 target: 0) +grep -rln 'cte_events' --include='*.py' services server | grep -v tests | grep -v 'cte_persistence\|legacy_dual_write' +``` + +```bash +# A14: modules over 1,000 lines (WI-14) +find services server regengine packages scripts -name '*.py' -not -path '*/tests/*' -not -name 'test_*' | xargs wc -l | awk '$1>1000 && $2!="total"' | sort -rn +``` + +```bash +# A3b: lint suppression counts (WI-03) +grep -rn 'noqa' --include='*.py' services server regengine packages | wc -l +grep -rn 'type: ignore' --include='*.py' services server regengine packages | wc -l +grep -rnE 'noqa(?!:.*--)' --include='*.py' services 2>/dev/null | wc -l +``` + +## Appendix B — Baseline values on 2026-10-28 (commit `9f6efb3`) + +| Measure | Value | +|---|--:| +| Workflows | 33 | +| Bare `app` import files (non-test) | 20 | +| Kafka-path production modules | 13 | +| Orphan top-level scripts | 31 of 87 | +| pytest skip sites | 170 | +| `noqa` markers (backend, non-test) | 766 | +| `type: ignore` markers | 223 | +| Frontend files with bare `fetch(` | 33 | +| Hand-rolled API route handlers | 8 of 30 | +| Non-legacy `cte_events` readers | 12 | +| Modules over 1,000 lines | 25 (20 under `services/` + `server/`) | +| Alembic revisions / head | 105 / v137 | +| `supabase/migrations/` files | 45 | +| Open GitHub issues | 3 | diff --git a/docs/runbooks/ALEMBIC_VERSION_RECONCILE.md b/docs/runbooks/ALEMBIC_VERSION_RECONCILE.md new file mode 100644 index 000000000..669be99c9 --- /dev/null +++ b/docs/runbooks/ALEMBIC_VERSION_RECONCILE.md @@ -0,0 +1,153 @@ +# Alembic version-row reconcile runbook + +**Severity:** SEV2 if a deploy is pending (a deploy runs `alembic upgrade head`) +**Owner:** Christopher Sellers +**Last updated:** 2026-10-30 +**Decision record:** `docs/decisions/2026-10-30-schema-change-channel.md` + +> The agent that wrote this did not run anything against a database. Every +> command below is for Christopher to run. The only script that touches a +> database here is read-only. + +## 0. When to use this + +Use it when `public.alembic_version` on the production database does not +match what the catalog shows. Today that is the case: the row reads +`267d8053ced5` (v079) while effects of v128, v131 and v134 to v137 are +present, because those reached production through `supabase/migrations/` +and `sql/prod-apply/`. + +Do not use it to "get the row to head". The row is reconciled to what the +catalog proves, nothing more. + +## 1. Preconditions + +- A current backup with a tested restore, per + `docs/operations/DISASTER_RECOVERY.md`. +- A read-only database session. Any of: the Supabase SQL editor on a + read-only role, or `psql` with a role that has `SELECT` on `pg_catalog` + and `public.alembic_version`. +- The repository checked out at the commit you are reconciling against. + Note its SHA; the precheck's signature table is tied to the revision files + in that tree. + +## 2. Run the precheck + +```bash +psql -X -v ON_ERROR_STOP=1 -f sql/prod-apply/PRECHECK_alembic_version_reconcile.sql +``` + +The script opens a `BEGIN READ ONLY` transaction with a 15-second statement +timeout, reads `pg_catalog` and the version row, and rolls back. It writes +nothing. It returns three result sets: + +1. `alembic_version`: the row as recorded. +2. One line per revision after v079, in chain order, with `present` = + `true`, `false` or `NULL` (not observable: merge revisions, data-only + revisions, or a role the check needs is absent). +3. A summary: `recorded_version`, `supported_through_revision` (the last + revision before the first `false`), counts, and + `present_beyond_first_gap` (effects that exist past that point). + +The three `absent_*` signatures (v088, v090 `4c5d6e7f8a9b`, v132) read +`true` on any database that never had the object, so they appear in +`present_beyond_first_gap` on a v079 database. That is expected; read them +with their neighbours. Verified on a scratch PostgreSQL 16 cluster on +2026-10-30: the script runs with `ON_ERROR_STOP`, returns 71 signature rows +and the summary, and leaves no writes. + +Save the output next to the run date under +`docs/operations/` as ALEMBIC_RECONCILE_(date).txt. It contains no +customer rows. + +## 3. Read the result + +Expected pattern on 2026-10-30, from the evidence in the decision record: + +| Observation | Meaning | +|---|---| +| `recorded_version = 267d8053ced5` | Row is at v079. | +| `supported_through_revision` is v079 or shortly after | The chain was not run past v079. Most v080 to v127 effects are absent. | +| `present_beyond_first_gap` lists v128, v131, v134, v135, v136, v137 | Those revisions were hand-applied (decision record, Appendix A). | + +Three outcomes: + +**A. The expected pattern.** Do not change the row. v079 is the truthful +chain position. Go to §4. + +**B. Every observable revision through some revision X is `true`, and +nothing is `true` beyond X.** The chain was run to X and the row was not +updated (for example a crashed `run-migrations.sh` after the last +revision committed). Go to §5 with X. + +**C. Anything else** (gaps in the middle, `false` on a revision whose +later neighbours are `true` without a HAND-APPLIED note). Stop. Record the +output and raise it in the PR or with the engine that prepared the +migration. Do not edit the row. + +## 4. Outcome A: run the chain, do not stamp + +Reconciling the row means executing the pending revisions through the +supported runner, not editing the row. The procedure is already written: + +1. `docs/operations/DATABASE_ROLE_CUTOVER.md` sections 1 to 4: roles, + ownership, extension ownership, memberships. +2. `docs/operations/MIGRATION_V079_TO_V120_2026_09_05.md` "Controlled + execution sequence": steps 1 to 8, with the target now + `a3f7ce361b0e` (v137) instead of `c6a120b4e901`. +3. Before step 4 of that sequence, resolve decisions B1 to B7 in the + decision record. The hand-applied changes without an Alembic + counterpart must be back-ported first, or the chain's last word on + `tenant_isolation_tasks` and `data_gap_events` will differ from + production. +4. On the staging restore, confirm the hand-applied revisions pass over + their already-present effects. Expected: v128, v131, v134 to v137 complete + with no error because each `upgrade()` is guarded (decision record, + "Hand-applied revisions and their idempotency"). If any raises + "already exists", repair that revision's guard in a PR before touching + production. +5. After the production run, `alembic current` must print `a3f7ce361b0e`, + and a second run of `scripts/run-migrations.sh` must be a no-op. + +## 5. Outcome B: set the row to X + +Only when §3 outcome B holds. One transaction, one statement, then the +precheck again. + +```sql +BEGIN; +SET LOCAL statement_timeout = '5s'; +UPDATE public.alembic_version SET version_num = '' WHERE version_num = '267d8053ced5'; +-- Expect: UPDATE 1. If 0, another session changed the row; ROLLBACK and re-run §2. +COMMIT; +``` + +Then: + +```bash +psql -X -v ON_ERROR_STOP=1 -f sql/prod-apply/PRECHECK_alembic_version_reconcile.sql +``` + +`recorded_version` must equal `supported_through_revision`. Record the +before and after outputs in `DEVOPS_NOTES.md` with the date. + +## 6. What never to do + +- `alembic stamp head`, or any `UPDATE` of the row to a revision whose + effects the precheck shows absent. +- `alembic upgrade head` against production before + `DATABASE_ROLE_CUTOVER.md` is complete and the staging rehearsal in + §4 step 4 has passed. +- Applying another file from `supabase/migrations/` or `sql/prod-apply/` + without a paired Alembic revision (decision record, rule 4). +- Running the precheck with a role that can write. It cannot write by + construction, but the habit matters. + +## 7. Keeping the precheck current + +The signature table inside the precheck names one object per revision. +When a revision is added to `alembic/versions/`, add one line to the +`VALUES` list in `sql/prod-apply/PRECHECK_alembic_version_reconcile.sql` +in the same PR. `scripts/check_schema_channels.py` guards the two channel directories but +not this file's table; adding the line is part of the PR checklist for a new +revision. diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..4ae313434 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,136 @@ +# scripts/ registry + +Every top-level file in `scripts/` has one line here saying what it is for and +what invokes it. `scripts/check_scripts_registered.py` fails CI when a file is +added without a line, so the directory cannot silently collect orphans again +(tech-debt WI-08, 2026-10-30). Subdirectories are described once each. + +Status values: **used** (a workflow, image, runbook or document names it), +**data** (a non-executable file a script reads), +**manual** (a documented hand-run procedure), **one-time** (done, kept for the +record), **retire with WI-12**, **archive candidate** (no known use; moving it +to `scripts/_archive/` needs the owner's approval by name, per the +remediation PRD). The 14 candidates stay in place until then; the move is +`git mv scripts/ scripts/_archive/` plus a status change here. + +## Top-level scripts + +| Script | Status | Purpose / invoked by | +|---|---|---| +| `apply_runtime_role_cutover.py` | used | Idempotently reapply the v111 runtime-role cutover as schema owner. — invoked by `Dockerfile`, `docs/operations/DATABASE_ROLE_CUTOVER.md` | +| `audit-issues-pass2.py` | archive candidate | one-off GitHub issue creator, second pass, March 2026 | +| `audit-issues.py` | archive candidate | one-off GitHub issue creator from the March 2026 audit | +| `audit_api_key_scopes.py` | used | Audit RegEngine API keys for over-broad scopes. — invoked by `docs/runbooks/api_key_scope_audit.md` | +| `audit_fsma_rule_catalog.py` | manual | manual: regenerate `docs/audits/FSMA_RULE_CATALOG.md` and `.json` after a rule seed change (tested in `tests/scripts/`) | +| `backfill_canonical_from_legacy.py` | retire with WI-12 | retire with WI-12: WS-1c backfill of canonical rows from legacy `cte_events` | +| `backfill_obligation_catalog.py` | one-time | one-time, run 2026-10-08 (Decision 3 obligation quarantine); kept for the record | +| `backup_neo4j.sh` | used | backup_neo4j.sh — Offline Neo4j backup via neo4j-admin (#2939) — invoked by `docs/operations/DISASTER_RECOVERY.md` | +| `backup_postgres.sh` | used | backup_postgres.sh — Automated PostgreSQL backup with retention (#1003) — invoked by `docs/operations/DISASTER_RECOVERY.md`, `docs/operations/MIGRATION_V079_TO_V120_2026_09_05.md` | +| `check_alembic_revisions.py` | used | CI guard — reject Alembic migrations that would corrupt the revision DAG. — invoked by `.github/workflows/backend-ci.yml`, `README.md` | +| `check_api_host_strings.py` | used | Guard against api.regengine.co (no DNS) in user-visible code samples. — invoked by `.github/workflows/frontend-ci.yml` | +| `check_dep_versions.py` | used | CI guard — reject dependency version drift between the declared floors and the lock. — invoked by `.github/workflows/backend-ci.yml` | +| `check_docs_links.py` | used | Fail CI when documentation references a repo path that does not exist. — invoked by `docs/CODEBASE_MAP.md` | +| `check_enabled_outbound_http.py` | used | Reject raw HTTP clients in release-enabled outbound-fetch paths. — invoked by `.github/workflows/backend-ci.yml` | +| `check_fda_spreadsheet_headers.py` | manual | manual: compare our FDA sortable-spreadsheet headers with FDA's template when FDA republishes it | +| `check_legacy_dead_code.py` | used | Guardrail for high-confidence legacy and dead-code drift. — invoked by `.github/workflows/legacy-code-scan.yml` | +| `check_no_kernel_control_imports.py` | used | Guard: forbid imports of ``kernel.control`` from production code. — invoked by `.github/workflows/backend-ci.yml` | +| `check_orphan_migrations.py` | used | CI guard: keep orphan migrations at zero. — invoked by `.github/workflows/backend-ci.yml`, `README.md` | +| `check_public_api_surface.mjs` | used | check_public_api_surface.mjs — invoked by `.github/workflows/public-api-surface.yml` | +| `check_rls_fallback.py` | used | CI guard — reject RLS policies that fall back to a constant tenant. — invoked by `.github/workflows/backend-ci.yml`, `docs/engineering/TEARDOWN_FOLLOWUP_2026_09_05.md` | +| `check_schema_channels.py` | used | CI guard — every SQL file in the two non-Alembic schema channels names the — invoked by `.github/workflows/backend-ci.yml`, `docs/runbooks/ALEMBIC_VERSION_RECONCILE.md` | +| `check_scripts_registered.py` | used | this registry's guard — invoked by `.github/workflows/backend-ci.yml` (lint-repo-roots) | +| `check_tenant_id_uuid_only.py` | used | CI guard — reject new migrations that drift tenant isolation. — invoked by `.github/workflows/backend-ci.yml`, `README.md` | +| `check_trivyignore_sync.py` | used | CI guard — keep .trivyignore in lockstep with the pip-audit ignore set. — invoked by `.github/workflows/security.yml` | +| `check_trust_safety_copy.py` | used | Fail CI when trust-and-safety outcome claims drift back into UI copy. — invoked by `.github/workflows/frontend-ci.yml`, `.github/workflows/pr-gate.yml` | +| `check_workflow_inventory.py` | used | CI guard: keep workflows and branch-protection required contexts in sync. — invoked by `.github/workflows/pr-gate.yml`, `docs/runbooks/BRANCH_PROTECTION.md` | +| `clear_fsma_data.sql` | manual | dev/test helper: clear FSMA demo rows for the zero-UUID tenant in one transaction; header says never against production | +| `compliance_sweep.py` | used | self-described legacy sweep; named by `tests/swarm/test_swarm_guards.py`, so it stays (PRD WI-08 requirement 4) | +| `create_copilot_issues.sh` | archive candidate | one-off issue batch for retired vertical development | +| `create_neo4j_indexes.sh` | manual | create Neo4j indexes for the optional graph tier; its secret handling is covered by `services/shared/tests/test_secret_redaction.py` | +| `create_test_api_key.py` | used | Mint a tenant-scoped RegEngine API key for local/dev testing. — invoked by `README.md` | +| `demo_freshness.py` | manual | manual: check the demo narrative dates before a demo | +| `deploy_railway.sh` | manual | Railway deploy wrapper for the monolith; documented in its own header (`docs/operations/PRODUCTION_DEPLOYMENT.md` is the procedure) | +| `deploy_rls.py` | used | Retired direct RLS SQL deploy helper. — invoked by `docs/runbooks/MIGRATION_ROLLBACK.md` | +| `deploy_rls.sh` | used | deploy_rls.sh — invoked by `docs/operations/rls_deployment_runbook.md`, `docs/runbooks/MIGRATION_ROLLBACK.md` | +| `e2e_brutal_scenario.py` | used | Brutal End-to-End Scenario — invoked by `docs/engineering/security-findings-2026-09-10/dispositions.json` | +| `e2e_tests.js` | archive candidate | targets the retired multi-service ports (8400, 8000, 8500); superseded by `tests/` e2e suites and `frontend/` Playwright | +| `env_preflight.sh` | manual | manual: local environment preflight before starting the stack | +| `export-openapi.sh` | manual | legacy helper for local inspection of non-production services (its header says so) | +| `export_env_bundle.py` | used | Validate the local secret set, and optionally emit it to stdout for piping. — invoked by `docs/engineering/TEARDOWN_FOLLOWUP_2026_09_05.md` | +| `extract_document.py` | used | Extract structured data from a local document with LlamaIndex Extract v2.5. — invoked by `docs/engineering/DOCUMENT_EXTRACT.md` | +| `format_chaos_results.py` | manual | manual: after a `scripts/chaos/` run, format results for a GitHub step summary | +| `generate_redis_certs.sh` | manual | local-dev self-signed Redis TLS certs (#999); output dir is gitignored | +| `generate_route_security_manifest.py` | used | Generate or verify the route-complete production security manifest. — invoked by `.github/workflows/backend-ci.yml`, `docs/engineering/ROUTER_SURFACE.md` | +| `health_check.py` | manual | manual: founder's health check against a running stack (tested in `tests/scripts/`) | +| `ingest_document.py` | used | RegEngine Regulatory Document Ingestion Script — invoked by `docs/engineering/CONTENT_INGESTION.md` | +| `init-postgres.sh` | used | init-postgres.sh — Create additional databases required by services — invoked by `.github/workflows/backend-ci.yml`, `.github/workflows/inflow-contract-ci.yml` | +| `install_pkgs.sh` | used | Non-interactive bootstrap for Claude Code on the web sessions, whose — invoked by `.github/workflows/sentinel.yml` | +| `jev_ask.py` | used | Ask TypeSafe Jev typed questions from the command line. — invoked by `docs/engineering/JEV_HEADER_MAPPING.md`, `docs/engineering/JEV_LAB.md` | +| `jev_eval.py` | used | Measure TypeSafe Jev on RegEngine's own CSV header rows. — invoked by `docs/engineering/JEV_HEADER_MAPPING.md`, `docs/engineering/JEV_LAB.md` | +| `jev_header_preview.py` | used | Preview TypeSafe Jev header-mapping suggestions for a local CSV file. — invoked by `docs/engineering/JEV_HEADER_MAPPING.md` | +| `lifecycle_test_results.json` | data | captured output of an old lifecycle run; named in `scripts/check_trust_safety_copy.py`; archive candidate once that reference is dropped | +| `lint_response_models.py` | manual | manual lint: every FastAPI route declares `response_model`; candidate for CI (WI-03 follow-up) | +| `okr_pilots_ingesting_weekly.sql` | used | okr_pilots_ingesting_weekly.sql — invoked by `docs/runbooks/design_partner_funnel_operating_rhythm.md`, `docs/runbooks/first_pilot_session.md` | +| `orphan_migrations_allowlist.txt` | data | allowlist read by `scripts/check_orphan_migrations.py` (backend-ci) | +| `phone_test_quickstart.sh` | manual | run the published quickstart curl against prod to catch docs/contract drift; companion to `scripts/smoke_test_ingest.sh` | +| `qa_health_summary.py` | used | Build a redacted QA health dashboard artifact from CI test outputs. — invoked by `.github/workflows/qa-health.yml` | +| `render_ingestion_ignores.py` | used | Render ``--ignore=`` flags for the ingestion suite-runs gate. — invoked by `.github/workflows/backend-ci.yml` | +| `run-migrations.sh` | used | run-migrations.sh — Idempotent Alembic migration runner with distributed lock — invoked by `.github/workflows/backend-ci.yml`, `.github/workflows/security.yml` | +| `run-performance-baseline.sh` | used | Performance Baseline Test Script — invoked by `docs/operations/performance-baseline.md` | +| `run_verification.sh` | archive candidate | ad-hoc verification runner with no documented procedure | +| `scale_benchmark.py` | archive candidate | 42-line tenant onboarding benchmark, no caller | +| `schema_parity_audit.py` | used | Schema parity audit (WS-2c) — fresh-DB `alembic upgrade head` ≡ expected schema. — invoked by `.github/workflows/backend-ci.yml`, `README.md` | +| `seed_agent_review_demo.py` | manual | manual: seed synthetic agent-review artifacts for a demo (tested in `tests/scripts/`) | +| `seed_demo.js` | used | seeds the demo tenant via the Admin API; named by `tests/test_e2e_fda_request.py` and `docs/persistence/ws1d-retirement-plan.md` | +| `seed_demo_data.py` | manual | manual: seed demo tenants and events into a local stack | +| `seed_passwordless_e2e_user.py` | used | Seed the local Playwright passwordless auth user. — invoked by `.github/workflows/frontend-ci.yml`, `.github/workflows/pr-gate.yml` | +| `seed_phase2b_key.py` | used | deprecated by its own docstring (use `scripts/create_test_api_key.py`), but `tests/scripts/test_seed_phase2b_key_masking.py` exercises its masking, so it stays (PRD WI-08 requirement 4) | +| `seed_v1_data.py` | used | payload = {"text": content, "source_url": unique_url, "source_system": "seed_script", "ver — invoked by `scripts/run_verification.sh` | +| `sentinel-backfill.py` | archive candidate | one-off backfill of closing-PR references for closed issues | +| `setup_dev.sh` | used | RegEngine Local Development Setup Script — invoked by `scripts/ci/supply_chain_baseline.json`, `scripts/install_pkgs.sh` | +| `setup_e2e_postgres.py` | used | Migrate the isolated local Playwright database before starting its servers. — invoked by `.github/workflows/frontend-ci.yml` | +| `setup_production.sh` | archive candidate | superseded by `docs/operations/PRODUCTION_DEPLOYMENT.md` and Railway config | +| `smoke_test_ingest.sh` | used | Smoke test for /v1/ingest/* endpoints. — invoked by `docs/runbooks/r2_provisioning.md` | +| `start-e2e-backend.sh` | used | start-e2e-backend.sh — invoked by `.github/workflows/frontend-ci.yml`, `.github/workflows/pr-gate.yml` | +| `start-fsma.sh` | used | FSMA-only runtime: keep local stack small during pilot work. — invoked by `scripts/start-stack.sh` | +| `start-stack.sh` | used | RegEngine Stack Startup Script — invoked by `docs/operations/OPERATIONS.md` | +| `stop-fsma.sh` | used | Shut down FSMA-only runtime. — invoked by `scripts/stop-stack.sh` | +| `stop-stack.sh` | used | RegEngine Stack Shutdown Script — invoked by `docs/operations/OPERATIONS.md` | +| `stress_test.py` | used | RegEngine Massive Stress Test — invoked by `docs/engineering/security-findings-2026-09-10/README.md`, `docs/engineering/security-findings-2026-09-10/dispositions.json` | +| `summon_agent.py` | used | RegEngine agent prompt helper. — invoked by `.github/workflows/legacy-code-scan.yml` | +| `test-all.sh` | used | RegEngine Full Test Suite Runner — invoked by `docs/engineering/TESTING_REQUIREMENTS.md` | +| `test_connectors.sh` | archive candidate | 10-line connector smoke with no documented procedure | +| `test_fsma_endpoints.js` | archive candidate | targets the retired graph service on port 8200; superseded by `tests/` FSMA endpoint suites | +| `test_id_003.sh` | archive candidate | hard-codes a demo tenant id; superseded by `scripts/regctl/` | +| `trust_safety_copy_allowlist.txt` | data | allowlist read by `scripts/check_trust_safety_copy.py` (frontend-ci, pr-gate) | +| `validate_data_integrity.py` | manual | manual: after a `scripts/chaos/` run, validate data integrity | +| `validate_env.py` | manual | manual: validate a `.env` against the startup contract (#533) | +| `validate_release_evidence.py` | used | Validate sanitized, deployment-bound security release evidence. — invoked by `.github/workflows/backend-ci.yml` | +| `verify-health.sh` | used | RegEngine Health Verification Script — invoked by `docs/operations/OPERATIONS.md` | +| `verify_canonical_parity_live.py` | retire with WI-12 | retire with WI-12: live dual-write parity verifier (#1335) | +| `verify_ciam_models.py` | archive candidate | 28-line import check for retired CIAM models | +| `verify_compliance_lineage.py` | used | Configuration — invoked by `.github/workflows/backend-ci.yml` | +| `verify_fda_export.py` | archive candidate | ad-hoc check superseded by `tests/` FDA export suites | +| `verify_fsma_only.py` | archive candidate | one-off check that non-FSMA vertical stubs were deleted (done) | +| `verify_inflow_to_regengine_e2e.py` | used | Hard-fail local Inflow-to-RegEngine evidence proof. — invoked by `README.md`, `docs/runbooks/inflow_workbench_design_partner_demo.md` | +| `verify_live_freshness.py` | used | verify_live_freshness.py — invoked by `scripts/run_verification.sh` | +| `verify_production_readiness.py` | manual | manual: release-day readiness verification | +| `workflow_inventory.txt` | data | the committed workflow inventory read by `scripts/check_workflow_inventory.py` (supply-chain, inflow-contract-ci) | + +## Subdirectories + +| Directory | Purpose | +|---|---| +| `scripts/chaos/` | chaos and resilience drills (see its README or headers); results feed `format_chaos_results.py` | +| `scripts/ci/` | helpers invoked by workflows under `.github/workflows/` | +| `scripts/copy-lint/` | the SEL-452 copy gate (`copy-gate.yml`, `copy-lint.yml`) | +| `scripts/fixtures/` | fixture files consumed by `tests/scripts/` (for example the RLS fail-open cases) | +| `scripts/jev_lab/` | Jev Lab: `venv/bin/python scripts/jev_lab --help` (see `docs/engineering/JEV_LAB.md`) | +| `scripts/railway/` | Railway bootstrap and staging helpers referenced by `docs/operations/` | +| `scripts/regctl/` | tenant management CLI (`python scripts/regctl/tenant.py --help`) | +| `scripts/scheduler/` | standalone scheduler entrypoints; production runs the embedded scheduler instead (`ASYNC_PROCESSES.md`) | +| `scripts/security/` | release-gate and scanning helpers referenced by `security.yml` and `docs/security/` | +| `scripts/test/` | test database bootstrap SQL used by `docker-compose.dev.yml` and CI | +| `scripts/utilities/` | local diagnostics; none is referenced by CI or a runbook | + +Counts on 2026-10-30: 60 used, 4 data, 21 manual/one-time/retire, 14 archive candidates. diff --git a/scripts/check_schema_channels.py b/scripts/check_schema_channels.py new file mode 100644 index 000000000..760d5af48 --- /dev/null +++ b/scripts/check_schema_channels.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""CI guard — every SQL file in the two non-Alembic schema channels names the +Alembic revision it renders. + +Decision record: docs/decisions/2026-10-30-schema-change-channel.md. The +Alembic chain under ``alembic/versions/`` is the only schema authority. +``supabase/migrations/`` and ``sql/prod-apply/`` may hold rendered copies of +Alembic revisions, and nothing else. A rendered copy starts with: + + -- alembic-revision: <12-hex revision id> + -- rendered-from: alembic/versions/.py (upgrade|downgrade|precheck) + +Checks, per file under the two directories: + +1. The ``alembic-revision`` header is present (within the first 20 lines) + and names a revision id that exists in ``alembic/versions/``. +2. The ``rendered-from`` header is present and names a file that exists in + ``alembic/versions/`` and whose ``revision`` matches the id in line 1. + +Files that predate the decision carry no header and are grandfathered by +name below; the list only shrinks. ``PRECHECK_alembic_version_reconcile.sql`` +is exempt because it checks every revision rather than rendering one. + +Exit code 0 = clean. 1 = violation. 2 = repository layout not found. + +Usage: + python scripts/check_schema_channels.py + python scripts/check_schema_channels.py --repo-root /path/to/checkout +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +CHANNEL_DIRS = ("supabase/migrations", "sql/prod-apply") +HEADER_WINDOW_LINES = 20 + +_REVISION_HEADER = re.compile(r"^--\s*alembic-revision:\s*([0-9a-fA-F]{12})\s*$") +_RENDERED_FROM_HEADER = re.compile( + r"^--\s*rendered-from:\s*(alembic/versions/[\w.\-]+\.py)\s*\((upgrade|downgrade|precheck)\)\s*$" +) +_REVISION_ASSIGNMENT = re.compile(r"""^revision(?::\s*str)?\s*=\s*['"]([0-9a-fA-F]+)['"]""", re.M) + +# Files that existed before the 2026-10-30 decision without a header. Each is +# inventoried in the decision record's Appendix A. Remove an entry when the +# file gains its header or is moved out of the channel directory. Never add +# to this list: a new file must carry the header. +GRANDFATHERED: frozenset[str] = frozenset( + { + "supabase/migrations/20260314053557_create_waitlist_leads.sql", + "supabase/migrations/20260314073855_create_developer_portal_tables.sql", + "supabase/migrations/20260314074042_create_increment_invite_usage_rpc.sql", + "supabase/migrations/20260314093510_create_assessment_submissions.sql", + "supabase/migrations/20260317074952_create_fsma_organizations.sql", + "supabase/migrations/20260319014551_create_search_queries_table.sql", + "supabase/migrations/20260320220954_extend_organizations_schema.sql", + "supabase/migrations/20260321014327_seed_fsma_204_obligations.sql", + "supabase/migrations/20260321014418_seed_obligation_cte_rules.sql", + "supabase/migrations/20260330002451_enable_rls_missing_tables.sql", + "supabase/migrations/20260330031620_developer_portal_schema_doc.sql", + "supabase/migrations/20260403234813_create_traceability_events_and_ingestion_runs.sql", + "supabase/migrations/20260403235145_drop_hash_chain_fk_to_cte_events.sql", + "supabase/migrations/20260403235708_create_rules_engine_tables.sql", + "supabase/migrations/20260404000051_fix_cte_events_idempotency_constraint.sql", + "supabase/migrations/20260404011602_create_transformation_links_adjacency.sql", + "supabase/migrations/20260413044744_add_last_login_at_to_users.sql", + "supabase/migrations/20260730025741_lockdown_anon_lead_capture_and_definer_rpcs.sql", + "supabase/migrations/20260730025751_revoke_pgaudit_event_trigger_fn_execute.sql", + "supabase/migrations/20260730033519_drop_transitional_anon_insert_on_assessment_submissions.sql", + "supabase/migrations/20261001054030_security_fsma_rls_and_function_search_path_20260930.sql", + "supabase/migrations/20261016072839_search_path_extensions.sql", + "supabase/migrations/20261016073047_a_move_extensions_to_extensions_schema.sql", + "supabase/migrations/20261016073057_b_drop_duplicate_indexes.sql", + "supabase/migrations/20261016073107_c_rls_initplan_select_wrap.sql", + "supabase/migrations/20261016073122_d_index_unindexed_foreign_keys.sql", + "supabase/migrations/20261017021229_e_wrap_get_tenant_context.sql", + "supabase/migrations/20261017021244_f_tenant_id_indexes.sql", + "supabase/migrations/20261020072328_assessment_lead_abuse_guard.sql", + "supabase/migrations/20261022101703_lock_down_submit_assessment_lead.sql", + "supabase/migrations/20261022103548_enable_rls_internal_tables.sql", + "supabase/migrations/20261023050348_sel401_notify_new_lead_secret.sql", + "supabase/migrations/20261025100000_fix_obligation_1455_regulation_refs.sql", + "supabase/migrations/20261025100100_waitlist_leads_drop_authenticated_read.sql", + "supabase/migrations/20261025100200_task_queue_fail_closed_tenant_policy.sql", + "supabase/migrations/20261025100300_data_gap_close_receipt_not_null.sql", + "supabase/migrations/20261026132000_sel399_rls_split_api_keys_sessions_users.sql", + "sql/prod-apply/rollback_sel399_rls_split_api_keys_sessions_users.sql", + } +) + +EXEMPT: frozenset[str] = frozenset({"sql/prod-apply/PRECHECK_alembic_version_reconcile.sql"}) + + +def collect_revisions(versions_dir: Path) -> dict[str, str]: + """Map revision id -> file name (relative to the repo root) for every + migration under ``versions_dir``.""" + found: dict[str, str] = {} + for path in sorted(versions_dir.glob("*.py")): + match = _REVISION_ASSIGNMENT.search(path.read_text(encoding="utf-8", errors="replace")) + if match: + found[match.group(1).lower()] = f"alembic/versions/{path.name}" + return found + + +def read_headers(path: Path) -> tuple[str | None, str | None, str | None]: + """Return (revision_id, rendered_from_file, rendered_from_kind) from the + first ``HEADER_WINDOW_LINES`` lines, or ``None`` for each missing part.""" + revision = rendered_file = rendered_kind = None + with path.open(encoding="utf-8", errors="replace") as handle: + for index, line in enumerate(handle): + if index >= HEADER_WINDOW_LINES: + break + stripped = line.rstrip("\n") + if revision is None: + match = _REVISION_HEADER.match(stripped) + if match: + revision = match.group(1).lower() + continue + if rendered_file is None: + match = _RENDERED_FROM_HEADER.match(stripped) + if match: + rendered_file, rendered_kind = match.group(1), match.group(2) + return revision, rendered_file, rendered_kind + + +def check_file(rel: str, path: Path, revisions: dict[str, str]) -> list[str]: + """Return the list of violations for one channel file (empty when clean).""" + if rel in EXEMPT or rel in GRANDFATHERED: + return [] + problems: list[str] = [] + revision, rendered_file, _kind = read_headers(path) + if revision is None: + problems.append(f"{rel}: missing '-- alembic-revision: ' header in the first {HEADER_WINDOW_LINES} lines") + elif revision not in revisions: + problems.append(f"{rel}: alembic-revision {revision} does not exist in alembic/versions/") + if rendered_file is None: + problems.append( + f"{rel}: missing '-- rendered-from: alembic/versions/.py (upgrade|downgrade|precheck)' header" + ) + elif revision is not None and revision in revisions and revisions[revision] != rendered_file: + problems.append( + f"{rel}: rendered-from names {rendered_file} but revision {revision} lives in {revisions[revision]}" + ) + return problems + + +def scan(repo_root: Path) -> tuple[list[str], int]: + versions_dir = repo_root / "alembic" / "versions" + if not versions_dir.is_dir(): + print(f"!! alembic/versions not found under {repo_root}", file=sys.stderr) + sys.exit(2) + revisions = collect_revisions(versions_dir) + problems: list[str] = [] + scanned = 0 + for channel in CHANNEL_DIRS: + channel_dir = repo_root / channel + if not channel_dir.is_dir(): + continue + for path in sorted(channel_dir.rglob("*.sql")): + rel = path.relative_to(repo_root).as_posix() + scanned += 1 + problems.extend(check_file(rel, path, revisions)) + return problems, scanned + + +def stale_grandfathered(repo_root: Path) -> list[str]: + """Grandfathered names that no longer exist on disk; each must be removed + from ``GRANDFATHERED`` so the list only shrinks.""" + return sorted(name for name in GRANDFATHERED if not (repo_root / name).exists()) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument( + "--repo-root", + default=str(Path(__file__).resolve().parent.parent), + help="repository root (default: parent of scripts/)", + ) + args = parser.parse_args(argv) + repo_root = Path(args.repo_root).resolve() + problems, scanned = scan(repo_root) + for name in stale_grandfathered(repo_root): + problems.append(f"GRANDFATHERED lists {name}, which no longer exists; remove the entry") + print(f"Scanned {scanned} file(s) under {', '.join(CHANNEL_DIRS)}") + if problems: + print("\nSchema channel guard FAILED:") + for problem in problems: + print(f" {problem}") + print( + "\nA file in these directories must be a render of an Alembic revision and " + "start with the two header lines described in " + "docs/decisions/2026-10-30-schema-change-channel.md." + ) + return 1 + print("OK — every non-grandfathered channel file names an existing Alembic revision.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_scripts_registered.py b/scripts/check_scripts_registered.py new file mode 100755 index 000000000..226c4b57a --- /dev/null +++ b/scripts/check_scripts_registered.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""CI guard — every top-level entry in ``scripts/`` has a row in ``scripts/README.md``. + +Tech-debt work item WI-08 (2026-10-30): ``scripts/`` had 148 scripts and 67 of +them were referenced by nothing. The registry in ``scripts/README.md`` gives +each top-level file one line (status, purpose, what invokes it) and each +subdirectory one line. This guard keeps the registry and the directory in +step, in both directions: + +1. A file or directory directly under ``scripts/`` that has no row fails. +2. A row whose file or directory no longer exists fails (the registry only + describes what is there). + +Only the first level is checked: files inside subdirectories are covered by +the subdirectory's row. Hidden entries, ``__pycache__``, ``README.md`` itself +and ``_archive/`` are ignored. + +Exit code 0 = clean. 1 = violation. 2 = repository layout not found. + +Usage: + python scripts/check_scripts_registered.py + python scripts/check_scripts_registered.py --repo-root /path/to/checkout +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +SCRIPTS_DIR = "scripts" +REGISTRY = "scripts/README.md" +IGNORED_NAMES: frozenset[str] = frozenset({"README.md", "__pycache__", "_archive"}) + +_FILE_ROW = re.compile(r"^\|\s*`([^`/]+)`\s*\|") +_DIR_ROW = re.compile(r"^\|\s*`scripts/([^`/]+)/`\s*\|") + + +def tree_entries(scripts_dir: Path) -> tuple[set[str], set[str]]: + """Return (files, directories) directly under ``scripts_dir``, minus ignored names.""" + files: set[str] = set() + dirs: set[str] = set() + for entry in scripts_dir.iterdir(): + if entry.name.startswith(".") or entry.name in IGNORED_NAMES: + continue + (dirs if entry.is_dir() else files).add(entry.name) + return files, dirs + + +def registry_entries(readme: Path) -> tuple[set[str], set[str]]: + """Return (files, directories) that have a table row in the registry.""" + files: set[str] = set() + dirs: set[str] = set() + for line in readme.read_text(encoding="utf-8").splitlines(): + match = _DIR_ROW.match(line) + if match: + dirs.add(match.group(1)) + continue + match = _FILE_ROW.match(line) + if match: + files.add(match.group(1)) + return files, dirs + + +def scan(repo_root: Path) -> list[str]: + scripts_dir = repo_root / SCRIPTS_DIR + readme = repo_root / REGISTRY + if not scripts_dir.is_dir() or not readme.is_file(): + print(f"!! {SCRIPTS_DIR}/ or {REGISTRY} not found under {repo_root}", file=sys.stderr) + sys.exit(2) + tree_files, tree_dirs = tree_entries(scripts_dir) + reg_files, reg_dirs = registry_entries(readme) + problems: list[str] = [] + for name in sorted(tree_files - reg_files): + problems.append(f"{SCRIPTS_DIR}/{name}: no row in {REGISTRY}; add one (status, purpose, invoked by)") + for name in sorted(tree_dirs - reg_dirs): + problems.append(f"{SCRIPTS_DIR}/{name}/: no row in the Subdirectories table of {REGISTRY}") + for name in sorted(reg_files - tree_files): + problems.append(f"{REGISTRY} lists `{name}`, which is not in {SCRIPTS_DIR}/; remove or correct the row") + for name in sorted(reg_dirs - tree_dirs): + problems.append(f"{REGISTRY} lists `scripts/{name}/`, which does not exist; remove or correct the row") + return problems + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n", 1)[0]) + parser.add_argument( + "--repo-root", + default=str(Path(__file__).resolve().parent.parent), + help="repository root (default: parent of scripts/)", + ) + args = parser.parse_args(argv) + repo_root = Path(args.repo_root).resolve() + problems = scan(repo_root) + tree_files, tree_dirs = tree_entries(repo_root / SCRIPTS_DIR) + print(f"Checked {len(tree_files)} file(s) and {len(tree_dirs)} director(y/ies) under {SCRIPTS_DIR}/") + if problems: + print(f"\n{SCRIPTS_DIR}/ registry guard FAILED:") + for problem in problems: + print(f" {problem}") + print(f"\nEvery top-level entry in {SCRIPTS_DIR}/ needs one row in {REGISTRY} (WI-08).") + return 1 + print(f"OK — {REGISTRY} and {SCRIPTS_DIR}/ agree.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/server/workers/task_processor.py b/server/workers/task_processor.py index a0594b64d..722fe422d 100644 --- a/server/workers/task_processor.py +++ b/server/workers/task_processor.py @@ -1,11 +1,15 @@ """PostgreSQL-backed task processor — replaces Kafka consumers. -Polls the ``fsma.task_queue`` table for new work. Handles three task +Polls the ``fsma.task_queue`` table for new work. Handles two task types that previously required separate Kafka consumers: 1. nlp_extraction — extract entities from ingested documents - 2. graph_update — upsert extracted entities into the knowledge graph - 3. review_item — record low-confidence extractions for human review + 2. review_item — record low-confidence extractions for human review + +There is no ``graph_update`` task type on the pg backbone: nothing enqueues +one and no handler exists (retired 2026-10-30, tech-debt WI-15). High-tier +extractions are logged (``fsma_high_confidence_no_graph_sink``) until a graph +sink is built. Delivery mechanism ------------------ @@ -72,7 +76,6 @@ # (which costs duplicate LLM calls / duplicate graph upserts). TASK_TIMEOUTS_SECONDS: Dict[str, int] = { "nlp_extraction": 300, # up to 5 min for a long document - "graph_update": 120, # Neo4j cluster hiccups "review_item": 30, # near-instant, trivial DB write } @@ -156,9 +159,9 @@ def _run_hardened_extraction( 3. Three-tier confidence routing (#1258): review-tier extractions are enqueued as ``review_item`` tasks — the pg replacement for the ``nlp.needs_review`` topic. High-tier results auto-approve to - ``graph.update`` on the Kafka backbone, but the pg worker registers - no ``graph_update`` handler (an enqueued row would sit unclaimed - forever), so they are logged loudly instead. + ``graph.update`` on the Kafka backbone; the pg backbone has no + graph sink and no ``graph_update`` task type, so they are logged + loudly instead. Kafka-only stages (HMAC signing/verification, DLQ topics, offset commits, inbound schema validation) have no pg analog: the task queue @@ -233,8 +236,8 @@ def _run_hardened_extraction( summary["review_items_enqueued"] += 1 else: # High-confidence FSMA result. On the Kafka backbone this is - # emitted to graph.update; the pg worker has no graph sink, so - # log loudly until a pg graph_update handler exists. + # emitted to graph.update; the pg backbone has no graph sink, so + # log loudly until one is built. logger.warning( "fsma_high_confidence_no_graph_sink", document_id=doc_id, @@ -906,7 +909,6 @@ def enqueue_task( Recommended keys: * NLP extraction: ``f"nlp_extraction:{document_id}"`` - * Graph update: ``f"graph_update:{document_id}"`` * Review item: ``f"review_item:{document_id}:{entity_type}"`` """ try: diff --git a/services/shared/task_types.py b/services/shared/task_types.py index e64db9f52..b3253d3ca 100644 --- a/services/shared/task_types.py +++ b/services/shared/task_types.py @@ -72,7 +72,6 @@ { # server/workers/task_processor.py "nlp_extraction", - "graph_update", "review_item", # services/ingestion/app/task_handlers_sources.py "federal_register_ingest", diff --git a/sql/prod-apply/PRECHECK_alembic_version_reconcile.sql b/sql/prod-apply/PRECHECK_alembic_version_reconcile.sql new file mode 100644 index 000000000..c98b9b755 --- /dev/null +++ b/sql/prod-apply/PRECHECK_alembic_version_reconcile.sql @@ -0,0 +1,332 @@ +-- Read-only reconcile precheck for public.alembic_version. +-- Companion to docs/runbooks/ALEMBIC_VERSION_RECONCILE.md and +-- docs/decisions/2026-10-30-schema-change-channel.md. +-- +-- What it does: for every Alembic revision after 267d8053ced5 (v079, the +-- revision production was observed at on 2026-10-20), test one catalog +-- signature that the revision's upgrade() leaves behind, in chain order. +-- It then reports the highest revision through which every observable +-- signature is present, and every revision whose signature is present +-- beyond that point (effects that reached the database without the chain). +-- +-- What it never does: write. Only BEGIN READ ONLY, SET LOCAL, WITH/SELECT and +-- ROLLBACK appear below. It reads pg_catalog only; no customer rows. +-- +-- Run: psql -X -v ON_ERROR_STOP=1 -f sql/prod-apply/PRECHECK_alembic_version_reconcile.sql +-- +-- Reading the result: +-- present = true the signature exists (or, for absent_* kinds, is gone) +-- present = false the revision's effect is not in this database +-- present = NULL not observable from the catalog (merge or data revision, +-- or a role the check needs does not exist) +-- Kinds absent_relation, absent_fk and absent_policy read true on any +-- database that never had the object, so they cannot distinguish "never +-- created" from "removed by the revision"; weigh them with their neighbours. +-- Signatures marked HAND-APPLIED in note are expected present even on a +-- database whose version row is far behind: they reached production through +-- supabase/migrations or sql/prod-apply, not through Alembic. + +BEGIN READ ONLY; +SET LOCAL statement_timeout = '15s'; +SET LOCAL search_path = pg_catalog; + +-- 1. The version row as recorded. +SELECT 'alembic_version' AS object_type, version_num AS value +FROM public.alembic_version; + +-- 2. One signature per revision, in chain order. +WITH sig(ord, revision, label, kind, arg1, arg2, note) AS ( + VALUES + (1, '8a9b0c1d2e3f', 'v080', 'relation', 'fsma.rule_rejections', '', ''), + (2, 'c1d2e3f4a5b6', 'v079', 'relation', 'fsma.ingestion_run_receipts', '', 'second v079 label; use revision ids'), + (3, 'd2e3f4a5b6c7', 'v080', 'relation', 'fsma.agent_artifacts', '', ''), + (4, '3e7a1c5b9f24', 'v081', 'column', 'fsma.key_data_elements', 'tenant_id', ''), + (5, 'a7b8c9d0e1f2', 'v081', 'index', 'fsma.idx_cte_events_tenant_sha', '', ''), + (6, '70d5e59e3257', 'v082', 'merge', '', '', 'merge revision; nothing to observe'), + (7, 'e3f4a5b6c7d8', 'v083', 'column', 'fsma.cte_kdes_backfill_log_v059', 'tenant_id', ''), + (8, 'da5d93004161', 'v084', 'relation', 'public.user_emails', '', 'passwordless auth schema'), + (9, 'f8a9b0c1d2e3', 'v085', 'column', 'public.organizations', 'domain_name', ''), + (10, 'a9b0c1d2e3f4', 'v086', 'column', 'public.organizations', 'domain_verified', ''), + (11, 'b0c1d2e3f4a5', 'v087', 'relation', 'public.webauthn_credentials', '', ''), + (12, 'bdb2bec8bcc6', 'v088', 'absent_relation', 'public.password_reset_tokens', '', 'destructive: table dropped'), + (13, '38e02636e340', 'v089', 'column', 'public.sessions', 'authenticated_at', ''), + (14, '44225a26ed58', 'v090', 'column', 'public.audit_logs', 'sequence_num', ''), + (15, '4c5d6e7f8a9b', 'v090', 'absent_fk', 'fsma.hash_chain', 'fsma.cte_events', 'also hand-applied by supabase 20260403235145'), + (16, 'd1bdfe74d1fd', 'v090', 'merge', '', '', ''), + (17, 'a128f7f95db8', 'v090', 'policy', 'tenant_isolation_tasks', 'fsma.task_queue', 'AMBIGUOUS: supabase 20261025100200 recreates a policy of the same name'), + (18, '127a24ffb54c', 'v091', 'constraint', 'uq_request_signoffs_tenant_case_type', 'fsma.request_signoffs', ''), + (19, 'b3c8e1f4a627', 'v092', 'privilege_absent', 'anon', 'fsma.cte_events', 'client-role lockdown; NULL when role anon is absent'), + (20, '16cb29a3938c', 'v093', 'relation', 'public.scheduler_seen_items', '', ''), + (21, 'feb267263e0a', 'v094', 'data_user_emails_fingerprint', '', '', 'data backfill; NULL when public.user_emails is absent'), + (22, 'c7a1e5f2b9d4', 'v095', 'relation', 'fsma.remediation_cases', '', ''), + (23, 'c3a51e2f7b90', 'v096', 'policy', 'tenant_isolation_audit', 'fsma.fsma_audit_trail', ''), + (24, 'e4d7b2a9c561', 'v097', 'func_config_search_path', 'public.get_tenant_context', '', ''), + (25, '8b2f4e6a9c31', 'v098', 'relation', 'fsma.exchange_packages', '', ''), + (26, '2aa607050c84', 'v099', 'function', 'public.set_admin_context', '', ''), + (27, '3ba0291a69f2', 'v100', 'policy', 'memberships_sysadmin_read', 'public.memberships', ''), + (28, '7c4d9a1f2e8b', 'v101', 'column', 'fsma.notification_outbox', 'attempts', ''), + (29, '1b10a708e36a', 'v102', 'relation', 'fsma.mock_audit_drills', '', ''), + (30, 'fcf558aeb1e2', 'v103', 'relation', 'fsma.compliance_evidence_snapshots', '', ''), + (31, '7c1f2c8ffe05', 'v104', 'policy', 'obligation_cte_rules_tenant_isolation', 'public.obligation_cte_rules', ''), + (32, '323c945cf68d', 'v105', 'relation', 'fsma.rule_evaluation_deadletter', '', ''), + (33, '8633e0a7a84e', 'v106', 'relation', 'fsma.tenant_seed_state', '', ''), + (34, 'c35358cae7d4', 'v107', 'func_src', 'fsma.prevent_cte_mutation', 'RETURN NEW', ''), + (35, 'a43b0fb1edd0', 'v108', 'column', 'fsma.traceability_events', 'evaluation_error', ''), + (36, '51701aedee57', 'v109', 'function', 'fsma.get_team_roster', '', ''), + (37, '6f8c2d1a4b90', 'v110', 'policy', 'developers_insert_own_profile', 'public.developer_profiles', ''), + (38, 'df664d92adba', 'v111', 'force_rls', 'public.controls', '', ''), + (39, 'd8b2056471a7', 'v112', 'func_secdef', 'seed_obligations_for_tenant', '', 'any schema'), + (40, '4e9c1f7a02b6', 'v113', 'function', 'public.get_invite_tenant_by_token_hash', '', ''), + (41, 'c4f1a7e93b28', 'v114', 'column', 'fsma.fda_export_log', 'include_pii', ''), + (42, 'c1d4e8f20a37', 'v115', 'index', 'fsma.idx_task_queue_idempotency', '', ''), + (43, 'e4a7c2d91b06', 'v116', 'data', '', '', 'rule seed integrity; not catalog-observable'), + (44, '6f4c2a9d8b17', 'v110', 'data_roles_review_decide', '', '', 'NULL when public.roles is absent'), + (45, 'b71d9e4c2a60', 'v111', 'force_rls', 'fsma.organizations', '', 'runtime role grants; FORCE RLS is the observable part'), + (46, '8b866e4b4798', 'v119', 'data', '', '', 'seed reconcile; not catalog-observable'), + (47, 'c6a120b4e901', 'v120', 'data', '', '', 'rule citations; not catalog-observable'), + (48, '7551be5c64ac', 'v121', 'index', 'fsma.idx_label_charges_tenant_created', '', ''), + (49, '8301f3cad96a', 'v121', 'relation', 'fsma.tenant_inbound_email_addresses', '', ''), + (50, 'ceb552c6d36d', 'v121', 'relation', 'billing.tenant_usage_counters', '', ''), + (51, '5541fb62ee2d', 'v121', 'merge', '', '', ''), + (52, 'bd0c6549153a', 'v121', 'relation', 'agent_org.agent_pods', '', ''), + (53, 'ccaa01e4fd7c', 'v121', 'relation', 'fsma.inbound_email_audit', '', ''), + (54, 'd14f8a2b9c31', 'v122', 'function', 'fsma.claim_inbound_email_audit', '', ''), + (55, 'e9f2a4c71b05', 'v123', 'merge', '', '', ''), + (56, '26d215418784', 'v124', 'data', '', '', 'rule field paths; not catalog-observable'), + (57, '22a278f6dc78', 'v125', 'column', 'fsma.traceability_events', 'evaluation_verdict', ''), + (58, 'c51d4898e0a2', 'v126', 'data', '', '', 'grants only; not checked here'), + (59, 'd2e7a7319b33', 'v127', 'data', '', '', 'function owner RLS setting; not checked here'), + (60, '2db4f1d7b4ca', 'v126', 'data', '', '', 'composite FK rewrite; not checked here'), + (61, '486eb6809921', 'v126', 'data', '', '', 'drops a function; not checked here'), + (62, 'ed902d03df45', 'v127', 'merge', '', '', ''), + (63, 'a5e7c9b2d341', 'v128', 'policy', 'v128_cte_events_select_regengine', 'fsma.cte_events', 'HAND-APPLIED: supabase 20261016234347 / sql/prod-apply/apply_v128_fsma_rls.sql'), + (64, '93eb33f262a1', 'v128', 'data', '', '', 'HAND-APPLIED: supabase 20261016234352 / apply_v128_policy_merges.sql; merged policy names vary per table'), + (65, '0a329588b824', 'v131', 'column', 'public.tool_leads', 'utm_source', 'HAND-APPLIED: sql/prod-apply/apply_v131_tool_leads_utm.sql'), + (66, 'a9ea09050a30', 'v132', 'absent_policy', 'v128_client_guard_ingestion_runs', 'fsma.ingestion_runs', 'AMBIGUOUS: absent both before v128 and after v132'), + (67, 'c4d8f1a2b6e7', 'v133', 'privilege_column_update', 'regengine_sysadmin', 'public.tenants', 'NULL when the role is absent'), + (68, '1faec7148e78', 'v134', 'policy', 'portal_links_select', 'fsma.tenant_portal_links', 'HAND-APPLIED: sql/prod-apply/apply_sel314_rls_dedupe.sql'), + (69, '41f96588d87c', 'v135', 'relation', 'public.data_gap_events', '', 'HAND-APPLIED: supabase 20261025033803'), + (70, 'b005ddbf1dba', 'v136', 'column', 'public.supplier_facilities', 'agreed_sending_hours', 'HAND-APPLIED: supabase 20261025090000'), + (71, 'a3f7ce361b0e', 'v137', 'relation', 'fsma.epcis_document_receipts', '', 'HAND-APPLIED: supabase 20261025090100; current head') +), +checked AS ( + SELECT ord, revision, label, kind, arg1, arg2, note, + CASE kind + WHEN 'relation' THEN pg_catalog.to_regclass(arg1) IS NOT NULL + WHEN 'index' THEN pg_catalog.to_regclass(arg1) IS NOT NULL + WHEN 'absent_relation' THEN pg_catalog.to_regclass(arg1) IS NULL + WHEN 'column' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_attribute a + WHERE a.attrelid = pg_catalog.to_regclass(arg1) + AND a.attname = arg2 AND NOT a.attisdropped AND a.attnum > 0) + WHEN 'function' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1) + WHEN 'func_src' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1 + AND p.prosrc ILIKE '%' || arg2 || '%') + WHEN 'func_secdef' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + WHERE p.proname = arg1 AND p.prosecdef) + WHEN 'func_config_search_path' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1 + AND p.proconfig IS NOT NULL + AND array_to_string(p.proconfig, ',') ILIKE '%search_path%') + WHEN 'policy' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = pg_catalog.to_regclass(arg2) AND pol.polname = arg1) + WHEN 'absent_policy' THEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = pg_catalog.to_regclass(arg2) AND pol.polname = arg1) + WHEN 'constraint' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint c + WHERE c.conname = arg1 AND c.conrelid = pg_catalog.to_regclass(arg2)) + WHEN 'absent_fk' THEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint c + WHERE c.contype = 'f' + AND c.conrelid = pg_catalog.to_regclass(arg1) + AND c.confrelid = pg_catalog.to_regclass(arg2)) + WHEN 'force_rls' THEN ( + SELECT c.relforcerowsecurity FROM pg_catalog.pg_class c + WHERE c.oid = pg_catalog.to_regclass(arg1)) + WHEN 'privilege_absent' THEN ( + CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = arg1) + AND pg_catalog.to_regclass(arg2) IS NOT NULL + THEN NOT pg_catalog.has_table_privilege(arg1, arg2, 'SELECT') + ELSE NULL END) + WHEN 'privilege_column_update' THEN ( + CASE WHEN EXISTS (SELECT 1 FROM pg_catalog.pg_roles r WHERE r.rolname = arg1) + AND pg_catalog.to_regclass(arg2) IS NOT NULL + THEN pg_catalog.has_column_privilege(arg1, arg2, 'name', 'UPDATE') + ELSE NULL END) + WHEN 'data_user_emails_fingerprint' THEN ( + CASE WHEN pg_catalog.to_regclass('public.user_emails') IS NULL THEN NULL + WHEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_attribute a + WHERE a.attrelid = pg_catalog.to_regclass('public.user_emails') + AND a.attname = 'email_fingerprint' AND NOT a.attisdropped) THEN FALSE + ELSE NULL END) + WHEN 'data_roles_review_decide' THEN ( + CASE WHEN pg_catalog.to_regclass('public.roles') IS NULL THEN NULL + WHEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_attribute a + WHERE a.attrelid = pg_catalog.to_regclass('public.roles') + AND a.attname = 'permissions' AND NOT a.attisdropped) THEN FALSE + ELSE NULL END) + ELSE NULL + END AS present + FROM sig +) +SELECT ord, revision, label, kind, arg1, arg2, present, note +FROM checked +ORDER BY ord; + +-- 3. Summary: the chain position the catalog supports, and what lies beyond it. +WITH sig(ord, revision, label, kind, arg1, arg2, note) AS ( + VALUES + (1, '8a9b0c1d2e3f', 'v080', 'relation', 'fsma.rule_rejections', '', ''), + (2, 'c1d2e3f4a5b6', 'v079', 'relation', 'fsma.ingestion_run_receipts', '', 'second v079 label; use revision ids'), + (3, 'd2e3f4a5b6c7', 'v080', 'relation', 'fsma.agent_artifacts', '', ''), + (4, '3e7a1c5b9f24', 'v081', 'column', 'fsma.key_data_elements', 'tenant_id', ''), + (5, 'a7b8c9d0e1f2', 'v081', 'index', 'fsma.idx_cte_events_tenant_sha', '', ''), + (6, '70d5e59e3257', 'v082', 'merge', '', '', 'merge revision; nothing to observe'), + (7, 'e3f4a5b6c7d8', 'v083', 'column', 'fsma.cte_kdes_backfill_log_v059', 'tenant_id', ''), + (8, 'da5d93004161', 'v084', 'relation', 'public.user_emails', '', 'passwordless auth schema'), + (9, 'f8a9b0c1d2e3', 'v085', 'column', 'public.organizations', 'domain_name', ''), + (10, 'a9b0c1d2e3f4', 'v086', 'column', 'public.organizations', 'domain_verified', ''), + (11, 'b0c1d2e3f4a5', 'v087', 'relation', 'public.webauthn_credentials', '', ''), + (12, 'bdb2bec8bcc6', 'v088', 'absent_relation', 'public.password_reset_tokens', '', 'destructive: table dropped'), + (13, '38e02636e340', 'v089', 'column', 'public.sessions', 'authenticated_at', ''), + (14, '44225a26ed58', 'v090', 'column', 'public.audit_logs', 'sequence_num', ''), + (15, '4c5d6e7f8a9b', 'v090', 'absent_fk', 'fsma.hash_chain', 'fsma.cte_events', 'also hand-applied by supabase 20260403235145'), + (16, 'd1bdfe74d1fd', 'v090', 'merge', '', '', ''), + (17, 'a128f7f95db8', 'v090', 'policy', 'tenant_isolation_tasks', 'fsma.task_queue', 'AMBIGUOUS: supabase 20261025100200 recreates a policy of the same name'), + (18, '127a24ffb54c', 'v091', 'constraint', 'uq_request_signoffs_tenant_case_type', 'fsma.request_signoffs', ''), + (19, 'b3c8e1f4a627', 'v092', 'privilege_absent', 'anon', 'fsma.cte_events', 'client-role lockdown; NULL when role anon is absent'), + (20, '16cb29a3938c', 'v093', 'relation', 'public.scheduler_seen_items', '', ''), + (21, 'feb267263e0a', 'v094', 'data_user_emails_fingerprint', '', '', 'data backfill; NULL when public.user_emails is absent'), + (22, 'c7a1e5f2b9d4', 'v095', 'relation', 'fsma.remediation_cases', '', ''), + (23, 'c3a51e2f7b90', 'v096', 'policy', 'tenant_isolation_audit', 'fsma.fsma_audit_trail', ''), + (24, 'e4d7b2a9c561', 'v097', 'func_config_search_path', 'public.get_tenant_context', '', ''), + (25, '8b2f4e6a9c31', 'v098', 'relation', 'fsma.exchange_packages', '', ''), + (26, '2aa607050c84', 'v099', 'function', 'public.set_admin_context', '', ''), + (27, '3ba0291a69f2', 'v100', 'policy', 'memberships_sysadmin_read', 'public.memberships', ''), + (28, '7c4d9a1f2e8b', 'v101', 'column', 'fsma.notification_outbox', 'attempts', ''), + (29, '1b10a708e36a', 'v102', 'relation', 'fsma.mock_audit_drills', '', ''), + (30, 'fcf558aeb1e2', 'v103', 'relation', 'fsma.compliance_evidence_snapshots', '', ''), + (31, '7c1f2c8ffe05', 'v104', 'policy', 'obligation_cte_rules_tenant_isolation', 'public.obligation_cte_rules', ''), + (32, '323c945cf68d', 'v105', 'relation', 'fsma.rule_evaluation_deadletter', '', ''), + (33, '8633e0a7a84e', 'v106', 'relation', 'fsma.tenant_seed_state', '', ''), + (34, 'c35358cae7d4', 'v107', 'func_src', 'fsma.prevent_cte_mutation', 'RETURN NEW', ''), + (35, 'a43b0fb1edd0', 'v108', 'column', 'fsma.traceability_events', 'evaluation_error', ''), + (36, '51701aedee57', 'v109', 'function', 'fsma.get_team_roster', '', ''), + (37, '6f8c2d1a4b90', 'v110', 'policy', 'developers_insert_own_profile', 'public.developer_profiles', ''), + (38, 'df664d92adba', 'v111', 'force_rls', 'public.controls', '', ''), + (39, 'd8b2056471a7', 'v112', 'func_secdef', 'seed_obligations_for_tenant', '', 'any schema'), + (40, '4e9c1f7a02b6', 'v113', 'function', 'public.get_invite_tenant_by_token_hash', '', ''), + (41, 'c4f1a7e93b28', 'v114', 'column', 'fsma.fda_export_log', 'include_pii', ''), + (42, 'c1d4e8f20a37', 'v115', 'index', 'fsma.idx_task_queue_idempotency', '', ''), + (43, 'e4a7c2d91b06', 'v116', 'data', '', '', 'rule seed integrity; not catalog-observable'), + (44, '6f4c2a9d8b17', 'v110', 'data_roles_review_decide', '', '', 'NULL when public.roles is absent'), + (45, 'b71d9e4c2a60', 'v111', 'force_rls', 'fsma.organizations', '', 'runtime role grants; FORCE RLS is the observable part'), + (46, '8b866e4b4798', 'v119', 'data', '', '', 'seed reconcile; not catalog-observable'), + (47, 'c6a120b4e901', 'v120', 'data', '', '', 'rule citations; not catalog-observable'), + (48, '7551be5c64ac', 'v121', 'index', 'fsma.idx_label_charges_tenant_created', '', ''), + (49, '8301f3cad96a', 'v121', 'relation', 'fsma.tenant_inbound_email_addresses', '', ''), + (50, 'ceb552c6d36d', 'v121', 'relation', 'billing.tenant_usage_counters', '', ''), + (51, '5541fb62ee2d', 'v121', 'merge', '', '', ''), + (52, 'bd0c6549153a', 'v121', 'relation', 'agent_org.agent_pods', '', ''), + (53, 'ccaa01e4fd7c', 'v121', 'relation', 'fsma.inbound_email_audit', '', ''), + (54, 'd14f8a2b9c31', 'v122', 'function', 'fsma.claim_inbound_email_audit', '', ''), + (55, 'e9f2a4c71b05', 'v123', 'merge', '', '', ''), + (56, '26d215418784', 'v124', 'data', '', '', 'rule field paths; not catalog-observable'), + (57, '22a278f6dc78', 'v125', 'column', 'fsma.traceability_events', 'evaluation_verdict', ''), + (58, 'c51d4898e0a2', 'v126', 'data', '', '', 'grants only; not checked here'), + (59, 'd2e7a7319b33', 'v127', 'data', '', '', 'function owner RLS setting; not checked here'), + (60, '2db4f1d7b4ca', 'v126', 'data', '', '', 'composite FK rewrite; not checked here'), + (61, '486eb6809921', 'v126', 'data', '', '', 'drops a function; not checked here'), + (62, 'ed902d03df45', 'v127', 'merge', '', '', ''), + (63, 'a5e7c9b2d341', 'v128', 'policy', 'v128_cte_events_select_regengine', 'fsma.cte_events', 'HAND-APPLIED: supabase 20261016234347 / sql/prod-apply/apply_v128_fsma_rls.sql'), + (64, '93eb33f262a1', 'v128', 'data', '', '', 'HAND-APPLIED: supabase 20261016234352 / apply_v128_policy_merges.sql; merged policy names vary per table'), + (65, '0a329588b824', 'v131', 'column', 'public.tool_leads', 'utm_source', 'HAND-APPLIED: sql/prod-apply/apply_v131_tool_leads_utm.sql'), + (66, 'a9ea09050a30', 'v132', 'absent_policy', 'v128_client_guard_ingestion_runs', 'fsma.ingestion_runs', 'AMBIGUOUS: absent both before v128 and after v132'), + (67, 'c4d8f1a2b6e7', 'v133', 'privilege_column_update', 'regengine_sysadmin', 'public.tenants', 'NULL when the role is absent'), + (68, '1faec7148e78', 'v134', 'policy', 'portal_links_select', 'fsma.tenant_portal_links', 'HAND-APPLIED: sql/prod-apply/apply_sel314_rls_dedupe.sql'), + (69, '41f96588d87c', 'v135', 'relation', 'public.data_gap_events', '', 'HAND-APPLIED: supabase 20261025033803'), + (70, 'b005ddbf1dba', 'v136', 'column', 'public.supplier_facilities', 'agreed_sending_hours', 'HAND-APPLIED: supabase 20261025090000'), + (71, 'a3f7ce361b0e', 'v137', 'relation', 'fsma.epcis_document_receipts', '', 'HAND-APPLIED: supabase 20261025090100; current head') +), +checked AS ( + SELECT ord, revision, label, note, + CASE kind + WHEN 'relation' THEN pg_catalog.to_regclass(arg1) IS NOT NULL + WHEN 'index' THEN pg_catalog.to_regclass(arg1) IS NOT NULL + WHEN 'absent_relation' THEN pg_catalog.to_regclass(arg1) IS NULL + WHEN 'column' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_attribute a + WHERE a.attrelid = pg_catalog.to_regclass(arg1) + AND a.attname = arg2 AND NOT a.attisdropped AND a.attnum > 0) + WHEN 'function' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1) + WHEN 'func_src' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1 + AND p.prosrc ILIKE '%' || arg2 || '%') + WHEN 'func_secdef' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p WHERE p.proname = arg1 AND p.prosecdef) + WHEN 'func_config_search_path' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname || '.' || p.proname = arg1 + AND p.proconfig IS NOT NULL + AND array_to_string(p.proconfig, ',') ILIKE '%search_path%') + WHEN 'policy' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = pg_catalog.to_regclass(arg2) AND pol.polname = arg1) + WHEN 'absent_policy' THEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_policy pol + WHERE pol.polrelid = pg_catalog.to_regclass(arg2) AND pol.polname = arg1) + WHEN 'constraint' THEN EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint c + WHERE c.conname = arg1 AND c.conrelid = pg_catalog.to_regclass(arg2)) + WHEN 'absent_fk' THEN NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint c + WHERE c.contype = 'f' + AND c.conrelid = pg_catalog.to_regclass(arg1) + AND c.confrelid = pg_catalog.to_regclass(arg2)) + WHEN 'force_rls' THEN ( + SELECT c.relforcerowsecurity FROM pg_catalog.pg_class c + WHERE c.oid = pg_catalog.to_regclass(arg1)) + ELSE NULL + END AS present + FROM sig +), +observable AS ( + SELECT * FROM checked WHERE present IS NOT NULL +), +first_gap AS ( + SELECT min(ord) AS ord FROM observable WHERE present = FALSE +) +SELECT + (SELECT version_num FROM public.alembic_version LIMIT 1) AS recorded_version, + (SELECT revision FROM observable + WHERE present AND ord < COALESCE((SELECT ord FROM first_gap), 2147483647) + ORDER BY ord DESC LIMIT 1) AS supported_through_revision, + (SELECT label FROM observable + WHERE present AND ord < COALESCE((SELECT ord FROM first_gap), 2147483647) + ORDER BY ord DESC LIMIT 1) AS supported_through_label, + (SELECT count(*) FROM observable WHERE present) AS observable_present, + (SELECT count(*) FROM observable WHERE NOT present) AS observable_absent, + (SELECT string_agg(label || ':' || revision, ', ' ORDER BY ord) + FROM observable + WHERE present AND ord > COALESCE((SELECT ord FROM first_gap), 0)) AS present_beyond_first_gap; + +ROLLBACK; diff --git a/sql/prod-apply/PRECHECK_v128_fsma_rls.sql b/sql/prod-apply/PRECHECK_v128_fsma_rls.sql index bec86a2ed..c2a124efb 100644 --- a/sql/prod-apply/PRECHECK_v128_fsma_rls.sql +++ b/sql/prod-apply/PRECHECK_v128_fsma_rls.sql @@ -1,3 +1,5 @@ +-- alembic-revision: a5e7c9b2d341 +-- rendered-from: alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py (precheck) -- Read-only object inventory for apply_v128_fsma_rls.sql. -- Check the public.get_tenant_context() function used by APPLY. -- PUBLIC is a pseudo-role. diff --git a/sql/prod-apply/PRECHECK_v128_policy_merges.sql b/sql/prod-apply/PRECHECK_v128_policy_merges.sql index e7cfcb16f..149ca6fc9 100644 --- a/sql/prod-apply/PRECHECK_v128_policy_merges.sql +++ b/sql/prod-apply/PRECHECK_v128_policy_merges.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 93eb33f262a1 +-- rendered-from: alembic/versions/20261016_93eb33f262a1_v128_consolidate_permissive_policies.py (precheck) -- Read-only inventory for apply_v128_policy_merges.sql. -- PUBLIC is a PostgreSQL pseudo-role, so it has no pg_roles row. -- get_tenant_context() resolves through the session search_path, as APPLY does. diff --git a/sql/prod-apply/PRECHECK_v131_tool_leads_utm.sql b/sql/prod-apply/PRECHECK_v131_tool_leads_utm.sql index 5f77136f3..43b1cc49f 100644 --- a/sql/prod-apply/PRECHECK_v131_tool_leads_utm.sql +++ b/sql/prod-apply/PRECHECK_v131_tool_leads_utm.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 0a329588b824 +-- rendered-from: alembic/versions/20261016_0a329588b824_v131_tool_leads_utm.py (precheck) -- Read-only check before adding campaign attribution to public.tool_leads. -- Existing columns, if any, must be character varying(100). SELECT to_regclass('public.tool_leads') AS tool_leads_table; diff --git a/sql/prod-apply/apply_sel314_rls_dedupe.sql b/sql/prod-apply/apply_sel314_rls_dedupe.sql index bd8389a9f..24d594e7c 100644 --- a/sql/prod-apply/apply_sel314_rls_dedupe.sql +++ b/sql/prod-apply/apply_sel314_rls_dedupe.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 1faec7148e78 +-- rendered-from: alembic/versions/20261020_1faec7148e78_v134_sel314_rls_policy_dedupe.py (upgrade) -- SEL-314: remove duplicate permissive RLS policies (Supabase advisor -- "multiple_permissive_policies", 46 warnings across 12 tables). -- diff --git a/sql/prod-apply/apply_v128_fsma_rls.sql b/sql/prod-apply/apply_v128_fsma_rls.sql index a2135bd76..f82f0bf70 100644 --- a/sql/prod-apply/apply_v128_fsma_rls.sql +++ b/sql/prod-apply/apply_v128_fsma_rls.sql @@ -1,3 +1,5 @@ +-- alembic-revision: a5e7c9b2d341 +-- rendered-from: alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py (upgrade) -- v128 FSMA RLS policy repair, rendered from upgrade() in -- alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py. -- DROP IF EXISTS stays within each original creation branch so existing diff --git a/sql/prod-apply/apply_v128_policy_merges.sql b/sql/prod-apply/apply_v128_policy_merges.sql index 4c0472565..a20999178 100644 --- a/sql/prod-apply/apply_v128_policy_merges.sql +++ b/sql/prod-apply/apply_v128_policy_merges.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 93eb33f262a1 +-- rendered-from: alembic/versions/20261016_93eb33f262a1_v128_consolidate_permissive_policies.py (upgrade) -- Rendered from revision 93eb33f262a1, upgrade(). BEGIN; diff --git a/sql/prod-apply/apply_v131_tool_leads_utm.sql b/sql/prod-apply/apply_v131_tool_leads_utm.sql index 62453a276..963ccee1f 100644 --- a/sql/prod-apply/apply_v131_tool_leads_utm.sql +++ b/sql/prod-apply/apply_v131_tool_leads_utm.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 0a329588b824 +-- rendered-from: alembic/versions/20261016_0a329588b824_v131_tool_leads_utm.py (upgrade) -- Manual equivalent of revision 0a329588b824. Do not run after Alembic applied it. BEGIN; diff --git a/sql/prod-apply/rollback_sel314_rls_dedupe.sql b/sql/prod-apply/rollback_sel314_rls_dedupe.sql index ef32e252f..d99d0a223 100644 --- a/sql/prod-apply/rollback_sel314_rls_dedupe.sql +++ b/sql/prod-apply/rollback_sel314_rls_dedupe.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 1faec7148e78 +-- rendered-from: alembic/versions/20261020_1faec7148e78_v134_sel314_rls_policy_dedupe.py (downgrade) -- Rollback for apply_sel314_rls_dedupe.sql. Recreates the eleven out-of-band -- `authenticated` policies exactly as they were observed on 2026-10-20 and -- restores the two original fsma.tenant_portal_links policies. diff --git a/sql/prod-apply/rollback_v128_fsma_rls.sql b/sql/prod-apply/rollback_v128_fsma_rls.sql index f0f71e7a4..7755fbce0 100644 --- a/sql/prod-apply/rollback_v128_fsma_rls.sql +++ b/sql/prod-apply/rollback_v128_fsma_rls.sql @@ -1,3 +1,5 @@ +-- alembic-revision: a5e7c9b2d341 +-- rendered-from: alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py (downgrade) -- v128 FSMA RLS policy repair, rendered from downgrade() in -- alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py. diff --git a/sql/prod-apply/rollback_v128_policy_merges.sql b/sql/prod-apply/rollback_v128_policy_merges.sql index 1f900779f..ec5e599d5 100644 --- a/sql/prod-apply/rollback_v128_policy_merges.sql +++ b/sql/prod-apply/rollback_v128_policy_merges.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 93eb33f262a1 +-- rendered-from: alembic/versions/20261016_93eb33f262a1_v128_consolidate_permissive_policies.py (downgrade) -- Rendered from revision 93eb33f262a1, downgrade(). BEGIN; diff --git a/sql/prod-apply/rollback_v131_tool_leads_utm.sql b/sql/prod-apply/rollback_v131_tool_leads_utm.sql index 8c31f21d1..5b7512625 100644 --- a/sql/prod-apply/rollback_v131_tool_leads_utm.sql +++ b/sql/prod-apply/rollback_v131_tool_leads_utm.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 0a329588b824 +-- rendered-from: alembic/versions/20261016_0a329588b824_v131_tool_leads_utm.py (downgrade) -- Manual rollback of revision 0a329588b824. Attribution values will be lost. BEGIN; diff --git a/supabase/migrations/20260729012459_v092_lockdown_rls_and_revokes_only.sql b/supabase/migrations/20260729012459_v092_lockdown_rls_and_revokes_only.sql index de612f4dd..f71dc6356 100644 --- a/supabase/migrations/20260729012459_v092_lockdown_rls_and_revokes_only.sql +++ b/supabase/migrations/20260729012459_v092_lockdown_rls_and_revokes_only.sql @@ -1,3 +1,5 @@ +-- alembic-revision: b3c8e1f4a627 +-- rendered-from: alembic/versions/20260709_b3c8e1f4a627_v092_client_role_lockdown.py (upgrade) DO $$ DECLARE tbl record; diff --git a/supabase/migrations/20261016234347_v128_fsma_rls_policy_repair.sql b/supabase/migrations/20261016234347_v128_fsma_rls_policy_repair.sql index afed41062..f4d9d258e 100644 --- a/supabase/migrations/20261016234347_v128_fsma_rls_policy_repair.sql +++ b/supabase/migrations/20261016234347_v128_fsma_rls_policy_repair.sql @@ -1,3 +1,5 @@ +-- alembic-revision: a5e7c9b2d341 +-- rendered-from: alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py (upgrade) -- v128 FSMA RLS policy repair, rendered from upgrade() in -- alembic/versions/20261016_a5e7c9b2d341_v128_fsma_rls_policy_repair.py. -- DROP IF EXISTS stays within each original creation branch so existing diff --git a/supabase/migrations/20261016234352_v128_consolidate_permissive_policies.sql b/supabase/migrations/20261016234352_v128_consolidate_permissive_policies.sql index d89ceceb6..1fa98a369 100644 --- a/supabase/migrations/20261016234352_v128_consolidate_permissive_policies.sql +++ b/supabase/migrations/20261016234352_v128_consolidate_permissive_policies.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 93eb33f262a1 +-- rendered-from: alembic/versions/20261016_93eb33f262a1_v128_consolidate_permissive_policies.py (upgrade) -- Rendered from revision 93eb33f262a1, upgrade(). BEGIN; diff --git a/supabase/migrations/20261020085506_sel314_ftl_read_only_drop_sysadmin_overlap.sql b/supabase/migrations/20261020085506_sel314_ftl_read_only_drop_sysadmin_overlap.sql index 7fbc0e32b..11baf3623 100644 --- a/supabase/migrations/20261020085506_sel314_ftl_read_only_drop_sysadmin_overlap.sql +++ b/supabase/migrations/20261020085506_sel314_ftl_read_only_drop_sysadmin_overlap.sql @@ -1 +1,3 @@ +-- alembic-revision: 1faec7148e78 +-- rendered-from: alembic/versions/20261020_1faec7148e78_v134_sel314_rls_policy_dedupe.py (upgrade) alter policy ftl_read_only on public.food_traceability_list to regengine;; diff --git a/supabase/migrations/20261020094817_sel314_drop_fallback_tenant_policies.sql b/supabase/migrations/20261020094817_sel314_drop_fallback_tenant_policies.sql index a1826f2b7..4a54d7318 100644 --- a/supabase/migrations/20261020094817_sel314_drop_fallback_tenant_policies.sql +++ b/supabase/migrations/20261020094817_sel314_drop_fallback_tenant_policies.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 1faec7148e78 +-- rendered-from: alembic/versions/20261020_1faec7148e78_v134_sel314_rls_policy_dedupe.py (upgrade) DROP POLICY IF EXISTS compliance_alerts_tenant_isolation ON public.compliance_alerts; DROP POLICY IF EXISTS compliance_status_log_tenant_isolation ON public.compliance_status_log; DROP POLICY IF EXISTS invites_tenant_isolation ON public.invites; diff --git a/supabase/migrations/20261025033803_data_gap_time_to_fix.sql b/supabase/migrations/20261025033803_data_gap_time_to_fix.sql index da6c181a2..e8eff076c 100644 --- a/supabase/migrations/20261025033803_data_gap_time_to_fix.sql +++ b/supabase/migrations/20261025033803_data_gap_time_to_fix.sql @@ -1,3 +1,5 @@ +-- alembic-revision: 41f96588d87c +-- rendered-from: alembic/versions/20261024_41f96588d87c_v135_data_gap_time_to_fix.py (upgrade) -- Data gap time to fix. -- -- One row per data gap on a traceability lot: a required KDE missing from a diff --git a/supabase/migrations/20261025090000_supplier_agreed_sending_hours.sql b/supabase/migrations/20261025090000_supplier_agreed_sending_hours.sql index 526fd8f2d..7e7101ea1 100644 --- a/supabase/migrations/20261025090000_supplier_agreed_sending_hours.sql +++ b/supabase/migrations/20261025090000_supplier_agreed_sending_hours.sql @@ -1,3 +1,5 @@ +-- alembic-revision: b005ddbf1dba +-- rendered-from: alembic/versions/20261024_b005ddbf1dba_v136_supplier_agreed_sending_hours.py (upgrade) -- Agreed sending time per supplier facility. -- -- agreed_sending_hours is how many hours after a CTE's event time the diff --git a/supabase/migrations/20261025090100_epcis_document_receipts.sql b/supabase/migrations/20261025090100_epcis_document_receipts.sql index 2ca99c03c..fd806fdfd 100644 --- a/supabase/migrations/20261025090100_epcis_document_receipts.sql +++ b/supabase/migrations/20261025090100_epcis_document_receipts.sql @@ -1,3 +1,5 @@ +-- alembic-revision: a3f7ce361b0e +-- rendered-from: alembic/versions/20261024_a3f7ce361b0e_v137_epcis_document_receipts.py (upgrade) -- EPCIS document receipts. -- -- One row per EPCIS document the ingest routes receive (POST diff --git a/tests/scripts/test_check_schema_channels.py b/tests/scripts/test_check_schema_channels.py new file mode 100644 index 000000000..9974cf24a --- /dev/null +++ b/tests/scripts/test_check_schema_channels.py @@ -0,0 +1,181 @@ +"""Tests for scripts/check_schema_channels.py. + +Each test builds a tiny repository layout in a temp directory (an +``alembic/versions`` file plus channel files) and calls the module's +functions directly, so the suite stays fast and needs no git or database. +""" + +from __future__ import annotations + +import importlib.util +import re +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +_SCRIPT = _REPO_ROOT / "scripts" / "check_schema_channels.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("check_schema_channels", _SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_mod = _load_module() + +_REV = "0a329588b824" +_REV_FILE = "20261016_0a329588b824_v131_tool_leads_utm.py" + + +def _fake_repo(tmp_path: Path) -> Path: + versions = tmp_path / "alembic" / "versions" + versions.mkdir(parents=True) + (versions / _REV_FILE).write_text(f'revision = "{_REV}"\ndown_revision = "a5e7c9b2d341"\n') + (tmp_path / "supabase" / "migrations").mkdir(parents=True) + (tmp_path / "sql" / "prod-apply").mkdir(parents=True) + return tmp_path + + +def _write(repo: Path, rel: str, body: str) -> None: + path = repo / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(body) + + +_GOOD_HEADER = f"-- alembic-revision: {_REV}\n-- rendered-from: alembic/versions/{_REV_FILE} (upgrade)\n" + + +def test_headered_render_passes(tmp_path): + repo = _fake_repo(tmp_path) + body = _GOOD_HEADER + "ALTER TABLE public.tool_leads ADD COLUMN utm_source VARCHAR(100);\n" + _write(repo, "sql/prod-apply/apply_v131_tool_leads_utm.sql", body) + problems, scanned = _mod.scan(repo) + assert scanned == 1 + assert problems == [] + + +def test_missing_both_headers_fails(tmp_path): + repo = _fake_repo(tmp_path) + _write(repo, "supabase/migrations/20261101000000_new_thing.sql", "CREATE TABLE public.new_thing (id int);\n") + problems, _ = _mod.scan(repo) + assert len(problems) == 2 + assert "missing '-- alembic-revision" in problems[0] + assert "missing '-- rendered-from" in problems[1] + + +def test_unknown_revision_fails(tmp_path): + repo = _fake_repo(tmp_path) + _write( + repo, + "sql/prod-apply/apply_mystery.sql", + "-- alembic-revision: ffffffffffff\n" + + f"-- rendered-from: alembic/versions/{_REV_FILE} (upgrade)\nSELECT 1;\n", + ) + problems, _ = _mod.scan(repo) + assert any("does not exist in alembic/versions" in p for p in problems) + + +def test_rendered_from_must_match_revision_file(tmp_path): + repo = _fake_repo(tmp_path) + _write( + repo, + "sql/prod-apply/apply_v131_tool_leads_utm.sql", + f"-- alembic-revision: {_REV}\n-- rendered-from: alembic/versions/20260101_other.py (upgrade)\nSELECT 1;\n", + ) + problems, _ = _mod.scan(repo) + assert len(problems) == 1 + assert "rendered-from names" in problems[0] + + +def test_precheck_and_downgrade_kinds_accepted(tmp_path): + repo = _fake_repo(tmp_path) + for kind in ("precheck", "downgrade"): + _write( + repo, + f"sql/prod-apply/{kind}_v131.sql", + f"-- alembic-revision: {_REV}\n-- rendered-from: alembic/versions/{_REV_FILE} ({kind})\nSELECT 1;\n", + ) + problems, scanned = _mod.scan(repo) + assert scanned == 2 + assert problems == [] + + +def test_header_outside_window_is_missing(tmp_path): + repo = _fake_repo(tmp_path) + body = "-- filler\n" * _mod.HEADER_WINDOW_LINES + _GOOD_HEADER + _write(repo, "sql/prod-apply/late_header.sql", body) + problems, _ = _mod.scan(repo) + assert len(problems) == 2 + + +def test_exempt_reconcile_precheck_needs_no_header(tmp_path): + repo = _fake_repo(tmp_path) + _write(repo, "sql/prod-apply/PRECHECK_alembic_version_reconcile.sql", "BEGIN READ ONLY;\nSELECT 1;\nROLLBACK;\n") + problems, _ = _mod.scan(repo) + assert problems == [] + + +def test_grandfathered_file_needs_no_header(tmp_path): + repo = _fake_repo(tmp_path) + name = sorted(_mod.GRANDFATHERED)[0] + _write(repo, name, "SELECT 1;\n") + problems, scanned = _mod.scan(repo) + assert scanned == 1 + assert problems == [] + + +def test_stale_grandfathered_entries_are_reported(tmp_path): + repo = _fake_repo(tmp_path) + name = sorted(_mod.GRANDFATHERED)[0] + _write(repo, name, "SELECT 1;\n") + stale = _mod.stale_grandfathered(repo) + assert name not in stale + assert len(stale) == len(_mod.GRANDFATHERED) - 1 + + +def test_main_fails_on_stale_grandfather_entry(tmp_path, capsys): + repo = _fake_repo(tmp_path) + assert _mod.main(["--repo-root", str(repo)]) == 1 + assert "no longer exists" in capsys.readouterr().out + + +def test_grandfather_list_matches_tree(): + """Every grandfathered name exists, and no grandfathered file carries a header + (if one gains a header, drop it from the list).""" + for name in _mod.GRANDFATHERED: + path = _REPO_ROOT / name + assert path.exists(), f"stale GRANDFATHERED entry: {name}" + revision, rendered, _ = _mod.read_headers(path) + assert revision is None and rendered is None, f"{name} has a header; remove it from GRANDFATHERED" + + +def test_real_repository_is_clean(): + problems, scanned = _mod.scan(_REPO_ROOT) + assert scanned >= 50 + assert problems == [], "\n".join(problems) + assert _mod.stale_grandfathered(_REPO_ROOT) == [] + + +def test_revision_regex_matches_both_assignment_styles(): + assert _mod._REVISION_ASSIGNMENT.search('revision: str = "abc123abc123"').group(1) == "abc123abc123" + assert _mod._REVISION_ASSIGNMENT.search("revision = 'abc123abc123'").group(1) == "abc123abc123" + assert re.fullmatch(r"[0-9a-f]{12}", _REV) + + +@pytest.mark.parametrize( + "line,expected", + [ + ("-- alembic-revision: 0a329588b824", "0a329588b824"), + ("-- alembic-revision: 0A329588B824 ", "0a329588b824"), + ("-- alembic-revision: 0a32", None), + ("# alembic-revision: 0a329588b824", None), + ], +) +def test_revision_header_regex(line, expected, tmp_path): + path = tmp_path / "f.sql" + path.write_text(line + "\n") + revision, _, _ = _mod.read_headers(path) + assert revision == expected diff --git a/tests/scripts/test_check_scripts_registered.py b/tests/scripts/test_check_scripts_registered.py new file mode 100644 index 000000000..41ebda1c1 --- /dev/null +++ b/tests/scripts/test_check_scripts_registered.py @@ -0,0 +1,119 @@ +"""Tests for scripts/check_scripts_registered.py. + +Each test builds a tiny ``scripts/`` tree plus a registry in a temp directory +and calls the module's functions directly, so the suite needs no git. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent.parent +_SCRIPT = _REPO_ROOT / "scripts" / "check_scripts_registered.py" + + +def _load_module(): + spec = importlib.util.spec_from_file_location("check_scripts_registered", _SCRIPT) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +_mod = _load_module() + +_REGISTRY = """# scripts/ registry + +| Script | Status | Purpose / invoked by | +|---|---|---| +| `alpha.py` | used | invoked by `.github/workflows/x.yml` | +| `beta.sh` | manual | run by hand | + +## Subdirectories + +| Directory | Purpose | +|---|---| +| `scripts/ci/` | workflow helpers | +""" + + +def _fake_repo(tmp_path: Path, registry: str = _REGISTRY) -> Path: + scripts = tmp_path / "scripts" + (scripts / "ci").mkdir(parents=True) + (scripts / "alpha.py").write_text("print(1)\n") + (scripts / "beta.sh").write_text("echo 1\n") + (scripts / "README.md").write_text(registry) + return tmp_path + + +def test_clean_tree_passes(tmp_path): + assert _mod.scan(_fake_repo(tmp_path)) == [] + + +def test_unregistered_file_fails(tmp_path): + repo = _fake_repo(tmp_path) + (repo / "scripts" / "gamma.py").write_text("print(3)\n") + problems = _mod.scan(repo) + assert len(problems) == 1 + assert problems[0].startswith("scripts/gamma.py: no row") + + +def test_unregistered_directory_fails(tmp_path): + repo = _fake_repo(tmp_path) + (repo / "scripts" / "newdir").mkdir() + problems = _mod.scan(repo) + assert problems == ["scripts/newdir/: no row in the Subdirectories table of scripts/README.md"] + + +def test_stale_file_row_fails(tmp_path): + repo = _fake_repo(tmp_path, _REGISTRY + "| `gone.py` | used | nothing |\n") + problems = _mod.scan(repo) + assert len(problems) == 1 + assert "lists `gone.py`, which is not in scripts/" in problems[0] + + +def test_stale_directory_row_fails(tmp_path): + repo = _fake_repo(tmp_path, _REGISTRY + "| `scripts/gone/` | nothing |\n") + problems = _mod.scan(repo) + assert len(problems) == 1 + assert "lists `scripts/gone/`, which does not exist" in problems[0] + + +def test_ignored_entries_need_no_row(tmp_path): + repo = _fake_repo(tmp_path) + (repo / "scripts" / "__pycache__").mkdir() + (repo / "scripts" / "_archive").mkdir() + (repo / "scripts" / ".hidden").write_text("") + assert _mod.scan(repo) == [] + + +def test_files_inside_subdirectories_are_not_checked(tmp_path): + repo = _fake_repo(tmp_path) + (repo / "scripts" / "ci" / "helper.py").write_text("print(1)\n") + assert _mod.scan(repo) == [] + + +def test_main_reports_and_fails(tmp_path, capsys): + repo = _fake_repo(tmp_path) + (repo / "scripts" / "gamma.py").write_text("print(3)\n") + assert _mod.main(["--repo-root", str(repo)]) == 1 + out = capsys.readouterr().out + assert "registry guard FAILED" in out + assert "scripts/gamma.py" in out + + +def test_main_passes_on_clean_tree(tmp_path, capsys): + assert _mod.main(["--repo-root", str(_fake_repo(tmp_path))]) == 0 + assert "OK" in capsys.readouterr().out + + +def test_missing_layout_exits_2(tmp_path): + import pytest + + with pytest.raises(SystemExit) as excinfo: + _mod.scan(tmp_path) + assert excinfo.value.code == 2 + + +def test_real_repository_is_clean(): + assert _mod.scan(_REPO_ROOT) == [] diff --git a/tests/shared/test_task_queue_hardening.py b/tests/shared/test_task_queue_hardening.py index 65481a2c4..c6aa91e6d 100644 --- a/tests/shared/test_task_queue_hardening.py +++ b/tests/shared/test_task_queue_hardening.py @@ -217,7 +217,7 @@ def test_claim_does_not_raise_nameerror(self): dict without ever being assigned in the function body. Every call raised NameError which the outer `_worker_loop` try/except swallowed — with the net effect that the monolith's task worker - for `nlp_extraction` / `graph_update` / `review_item` claimed + for `nlp_extraction` / `review_item` claimed zero tasks in production. This test ensures the function at least reaches the `db.execute` call cleanly. """ diff --git a/tests/shared/test_visibility_timeout_2918.py b/tests/shared/test_visibility_timeout_2918.py index 0e987e8d3..95167eab6 100644 --- a/tests/shared/test_visibility_timeout_2918.py +++ b/tests/shared/test_visibility_timeout_2918.py @@ -268,7 +268,7 @@ def _sql_resolution(self, params, task_type, row_value): return row_value return mapping.get(task_type, params["default_timeout"]) - @pytest.mark.parametrize("task_type", ["nlp_extraction", "graph_update", "review_item", "unmapped_type"]) + @pytest.mark.parametrize("task_type", ["nlp_extraction", "review_item", "unmapped_type"]) @pytest.mark.parametrize("row_value", [None, 45]) def test_resolutions_match(self, task_type, row_value): row = (1, task_type, {}, 0, 3, row_value, None)