diff --git a/docs/DATA_GOVERNANCE.md b/docs/DATA_GOVERNANCE.md new file mode 100644 index 000000000..43f9f746f --- /dev/null +++ b/docs/DATA_GOVERNANCE.md @@ -0,0 +1,80 @@ +# Data governance + +## Authority + +This contract maps protected-main data classes, owners, tenant authority, +retention, deletion, and privacy boundaries. SQL, package code, and the +canonical PRD/TRD remain stronger authority. It is evidence readiness, not a +certification and not a claim that a deployment has a complete records program. + +## What to do next + +1. Decide which host identity is allowed to select `tenant_scope` before any + package call. +2. Do not mask, tokenize away, or truncate authorized business payloads inside + this package. If your policy requires transformation, do it in an explicit + host boundary with provenance and acceptance tests. +3. Treat Fernet as an optional host/deployment policy on protected main. + `SecretStore(require_encryption=False)` permits base64-obfuscated + compatibility rows with `com_secrets.is_encrypted = FALSE`; that mode is not + a mandatory encryption-at-rest guarantee. Historical-row migration, key + rotation/recovery, and external key custody remain separate responsibilities. +4. Own backup copies, replica retention, log/telemetry retention, and + destructive deletion yourself. The package will not invent a general purge. +5. Use `standalone` for a single-tenant operator. Do not reuse that scope as an + anonymous public bucket. + +## Data classes + +| Class | Principal objects | Owner | Package duty | +| --- | --- | --- | --- | +| Authorized business payloads | `llm_requests` prompts/results, `llm_batch_file_payloads`, `llm_jsonl_lines` | Embedding host / business process | Persist and replay exactly. Do not mask. | +| Durable lifecycle projection | `llm_remote_batch_jobs` | Host-selected `tenant_scope` plus package recorder | Tenant-qualify identity, bind `set_config`, force RLS. | +| Result checkpoints | `llm_result_stream_checkpoints` | Host-selected consumer name plus tenant | Store prefix evidence only. | +| Standalone configuration | `com_config` | Operator | Key/value settings. Not tenant authorization. | +| Standalone secrets | `com_secrets` | Operator / secret-manager host | Optional Fernet or explicit compatibility mode. Compatibility rows are base64-obfuscated, not an encryption-at-rest claim. | +| Provider credentials | Host credential provider | Deployment | Resolve after tenant validation. Never tenant-keyed by this package. | +| Recovery evidence | Receipts, artifact hashes, schema hashes | Operator | Content-free identity. Not restorability. | +| Operational diagnostics | Errors, logs, readiness, telemetry | Package | Omit payloads, DSNs, credentials, and dynamic exception text. | + +## Tenant authority + +`tenant_scope` is selected only by a trusted authenticated/authorized host +boundary. Provider metadata, remote identifiers, request bodies, model output, +endpoint aliases, and transport headers are never tenant authorities. The +embedding host owns the identity-to-tenant map. + +## Retention and deletion + +Protected main does not define a universal business-data retention duration. +The embedding host owns purpose, retention period, deletion authorization, and +evidence that the policy ran. The deployment owner separately owns PostgreSQL +backup/replica, WAL, and infrastructure log retention. Provider-side retention +remains a provider/account policy unless a reviewed adapter implements it. + +The package will not silently delete unknown operator objects, rewrite history, +or `CASCADE` through unrelated schemas as a recovery shortcut. + +## Privacy without paralysis + +ISO/IEC 29100 treats purpose specification and data minimization as +organization policy, not as an excuse to destroy the meaning of a processing +record (ISO/IEC, 2024). NIST SP 800-53 Revision 5 likewise places confidentiality +controls at authorization, access, transmission, and audit boundaries (Joint +Task Force, 2020). This package therefore: + +- preserves authorized business payloads; +- redacts operational surfaces; +- fails closed on untrusted provider and path input; +- refuses to treat RLS or diagnostic redaction as proof that persisted + business content was masked. + +## References + +International Organization for Standardization. (2024). *Information +technology — Security techniques — Privacy framework* (ISO/IEC 29100:2024). +https://www.iso.org/standard/85938.html + +Joint Task Force. (2020). *Security and privacy controls for information systems +and organizations* (NIST Special Publication 800-53, Revision 5). National +Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-53r5 \ No newline at end of file diff --git a/docs/DOCUMENTATION_FITNESS.md b/docs/DOCUMENTATION_FITNESS.md new file mode 100644 index 000000000..c6a46d464 --- /dev/null +++ b/docs/DOCUMENTATION_FITNESS.md @@ -0,0 +1,59 @@ +# Documentation Fitness + +## Authority and status model + +This inventory evaluates canonical documentation against live protected-default-branch behavior. It deliberately does not freeze an exact protected SHA: exact heads and run IDs belong in PR/review evidence, while durable documentation records capability contracts. Status vocabulary is **IMPLEMENTED-ON-PROTECTED-MAIN**, **ACTIVE-PR**, **PARTIAL**, **PLANNED**, and **SUPERSEDED**. + +A document is fit only when it agrees with protected code/schema/tests, preserves non-guarantees, keeps branch evidence out of shipped claims, and gives operators/reviewers enough information to use or reject the behavior safely. + +Protected main contains the tenant lifecycle/RLS contract, bounded reconciliation, tenant-qualified transient session single-flight integrated through #191, bounded recovery-evidence primitives integrated through #205/#206/#207, bounded direct logical restore integrated through #212, and bounded restore-target name+cluster-identity verification integrated through merged #228. These are narrower than a complete worker or recovery product. + +## Current fitness matrix + +| Documentation surface | Status | Fitness assessment | Required next action | +| --- | --- | --- | --- | +| `README.md` | IMPLEMENTED-ON-PROTECTED-MAIN | Public entry point for standalone/embedded operation. | Change through its live owner when protected behavior changes. | +| `ARCHITECTURE.md` | IMPLEMENTED-ON-PROTECTED-MAIN | Root architecture is separately owned by the active root-documentation lane. | Keep this PR out of that path. | +| `docs/product/PRD.md` | ACTIVE-PR | Canonical product contract records merged #191, #212, and #228 only at their bounded protected scope. | Revalidate from protected source after every integration. | +| `docs/product/TRD.md` | ACTIVE-PR | Separates transient session exclusion from durable leasing; restore execution from backup/application/PITR; and cluster-identity comparison from connection provenance/authorization. | Keep source/test authority stronger than prose. | +| ADR set | IMPLEMENTED-ON-PROTECTED-MAIN / record-local | ADR 0016 governs custom-format restore seek semantics; ADR 0022 governs restore-target name+cluster identity separation. Other ADRs retain their own status. | Preserve record-local status and collision-free identifiers. | +| `docs/adr/README.md` | ACTIVE-PR | Navigation/status index without exact-head authority. | Keep synchronized with protected ADR files. | +| Tenant lifecycle operator material | IMPLEMENTED-ON-PROTECTED-MAIN | Trusted tenant selection, standalone compatibility, forced RLS, direct-SQL limits, migration, and rollback are documented. | Do not duplicate into competing operator guides. | +| PostgreSQL recovery evidence | IMPLEMENTED-ON-PROTECTED-MAIN / end-to-end PARTIAL | Receipt/artifact/schema evidence is bounded content-free identity/integrity evidence. | Do not infer restorability, provenance, PITR, or RPO/RTO. | +| PostgreSQL logical backup | ACTIVE-PR | #208 remains a `pg_dump` candidate. | Keep unshipped until normal integration. | +| PostgreSQL logical restore | IMPLEMENTED-ON-PROTECTED-MAIN / end-to-end PARTIAL | The logical restore executor is protected-main behavior through merged #212; #209 is historical EOF-defect evidence. | Preserve source trust, environment, transaction, metadata, target, and application-readiness boundaries. | +| PostgreSQL restore-target cluster identity verification | IMPLEMENTED-ON-PROTECTED-MAIN / end-to-end PARTIAL | Merged #228 supplies exact service-name plus caller-owned `system_identifier` separation. Same-cluster aliases fail closed. The package does not open/authenticate the connections, execute restore, or prove application/PITR/RPO-RTO readiness. Closed #225 is predecessor lineage only. | Treat the verifier as a bounded precondition, not end-to-end restore authorization. | +| Effective PITR target configuration observation | ACTIVE-PR | #299 reads exactly eight PostgreSQL recovery-target settings plus `pg_is_in_recovery()` from a caller-owned isolated target and compares them with reviewed target authority. The functions remain module-scoped rather than root-package exports. | Keep fixed-query, timeout-ownership, public-surface, README/CHANGELOG/ADR/doctoring work with their live owners; do not promote branch evidence to shipped PITR proof. | +| Recovery evidence binding / live reinspection | ACTIVE-PR | Candidate composition/reinspection is not provenance or restore proof. | Keep branch evidence distinct from shipped truth. | +| Post-restore catalog/application acceptance | ACTIVE-PR | #296 is an application-readiness candidate. | Require protected integration before shipped claims. | +| Permanent live PostgreSQL integration acceptance | ACTIVE-PR | #341 owns the branch-level full integration-marker lane and currently tests #296 as a child. | Preserve the lane through normal integration; no mocks/deselection substitution. | +| Physical/WAL/PITR recovery | ACTIVE-PR / PARTIAL | Intent/evidence does not prove replay/promotion or achieved objectives. | Require deployment-specific execution and measurement. | +| Existing-volume legacy extension retirement | ACTIVE-PR | Separate migration/operator work. | Do not conflate with tenant lifecycle. | +| Durable reconciliation discovery | ACTIVE-PR | Discovery remains unshipped. | Keep tenant-qualified/bounded/deterministic authority. | +| Tenant-qualified reconciliation single-flight | IMPLEMENTED-ON-PROTECTED-MAIN | #191 is transient PostgreSQL session advisory locking only. | Never call it a scheduler, durable lease, result-app transaction, terminal-retirement authority, or distributed exactly-once mechanism. | +| Atomic durable result application | ACTIVE-PR / end-to-end PARTIAL | Checkpoint/stream primitives do not prove complete result application. | Keep external-effect limits explicit. | +| Runtime config/schema provisioning and secret policy | ACTIVE-PR / protected compatibility baseline | Protected main permits optional Fernet and `is_encrypted = FALSE` compatibility rows; #210 is stricter active work. | Do not claim mandatory encryption, historical-row migration, rotation/recovery, or external custody. | +| Canonical traceability | ACTIVE-PR | This overlay is the current canonical documentation landing vehicle; #226 and superseded #214 are historical predecessors. | Use stable implementation/test/doc authorities, not exact heads. | +| `docs/THREAT_MODEL.md` | ACTIVE-PR | Assets, boundaries, mitigations, residual risk, and NIST evidence are documented without certification claims. | Keep residual risk synchronized with protected authority. | +| `docs/DATA_GOVERNANCE.md` | ACTIVE-PR | Data classes, owners, retention/deletion, content fidelity, and optional Fernet compatibility are explicit. | Do not turn evidence readiness into certification. | +| `docs/uml/component-and-sequence.md` | ACTIVE-PR | Standalone/embedded and tenant-validation views exist. | Keep branch-only components off shipped diagrams. | +| `docs/erd/package-owned-schema.md` | ACTIVE-PR | Packaged schema and migration-owned checkpoint identity are mapped. | SQL remains stronger authority. | +| Release governance | PARTIAL | Release evidence exists; immutable publication requires the exact accepted protected head. | Tie version, CHANGELOG, package, SBOM, provenance, rollback, tag, and publication verification together through release ownership. | + +## Non-negotiable documentation invariants + +- Protected-main behavior is shipped authority; active PRs and historical branches are not. +- Exact SHAs, generated merge commits, run IDs, and queue state stay in PR/review evidence. +- Standalone and modular embedding remain co-equal boundaries. +- `tenant_scope` comes from a trusted authenticated/authorized host; RLS is defense in depth, not authentication. +- #191 proves transient tenant-qualified session single-flight, not durable leasing or exactly-once. +- #212 proves bounded direct logical restore with corrected custom-format seek semantics, not backup, application readiness, PITR, or RPO/RTO. +- #228 proves only that supplied exact service names and caller-owned PostgreSQL `system_identifier` values differ. It does not authenticate the connections or collector, execute restore, or prove post-restore application readiness. +- #299 is an ACTIVE-PR fixed-query observation of effective recovery-target settings on a caller-owned isolated target. Its functions are module-scoped, it owns no connection timeout, and it does not prove WAL completeness, target attainment, promotion, application readiness, PITR success, or achieved RPO/RTO. +- Optional Fernet plus explicit compatibility mode is protected behavior; mandatory encryption/migration/rotation/custody is not inferred. +- Recovery evidence and command success are not equivalent to recovery success. +- SOC 2/CSAP/security/privacy material remains evidence readiness absent external certification. + +## Fitness gate + +Before changing a canonical surface, refetch protected main, open PRs, affected source/schema/tests, current ADRs, and adjacent writers. Repair the earliest stale authority boundary without widening ownership. After a merge, refresh status only from the resulting protected tree; after supersession, retain predecessor context only where it explains a live constraint. Every changed documentation head must reacquire then-required exact-head quality/security/release evidence and qualifying review. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md new file mode 100644 index 000000000..36666a2e5 --- /dev/null +++ b/docs/THREAT_MODEL.md @@ -0,0 +1,86 @@ +# Threat model + +## Authority + +This model describes protected-main `pg-llm-batch` assets, trust boundaries, attacker capabilities, mitigations, and residual risk. It is evidence readiness for SOC 2 / CSAP preparation, not certification, penetration-test evidence, or production authorization. + +The methodology follows data-centric threat modeling and NIST risk assessment: identify data, systems, threat sources, preconditions, controls, and residual risk. Controls are aligned to NIST SP 800-53 Revision 5. + +## What operators must do + +1. Select `tenant_scope` only behind an authenticated and authorized host boundary. +2. Do not grant arbitrary SQL, `SUPERUSER`, or `BYPASSRLS` to ordinary application roles. +3. Preserve authorized business payloads unless an explicit host policy transforms them with provenance and acceptance tests. +4. Treat recovery receipt/hash evidence as identity/integrity metadata, not restorability. +5. Treat Fernet as optional protected-main behavior. Compatibility mode can persist `is_encrypted = FALSE`; production encryption policy, migration, rotation/recovery, and external key custody remain separate responsibilities. +6. Before logical restore, obtain live and restore `system_identifier` values from caller-opened connections and use the protected restore-target verifier. Do not treat that comparison as authentication of the connections or as post-restore application readiness. +7. Use the explicit `standalone` scope for a single-tenant operator; do not reinterpret it as anonymous public tenancy. + +## Assets + +| Asset | Location | Buyer relevance | +| --- | --- | --- | +| Authorized prompts, JSONL, provider results | package payload/request tables and provider files | Business meaning; silent transformation invalidates accounting/replay. | +| Durable lifecycle identity | `llm_remote_batch_jobs`, keyed by `(tenant_scope, endpoint_alias, remote_batch_id)` | Cross-tenant lifecycle isolation. | +| Result checkpoints | `llm_result_stream_checkpoints`, keyed by tenant + consumer + provider identity | Resumable prefix evidence; not whole-stream authenticity. | +| Standalone config/secrets | `com_config`, `com_secrets` | Optional Fernet plus explicit base64 compatibility mode. | +| Provider credentials | host credential provider or standalone store | Never tenant-selected authority or telemetry content. | +| Recovery evidence | receipts/hashes/schema evidence | Content-free identity/integrity, not restore success. | +| Restore target identity | exact libpq service names + caller-owned PostgreSQL `system_identifier` values | Detects same-name and same-cluster aliasing before a host proceeds. | + +## Trust boundaries + +```text +[Caller / operator] + | authentication + authorization + connection provenance (host-owned) + v +[Host control plane] -- chooses tenant, credentials, DSN/service, recovery target + | package Python API / CLI + v +[pg-llm-batch] -- validates and bounds inputs/effects + | parameterized SQL / bounded subprocess seams + v +[PostgreSQL] -- forced tenant RLS + caller-opened recovery connections + | + +--> [Provider Batch API] untrusted external data + +--> [Caller-owned backup/WAL/key infrastructure] +``` + +`tenant_scope` is routing context written through parameterized transaction-local `set_config`; it is not a credential. A role with arbitrary SQL can choose a custom-setting value. RLS therefore augments, rather than replaces, host authentication/authorization and SQL-injection prevention. + +The restore-target verifier has a similarly bounded trust model. `postgres_restore_target.py` does not open either database connection. The caller supplies service names and `PostgresRestoreTargetIdentity(system_identifier=...)` values collected from already-opened connections. Distinct exact names and identifiers reject obvious same-cluster aliases, but the package does not authenticate the collector, the connection setup, or the mapping from a service name to the supplied identifier. + +## Threat sources and controls + +| Threat source | Needed precondition | Protected-main mitigation | Residual risk | +| --- | --- | --- | --- | +| Confused-deputy tenant selection | Host accepts provider/model/transport data as tenant authority | Tenant clients validate host-selected scope before credentials/provider/DB effects | Host identity-mapping bugs remain external. | +| Cross-tenant lifecycle access | Missing/wrong transaction-local scope | Forced RLS, tenant-qualified keys/indexes | Administrative SQL roles bypass the guarantee. | +| SQL injection / generic tenant SQL | Application exposes arbitrary SQL | Parameterized package SQL; generic SQL is outside supported tenant boundary | Host must prevent injection and generic SQL authority. | +| Provider spoofing / oversized input | Untrusted network/provider response | HTTPS production destination policy, finite decode/download budgets, closed reviewed GET retry set | Payload validation does not prove provider authenticity. | +| Secret reflection / weak at-rest policy | Diagnostics expose values or operator mistakes compatibility mode for encryption | Bounded diagnostics; optional Fernet can be explicitly required | Default compatibility rows are not encrypted-at-rest proof; migration/rotation/custody remain external. | +| Restore into live cluster through alias | Different service labels resolve to same PostgreSQL cluster | Merged #228 verifier requires distinct exact service names and distinct caller-owned `pg_control_system().system_identifier` values | Caller may collect/misassociate identity evidence incorrectly; package does not authenticate the connection or authorize restore. | +| Unsafe or semantically incomplete restore | Command succeeds but target/catalog/application is wrong | Merged #212 bounds direct restore execution and archive metadata semantics; #228 bounds target cluster separation | Backup provenance, application/catalog acceptance, migration/key/WAL/PITR and RPO/RTO remain separate. | +| Checkpoint fork/replay | Concurrent consumers mutate same identity | PostgreSQL CAS/locking semantics | DB atomicity does not extend to provider/network effects. | +| Reconciliation overlap | Concurrent workers target same bounded identity | Tenant-qualified transient session advisory single-flight | Session lock is not durable leasing, scheduling, terminal retirement, or exactly-once. | +| Content-fidelity sabotage | Privacy layer silently rewrites authorized payload | Package paths preserve authorized content absent explicit policy | Host transformations require provenance and acceptance tests. | + +## Explicit non-guarantees + +- This document does not claim SOC 2, CSAP, ISO/IEC 27001, or another certification. +- RLS does not replace authentication, authorization, or SQL-injection prevention. +- Optional Fernet and redacted diagnostics do not prove all persisted secrets are encrypted or that key lifecycle/custody is solved. +- A receipt/hash does not prove backup provenance or restorability. +- A successful direct restore does not prove application readiness, PITR, or achieved recovery objectives. +- Distinct service names and `system_identifier` values prove only the bounded evidence supplied to the verifier differs; they do not authenticate connection provenance or authorize destructive recovery. +- Prefix checkpoints and session advisory locks are not distributed exactly-once guarantees. + +## References + +Joint Task Force. (2012). *Guide for conducting risk assessments* (NIST Special Publication 800-30, Revision 1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-30r1 + +Joint Task Force. (2020). *Security and privacy controls for information systems and organizations* (NIST Special Publication 800-53, Revision 5). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-53r5 + +Scarfone, K., & Souppaya, M. (2016). *Guide to data-centric system threat modeling* (NIST Special Publication 800-154, Initial Public Draft). National Institute of Standards and Technology. https://csrc.nist.gov/pubs/sp/800/154/ipd + +Swanson, M., Bowen, P., Phillips, A., Gallup, D., & Lynes, D. (2010). *Contingency planning guide for federal information systems* (NIST Special Publication 800-34 Rev. 1). National Institute of Standards and Technology. https://doi.org/10.6028/NIST.SP.800-34r1 diff --git a/docs/TRACEABILITY.md b/docs/TRACEABILITY.md new file mode 100644 index 000000000..2dd42737a --- /dev/null +++ b/docs/TRACEABILITY.md @@ -0,0 +1,94 @@ +# Requirements Traceability + +## Purpose and authority + +This map ties canonical PRD/TRD requirements to durable protected-main implementation, tests, ADRs, and operator evidence. It avoids workflow-run IDs, exact SHAs, generated merge commits, and transient review/check state. **ACTIVE-PR** and **PARTIAL** rows are not shipped claims. + +## Product-to-technical traceability + +| Requirement | Status | Protected-main authority | Durable evidence | Residual boundary / active overlay | +| --- | --- | --- | --- | --- | +| FR-1 bounded batch preparation | IMPLEMENTED-ON-PROTECTED-MAIN | `orchestrator.py`, `token_counter.py`, package schema | preparation/token/schema tests | No additional shipped claim inferred. | +| FR-2 bounded provider interaction | IMPLEMENTED-ON-PROTECTED-MAIN | `batch_api_client.py` | endpoint/resource/retry/response-budget tests; ADR 0015 | Provider-specific widening requires review. | +| FR-3 durable lifecycle and tenancy | IMPLEMENTED-ON-PROTECTED-MAIN | `durable_client.py`, `db.py`, schema/RLS | tenant lifecycle/live PostgreSQL tests; ADR 0002 | Arbitrary SQL, `SUPERUSER`, and `BYPASSRLS` remain outside tenant isolation. | +| FR-4 bounded reconciliation | IMPLEMENTED-ON-PROTECTED-MAIN | `reconciliation.py` | reconciliation tests | Durable candidate discovery remains ACTIVE-PR. | +| FR-4 tenant-qualified cross-process single-flight | IMPLEMENTED-ON-PROTECTED-MAIN | `reconciliation_single_flight.py` | focused exact-type/lock/traceback tests; merged #191 is historical integration evidence | Session advisory lock only: not scheduler, durable lease, result application, terminal retirement, or distributed exactly-once. | +| FR-4 durable result application | PARTIAL | `result_streaming.py`, `checkpoint_store.py` primitives | ADR 0006/0007 and tests | #194 remains active; no end-to-end exactly-once claim. | +| FR-5 persistence integrity | IMPLEMENTED-ON-PROTECTED-MAIN | `db.py`, `schema.sql`, Docker schema mirror | schema/payload/lifecycle/checkpoint tests | Existing-volume legacy-extension retirement remains active. | +| FR-5 bounded recovery evidence | IMPLEMENTED-ON-PROTECTED-MAIN | `postgres_recovery_receipt.py`, `postgres_backup_evidence.py`, `postgres_schema_evidence.py` | focused evidence tests; merged #205/#206/#207 are historical integration evidence | Identity/integrity only; not backup execution, provenance, restorability, live parity, PITR, or RPO/RTO. | +| FR-5 executable PostgreSQL logical backup | ACTIVE-PR | no protected `pg_dump` executor | #208 branch evidence | Protected main must not be described as creating a restorable backup. | +| FR-5 executable PostgreSQL logical restore | IMPLEMENTED-ON-PROTECTED-MAIN | `postgres_logical_restore.py` | logical-restore regressions; ADR 0016; merged #212 historical integration evidence; #209 predecessor defect evidence | Custom-format seek + metadata verification is protected. No backup, target-authentication, application-readiness, PITR, or RPO/RTO guarantee. | +| FR-5 restore-target cluster identity verification | IMPLEMENTED-ON-PROTECTED-MAIN | `postgres_restore_target.py` | focused service-name/system-identifier tests; ADR 0022; merged #228 historical integration evidence | Requires distinct exact service names and caller-owned `pg_control_system().system_identifier` values. It does not open/authenticate connections, accept DSNs, execute restore, or prove catalog/application/PITR/RPO-RTO readiness. | +| FR-5 effective PITR target configuration observation | ACTIVE-PR | no protected recovery-target configuration observer | #299 fixed-query source/tests plus canonical documentation overlay | Exactly eight PostgreSQL recovery-target settings plus `pg_is_in_recovery()` are observed from a caller-owned isolated target. The APIs remain module-scoped; no connection timeout is imposed; no WAL completeness, target attainment, promotion, application readiness, PITR success, or achieved RPO/RTO is proved. | +| FR-5 recovery evidence binding/reinspection | ACTIVE-PR | underlying protected evidence primitives only | active binding/reinspection branches | Composition/reinspection is not provenance or restore proof. | +| FR-5 post-restore catalog/application acceptance | ACTIVE-PR | protected schema/recovery/restore/target primitives only | active catalog work and #296 application-readiness branch | Exact catalog/function/privilege/live behavior must integrate before becoming shipped acceptance. | +| FR-5 permanent live PostgreSQL integration lane | ACTIVE-PR | current protected CI does not yet contain #341 lane | #341 branch executes the complete integration marker; #296 is its tested child | Branch GREEN is not protected-main workflow authority. | +| FR-5 physical/WAL/PITR recovery | ACTIVE-PR / PARTIAL | bounded protected evidence only | active physical/WAL/PITR branch evidence | Does not prove basebackup, WAL replay, promotion, or achieved objectives. | +| FR-5 end-to-end recovery readiness | PARTIAL | evidence primitives + logical restore + target cluster-identity verifier | protected focused tests plus active acceptance work | Still missing integrated backup execution, connection provenance/authorization, application/catalog parity, migration/key custody, physical/WAL/PITR, and measured RPO/RTO/HA/DR. | +| FR-6 PostgreSQL-backed config/secrets | IMPLEMENTED-ON-PROTECTED-MAIN | `config.py`, schema | config/secret/bootstrap tests | Optional Fernet + base64 compatibility are protected; mandatory policy is not. | +| FR-6 production secret-at-rest lifecycle | PARTIAL | opt-in `require_encryption=True` path | config/secret tests; #210 active work | Compatibility-row migration, rotation/recovery, external key custody, and mandatory default remain unshipped. | +| FR-7 bounded diagnostics/readiness | IMPLEMENTED-ON-PROTECTED-MAIN | `health.py`, bounded error surfaces | health/confidentiality tests | Broader rejected-value confidentiality remains active work. | +| FR-7 opt-in telemetry | IMPLEMENTED-ON-PROTECTED-MAIN / packaging PARTIAL | `observability.py` | observability tests | First-class locked installation extra remains ACTIVE-PR. | +| FR-8 standalone + modular embedding | IMPLEMENTED-ON-PROTECTED-MAIN | package/CLI/container seams | package/container/docs tests | CWL hosts are optional integrations. | +| Exact owned production coverage/docstrings | IMPLEMENTED-ON-PROTECTED-MAIN governance contract | repository CI | exact-head coverage/docstring gates | Re-prove on every changed head. | +| Python 3.10/3.12/3.14 | IMPLEMENTED-ON-PROTECTED-MAIN governance contract | package/workflow matrix | exact-head CI | Queued/skipped/infrastructure failure is not proof. | +| Reproducible release + SBOM/provenance | IMPLEMENTED-ON-PROTECTED-MAIN governance contract | release-evidence code/workflows | release acceptance/artifact tests | Publication is authoritative only from an accepted protected head. | +| SOC 2 / CSAP evidence readiness | PARTIAL | tenancy/security/logging/release controls | PRD/TRD/security/ADR evidence | No external certification claim. | + +## Security and privacy traceability + +| Control objective | Protected-main authority | Verification | Residual boundary | +| --- | --- | --- | --- | +| Trusted tenant selection | tenant validation + host-boundary docs | tenant/RLS tests | Host authentication/authorization remains external. | +| RLS defense in depth | forced RLS + transaction-local scope | live PostgreSQL isolation/migration tests | Administrative SQL identities bypass the guarantee. | +| Provider destination/input bounds | `batch_api_client.py` | URL/resource/response/download tests | Payload validation does not prove provider authenticity. | +| Reconciliation exclusion | `reconciliation_single_flight.py` | lock/release/exact-type/traceback tests | Session lifetime is not durable leasing. | +| Secret/config boundary | `config.py` | config/secret/bootstrap tests | Optional Fernet does not imply mandatory encryption, rotation, or custody. | +| Diagnostic confidentiality | health/error/logging contracts | health/redaction/traceback tests | Broader generic validation hardening remains active. | +| Checkpoint integrity | `checkpoint_store.py` | CAS/concurrency/RLS/rollback tests | PostgreSQL atomicity does not span external systems. | +| Recovery evidence integrity | receipt/artifact/schema modules | focused evidence tests | No operator authentication, target authority, or restorability proof. | +| Logical restore execution | `postgres_logical_restore.py` | seek/metadata/environment/transaction tests | Command success is not application/PITR/RPO-RTO proof. | +| Restore-target cluster separation | `postgres_restore_target.py` | exact service-name/system-identifier tests | Caller supplies identities from already-opened connections; the package does not authenticate connection provenance or authorize restore. | +| Recovery-target configuration observation | no protected-main implementation | #299 fixed-query branch tests and canonical docs | Caller owns connection/timeouts. Module-scoped branch APIs do not prove WAL completeness, exact stop semantics, promotion, application readiness, PITR success, or achieved RPO/RTO. | +| Release artifact integrity | release-evidence contracts | reproducibility/artifact identity | Publication credentials/registry availability remain operational dependencies. | + +## Data and persistence traceability + +| Data family | Durable identity / authority | Principal docs | Non-guarantee | +| --- | --- | --- | --- | +| Tenant lifecycle | `(tenant_scope, endpoint_alias, remote_batch_id)` | ADR 0002; lifecycle docs | Host must select tenant authority. | +| Result checkpoint | `(tenant_scope, checkpoint_consumer_name, endpoint_alias, remote_batch_id)` | ADR 0006/0007 | Prefix evidence is not provider authentication or exactly-once. | +| Package payload state | package schema identities | PRD/TRD/schema docs | Canonical persisted business state, not disposable cache. | +| Configuration/secrets | `com_config`, `com_secrets` | PRD/TRD/config tests | Compatibility mode can persist `is_encrypted = FALSE`; optional Fernet is not mandatory policy. | +| Recovery evidence | bounded receipt/hash/size values | recovery modules/tests | Does not persist backups or establish restore/PITR/RPO-RTO. | +| Restore-target identity evidence | exact service names + caller-owned `system_identifier` values | ADR 0022; target module/tests | Difference proves bounded name/cluster separation only, not connection provenance, authorization, or application readiness. | +| Release evidence | descriptor/artifact identity | ADR 0003/0004 | Does not establish organizational certification. | + +## Active overlay register + +The following open PRs are overlays only; their existence does not make behavior protected-main truth: + +- **#175** — first-class OpenTelemetry packaging extra. +- **#184** — existing-volume legacy PostgreSQL extension retirement. +- **#190** — durable reconciliation candidate discovery. +- **#194** — atomic local result-effect/checkpoint application. +- **#202** — generic rejected-value confidentiality hardening. +- **#208** — bounded logical PostgreSQL backup executor using `pg_dump`. +- **#210** — stricter configuration/secrets runtime/operator contract. +- **#215** — recovery receipt evidence binding. +- **#219** — physical/WAL/PITR recovery profile. +- **#221** — recovery receipt live reinspection. +- **#222** — read-only workflow-registry audit. +- **#223** — live PostgreSQL restore-catalog acceptance. +- **#229** — current canonical documentation landing vehicle; it does not own root `ARCHITECTURE.md`, `CHANGELOG.md`, or the product-gap baseline. +- **#296** — isolated restore application-readiness candidate. +- **#299** — effective recovery-target configuration observation on a caller-owned isolated target; functions remain module-scoped pending explicit public-surface decision. +- **#341** — permanent live PostgreSQL integration-lane candidate. + +Merged #191, #212, and #228 are protected-main history rather than active overlays. Closed #225 is superseded restore-target predecessor lineage; closed #209 remains defect evidence for the invalid EOF restore postcondition. Closed documentation predecessors #214 and #226 are superseded historical lineage only. + +This register is descriptive, not a substitute for refetching live GitHub state. + +## Change-control rule + +After a capability merges, change canonical status only after reading the resulting protected tree. After abandonment or supersession, retain predecessor context only where it explains a current constraint. New requirements need an intended implementation authority and deterministic verification authority before they are acquisition-ready. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 000000000..0089183a7 --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,36 @@ +# Architecture Decision Record Index + +## Authority + +This index is a navigation/status aid for ADRs present on the protected default branch. Each ADR remains normative for its own context, decision, consequences, security boundary, and supersession rules. Exact protected/contributor SHAs, workflow runs, generated merge commits, and review comments are intentionally excluded from durable architecture authority. + +## Protected-main decisions + +| ADR | Decision | Document status | Protected-main applicability | +| --- | --- | --- | --- | +| [0002](0002-tenant-scoped-lifecycle.md) | Tenant-scoped durable lifecycle state | Accepted | Trusted host-selected `tenant_scope`, tenant-qualified identity, transaction-local RLS binding, and standalone compatibility. | +| [0003](0003-reproducible-release-evidence.md) | Reproducible release evidence before publication | Proposed | Retains record-local `Proposed` status even where related implementation exists. | +| [0004](0004-descriptor-pinned-release-artifact-verification.md) | Descriptor-pinned release artifact verification | Proposed | Retains record-local `Proposed` status; implementation presence does not silently accept the decision. | +| [0006](0006-resumable-result-checkpoints.md) | Resumable provider-result checkpoints | Accepted | Immutable prefix checkpoint evidence; not provider authentication or whole-stream immutability. | +| [0007](0007-durable-result-checkpoint-store.md) | Durable tenant-isolated result checkpoint store | Accepted | PostgreSQL persistence, tenant isolation, CAS concurrency, and caller-owned transaction seam without distributed exactly-once. | +| [0015](0015-http-425-too-early-retry.md) | HTTP 425 retry for bounded idempotent GETs | Accepted for the bounded retry slice | Keeps the automatic GET retry set closed and side-effecting POSTs single-attempt. | +| [0016](0016-postgres-logical-restore-seek.md) | PostgreSQL custom-format logical restore seek semantics | Accepted for the bounded direct-restore slice | Integrated with #212. Final descriptor EOF is not a valid completion invariant for random-access custom archives; metadata verification and transactional failure semantics remain bounded by the executor. No backup, target-authentication, application-readiness, PITR, or RPO/RTO/HA/DR guarantee follows. | +| [0022](0022-postgres-restore-target-isolation.md) | Restore-target service-name and PostgreSQL cluster-identity separation | Accepted for the bounded restore-target identity seam | Integrated with merged #228. Exact live/restore service names and caller-owned `pg_control_system().system_identifier` values must differ. The package does not open/authenticate connections, accept DSNs/passwords, execute restore, or prove application/PITR/RPO-RTO readiness. | + +Protected main also contains bounded recovery-receipt/artifact/schema evidence primitives. Their integration does not create implicit ADRs. Direct logical restore is governed by ADR 0016; bounded restore-target cluster separation is governed by ADR 0022. Logical backup, post-restore application acceptance, physical/WAL/PITR recovery, external key custody, and end-to-end recovery remain separately governed capabilities. + +## Numbering and missing identifiers + +ADR numbers are stable identifiers, not a promise of contiguous numbering. Missing numbers may reflect historical work, superseded proposals, unmerged branches, or reserved identifiers. A gap does not authorize reconstruction from an old branch. New records use a repository-approved stable identifier; renaming an integrated ADR is an architecture-governance migration rather than cosmetic cleanup. + +## Decision status versus implementation status + +ADR status is not product implementation status. Canonical product documents use `IMPLEMENTED-ON-PROTECTED-MAIN`, `ACTIVE-PR`, `PARTIAL`, `PLANNED`, and `SUPERSEDED`. A `Proposed` ADR may coexist with code on protected main, while an `Accepted` ADR may govern only a bounded slice of a larger partial capability. When ADR and protected implementation appear inconsistent, reconcile the defect explicitly rather than hiding it in the index. + +## Supersession and amendments + +A decision is superseded only by an explicit reviewed record/amendment. Later code, issues, PRs, or historical branches do not implicitly supersede it. Material changes require a fresh read of protected source and adjacent writers, explicit alternatives/rejected reasons/consequences, correct active-vs-shipped classification, and traceability refresh after integration. + +## Non-guarantees + +The ADR set does not establish authentication from RLS, provider authenticity from checkpointing, distributed exactly-once processing, arbitrary HTTP retry permission, backup restorability, authenticated connection provenance, post-restore application readiness, PITR/RPO/RTO/HA/DR from recovery evidence/direct restore/`system_identifier` comparison alone, or organizational certification. diff --git a/docs/erd/package-owned-schema.md b/docs/erd/package-owned-schema.md new file mode 100644 index 000000000..d5149aa7f --- /dev/null +++ b/docs/erd/package-owned-schema.md @@ -0,0 +1,159 @@ +# Package-owned schema ERD + +## Authority + +This diagram summarizes protected-main package tables from +`pg_llm_batch/schema.sql` plus the migration-owned checkpoint table from +`pg_llm_batch/migrations/0007_result_stream_checkpoints.sql`. SQL and schema +tests remain stronger authority than this picture. + +Use it to see identities, foreign keys, and tenant qualification before you +write a report, a restore drill, or an embedding mapping. Do not treat the +diagram as proof that backup/restore, candidate discovery, or distributed +exactly-once processing is shipped. + +## What to do next + +1. Keep new database objects as two-or-more-word `snake_case` names. +2. Bind `tenant_scope` before reading `llm_remote_batch_jobs` or + `llm_result_stream_checkpoints`. +3. Treat `llm_result_stream_checkpoints` as prefix evidence, not a distributed + exactly-once claim. +4. Leave ACTIVE-PR recovery and reconciliation tables off this diagram until + they exist on protected main. + +## Identities + +| Table | Durable identity | Notes | +| --- | --- | --- | +| `llm_remote_batch_jobs` | `tenant_scope, endpoint_alias, remote_batch_id` | Forced RLS. Provider data never selects the tenant. | +| `llm_result_stream_checkpoints` | `tenant_scope, checkpoint_consumer_name, endpoint_alias, remote_batch_id` | Added by migration 0007. Forced RLS. | +| `llm_queues` | `queue_uuid` / unique `queue_name` | Standalone preparation owner. | +| `llm_batches` | `batch_uuid` | Child of one queue. | +| `llm_batch_file_payloads` | `file_id` | Canonical JSONL bytes in PostgreSQL. | +| `llm_batch_files` | `(batch_uuid, part_index)` | Virtual payload via `payload_file_id`. | +| `llm_requests` | `request_uuid` | Authorized prompt/result content. | +| `llm_jsonl_lines` | `(payload_file_id, sequence_no)` | JOIN-only reconstruction. | +| `llm_endpoints` | `endpoint_alias` | Configuration, not tenant authority. | +| `com_config` / `com_secrets` | `config_key` / `secret_key` | KV stores. Compatibility secrets may be plaintext. | + +## Entity-relationship diagram + +```mermaid +erDiagram + com_config { + text config_key PK + text config_value + timestamptz updated_at + } + + com_secrets { + text secret_key PK + text secret_value + boolean is_encrypted + timestamptz updated_at + } + + llm_queues { + uuid queue_uuid PK + text queue_name UK + text queue_status + timestamptz created_at + } + + llm_batches { + uuid batch_uuid PK + uuid queue_uuid FK + text batch_name + text batch_status + text model_name + timestamptz created_at + } + + llm_batch_file_payloads { + uuid file_uuid PK + text file_id UK + jsonb content + timestamptz created_at + } + + llm_batch_files { + uuid file_uuid PK + uuid batch_uuid FK + uuid queue_uuid FK + text payload_file_id FK + int part_index + } + + llm_requests { + uuid request_uuid PK + uuid batch_uuid FK + uuid batch_file_uuid FK + text user_prompt + text response_content + text request_status + } + + llm_jsonl_lines { + uuid line_uuid PK + text payload_file_id FK + uuid request_uuid FK + int sequence_no + text line_text + } + + llm_endpoints { + uuid endpoint_uuid PK + text endpoint_alias UK + text base_url + boolean active + } + + llm_endpoint_models { + uuid endpoint_uuid PK,FK + text model_id PK + text tokenizer_model + } + + llm_remote_batch_jobs { + uuid remote_job_uuid PK + text tenant_scope + text endpoint_alias + text remote_batch_id + bigint observation_order + text batch_status + } + + llm_result_stream_checkpoints { + uuid result_checkpoint_uuid PK + text tenant_scope + text checkpoint_consumer_name + text endpoint_alias + text remote_batch_id + text prefix_sha256 + } + + llm_queues ||--o{ llm_batches : contains + llm_queues ||--o{ llm_batch_files : owns + llm_batches ||--o{ llm_batch_files : partitions + llm_batch_file_payloads ||--o{ llm_batch_files : stores + llm_batches ||--o{ llm_requests : queues + llm_batch_files ||--o{ llm_requests : assigns + llm_batch_file_payloads ||--o{ llm_jsonl_lines : lines + llm_requests ||--o{ llm_jsonl_lines : reconstructs + llm_endpoints ||--o{ llm_endpoint_models : maps +``` + +`llm_remote_batch_jobs` and `llm_result_stream_checkpoints` are intentionally +unrelated to the preparation graph. Their tenant-qualified unique keys are the +lifecycle and checkpoint identities. Migration 0007 owns the checkpoint table; +the packaged `schema.sql` / Docker init mirror do not create it. + +## Third-normal-form boundary + +Preparation tables keep queue, batch, payload, request, and line facts in +separate relations with explicit keys. Endpoint/model mapping is a separate +relation from lifecycle projection. The KV tables `com_config` and `com_secrets` +are key/value stores by design: each row is one setting, not a repeating group +inside another entity. Do not add single-word table names or embed tenant +authority in provider columns. diff --git a/docs/product/PRD.md b/docs/product/PRD.md new file mode 100644 index 000000000..9d3de3c74 --- /dev/null +++ b/docs/product/PRD.md @@ -0,0 +1,107 @@ +# Product Requirements Document + +## Document authority + +This PRD is the canonical product-status contract for `pg-llm-batch`. Protected default-branch code, schema, tests, and accepted decisions are shipped authority. Pull requests, issue plans, historical branches, generated merge commits, exact SHAs, run IDs, and review commentary are evidence or work-in-progress rather than durable product truth. + +Status vocabulary: **IMPLEMENTED-ON-PROTECTED-MAIN**, **ACTIVE-PR**, **PARTIAL**, **PLANNED**, and **SUPERSEDED**. + +## Product purpose and users + +`pg-llm-batch` is a standalone and embeddable PostgreSQL-centered engine for durable/asynchronous LLM Batch API workloads. It owns deterministic token/resource accounting, bounded JSONL preparation, package-owned PostgreSQL persistence, validated provider Batch API interaction, durable lifecycle/checkpoint state, tenant/RLS boundaries, bounded reconciliation, and bounded recovery controls without requiring a particular CWL host service. + +Primary users are platform operators, application/platform engineers, multi-tenant hosts that authenticate and authorize callers before choosing `tenant_scope`, and reliability/security reviewers who need deterministic failure, migration, rollback, release, and evidence boundaries. + +## Protected-main capability contract + +| Capability | Status | Product boundary | +| --- | --- | --- | +| PostgreSQL token counting and bounded batch preparation | IMPLEMENTED-ON-PROTECTED-MAIN | Deterministic token/resource accounting and finite batch partitioning. | +| Disk-free package payload persistence | IMPLEMENTED-ON-PROTECTED-MAIN | Package JSONL/file/line/request state is durable PostgreSQL state, not a disposable local cache. | +| OpenAI-compatible upload/create/poll/wait/cancel/retrieve | IMPLEMENTED-ON-PROTECTED-MAIN | Provider destinations, identifiers, responses, downloads, retries, and waits are validated and bounded. | +| Standalone durable lifecycle | IMPLEMENTED-ON-PROTECTED-MAIN | `DurableBatchAPIClient` retains its four-argument recorder seam and explicit `standalone` scope. | +| Tenant-qualified lifecycle with forced RLS | IMPLEMENTED-ON-PROTECTED-MAIN | Trusted host-selected `tenant_scope` qualifies durable identity and is bound transaction-locally; arbitrary SQL, `SUPERUSER`, and `BYPASSRLS` remain outside the isolation guarantee. | +| Durable result checkpoint/CAS storage | IMPLEMENTED-ON-PROTECTED-MAIN | Tenant-qualified checkpoint authority and conflict detection; not distributed exactly-once. | +| Scheduler-independent bounded reconciliation | IMPLEMENTED-ON-PROTECTED-MAIN | Finite host-selected candidates use the reviewed provider client; discovery and scheduling remain separate. | +| Cross-process reconciliation single-flight | IMPLEMENTED-ON-PROTECTED-MAIN | Merged #191 provides tenant-qualified PostgreSQL session advisory-lock exclusion. It is not a scheduler, durable lease, result-application transaction, terminal-work-retirement mechanism, or distributed exactly-once guarantee. | +| Bounded recovery receipt / backup-artifact / packaged-schema evidence | IMPLEMENTED-ON-PROTECTED-MAIN | Content-free identity/integrity evidence only; not backup execution, provenance, restorability, live-schema parity, PITR, or RPO/RTO. | +| PostgreSQL logical backup execution | ACTIVE-PR | #208 remains a bounded `pg_dump` candidate; protected main must not be described as creating a restorable backup from evidence primitives alone. | +| PostgreSQL logical restore execution | IMPLEMENTED-ON-PROTECTED-MAIN | Merged #212 provides the bounded direct custom-format `pg_restore` executor. It accepts random-access seek behavior and verifies archive metadata instead of requiring final EOF. Closed #209 remains historical defect evidence. No `pg_dump`, target-authentication, application-readiness, PITR, or RPO/RTO/HA/DR guarantee follows. | +| PostgreSQL restore-target cluster identity verification | IMPLEMENTED-ON-PROTECTED-MAIN | Merged #228 provides `verify_postgres_restore_target_isolation(...)`: live and restore libpq service names must differ and caller-owned `pg_control_system().system_identifier` values must differ. The package does not open either connection, authenticate who collected the identities, accept a DSN, execute restore, or prove post-restore application/catalog/PITR/RPO-RTO readiness. | +| Recovery evidence binding / live reinspection | ACTIVE-PR | Composition or later re-hashing is not provenance, restorability, or target isolation. | +| Post-restore catalog/application readiness | ACTIVE-PR | #296 is an unshipped application-readiness candidate; branch GREEN is not protected-main truth. | +| Permanent live PostgreSQL integration lane | ACTIVE-PR | #341 executes the full integration marker on its branch and carries #296 as a tested child; the workflow is not protected authority until integration. | +| Physical/WAL/PITR recovery | ACTIVE-PR / PARTIAL | Intent/evidence candidates do not prove `pg_basebackup`, WAL replay, promotion, or achieved objectives. | +| End-to-end PostgreSQL recovery readiness | PARTIAL | Evidence, direct logical restore, and bounded name+cluster separation still do not prove an isolated restore with schema/RLS/constraint/extension parity, migration compatibility, external key/config custody, physical/WAL/PITR recovery, application readiness, or measured RPO/RTO/HA/DR. | +| Durable reconciliation candidate discovery | ACTIVE-PR | Discovery must be tenant-qualified, bounded, deterministic, and database-authoritative. | +| Autonomous package worker | PARTIAL | Reconciliation, transient single-flight, and durable state exist; a complete scheduler/crash-recovery/terminal-retirement control plane does not. | +| Durable result application coupled to checkpoint advancement | PARTIAL | Existing checkpoint/retrieval primitives do not establish end-to-end exactly-once result application. | +| Existing-volume legacy `http` / `pg_cron` retirement | ACTIVE-PR | Existing deployments require fail-closed, reversible migration evidence before legacy authority is retired. | +| PostgreSQL-backed configuration and secret storage | IMPLEMENTED-ON-PROTECTED-MAIN | Protected main supports optional Fernet plus explicit compatibility mode. `SecretStore(require_encryption=False)` can persist base64-obfuscated `is_encrypted = FALSE` rows. | +| Production secret-at-rest policy lifecycle | PARTIAL | Mandatory encryption, historical compatibility-row migration, key rotation/recovery, and external key custody are not protected guarantees; #210 remains stricter active work. | +| Redacted readiness and bounded diagnostics | IMPLEMENTED-ON-PROTECTED-MAIN | Public/operational evidence omits content-bearing lower-layer details. | +| First-class OpenTelemetry installation extra | ACTIVE-PR | Base installs remain telemetry-dependency-free until the optional graph integrates with lock/release evidence. | +| Standalone + modular embedding | IMPLEMENTED-ON-PROTECTED-MAIN | CWL services may integrate, but are not hidden runtime dependencies. | + +## Functional requirements + +### FR-1 — Batch preparation + +Resolve a valid package batch identity, select eligible requests, count tokens through the reviewed PostgreSQL tokenizer boundary, partition under explicit token/byte/record/provider limits, and persist package-owned payload/file/line/request assignments atomically. Supported retries must not silently duplicate assignment or corrupt state. + +### FR-2 — Provider interaction + +Provider operations use validated endpoints and finite response/download budgets. Automatic retries are limited to reviewed idempotent GET behavior. Side-effecting POSTs do not gain implicit retry authority. Credentials and provider content are excluded from ordinary diagnostics. + +### FR-3 — Durable lifecycle and tenancy + +Tenant-aware clients validate trusted host-selected `tenant_scope` synchronously before observation reservation, credential lookup, provider I/O, or lifecycle SQL. The durable key is `(tenant_scope, endpoint_alias, remote_batch_id)` and tenant-aware writes/reads bind that scope through parameterized transaction-local PostgreSQL context with forced RLS. `pg_llm_batch.tenant_scope` is routing context, not authentication. A role capable of arbitrary SQL can set it arbitrarily, so host identity mapping, authorization, SQL-injection prevention, and administrative database roles remain outside the RLS guarantee. + +Standalone compatibility is normative: `DurableBatchAPIClient` retains `(postgres_dsn, endpoint_alias, provider_batch, observation_order)` and explicit `standalone` persistence/read scope. + +### FR-4 — Reconciliation and provider-effect recovery + +Reconciliation is finite, deterministic, payload-free in operational evidence, and reuses the validated provider-client boundary. Protected main supplies transient tenant-qualified session advisory single-flight. Candidate discovery, scheduling, durable leasing, result application, terminal-work retirement, and cross-system exactly-once semantics are separate capabilities. Provider-success/database-failure states remain observable rather than being rewritten as if the provider effect never occurred. + +### FR-5 — Persistence and PostgreSQL recovery + +Package-owned database objects use explicit durable identities, parameterized SQL, synchronized schema copies where required, and documented migration/rollback boundaries. Malformed durable state fails closed before downstream effects when correctness depends on it. + +Protected main contains bounded content-free recovery evidence, the bounded #212 logical restore executor, and the bounded #228 restore-target name+cluster-identity verifier. The verifier requires distinct exact libpq service names and distinct caller-owned PostgreSQL `system_identifier` values gathered from already-opened connections. It does not establish connection provenance or authorize destructive action. The logical restore executor does not create backups or prove target/application readiness. End-to-end recovery still requires independent backup authority, authenticated target provenance, post-restore schema/RLS/constraint/extension/application acceptance, migration compatibility, key/config custody, and physical/WAL/PITR evidence where claimed. + +### FR-6 — Configuration and secrets + +Configuration and secrets are PostgreSQL-backed. Fernet is optional on protected main. Compatibility mode may store base64-obfuscated rows with `is_encrypted = FALSE`; this is not an encryption-at-rest claim. Deployments may require Fernet explicitly, but mandatory-default policy, migration of historical compatibility rows, rotation/recovery, and external key custody require separately integrated contracts. Environment variables are bootstrap transport only where documented; embedding hosts may inject credential providers. + +### FR-7 — Observability and diagnostics + +Operational telemetry is opt-in. Finite operation/outcome vocabularies may be emitted; prompts, provider bodies, credentials, arbitrary endpoint aliases/resource IDs, and arbitrary exception text are not telemetry attributes. Public readiness uses bounded categories where lower-layer text could contain sensitive data. + +### FR-8 — Deployment + +The package remains usable without `contextual-orchestrator`, `naruon`, or another CWL repository. Host services may supply authentication, tenant routing, secret resolution, gateway/model routing, OpenTelemetry export, or scheduling through explicit seams. + +## Non-functional requirements + +Owned production Python maintains exact 100% statement/branch coverage and complete public docstrings under repository gates. Supported validation includes Python 3.10, 3.12, and 3.14 plus realistic PostgreSQL/container integration when behavior depends on PostgreSQL. + +Security-sensitive validation fails closed. Authorized business payloads are not silently masked or transformed: doing so changes token counts, provider semantics, persisted evidence, replay, and business meaning. Confidentiality is enforced at explicit authentication/authorization, database/service, transport/storage, retention, and operational-evidence boundaries. Any content transformation is an explicit host/business policy with provenance and acceptance evidence. + +Network, response, retry, wait, candidate-scan, payload, recovery, and release-evidence work is explicitly bounded. Queued, skipped, cancelled, absent, stale, predecessor-head, synthetic-only, or infrastructure-failed evidence is not exact-head success. + +Dependencies and release artifacts are locked/reproducible according to repository policy. Release acceptance covers the then-live quality, security, package/container, SBOM, provenance, artifact-identity, rollback/recovery, review, and governance gates on the exact integrated protected head. A version string or successful PR does not imply publication. + +## Explicit non-goals + +- Replacing host authentication or authorization. +- Treating RLS or `tenant_scope` as a credential or SQL-injection prevention. +- Making provider/model content an authority for tenant, endpoint, credential, or filesystem selection. +- Reintroducing provider networking or independent scheduling inside PostgreSQL. +- Inferring restorability, target provenance, PITR, RPO/RTO/HA/DR, or compliance from recovery hashes, a restore command, or `system_identifier` comparison alone. +- Claiming distributed exactly-once processing without an integrated transaction/recovery contract spanning every external effect. +- Requiring a particular CWL host service for standalone use. +- Claiming SOC 2, CSAP, or another certification from repository evidence alone. + +## Product acceptance boundary + +A capability moves to **IMPLEMENTED-ON-PROTECTED-MAIN** only after its unchanged source satisfies the live ruleset, required exact-head quality/security/coverage/package/provenance/release evidence, valid review findings, and protected-default-branch integration. Canonical documentation is then refreshed from the resulting protected tree; predecessor evidence never transfers automatically. diff --git a/docs/product/TRD.md b/docs/product/TRD.md new file mode 100644 index 000000000..cafb2d5b2 --- /dev/null +++ b/docs/product/TRD.md @@ -0,0 +1,81 @@ +# Technical Requirements Document + +## Document authority + +This TRD defines technical invariants for `pg-llm-batch`. Protected default-branch code, schema, tests, and accepted decisions are stronger authority than historical branches, stale PR prose, predecessor checks, exact SHAs, generated merge commits, or run IDs. Status vocabulary is shared with the PRD: **IMPLEMENTED-ON-PROTECTED-MAIN**, **ACTIVE-PR**, **PARTIAL**, **PLANNED**, and **SUPERSEDED**. + +## System boundary + +`pg-llm-batch` is a Python package plus PostgreSQL schema and container assets. It can run standalone or be embedded. It owns bounded batch preparation, PostgreSQL persistence, validated provider Batch API access, durable lifecycle/checkpoint state, tenant/RLS controls, bounded reconciliation, tenant-qualified transient single-flight, bounded recovery evidence, bounded logical restore, and bounded restore-target cluster-identity comparison. It does not own host authentication/business authorization, infrastructure TLS, external secret-manager choice, global telemetry policy, backup/WAL infrastructure, connection provenance, or cross-system distributed transactions. + +## Component contract + +| Component | Protected-main responsibility | Prohibited authority | +| --- | --- | --- | +| `token_counter.py` | reviewed PostgreSQL tokenizer boundary | provider credentials, tenant authorization, provider I/O | +| `orchestrator.py` | deterministic bounded preparation and package persistence | provider protocol/retry policy | +| `batch_api_client.py` | validated bounded provider upload/create/poll/wait/cancel/retrieve | tenant authentication, scheduler ownership | +| `durable_client.py` | standalone and tenant-qualified lifecycle composition | authenticating/authorizing host tenants | +| `db.py` | schema application and parameterized durable reads/writes/RLS context | database-side provider networking | +| `checkpoint_store.py` | tenant-qualified checkpoint/CAS operations | distributed exactly-once claims | +| `reconciliation.py` | finite host-selected reconciliation | discovery and scheduling | +| `reconciliation_single_flight.py` | transient tenant-qualified PostgreSQL session advisory-lock exclusion | durable lease, scheduler, result application, terminal retirement, distributed exactly-once | +| `postgres_recovery_receipt.py` | deterministic bounded content-free receipt | backup/restore execution or operator authentication | +| `postgres_backup_evidence.py` | descriptor-pinned finite artifact hash/size evidence | backup execution or restorability proof | +| `postgres_schema_evidence.py` | finite packaged-schema hash/size evidence | SQL execution or live-cluster parity proof | +| `postgres_logical_restore.py` | bounded custom-format direct restore with caller-owned source trust, constrained libpq environment, transactional failure handling, and archive metadata verification | backup creation, target authentication, application readiness, PITR/RPO-RTO | +| `postgres_restore_target.py` | require distinct exact service names and distinct caller-owned PostgreSQL `system_identifier` values before a host treats targets as separate | opening connections, accepting DSNs/passwords, authenticating identity collection, executing restore, application/catalog/PITR/RPO-RTO proof | +| `config.py` | PostgreSQL-backed config/secret storage with optional Fernet and explicit compatibility mode | mandatory encryption, compatibility-row migration, rotation/recovery, external key custody | +| `observability.py` | opt-in bounded telemetry | global SDK/exporter/resource policy | +| `health.py` | redacted readiness aggregation | arbitrary diagnostic reflection | +| `cli.py` | standalone operator composition | higher-level workflow orchestration | + +## Runtime architecture + +### Batch preparation and provider I/O + +Preparation resolves a supported package identity, reads eligible requests, counts tokens through `pg_tiktoken`, partitions under finite token/byte/record/provider limits, and commits package payload/file/line/request assignments atomically. Package payloads remain PostgreSQL-backed rather than package-owned local files. + +Provider URLs and identifiers are untrusted until validated. Production destinations require the reviewed secure URL policy. Control responses and result/error files are consumed under finite decoded-byte budgets. Automatic retry stays limited to reviewed idempotent GET semantics; side-effecting POSTs remain single-attempt unless a separately reviewed provider contract changes that rule. + +### Durable lifecycle tenancy + +Standalone lifecycle state uses the explicit `standalone` scope and preserves the four-argument `DurableBatchAPIClient` recorder seam. Tenant-aware clients validate trusted host-selected `tenant_scope` before reservation, credential resolution, provider I/O, or lifecycle SQL. Durable tenant identity is `(tenant_scope, endpoint_alias, remote_batch_id)` and package reads/writes bind that scope using parameterized transaction-local `set_config` under forced RLS. `tenant_scope` is routing context, not authentication. `SUPERUSER`, `BYPASSRLS`, and arbitrary SQL are administrative escape hatches outside the tenant guarantee. + +### Checkpoints and reconciliation + +Checkpoint storage is tenant-qualified and compare-and-swap based. PostgreSQL transactionality does not extend to provider/network effects. + +`reconcile_batch_candidates(...)` performs one finite host-selected reconciliation pass. Protected main also contains the tenant-qualified session advisory single-flight integrated through #191. That lock is session-lifetime transient state, not a scheduler, durable lease, result-application transaction, terminal-work-retirement authority, or distributed exactly-once mechanism. Durable discovery remains active work. + +### PostgreSQL recovery + +Protected main contains bounded receipt, backup-artifact, and packaged-schema evidence primitives. These do not execute backup/restore, establish provenance, prove live schema, or establish PITR/RPO/RTO/HA/DR. + +Direct `pg_restore` execution is **IMPLEMENTED-ON-PROTECTED-MAIN** through merged #212. The executor accepts PostgreSQL custom-format random-access seek semantics and verifies archive metadata rather than requiring final descriptor EOF. Closed #209 remains historical defect evidence for the invalid EOF postcondition. The executor retains caller-owned source-superuser trust, constrained libpq environment, and single-transaction failure semantics; it does not provide `pg_dump`, target identity authentication, application/catalog readiness, WAL/PITR, or recovery-objective proof. + +Restore-target cluster identity verification is also **IMPLEMENTED-ON-PROTECTED-MAIN** through merged #228. `postgres_restore_target.py` accepts exact live/restore libpq service names and exact `PostgresRestoreTargetIdentity` values; both names and both `pg_control_system().system_identifier` values must differ. Callers collect the identifiers from connections they already opened. The package does not open a connection, read `pg_service.conf`, accept a DSN/password/host/port, authenticate the collector or connection provenance, execute `pg_restore`, or prove post-restore schema/application readiness. Thus the seam rejects same-cluster aliases but is not end-to-end restore authorization. + +Logical `pg_dump` remains active work. #296 remains an unshipped application-readiness candidate, and #341 remains the active owner of a permanent hosted live-PostgreSQL integration lane. Their branch-level evidence does not become protected-main truth until integration. + +## Persistence, security, and privacy + +Package-owned persisted payload/state is canonical and validated before downstream effects when correctness depends on it. New database objects use descriptive snake_case naming where applicable, parameterized SQL, synchronized schema copies, explicit durable identities, and documented migration/rollback boundaries. + +Standalone config and secrets are PostgreSQL-backed. Fernet is optional on protected main. `SecretStore(require_encryption=False)` permits base64-obfuscated compatibility rows with `is_encrypted = FALSE`; callers may explicitly require Fernet. Mandatory encrypted-at-rest policy, migration of historical compatibility rows, key rotation/recovery, and external key custody are separate capabilities. + +Authorized prompts, requests, JSONL, and provider results are not silently masked or truncated merely because they may contain PII. Such transformation changes token counts, replay, provider semantics, and business meaning. Host authentication/authorization, tenant selection, least-privilege identities, transport/storage controls, retention/deletion policy, and redacted operational evidence are the confidentiality boundaries. Any content transformation requires an explicit host/business policy with provenance and acceptance tests. + +Errors, logs, telemetry, readiness, and review evidence omit DSNs, credentials, prompt/provider content, arbitrary SQL, untrusted identifiers, and dynamic lower-layer text where it is unnecessary. RLS remains defense in depth rather than authentication or SQL-injection prevention. + +## Concurrency, testing, and release + +Batch preparation and durable writes use PostgreSQL transactionality appropriate to their aggregate boundary. Session advisory locking is transient and must not be promoted to durable leasing. Recovery file inspection remains finite and fail-closed under mutation; a successful hash is evidence for the revalidated descriptor identity, not a lock on external storage. + +Every defect follows realistic RED → minimum causal fix → exact-head GREEN. Repository evidence includes Python 3.10/3.12/3.14, exact owned production statement/branch coverage, public docstrings, lint/static checks, PostgreSQL integration for SQL/RLS/migration/concurrency behavior, package/container validation, migration rollback where applicable, security/SAST, locked packaging, release acceptance, SBOM/provenance, and artifact identity. Queued, skipped, cancelled, absent, stale, predecessor, synthetic-only, or infrastructure-failed evidence is not success. + +A release originates only from an exact integrated protected head after all then-required quality, security, review, migration/recovery, packaging, provenance, and release gates pass. Version/CHANGELOG/package/tag/publication do not become authoritative until publication and artifact verification complete. Recovery evidence, logical restore execution, or `system_identifier` comparison alone does not make a release end-to-end recovery-ready. + +## Documentation requirements + +Canonical PRD/TRD/architecture/ADR/UML/ERD/security/operability/release/data-governance/traceability surfaces distinguish protected-main state from active work and avoid exact SHAs/run IDs. ADR 0016 records bounded restore-seek semantics. ADR 0022 records bounded restore-target name+cluster-identity separation. Broader direct-SQL, target-authentication, application-readiness, PITR, or recovery-objective changes require coordinated permanent documentation through their current owners. diff --git a/docs/uml/component-and-sequence.md b/docs/uml/component-and-sequence.md new file mode 100644 index 000000000..ec37a78fe --- /dev/null +++ b/docs/uml/component-and-sequence.md @@ -0,0 +1,77 @@ +# Component and sequence views + +## Authority + +These diagrams describe protected-main composition. They do not add runtime +authority and do not promote ACTIVE-PR backup, restore, discovery, or worker +lanes into shipped components. + +## What to do next + +1. Deploy this package alone, or embed it behind `contextual-orchestrator` / + `naruon` after those hosts authenticate and authorize. +2. Inject credentials and `tenant_scope` from the host. Do not read them from + provider responses. +3. Call `DurableBatchAPIClient` for `standalone`. Call + `TenantDurableBatchAPIClient` only after the host has chosen a scope. + +## Component view + +```mermaid +flowchart LR + subgraph hosts [Optional CWL hosts] + orchestrator[contextual-orchestrator] + naruon[naruon] + end + subgraph package [pg-llm-batch standalone or embedded] + cli[cli.py] + durable[DurableBatchAPIClient] + tenant[TenantDurableBatchAPIClient] + client[batch_api_client.py] + reconcile[reconciliation.py] + db[db.py] + checkpoints[checkpoint_store.py] + evidence[recovery evidence modules] + end + postgres[(PostgreSQL)] + provider[OpenAI-compatible Batch API] + orchestrator -->|authorized tenant_scope| tenant + naruon -->|authorized tenant_scope| tenant + cli --> durable + durable -->|standalone| db + tenant -->|set_config tenant_scope| db + durable --> client + tenant --> client + reconcile --> client + db --> postgres + checkpoints --> postgres + client --> provider + evidence -.->|hash and receipt only| postgres +``` + +`contextual-orchestrator` and `naruon` are optional. The package must keep +working when they are absent. + +## Tenant lifecycle sequence + +```mermaid +sequenceDiagram + participant Host + participant TenantClient as TenantDurableBatchAPIClient + participant DB as db.py + participant PG as PostgreSQL + participant Provider as Batch API + Host->>TenantClient: construct(tenant_scope) + TenantClient->>TenantClient: validate tenant_scope + Host->>TenantClient: observe or persist remote batch + TenantClient->>DB: parameterized set_config(tenant_scope) + DB->>PG: transaction-local scope plus RLS + TenantClient->>Provider: bounded upload/create/poll/retrieve + Provider-->>TenantClient: untrusted status and files + TenantClient->>DB: tenant-qualified upsert + DB->>PG: (tenant_scope, endpoint_alias, remote_batch_id) +``` + +A missing or malformed scope fails before credential lookup, provider I/O, or +lifecycle SQL. `DurableBatchAPIClient` keeps the four-argument recorder seam +and stores under the exact `standalone` scope. diff --git a/tests/test_canonical_status_vocabulary.py b/tests/test_canonical_status_vocabulary.py new file mode 100644 index 000000000..8d62eb13b --- /dev/null +++ b/tests/test_canonical_status_vocabulary.py @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Canonical product documents must track durable protected-main capability truth.""" + +from __future__ import annotations + +import inspect +import re +from pathlib import Path + +from pg_llm_batch.config import SecretStore + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +CANONICAL_STATUS_DOCUMENTS = ( + REPOSITORY_ROOT / "docs" / "product" / "PRD.md", + REPOSITORY_ROOT / "docs" / "product" / "TRD.md", + REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md", + REPOSITORY_ROOT / "docs" / "TRACEABILITY.md", + REPOSITORY_ROOT / "docs" / "adr" / "README.md", +) +_SECRET_POLICY_DOCUMENTS = ( + REPOSITORY_ROOT / "docs" / "product" / "PRD.md", + REPOSITORY_ROOT / "docs" / "product" / "TRD.md", + REPOSITORY_ROOT / "docs" / "DATA_GOVERNANCE.md", + REPOSITORY_ROOT / "docs" / "THREAT_MODEL.md", + REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md", + REPOSITORY_ROOT / "docs" / "TRACEABILITY.md", +) +_TRANSIENT_PR_STATE = re.compile( + r"(?i)\b(?:draft|ready)\s+#\d+\b|\bpr\s+#\d+\s+remains\s+draft\b" +) +_EXACT_HEAD_SHA = re.compile(r"\b[0-9a-f]{40}\b") +_ACTIVE_OVERLAY_ENTRY = re.compile(r"(?m)^- \*\*#(?P\d+)\*\*") + + +def _read(path: Path) -> str: + """Return one canonical Markdown document as UTF-8 text.""" + return path.read_text(encoding="utf-8") + + +def test_canonical_status_documents_omit_transient_pr_ready_or_draft_labels() -> None: + """Durable product/status prose must not freeze Draft/Ready pull-request labels.""" + for path in CANONICAL_STATUS_DOCUMENTS: + matches = _TRANSIENT_PR_STATE.findall(_read(path)) + assert matches == [], f"{path.name} persists transient PR state: {matches}" + + +def test_canonical_status_documents_do_not_embed_exact_heads() -> None: + """Exact contributor or protected SHAs belong in review evidence, not durable prose.""" + for path in CANONICAL_STATUS_DOCUMENTS: + matches = _EXACT_HEAD_SHA.findall(_read(path)) + assert matches == [], f"{path.name} embeds exact heads: {matches}" + + +def test_product_contract_records_integrated_logical_restore_boundary() -> None: + """Merged #212 is shipped only as the bounded direct logical-restore executor.""" + prd = _read(REPOSITORY_ROOT / "docs" / "product" / "PRD.md") + trd = _read(REPOSITORY_ROOT / "docs" / "product" / "TRD.md") + fitness = _read(REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md") + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + + assert "| PostgreSQL logical restore execution | IMPLEMENTED-ON-PROTECTED-MAIN |" in prd + assert "Direct `pg_restore` execution is **IMPLEMENTED-ON-PROTECTED-MAIN**" in trd + assert "logical restore executor is protected-main behavior" in fitness + assert "| FR-5 executable PostgreSQL logical restore | IMPLEMENTED-ON-PROTECTED-MAIN |" in traceability + for document in (prd, trd, fitness, traceability): + assert "#212" in document + assert "#209" in document + assert "EOF" in document + assert "RPO/RTO" in document or "RPO" in document + + +def test_product_contract_records_integrated_single_flight_without_lease_claim() -> None: + """Merged #191 is a transient session advisory lock, not a scheduler or durable lease.""" + prd = _read(REPOSITORY_ROOT / "docs" / "product" / "PRD.md") + trd = _read(REPOSITORY_ROOT / "docs" / "product" / "TRD.md") + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + + assert "| Cross-process reconciliation single-flight | IMPLEMENTED-ON-PROTECTED-MAIN |" in prd + assert "session advisory single-flight" in trd + assert "| FR-4 tenant-qualified cross-process single-flight | IMPLEMENTED-ON-PROTECTED-MAIN |" in traceability + for document in (prd, trd, traceability): + lowered = document.lower() + assert "durable lease" in lowered + assert "scheduler" in lowered + assert "distributed exactly-once" in lowered + + +def test_product_contract_records_integrated_restore_target_identity_boundary() -> None: + """Merged #228 proves bounded name-plus-cluster separation, not end-to-end restore safety.""" + prd = _read(REPOSITORY_ROOT / "docs" / "product" / "PRD.md") + trd = _read(REPOSITORY_ROOT / "docs" / "product" / "TRD.md") + fitness = _read(REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md") + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + adr_index = _read(REPOSITORY_ROOT / "docs" / "adr" / "README.md") + + assert "| PostgreSQL restore-target cluster identity verification | IMPLEMENTED-ON-PROTECTED-MAIN |" in prd + assert "`postgres_restore_target.py`" in trd + assert "merged #228" in fitness + assert "| FR-5 restore-target cluster identity verification | IMPLEMENTED-ON-PROTECTED-MAIN |" in traceability + assert "[0022](0022-postgres-restore-target-isolation.md)" in adr_index + for document in (prd, trd, fitness, traceability, adr_index): + lowered = document.lower() + assert "system_identifier" in document + assert "rpo/rto" in lowered or "rpo" in lowered + + +def test_secret_policy_docs_match_optional_fernet_compatibility_default() -> None: + """Canonical secret prose must not turn optional Fernet into a shipped mandate.""" + parameter = inspect.signature(SecretStore).parameters["require_encryption"] + assert parameter.default is False + + for path in _SECRET_POLICY_DOCUMENTS: + text = _read(path) + lowered = text.lower() + if "secret" not in lowered: + continue + assert "optional fernet" in lowered or "fernet" in lowered + assert "compatibility" in lowered + assert "encrypted secrets are postgresql-backed" not in lowered + assert "postgresql-backed configuration and encrypted secret storage" not in lowered + + +def test_active_overlay_register_excludes_merged_or_closed_recovery_predecessors() -> None: + """The active register must not retain merged #191/#212/#228 or closed #225 as live work.""" + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + active_numbers = { + int(match.group("number")) for match in _ACTIVE_OVERLAY_ENTRY.finditer(traceability) + } + assert 191 not in active_numbers + assert 212 not in active_numbers + assert 225 not in active_numbers + assert 228 not in active_numbers + assert 296 in active_numbers + assert 299 in active_numbers + assert 341 in active_numbers + + +def test_product_contract_names_active_recovery_capability_families() -> None: + """The recovery graph remains broader than the integrated logical restore executor.""" + prd = _read(REPOSITORY_ROOT / "docs" / "product" / "PRD.md") + fitness = _read(REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md") + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + + for document in (prd, fitness, traceability): + assert "evidence binding" in document.lower() or "receipt" in document.lower() + assert "catalog" in document.lower() + assert "pitr" in document.lower() + assert "target isolation" in document.lower() or "restore-target" in document.lower() + + +def test_canonical_overlay_register_preserves_superseded_lineage_without_live_status() -> None: + """Historical predecessors may remain as lineage without becoming active overlays.""" + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + fitness = _read(REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md") + + assert "#214" in traceability + assert "#226" in traceability + assert "superseded" in traceability.lower() + assert "keep it Draft" not in traceability + assert "keep it Ready" not in traceability + assert "#225" in traceability + assert "superseded restore-target predecessor" in traceability + assert "#226" in fitness + assert "superseded #214" in fitness + assert "current canonical documentation landing vehicle" in traceability + + +def test_recovery_target_configuration_overlay_stays_bounded_and_module_scoped() -> None: + """The #299 overlay must record fixed-query limits without inventing a public API.""" + fitness = _read(REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md") + traceability = _read(REPOSITORY_ROOT / "docs" / "TRACEABILITY.md") + package_init = _read(REPOSITORY_ROOT / "pg_llm_batch" / "__init__.py") + + for document in (fitness, traceability): + assert "#299" in document + assert "eight PostgreSQL recovery-target settings" in document + assert "pg_is_in_recovery()" in document + assert "module-scoped" in document + assert "RPO/RTO" in document + + assert "observe_postgres_recovery_target_configuration" not in package_init + assert "postgres_recovery_target_configuration_was_observed" not in package_init diff --git a/tests/test_data_governance_documentation.py b/tests/test_data_governance_documentation.py new file mode 100644 index 000000000..cc5ab232d --- /dev/null +++ b/tests/test_data_governance_documentation.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Static contracts for the data-governance overlay.""" + +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +GOVERNANCE = REPOSITORY_ROOT / "docs" / "DATA_GOVERNANCE.md" +FITNESS = REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md" + + +def _read(path: Path) -> str: + """Return one Markdown document as UTF-8 text.""" + return path.read_text(encoding="utf-8") + + +def test_data_governance_maps_classes_retention_and_deletion_limits() -> None: + """A buyer must see data classes, owners, retention, and what the package will not delete.""" + text = " ".join(_read(GOVERNANCE).split()) + assert "llm_requests" in text + assert "com_secrets" in text + assert "tenant_scope" in text + assert "authorized business payloads" in text.lower() + assert "Do not mask" in text or "do not mask" in text + assert "retention" in text.lower() + assert "deletion" in text.lower() + assert "host owns" in text.lower() or "embedding host owns" in text.lower() + + +def test_data_governance_cites_privacy_and_control_authorities() -> None: + """Doctoring must cite current privacy-control publications in APA 7th.""" + text = " ".join(_read(GOVERNANCE).split()) + assert "NIST Special Publication 800-53" in text + assert "https://doi.org/10.6028/NIST.SP.800-53r5" in text + assert "ISO/IEC 27701" in text or "ISO/IEC 29100" in text + + +def test_fitness_inventory_tracks_the_data_governance_overlay() -> None: + """The fitness matrix must stop calling data governance merely planned.""" + fitness = _read(FITNESS) + assert "docs/DATA_GOVERNANCE.md" in fitness + assert "| Data governance | PLANNED |" not in fitness diff --git a/tests/test_schema_erd_documentation.py b/tests/test_schema_erd_documentation.py new file mode 100644 index 000000000..039c2dc3b --- /dev/null +++ b/tests/test_schema_erd_documentation.py @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The package ERD must name every protected-main table and the checkpoint overlay.""" + +from __future__ import annotations + +import re +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +SCHEMA = REPOSITORY_ROOT / "pg_llm_batch" / "schema.sql" +CHECKPOINT_MIGRATION = ( + REPOSITORY_ROOT + / "pg_llm_batch" + / "migrations" + / "0007_result_stream_checkpoints.sql" +) +ERD = REPOSITORY_ROOT / "docs" / "erd" / "package-owned-schema.md" +FITNESS = REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md" +_TABLE_NAME = re.compile( + r"CREATE TABLE IF NOT EXISTS ([a-z][a-z0-9_]+)", + re.MULTILINE, +) + + +def _read(path: Path) -> str: + """Return one repository text file as UTF-8.""" + return path.read_text(encoding="utf-8") + + +def _table_names(sql_text: str) -> tuple[str, ...]: + """Return CREATE TABLE names in document order.""" + return tuple(_TABLE_NAME.findall(sql_text)) + + +def test_erd_names_every_packaged_schema_table() -> None: + """An acquisition reviewer can map SQL objects without reading schema.sql first.""" + erd = _read(ERD) + tables = _table_names(_read(SCHEMA)) + assert tables, "packaged schema must declare tables" + for table_name in tables: + assert table_name in erd, table_name + assert "_" in table_name + + +def test_erd_labels_checkpoint_table_as_migration_owned() -> None: + """Checkpoint storage is packaged by migration 0007, not the init schema copy.""" + erd = _read(ERD) + checkpoint_tables = _table_names(_read(CHECKPOINT_MIGRATION)) + assert checkpoint_tables == ("llm_result_stream_checkpoints",) + assert "llm_result_stream_checkpoints" in erd + assert "0007" in erd + assert "ACTIVE-PR" in erd or "not a distributed exactly-once" in erd.lower() + + +def test_erd_uses_tenant_qualified_identities() -> None: + """The diagram must show the tenant-qualified lifecycle and checkpoint keys.""" + erd = _read(ERD) + assert "tenant_scope, endpoint_alias, remote_batch_id" in erd + assert "checkpoint_consumer_name" in erd + assert "erDiagram" in erd + + +def test_fitness_inventory_tracks_the_erd_overlay() -> None: + """The fitness matrix must stop calling the ERD merely planned.""" + fitness = _read(FITNESS) + assert "docs/erd/package-owned-schema.md" in fitness + assert "| ERD / schema model | PLANNED |" not in fitness diff --git a/tests/test_threat_model_documentation.py b/tests/test_threat_model_documentation.py new file mode 100644 index 000000000..c792bd4da --- /dev/null +++ b/tests/test_threat_model_documentation.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Static contracts for the protected-main threat-model overlay.""" + +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +THREAT_MODEL = REPOSITORY_ROOT / "docs" / "THREAT_MODEL.md" +FITNESS = REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md" + + +def _read(path: Path) -> str: + """Return one Markdown document as UTF-8 text.""" + return path.read_text(encoding="utf-8") + + +def test_threat_model_identifies_assets_boundaries_and_residuals() -> None: + """A buyer must see assets, trust boundaries, mitigations, and residual risk.""" + text = _read(THREAT_MODEL) + assert "tenant_scope" in text + assert "llm_remote_batch_jobs" in text + assert "llm_result_stream_checkpoints" in text + assert "authorized business payloads" in text.lower() or "content fidelity" in text.lower() + assert "set_config" in text + assert "BYPASSRLS" in text + assert "Residual" in text + assert "does not claim" in text.lower() or "not a certification" in text.lower() + + +def test_threat_model_cites_current_risk_authorities() -> None: + """Doctoring must cite current NIST risk and control publications in APA 7th.""" + text = " ".join(_read(THREAT_MODEL).split()) + assert "NIST Special Publication 800-53" in text + assert "NIST Special Publication 800-30" in text + assert "800-154" in text + assert "https://doi.org/10.6028/NIST.SP.800-53r5" in text + assert "https://doi.org/10.6028/NIST.SP.800-30r1" in text + + +def test_threat_model_tells_the_operator_the_next_action() -> None: + """The document must tell a qualified host what to do next, not only what exists.""" + text = " ".join(_read(THREAT_MODEL).split()) + assert "Do not grant arbitrary SQL" in text or "do not grant arbitrary SQL" in text + assert "standalone" in text + + +def test_fitness_inventory_tracks_the_threat_model_overlay() -> None: + """The fitness matrix must stop calling the threat model merely planned.""" + fitness = _read(FITNESS) + assert "docs/THREAT_MODEL.md" in fitness + assert "| Threat model | PLANNED |" not in fitness diff --git a/tests/test_uml_documentation.py b/tests/test_uml_documentation.py new file mode 100644 index 000000000..b839c763d --- /dev/null +++ b/tests/test_uml_documentation.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Static contracts for the component and sequence UML overlay.""" + +from __future__ import annotations + +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[1] +UML = REPOSITORY_ROOT / "docs" / "uml" / "component-and-sequence.md" +FITNESS = REPOSITORY_ROOT / "docs" / "DOCUMENTATION_FITNESS.md" + + +def _read(path: Path) -> str: + """Return one Markdown document as UTF-8 text.""" + return path.read_text(encoding="utf-8") + + +def test_uml_shows_standalone_and_embedded_component_boundaries() -> None: + """A buyer must see standalone operation and MSA embedding as co-equal.""" + text = _read(UML) + assert "flowchart" in text or "C4Context" in text + assert "sequenceDiagram" in text + assert "DurableBatchAPIClient" in text + assert "TenantDurableBatchAPIClient" in text + assert "standalone" in text + assert "contextual-orchestrator" in text + assert "naruon" in text + assert "set_config" in text + + +def test_fitness_inventory_tracks_the_uml_overlay() -> None: + """The fitness matrix must stop calling UML merely planned.""" + fitness = _read(FITNESS) + assert "docs/uml/component-and-sequence.md" in fitness + assert "| UML/component/sequence views | PLANNED |" not in fitness