From cf501dfd75274bb889287626dfd80166865eca16 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:17:25 +0900 Subject: [PATCH 01/25] feat: persist proposed reconciliation matches --- CHANGELOG.md | 2 + README.md | 8 +- docs/ARCHITECTURE.md | 2 + docs/DATA_MODEL.md | 3 +- docs/ERD.md | 3 +- docs/OPERABILITY.md | 5 +- ...9-reconciliation-match-command-evidence.md | 54 +++ docs/doctoring/IMPLEMENTATION_SEQUENCE.md | 6 + docs/doctoring/STANDARD_TRACEABILITY.md | 1 + docs/product-technical-gap-baseline.md | 11 +- requirements-quality.txt | 5 +- scripts/validate_repository.py | 6 + .../__init__.py | 6 + .../http_api.py | 69 +++- .../persistence.py | 12 + .../reconciliation_match.py | 387 ++++++++++++++++++ ...st_foundation_install_manifest_contract.py | 28 ++ ...ation_data_model_documentation_contract.py | 3 +- tests/test_reconciliation_match_api.py | 303 ++++++++++++++ ...tion_multi_match_documentation_contract.py | 2 +- tests/test_repository_contracts.py | 9 + 21 files changed, 911 insertions(+), 14 deletions(-) create mode 100644 docs/adr/0059-reconciliation-match-command-evidence.md create mode 100644 src/accounting_information_platform/reconciliation_match.py create mode 100644 tests/test_reconciliation_match_api.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 04145b91..00b91267 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ## [Unreleased] +- Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. +- Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts are required, and the evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. - Added migration `0018_bank_statement_balance_evidence.sql` and exact normalized balance facts to the immutable camt.053 registry. Opening and closing amounts, currency, direction, sequence, locator, and source hash now survive persistence as forced-RLS evidence for a later exact reconciliation bridge; no balance row grants posting, reversal, approval, close, or accounting-policy authority. ADR 0057 records the decision. - Added migration `0019_reconciliation_run_command_evidence.sql` and the tenant-scoped `POST /reconciliation-runs` / `GET /reconciliation-runs?reconciliation_run_id=` boundary. A run opens only as `evaluating` from one persisted statement and active bank-account assignment; the command binds the raw artifact payload hash (distinct from the normalized statement hash), exact retries replay immutable command/source evidence, and changed key evidence fails closed. Distinct keys may create separately auditable runs for later policy or cutoff evaluation. This slice does not match, approve, close, post, or select chart accounts. ADR 0058 records the decision. - Reconciliation-run bank, book, and knowledge cutoffs now require an explicit zero-offset UTC timezone (`Z` or `+00:00`); timezone-naive and non-UTC values fail before persistence or command hashing, so an ambiguous wall clock cannot become authoritative evidence. diff --git a/README.md b/README.md index 12d17c9e..5eb20248 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,8 @@ What is present: - a stdlib HTTP surface for proposal acceptance, posting receipts, journals, reversals, period close/open, trial balances, financial statements, ledgers, aging, VAT/HomeTax rejection receipts, outbox, audit, catalog reads, and - tenant-scoped evaluating reconciliation-run creation/lookup; + tenant-scoped evaluating reconciliation-run and proposed reconciliation-match + creation/lookup; - product, architecture, security, and standards documents listed below. What is not present: an automatically started listener, gRPC or live event @@ -121,7 +122,7 @@ factory/runner and provide the tenant-bound host boundary explicitly. `unittest` discovery also runs `tests/test_postgres_posting.py`, which needs a reachable PostgreSQL 18 instance and `ACCOUNTING_DATABASE_URL` (CI uses `postgresql://postgres:postgres@127.0.0.1:5432/accounting_test` and applies -the checked-in migration chain through `database/migrations/0019_reconciliation_run_command_evidence.sql`). Persistence is still +the checked-in migration chain through `database/migrations/0020_reconciliation_match_command_evidence.sql`). Persistence is still local to this repository; it is not a Naruon or sibling checkout. Optional import smoke after the editable install above: @@ -174,6 +175,9 @@ in the PostgreSQL posting transaction; nothing in this tree publishes those events onto a live bus. The same boundary exposes `POST /journals` for AIS-owned adjustments and `GET /financial-statements`, `GET /trial-balances`, `GET /account-ledgers`, and the aging/reporting routes for buyer-facing reads. +The reconciliation boundary also exposes `POST /reconciliation-matches` and +`GET /reconciliation-matches?reconciliation_match_id=` for one exact 1:1 +proposed match; it records review evidence only and cannot approve or post. ## Standards already cited diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index dfefa3f6..46dd61bc 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,6 +36,7 @@ Metering and billing remain authoritative for usage, pricing, invoice intent, pa | `tax_interface` | VAT register and fail-closed HomeTax submission evidence; no NTS transport in this foundation | | `bank_statement_registry` | Immutable camt.053.001.14 statement/entry evidence, bank-account-to-book mapping, and host artifact locators | | `reconciliation_run_control` | Idempotent evaluating-run command identity over immutable statement evidence and active bank-account assignment; no matching, approval, close, or posting authority | +| `reconciliation_match_control` | Idempotent exact 1:1 proposed-match command evidence and allocations; no approval, close, chart-account, or posting authority | ## Persistence and migration order @@ -142,6 +143,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 17. `database/migrations/0017_reconciliation_approval_lock_order.sql` — repairs the approval-evidence trigger to acquire the parent match row before its snapshot advisory lock, closing the approval/allocation row-advisory deadlock cycle. 18. `database/migrations/0018_bank_statement_balance_evidence.sql` — preserves exact numeric camt.053 balance facts as immutable, tenant-scoped evidence for reconciliation bridge reads. 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. +20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. ## Durable soft-close command evidence diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index f11293cf..e569e823 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -40,7 +40,7 @@ The foundation ERD is maintained in [ERD.md](ERD.md). PostgreSQL migrations are ## Reconciliation control evidence -Migrations `0013_reconciliation_run_exception_evidence.sql` through `0019_reconciliation_run_command_evidence.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. +Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconciliation_match_command_evidence.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. - `reconciliation_run`: immutable evaluated scope for one tenant, legal entity, accounting book, bank-account assignment, ISO currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Only the run status may progress; evaluated scope changes require a new run. - `reconciliation_exception`: explicit operator-owned exception with an exception code, next action, effective/system time, and open/resolved/superseded resolution status. @@ -51,6 +51,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0019_reconc - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/ERD.md b/docs/ERD.md index 65645988..48dad56b 100644 --- a/docs/ERD.md +++ b/docs/ERD.md @@ -65,6 +65,7 @@ erDiagram reconciliation_run ||--o{ reconciliation_match : records reconciliation_match ||--o{ statement_match_allocation : consumes reconciliation_match ||--o{ journal_match_allocation : consumes + reconciliation_match ||--o| reconciliation_match_command : records reconciliation_match ||--o| reconciliation_approval : reviewed_by ``` @@ -78,7 +79,7 @@ erDiagram `bank_account_record` and `bank_account_assignment` map an opaque bank account onto one legal entity, book, and same-book cash chart account. The assignment composite foreign key requires that book to belong to the same legal entity. `bank_statement_record` and `bank_statement_entry` are append-only evidence. They retain `source_artifact_hash`, `normalized_payload_hash`, `ingestion_idempotency_key`, and `source_entry_hash` so a controller can prove which original artifact produced each entry without storing the raw XML in PostgreSQL. -`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. +`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_match_command` records the corresponding immutable proposed-match command identity and provenance over one exact 1:1 candidate/match/allocation chain; it is also evidence only. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. `reconciliation_candidate` records a deterministic statement/journal candidate and its exact source amounts; after INSERT it is append-only. `reconciliation_match` records the reviewable disposition. `statement_match_allocation` and `journal_match_allocation` preserve exact many-to-many consumption and are append-only regardless of later match status. Database-owned conservation guards serialize by immutable source identity and reject cross-run source-amount conflicts or over-consumption. Only an `approved` match consumes active source capacity; changing that match to `rejected` or `superseded` releases active capacity without deleting or rewriting the historical candidate/allocation evidence. These reconciliation relations provide audit and operator-control evidence only: they do not post, reverse, close, approve accounting policy, or mutate authoritative journals. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 8e7f8923..7839eda5 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -2,7 +2,7 @@ ## Deployment preconditions -Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0019_reconciliation_run_command_evidence.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. +Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0020_reconciliation_match_command_evidence.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. Required environment values are deployment-specific. At minimum, configure the accounting database URL and bind this AIS process to exactly one tenant reference. Secrets belong in an approved secret store; do not place database passwords, NTS credentials, bearer tokens or provider secrets in journal payloads, logs or outbox events. @@ -32,6 +32,7 @@ database/migrations/0016_reconciliation_approval_evidence.sql database/migrations/0017_reconciliation_approval_lock_order.sql database/migrations/0018_bank_statement_balance_evidence.sql database/migrations/0019_reconciliation_run_command_evidence.sql +database/migrations/0020_reconciliation_match_command_evidence.sql ``` Migration `0015_reconciliation_multi_match_conservation.sql` replaces the run-wide single-approved-match shortcut from `0014` with tenant/run-scoped match identity plus exact statement/journal allocation conservation. It permits multiple independently approved matches only when no authoritative source amount is over-consumed and grants no journal-posting authority. @@ -44,6 +45,8 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; the public run API opens only `evaluating` scope and does not match, approve, close, or post journals. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run and quoted positive equal decimal amounts. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. + Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. After installation, prove with the actual runtime login that supported reads and writes work for its tenant, another tenant is inaccessible, the login is not a migration owner / superuser / `BYPASSRLS`, and direct SQL cannot bypass journal immutability or period controls. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md new file mode 100644 index 00000000..98890f95 --- /dev/null +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -0,0 +1,54 @@ +# ADR 0059: Reconciliation match command evidence + +## Status + +Accepted in the current integration tree; protected-branch integration remains governed by the repository merge controls. + +## Context + +The reconciliation engine and migrations already define candidates, proposed +matches, exact statement/journal allocations, conservation guards, and later +human approval evidence. The system still lacked a public command boundary that +persisted one candidate and its source evidence atomically. A caller that wrote +those relations independently could leave a candidate without durable command +identity or make a retry ambiguous. + +## Decision + +Migration `0020_reconciliation_match_command_evidence.sql` adds the immutable, +forced-RLS `accounting_core.reconciliation_match_command` relation. It binds a +tenant, evaluating reconciliation run, candidate, and match to a tenant-scoped +candidate idempotency key, canonical command hash, source-payload hash, and +immutable object-storage reference. Composite foreign keys and uniqueness rules +prevent cross-scope evidence and duplicate command or match identities. + +`accept_reconciliation_match` is the smallest durable command boundary: it +accepts one exact 1:1 proposed match, requires quoted positive equal decimal +amounts, creates the candidate, proposed match, statement allocation, journal +allocation, and command evidence in one transaction, and grants no approval, +close, chart-account, reversal, or posting authority. Exact retries return the +stored document; reuse of the key with changed command or source evidence fails +closed. `POST /reconciliation-matches` exposes that command and +`GET /reconciliation-matches?reconciliation_match_id=` reads the tenant-scoped +document. + +The command intentionally does not replace the existing pure split/aggregate +allocation planner or the database approval workflow. It provides a bounded +buyer-facing persistence seam for the common 1:1 case; many-to-many planning, +human approval, and any accounting adjustment remain separate evidence and +authority boundaries. + +## Consequences + +Proposed reconciliation evidence now has a durable retry identity and an +atomic candidate-to-allocation provenance chain. Legacy candidates or matches +without command evidence are not synthesized by this read boundary. A proposed +match remains non-authoritative: only the existing approval controls can record +a human decision, and any adjustment must re-enter the authoritative journal +command boundary. + +## References + +See `docs/DATA_MODEL.md`, `docs/OPERABILITY.md`, and +`docs/doctoring/STANDARD_TRACEABILITY.md` for the relational, operational, and +standards traceability contracts. diff --git a/docs/doctoring/IMPLEMENTATION_SEQUENCE.md b/docs/doctoring/IMPLEMENTATION_SEQUENCE.md index 7f0c548d..0e1c8e14 100644 --- a/docs/doctoring/IMPLEMENTATION_SEQUENCE.md +++ b/docs/doctoring/IMPLEMENTATION_SEQUENCE.md @@ -32,6 +32,12 @@ This is review evidence only; it never becomes journal-posting authority. Any approved adjusting-journal proposal must re-enter the existing accounting command boundary with its own idempotency key, immutable source evidence, open-period checks, policy resolution, purpose-bound authorization, and authoritative posting receipt. +The current bounded persistence seam exposes `POST /reconciliation-matches` and +`GET /reconciliation-matches?reconciliation_match_id=` for one exact 1:1 +proposed match. It persists candidate, match, statement allocation, journal +allocation, and command provenance atomically; it remains review evidence only. +The existing many-to-many planner and human approval controls stay separate. + ## Later commercial increments After deterministic reconciliation is integrated, continue with purpose-bound accounting authorization, multi-currency/FX, consolidation and intercompany, fixed assets, revenue-recognition support, tax interfaces, management reporting, and operational/recovery hardening. Each increment must preserve tenant isolation, database-owned invariants where feasible, exact decimal arithmetic, append-only accounting facts, and source-to-receipt provenance. diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index cc50ebb8..428935ee 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,6 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP reconciliation-run regressions, ADR 0058 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 61f68d88..9f1b22fd 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -33,8 +33,9 @@ persistence and database-owned invariants, a bounded stdlib HTTP surface, versio JSON contracts, and a durable outbox. The accounting posting foundation and the immutable `camt.053.001.14` bank-statement evidence registry are integrated protected `develop` facts. The current integration tree also includes database-owned -candidate/match/allocation conservation and durable human approval snapshot controls -from migrations `0015` through `0017`; protected `develop` remains the release authority + candidate/match/allocation conservation, durable human approval snapshot controls, + and the proposed-match command boundary from migrations `0015` through `0020`; + protected `develop` remains the release authority until this exact integration candidate passes its live gates. It does not transmit HomeTax/NTS filings, enforce purpose-bound application authorization, or provide a controller UI. Those omissions are explicit product scope, not implied successes. @@ -89,8 +90,9 @@ that renumbering cannot silently drop a commitment. durable `reconciliation_run`/`reconciliation_exception`/ `reconciliation_evidence` rows with forced tenant RLS and the immutable evaluated-run scope guard. Delivered in the current integration tree: - candidate/match persistence, many-to-many exact allocation conservation, and - database-owned approval snapshot binding. Candidate: close-package provenance + candidate/match persistence, many-to-many exact allocation conservation, database-owned + approval snapshot binding, and the exact 1:1 proposed-match command/HTTP boundary. + Candidate: close-package provenance with complete approved match evidence, exact scope, and immutable run cutoff in PR #32; close-package provenance remains open until this candidate is integrated on the protected branch and revalidated. @@ -179,6 +181,7 @@ immutable bank statement artifact → durable run / exception / evidence rows [delivered; forced RLS + run-scope guard + immutable run-command idempotency/source evidence in current tree] → close-review projection [delivered; evidence eligibility only, same-scope deltas] → candidate/match allocation conservation [delivered in current tree; migration 0015] +→ proposed-match command evidence [delivered in current candidate; migration 0020; proposed only] → reconciliation approval and close package [approval snapshot delivered; structured close-package candidate in PR #32] ``` diff --git a/requirements-quality.txt b/requirements-quality.txt index 5d8f8766..9d38b3c5 100644 --- a/requirements-quality.txt +++ b/requirements-quality.txt @@ -1,6 +1,6 @@ # Hash-locked quality and packaging tools for --require-hashes --only-binary=:all:. # coverage 7.15.4: every CPython 3.13 wheel plus py3-none-any. -# psycopg 3.3.4 plus every CPython 3.13 psycopg-binary wheel. +# psycopg 3.3.4 plus every CPython 3.13 and 3.14 psycopg-binary wheel. coverage==7.15.4 \ --hash=sha256:c705b28feb2775dc82a25f1d473a370bc37ff93f5177f4e29ce2425f560f6921 \ --hash=sha256:3ff205ab5e3ecc670f6a4dd19d9cbf12ede53dd41cfc1e15716ec961ea6d314e \ @@ -37,4 +37,5 @@ psycopg-binary==3.3.4 \ --hash=sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf \ --hash=sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260 \ --hash=sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf \ - --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 + --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 \ + --hash=sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277 diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index 274a32c1..cac2c726 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -30,6 +30,7 @@ "src/accounting_information_platform/ingest.py", "src/accounting_information_platform/persistence.py", "src/accounting_information_platform/reconciliation_run.py", + "src/accounting_information_platform/reconciliation_match.py", "src/accounting_information_platform/py.typed", "schemas/accounting-journal-proposal.schema.json", "schemas/accounting-posting-receipt.schema.json", @@ -53,6 +54,7 @@ "database/migrations/0017_reconciliation_approval_lock_order.sql", "database/migrations/0018_bank_statement_balance_evidence.sql", "database/migrations/0019_reconciliation_run_command_evidence.sql", + "database/migrations/0020_reconciliation_match_command_evidence.sql", "docs/PRD.md", "docs/TRD.md", "docs/ARCHITECTURE.md", @@ -116,6 +118,7 @@ "docs/adr/0052-bank-statement-evidence-registry.md", "docs/adr/0057-bank-statement-balance-evidence.md", "docs/adr/0058-reconciliation-run-command-evidence.md", + "docs/adr/0059-reconciliation-match-command-evidence.md", "docs/doctoring/REFERENCES.md", "docs/doctoring/STANDARD_TRACEABILITY.md", "docs/superpowers/specs/2026-08-16-accounting-information-platform-design.md", @@ -137,6 +140,9 @@ COVERAGE_CP313_MANYLINUX_X86_64_WHEEL_HASH = ( "12b59c90084e3234fb11184886bf4a40f4f16a8c8f867be2e087b81f8e8868d4" ) +PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH = ( + "8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277" +) SCHEMA_NAME_PATTERN = re.compile( r"\bCREATE\s+SCHEMA(?:\s+IF\s+NOT\s+EXISTS)?\s+([A-Za-z_][A-Za-z0-9_]*)", re.IGNORECASE, diff --git a/src/accounting_information_platform/__init__.py b/src/accounting_information_platform/__init__.py index 797e111a..4eddf460 100644 --- a/src/accounting_information_platform/__init__.py +++ b/src/accounting_information_platform/__init__.py @@ -83,6 +83,10 @@ verify_reconciliation_close_package, ) from .reconciliation_run import accept_reconciliation_run, lookup_reconciliation_run +from .reconciliation_match import ( + accept_reconciliation_match, + lookup_reconciliation_match, +) __all__ = [ "AccountBalance", @@ -115,6 +119,7 @@ "accept_period_close", "accept_period_open", "accept_reconciliation_run", + "accept_reconciliation_match", "accept_pulled_proposals", "apply_foundation_migration", "build_reconciliation_close_package", @@ -146,6 +151,7 @@ "lookup_posted_journal", "lookup_published_receipt", "lookup_reconciliation_run", + "lookup_reconciliation_match", "lookup_receivable_aging", "lookup_trial_balance", "lookup_home_tax_submissions", diff --git a/src/accounting_information_platform/http_api.py b/src/accounting_information_platform/http_api.py index d389a0c4..0440c7f6 100644 --- a/src/accounting_information_platform/http_api.py +++ b/src/accounting_information_platform/http_api.py @@ -54,6 +54,7 @@ from .billing_pull import accept_billing_proposal_pull from .core import AccountingValidationError, IdempotencyConflictError, _require_reference from .reconciliation_run import accept_reconciliation_run, lookup_reconciliation_run +from .reconciliation_match import accept_reconciliation_match, lookup_reconciliation_match TENANT_HEADER = "X-CWL-Tenant-Reference" @@ -81,6 +82,7 @@ BANK_STATEMENT_PATH = "/bank-statements" BANK_STATEMENT_ENTRY_PATH = "/bank-statement-entries" RECONCILIATION_RUN_PATH = "/reconciliation-runs" +RECONCILIATION_MATCH_PATH = "/reconciliation-matches" RECEIVABLE_AGING_PATH = "/receivable-agings" PAYABLE_AGING_PATH = "/payable-agings" PERIOD_CLOSE_PACKAGE_PATH = "/period-close-packages" @@ -205,6 +207,9 @@ def do_GET(self) -> None: if parsed.path == RECONCILIATION_RUN_PATH: self._get_reconciliation_run(parsed.query) return + if parsed.path == RECONCILIATION_MATCH_PATH: + self._get_reconciliation_match(parsed.query) + return if parsed.path == RECEIVABLE_AGING_PATH: self._get_receivable_aging(parsed.query) return @@ -385,6 +390,9 @@ def do_POST(self) -> None: if parsed_path == RECONCILIATION_RUN_PATH: self._post_reconciliation_run(raw_body) return + if parsed_path == RECONCILIATION_MATCH_PATH: + self._post_reconciliation_match(raw_body) + return if parsed_path == BANK_STATEMENT_ENTRY_PATH: self._write_error( 405, @@ -419,7 +427,7 @@ def do_POST(self) -> None: "unknown path. POST /journal-proposals, POST /journals, POST /journal-reversals, " "POST /billing-proposal-pulls, POST /period-closes, POST /home-tax-submissions, " "POST /bank-accounts, POST /bank-account-assignments, POST /bank-statements, " - "POST /reconciliation-runs, POST /fiscal-periods, " + "POST /reconciliation-runs, POST /reconciliation-matches, POST /fiscal-periods, " "or POST /outbox-events/{outbox_event_id}/publish, then retry.", ) @@ -1679,6 +1687,55 @@ def _get_reconciliation_run(self, query: str) -> None: return self._write_json(200, document) + def _post_reconciliation_match(self, raw_body: bytes) -> None: + tenant_header = self._bound_tenant_header("reconciliation-match") + if tenant_header is None: + return + payload = self._read_json_object(raw_body, "a reconciliation-match command") + if payload is None: + return + if payload.get("tenant_reference") != tenant_header: + self._write_error( + 403, + "reconciliation-match tenant_reference does not match X-CWL-Tenant-Reference. " + "Send the match to that tenant's AIS endpoint, then retry.", + ) + return + try: + document = accept_reconciliation_match( + payload, self.server.database_url, tenant_header + ) + except IdempotencyConflictError as error: + self._write_error(409, str(error)) + return + except AccountingValidationError as error: + self._write_error(_reconciliation_match_status(error), str(error)) + return + self._write_json(200, document) + + def _get_reconciliation_match(self, query: str) -> None: + tenant_header = self._bound_tenant_header("reconciliation-match read") + if tenant_header is None: + return + match_id = _first_query( + parse_qs(query, keep_blank_values=True), "reconciliation_match_id" + ) + if not match_id: + self._write_error( + 400, + "reconciliation_match_id is required. " + "Supply that query key, then retry the reconciliation-match read.", + ) + return + try: + document = lookup_reconciliation_match( + self.server.database_url, tenant_header, match_id + ) + except AccountingValidationError as error: + self._write_error(_reconciliation_match_status(error), str(error)) + return + self._write_json(200, document) + def _post_billing_proposal_pull(self, raw_body: bytes) -> None: tenant_header = self._bound_tenant_header("pull") if tenant_header is None: @@ -1826,6 +1883,16 @@ def _reconciliation_run_status(error: AccountingValidationError) -> int: return 422 +def _reconciliation_match_status(error: AccountingValidationError) -> int: + """Map proposed-match validation to stable HTTP boundary statuses.""" + message = str(error) + if "is not recorded" in message: + return 404 + if "must be a UUID" in message or "is required" in message: + return 400 + return 422 + + def _query_validation_status(error: AccountingValidationError) -> int: if "UTC offset" in str(error): return 422 diff --git a/src/accounting_information_platform/persistence.py b/src/accounting_information_platform/persistence.py index 27cb0cfb..23762d49 100644 --- a/src/accounting_information_platform/persistence.py +++ b/src/accounting_information_platform/persistence.py @@ -6340,6 +6340,15 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: f"{run_command_migration_path}. Restore " "database/migrations/0019_reconciliation_run_command_evidence.sql, then retry." ) + match_command_migration_path = ( + migration_path.parent / "0020_reconciliation_match_command_evidence.sql" + ) + if not match_command_migration_path.is_file(): + raise AccountingValidationError( + "Reconciliation match-command evidence migration is missing at " + f"{match_command_migration_path}. Restore " + "database/migrations/0020_reconciliation_match_command_evidence.sql, then retry." + ) psycopg = _import_psycopg() try: with psycopg.connect( @@ -6378,6 +6387,9 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: connection.execute( run_command_migration_path.read_text(encoding="utf-8") ) + connection.execute( + match_command_migration_path.read_text(encoding="utf-8") + ) except Exception as error: raise AccountingValidationError( "Foundation migration failed. Inspect the PostgreSQL error, restore a clean " diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py new file mode 100644 index 00000000..9a5e4a05 --- /dev/null +++ b/src/accounting_information_platform/reconciliation_match.py @@ -0,0 +1,387 @@ +"""Persist and read proposed reconciliation matches without granting approval authority.""" + +from __future__ import annotations + +import hashlib +import json +from decimal import Decimal +from typing import Mapping +from uuid import UUID + +from .core import ( + AccountingValidationError, + IdempotencyConflictError, + _HASH_PATTERN, + _parse_amount, +) +from .persistence import PostgresPostingLedger, _exact_amount_text +from .reconciliation_run import _parse_uuid + + +def accept_reconciliation_match( + payload: object, database_url: str, tenant_reference: str +) -> dict[str, object]: + """Persist one exact 1:1 proposed match for an evaluating reconciliation run.""" + command = _require_command(payload, tenant_reference) + run_id = _parse_uuid( + str(command.get("reconciliation_run_id") or ""), + "reconciliation_run_id", + ) + statement_reference = _require_text( + command.get("statement_entry_reference"), "statement_entry_reference" + ) + journal_reference = _require_text(command.get("journal_reference"), "journal_reference") + rule_code = _require_text(command.get("rule_code"), "rule_code") + idempotency_key = _require_text( + command.get("candidate_idempotency_key"), "candidate_idempotency_key" + ) + source_payload_hash = _require_hash(command.get("source_payload_hash")) + source_payload_reference = _require_text( + command.get("source_payload_reference"), "source_payload_reference" + ) + statement_amount = _require_positive_amount( + command.get("statement_amount"), "statement_amount" + ) + journal_amount = _require_positive_amount(command.get("journal_amount"), "journal_amount") + if statement_amount != journal_amount: + raise AccountingValidationError( + "statement_amount and journal_amount must be equal for a 1:1 proposed match. " + "Supply exact equal source amounts, then retry the match." + ) + command_hash = _command_hash( + tenant_reference=tenant_reference, + reconciliation_run_id=run_id, + statement_entry_reference=statement_reference, + journal_reference=journal_reference, + statement_amount=statement_amount, + journal_amount=journal_amount, + rule_code=rule_code, + idempotency_key=idempotency_key, + source_payload_hash=source_payload_hash, + source_payload_reference=source_payload_reference, + ) + + ledger = PostgresPostingLedger(database_url, tenant_reference) + with ledger._session() as connection: + tenant_id = ledger._require_tenant(connection) + ledger._acquire_command_lock( + connection, f"reconciliation_match_command:{idempotency_key}" + ) + prior = connection.execute( + """ + SELECT reconciliation_candidate_id, reconciliation_match_id, + candidate_command_hash, source_payload_hash + FROM accounting_core.reconciliation_match_command + WHERE tenant_account_id = %s + AND candidate_idempotency_key = %s + """, + (tenant_id, idempotency_key), + ).fetchone() + if prior is not None: + if prior[2] != command_hash or prior[3] != source_payload_hash: + raise IdempotencyConflictError( + "candidate idempotency key was already used with different match evidence. " + "Supply a new candidate_idempotency_key, then retry the match." + ) + return _load_reconciliation_match_document( + connection, + tenant_id, + tenant_reference, + prior[1], + replayed=True, + ) + + run = connection.execute( + """ + SELECT run_status_code + FROM accounting_core.reconciliation_run + WHERE tenant_account_id = %s + AND reconciliation_run_id = %s + """, + (tenant_id, run_id), + ).fetchone() + if run is None: + raise AccountingValidationError( + "reconciliation run is not recorded for this tenant. " + "Supply an evaluating reconciliation_run_id, then retry the match." + ) + if run[0] != "evaluating": + raise AccountingValidationError( + "reconciliation matches can only be proposed on an evaluating run. " + "Open a new evaluating reconciliation run, then retry the match." + ) + + ledger._acquire_command_lock( + connection, + f"reconciliation_candidate:{run_id}:{statement_reference}:{journal_reference}", + ) + existing = connection.execute( + """ + SELECT reconciliation_candidate_id + FROM accounting_core.reconciliation_candidate + WHERE tenant_account_id = %s + AND reconciliation_run_id = %s + AND statement_entry_reference = %s + AND journal_reference = %s + """, + (tenant_id, run_id, statement_reference, journal_reference), + ).fetchone() + if existing is not None: + raise AccountingValidationError( + "the reconciliation candidate is already recorded for this run and source pair. " + "Use the existing proposed match or a new source pair, then retry." + ) + + candidate_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + run_id, + statement_reference, + journal_reference, + statement_amount, + journal_amount, + rule_code, + ), + ).fetchone()[0] + match_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (tenant_id, run_id, candidate_id), + ).fetchone()[0] + connection.execute( + """ + INSERT INTO accounting_core.statement_match_allocation ( + tenant_account_id, reconciliation_run_id, reconciliation_match_id, + statement_entry_reference, allocated_amount + ) + VALUES (%s, %s, %s, %s, %s) + """, + (tenant_id, run_id, match_id, statement_reference, statement_amount), + ) + connection.execute( + """ + INSERT INTO accounting_core.journal_match_allocation ( + tenant_account_id, reconciliation_run_id, reconciliation_match_id, + journal_reference, allocated_amount + ) + VALUES (%s, %s, %s, %s, %s) + """, + (tenant_id, run_id, match_id, journal_reference, journal_amount), + ) + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + tenant_id, + run_id, + candidate_id, + match_id, + idempotency_key, + command_hash, + source_payload_hash, + source_payload_reference, + ), + ) + return _load_reconciliation_match_document( + connection, + tenant_id, + tenant_reference, + match_id, + replayed=False, + ) + + +def lookup_reconciliation_match( + database_url: str, tenant_reference: str, reconciliation_match_id: str +) -> dict[str, object]: + """Read one tenant-scoped proposed reconciliation match and its exact allocation.""" + match_id = _parse_uuid(reconciliation_match_id, "reconciliation_match_id") + ledger = PostgresPostingLedger(database_url, tenant_reference) + with ledger._session() as connection: + tenant_id = ledger._require_tenant(connection) + return _load_reconciliation_match_document( + connection, + tenant_id, + tenant_reference, + match_id, + replayed=False, + ) + + +def _load_reconciliation_match_document( + connection: object, + tenant_id: UUID, + tenant_reference: str, + match_id: UUID, + *, + replayed: bool, +) -> dict[str, object]: + """Load one command-backed match with candidate, allocation, and provenance facts.""" + row = connection.execute( + """ + SELECT candidate.reconciliation_candidate_id, + match.reconciliation_match_id, + match.reconciliation_run_id, + candidate.statement_entry_reference, + candidate.journal_reference, + candidate.statement_amount, + candidate.journal_amount, + candidate.rule_code, + match.match_status_code, + command.candidate_idempotency_key, + command.candidate_command_hash, + command.source_payload_hash, + command.source_payload_reference, + statement_allocation.allocated_amount + FROM accounting_core.reconciliation_match AS match + JOIN accounting_core.reconciliation_candidate AS candidate + ON candidate.tenant_account_id = match.tenant_account_id + AND candidate.reconciliation_run_id = match.reconciliation_run_id + AND candidate.reconciliation_candidate_id = match.reconciliation_candidate_id + JOIN accounting_core.reconciliation_match_command AS command + ON command.tenant_account_id = match.tenant_account_id + AND command.reconciliation_run_id = match.reconciliation_run_id + AND command.reconciliation_match_id = match.reconciliation_match_id + JOIN accounting_core.statement_match_allocation AS statement_allocation + ON statement_allocation.tenant_account_id = match.tenant_account_id + AND statement_allocation.reconciliation_run_id = match.reconciliation_run_id + AND statement_allocation.reconciliation_match_id = match.reconciliation_match_id + WHERE match.tenant_account_id = %s + AND match.reconciliation_match_id = %s + """, + (tenant_id, match_id), + ).fetchone() + if row is None: + raise AccountingValidationError( + "reconciliation match is not recorded for this tenant. " + "Supply a persisted reconciliation_match_id, then retry the match read." + ) + return { + "tenant_reference": tenant_reference, + "reconciliation_candidate_id": str(row[0]), + "reconciliation_match_id": str(row[1]), + "reconciliation_run_id": str(row[2]), + "statement_entry_reference": row[3], + "journal_reference": row[4], + "statement_amount": _display_amount(row[5]), + "journal_amount": _display_amount(row[6]), + "allocated_amount": _display_amount(row[13]), + "rule_code": row[7], + "match_status_code": row[8], + "candidate_idempotency_key": row[9], + "candidate_command_hash": row[10], + "source_payload_hash": row[11], + "source_payload_reference": row[12], + "replayed": replayed, + } + + +def _require_command(payload: object, tenant_reference: str) -> Mapping[str, object]: + """Require a tenant-bound mapping for a proposed match command.""" + if not isinstance(payload, Mapping): + raise AccountingValidationError( + "reconciliation match payload must be a JSON object. " + "Supply a proposed reconciliation match, then retry." + ) + if payload.get("tenant_reference") != tenant_reference: + raise AccountingValidationError( + "reconciliation match tenant_reference does not match the bound tenant. " + "Send the match to that tenant's AIS endpoint, then retry." + ) + return payload + + +def _require_text(value: object, field_name: str) -> str: + """Require one canonical non-empty command text field.""" + if not isinstance(value, str) or not value or value.strip() != value: + raise AccountingValidationError( + f"{field_name} is required and must be a canonical non-empty string. " + f"Supply {field_name}, then retry the match." + ) + return value + + +def _require_hash(value: object) -> str: + """Require one immutable source-payload SHA-256 digest.""" + if not isinstance(value, str) or _HASH_PATTERN.fullmatch(value) is None: + raise AccountingValidationError( + "source_payload_hash must be a canonical sha256 digest. " + "Supply the immutable match-evidence hash, then retry the match." + ) + return value + + +def _require_positive_amount(value: object, field_name: str) -> Decimal: + """Require a quoted positive exact decimal for persisted match evidence.""" + if not isinstance(value, str): + raise AccountingValidationError( + f"{field_name} must be a quoted exact decimal string. " + f"Supply {field_name} as a decimal string, then retry the match." + ) + amount = _parse_amount(value) + if amount <= 0: + raise AccountingValidationError( + f"{field_name} must be greater than zero. Supply a positive exact amount, then retry the match." + ) + return amount + + +def _display_amount(value: Decimal) -> str: + """Render a database numeric as the canonical decimal string used by reads.""" + text = _exact_amount_text(value) + return text.rstrip("0").rstrip(".") if "." in text else text + + +def _command_hash( + *, + tenant_reference: str, + reconciliation_run_id: UUID, + statement_entry_reference: str, + journal_reference: str, + statement_amount: Decimal, + journal_amount: Decimal, + rule_code: str, + idempotency_key: str, + source_payload_hash: str, + source_payload_reference: str, +) -> str: + """Return the canonical hash for one proposed match command.""" + payload = { + "candidate_idempotency_key": idempotency_key, + "journal_amount": _exact_amount_text(journal_amount), + "journal_reference": journal_reference, + "reconciliation_run_id": str(reconciliation_run_id), + "rule_code": rule_code, + "source_payload_hash": source_payload_hash, + "source_payload_reference": source_payload_reference, + "statement_amount": _exact_amount_text(statement_amount), + "statement_entry_reference": statement_entry_reference, + "tenant_reference": tenant_reference, + } + serialized = json.dumps(payload, separators=(",", ":"), sort_keys=True) + return "sha256:" + hashlib.sha256(serialized.encode("utf-8")).hexdigest() + + +__all__ = ["accept_reconciliation_match", "lookup_reconciliation_match"] diff --git a/tests/test_foundation_install_manifest_contract.py b/tests/test_foundation_install_manifest_contract.py index 1b60e5b8..c5849d0a 100644 --- a/tests/test_foundation_install_manifest_contract.py +++ b/tests/test_foundation_install_manifest_contract.py @@ -147,6 +147,18 @@ def test_required_files_and_install_docs_include_run_command_evidence(self) -> N self.assertIn(migration_nineteen, text) self.assertLess(text.index(migration_eighteen), text.index(migration_nineteen)) + def test_required_files_and_install_docs_include_match_command_evidence(self) -> None: + """Proposed-match command evidence follows the immutable run command chain.""" + migration_nineteen = "database/migrations/0019_reconciliation_run_command_evidence.sql" + migration_twenty = "database/migrations/0020_reconciliation_match_command_evidence.sql" + self.assertIn(migration_twenty, set(REQUIRED_FILES)) + for relative_path in ("docs/OPERABILITY.md", "docs/ARCHITECTURE.md"): + with self.subTest(relative_path=relative_path): + text = (ROOT / relative_path).read_text(encoding="utf-8") + self.assertIn(migration_nineteen, text) + self.assertIn(migration_twenty, text) + self.assertLess(text.index(migration_nineteen), text.index(migration_twenty)) + def test_install_fails_closed_when_approval_snapshot_migration_is_missing(self) -> None: """The canonical loader may not silently stop before database-owned approval evidence.""" original_is_file = Path.is_file @@ -211,6 +223,22 @@ def is_file(path: Path) -> bool: ROOT / "database/migrations/0001_accounting_foundation.sql", ) + def test_install_fails_closed_when_match_command_migration_is_missing(self) -> None: + """The canonical loader may not persist proposed matches without command evidence.""" + original_is_file = Path.is_file + + def is_file(path: Path) -> bool: + if path.name == "0020_reconciliation_match_command_evidence.sql": + return False + return original_is_file(path) + + with patch.object(Path, "is_file", is_file): + with self.assertRaises(AccountingValidationError): + apply_foundation_migration( + "postgresql://unused", + ROOT / "database/migrations/0001_accounting_foundation.sql", + ) + def test_canonical_persistence_loader_fails_closed_when_conservation_is_missing(self) -> None: """Real PostgreSQL fixtures may not silently stop the authoritative chain at 0014.""" from accounting_information_platform.persistence import ( diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index a5cd893e..6079d56c 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -9,7 +9,7 @@ class ReconciliationDataModelDocumentationContractTests(unittest.TestCase): - """Keep diligence-facing data-model documents aligned with migrations 0013-0015.""" + """Keep diligence-facing data-model documents aligned with reconciliation migrations.""" def test_data_model_names_durable_reconciliation_relations(self) -> None: """The normalized reconciliation run/match/allocation facts must be explicit.""" @@ -21,6 +21,7 @@ def test_data_model_names_durable_reconciliation_relations(self) -> None: "`reconciliation_match`", "`statement_match_allocation`", "`journal_match_allocation`", + "`reconciliation_match_command`", ): with self.subTest(relation=relation): self.assertIn(relation, text) diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py new file mode 100644 index 00000000..f9823f58 --- /dev/null +++ b/tests/test_reconciliation_match_api.py @@ -0,0 +1,303 @@ +"""RED/GREEN contracts for the proposed reconciliation-match command API.""" + +from __future__ import annotations + +import hashlib +import unittest +import uuid + +import psycopg + +from accounting_information_platform import ( + CAMT053_MESSAGE_DEFINITION, + AccountingValidationError, + IdempotencyConflictError, + MemoryArtifactStore, + accept_bank_account_assignment, + accept_bank_account_record, + accept_bank_statement_evidence, + accept_reconciliation_match, + accept_reconciliation_run, + load_canonical_statement_fixture, + lookup_bank_statement_entries, + lookup_reconciliation_match, +) +from tests import test_postgres_posting as posting + + +class ReconciliationMatchApiTests(unittest.TestCase): + """Prove one exact 1:1 match is durable evidence, not approval or posting.""" + + @classmethod + def setUpClass(cls) -> None: + posting.PostgresPostingTests.setUpClass() + + def setUp(self) -> None: + self.case = posting.PostgresPostingTests("setUp") + self.case.setUp() + self.addCleanup(self.case.doCleanups) + self.addCleanup(self.case.tearDown) + self.store = MemoryArtifactStore() + self.account_reference = f"urn:cwl:bank_account:{uuid.uuid4().hex}" + accept_bank_account_record( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": self.account_reference, + "account_currency_code": "KRW", + "account_identifier": "acct-opaque-fixture-only", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + accept_bank_account_assignment( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": self.account_reference, + "legal_entity_reference": self.case.policy.legal_entity_reference, + "accounting_book_reference": self.case.policy.accounting_book_reference, + "chart_account_code": "110200", + "valid_from": "2026-01-01T00:00:00Z", + "assignment_idempotency_key": f"assign-match-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + + def _open_run(self) -> tuple[str, str]: + fixture = load_canonical_statement_fixture() + statement = accept_bank_statement_evidence( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": self.account_reference, + "message_definition_identifier": CAMT053_MESSAGE_DEFINITION, + "statement_payload": fixture.decode("utf-8"), + "ingestion_idempotency_key": f"statement-match-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + artifact_store=self.store, + ) + entries = lookup_bank_statement_entries( + posting.DATABASE_URL, + self.case.policy.tenant_reference, + str(statement["bank_statement_record_id"]), + )["bank_statement_entries"] + entry = entries[0] + statement_hash = "sha256:" + hashlib.sha256(fixture).hexdigest() + run = accept_reconciliation_run( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_statement_record_id": statement["bank_statement_record_id"], + "legal_entity_reference": self.case.policy.legal_entity_reference, + "accounting_book_reference": self.case.policy.accounting_book_reference, + "bank_cutoff_at": "2026-08-24T23:59:59Z", + "book_cutoff_at": "2026-08-24T23:59:59Z", + "matching_policy_version": "deterministic-v1", + "knowledge_cutoff_at": "2026-08-25T00:00:00Z", + "reconciliation_idempotency_key": f"run-match-{uuid.uuid4().hex}", + "source_payload_hash": statement_hash, + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + return str(run["reconciliation_run_id"]), str( + entry["source_entry_identity"] or entry["bank_statement_entry_id"] + ) + + def _command(self) -> dict[str, object]: + run_id, statement_reference = self._open_run() + return { + "tenant_reference": self.case.policy.tenant_reference, + "reconciliation_run_id": run_id, + "statement_entry_reference": statement_reference, + "journal_reference": "journal-match-fixture", + "statement_amount": "25000.00", + "journal_amount": "25000.00", + "rule_code": "provider_reference", + "candidate_idempotency_key": f"candidate-{uuid.uuid4().hex}", + "source_payload_hash": "sha256:" + "1" * 64, + "source_payload_reference": "urn:cwl:object:match-evidence", + } + + def test_proposed_match_is_persisted_and_replayed(self) -> None: + """An exact retry returns the same proposed match without approval authority.""" + command = self._command() + first = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + replay = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + loaded = lookup_reconciliation_match( + posting.DATABASE_URL, + self.case.policy.tenant_reference, + str(first["reconciliation_match_id"]), + ) + self.assertEqual(first["match_status_code"], "proposed") + self.assertFalse(first["replayed"]) + self.assertTrue(replay["replayed"]) + self.assertEqual(first["reconciliation_match_id"], replay["reconciliation_match_id"]) + self.assertEqual(loaded["reconciliation_candidate_id"], first["reconciliation_candidate_id"]) + self.assertEqual(loaded["allocated_amount"], "25000") + + def test_same_key_changed_source_fails_closed(self) -> None: + """A candidate key cannot be reused for changed immutable evidence.""" + command = self._command() + accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + changed = dict(command, source_payload_hash="sha256:" + "2" * 64) + with self.assertRaises(IdempotencyConflictError): + accept_reconciliation_match( + changed, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + + def test_match_command_evidence_is_immutable(self) -> None: + """The database trigger prevents mutation of recorded match command evidence.""" + command = self._command() + document = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + with psycopg.connect(posting.DATABASE_URL) as connection: + with self.assertRaises(psycopg.Error): + connection.execute( + """ + UPDATE accounting_core.reconciliation_match_command + SET source_payload_reference = 'urn:cwl:object:tampered' + WHERE reconciliation_match_id = %s + """, + (document["reconciliation_match_id"],), + ) + + def test_match_command_rejects_non_exact_or_unbalanced_amounts(self) -> None: + """The command rejects JSON numbers and non-conserving 1:1 evidence.""" + command = self._command() + for changed in ( + dict(command, statement_amount=25000.0), + dict(command, journal_amount="24999.99"), + ): + with self.subTest(changed=changed): + with self.assertRaisesRegex(AccountingValidationError, "amount|equal"): + accept_reconciliation_match( + changed, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + + def test_match_command_validation_and_run_lifecycle_fail_closed(self) -> None: + """Malformed, missing-run, and non-evaluating commands write no evidence.""" + command = self._command() + tenant = self.case.policy.tenant_reference + invalid_commands = ( + ([], "payload"), + (dict(command, tenant_reference="urn:cwl:tenant:other"), "tenant_reference"), + (dict(command, statement_entry_reference=""), "statement_entry_reference"), + (dict(command, source_payload_hash="not-a-hash"), "source_payload_hash"), + (dict(command, source_payload_reference=" evidence"), "source_payload_reference"), + (dict(command, statement_amount="0"), "greater than zero"), + ) + for invalid, message in invalid_commands: + with self.subTest(message=message): + with self.assertRaisesRegex(AccountingValidationError, message): + accept_reconciliation_match(invalid, posting.DATABASE_URL, tenant) + + missing_run = dict( + command, + reconciliation_run_id=str(uuid.uuid4()), + candidate_idempotency_key=f"missing-run-{uuid.uuid4().hex}", + ) + with self.assertRaisesRegex(AccountingValidationError, "not recorded"): + accept_reconciliation_match(missing_run, posting.DATABASE_URL, tenant) + + accept_reconciliation_match(command, posting.DATABASE_URL, tenant) + duplicate_source = dict( + command, + candidate_idempotency_key=f"duplicate-source-{uuid.uuid4().hex}", + ) + with self.assertRaisesRegex(AccountingValidationError, "already recorded"): + accept_reconciliation_match(duplicate_source, posting.DATABASE_URL, tenant) + + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + connection.execute( + """ + UPDATE accounting_core.reconciliation_run + SET run_status_code = 'review_required' + WHERE reconciliation_run_id = %s + """, + (command["reconciliation_run_id"],), + ) + non_evaluating = dict( + command, + candidate_idempotency_key=f"non-evaluating-{uuid.uuid4().hex}", + ) + with self.assertRaisesRegex(AccountingValidationError, "evaluating"): + accept_reconciliation_match(non_evaluating, posting.DATABASE_URL, tenant) + + def test_http_routes_persist_and_read_the_proposed_match(self) -> None: + """HTTP exposes the proposed match while preserving tenant and identity gates.""" + command = self._command() + server = self.case._start_http_server() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + status, created = self.case._http_json( + "POST", "/reconciliation-matches", command + ) + read_status, read = self.case._http_json( + "GET", + f"/reconciliation-matches?reconciliation_match_id={created['reconciliation_match_id']}", + None, + ) + conflict_status, _conflict = self.case._http_json( + "POST", + "/reconciliation-matches", + dict(command, source_payload_hash="sha256:" + "3" * 64), + ) + wrong_status, _wrong = self.case._http_json( + "POST", + "/reconciliation-matches", + dict(command, tenant_reference="urn:cwl:tenant:other"), + ) + missing_header_status, _missing_header = self.case._http_json( + "POST", "/reconciliation-matches", command, tenant_header=None + ) + invalid_body_status, _invalid_body = self.case._http_raw( + "POST", "/reconciliation-matches", b"[]", self.case.policy.tenant_reference + ) + invalid_command_status, _invalid_command = self.case._http_json( + "POST", + "/reconciliation-matches", + dict( + command, + candidate_idempotency_key=f"http-invalid-{uuid.uuid4().hex}", + statement_amount="0", + ), + ) + missing_id_status, _missing_id = self.case._http_json( + "GET", "/reconciliation-matches", None + ) + missing_get_header_status, _missing_get_header = self.case._http_json( + "GET", "/reconciliation-matches?reconciliation_match_id=not-a-uuid", None, + tenant_header=None, + ) + invalid_id_status, _invalid_id = self.case._http_json( + "GET", "/reconciliation-matches?reconciliation_match_id=not-a-uuid", None + ) + missing_status, _missing = self.case._http_json( + "GET", + f"/reconciliation-matches?reconciliation_match_id={uuid.uuid4()}", + None, + ) + self.assertEqual(status, 200) + self.assertEqual(read_status, 200) + self.assertEqual(read["reconciliation_match_id"], created["reconciliation_match_id"]) + self.assertEqual(conflict_status, 409) + self.assertEqual(wrong_status, 403) + self.assertEqual(missing_header_status, 400) + self.assertEqual(invalid_body_status, 400) + self.assertEqual(invalid_command_status, 422) + self.assertEqual(missing_id_status, 400) + self.assertEqual(missing_get_header_status, 400) + self.assertEqual(invalid_id_status, 400) + self.assertEqual(missing_status, 404) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_reconciliation_multi_match_documentation_contract.py b/tests/test_reconciliation_multi_match_documentation_contract.py index 7c94dc9e..150fde9c 100644 --- a/tests/test_reconciliation_multi_match_documentation_contract.py +++ b/tests/test_reconciliation_multi_match_documentation_contract.py @@ -41,7 +41,7 @@ def test_adr_describes_current_multi_match_persistence_contract(self) -> None: def test_product_baseline_distinguishes_current_tree_from_remaining_close_package(self) -> None: """The product gap queue must not call delivered 0015-0017 controls future work.""" text = BASELINE.read_text(encoding="utf-8") - self.assertIn("from migrations `0015` through `0017`", text) + self.assertIn("from migrations `0015` through `0020`", text) self.assertIn("close-package provenance remains open", text) self.assertIn("[delivered in current tree; migration 0015]", text) self.assertNotRegex( diff --git a/tests/test_repository_contracts.py b/tests/test_repository_contracts.py index 7709d66a..824c189b 100644 --- a/tests/test_repository_contracts.py +++ b/tests/test_repository_contracts.py @@ -17,6 +17,7 @@ APPEND_ONLY_JOURNAL_MUTATION_ERROR, COVERAGE_CP313_MANYLINUX_X86_64_WHEEL_HASH, COVERAGE_UNIVERSAL_WHEEL_HASH, + PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH, main, find_mutable_action_references, find_placeholder_tokens, @@ -369,6 +370,14 @@ def test_quality_requirements_require_ci_coverage_wheels_and_packaging_backend( ("coverage must pin the CPython 3.13 manylinux x86_64 wheel hash",), ) + quality_requirements = (ROOT / "requirements-quality.txt").read_text( + encoding="utf-8" + ) + self.assertIn( + f"--hash=sha256:{PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH}", + quality_requirements, + ) + orphan_and_unpinned = validate_quality_requirements( f"--hash=sha256:{COVERAGE_UNIVERSAL_WHEEL_HASH}\n" "not-a-pinned-requirement\n" From 75ca8ab024ecf3a7b38b7404d55d1b84e13dd02f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:18:01 +0900 Subject: [PATCH 02/25] feat: add reconciliation match command schema --- ..._reconciliation_match_command_evidence.sql | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 database/migrations/0020_reconciliation_match_command_evidence.sql diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql new file mode 100644 index 00000000..0f564957 --- /dev/null +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -0,0 +1,78 @@ +BEGIN; + +-- Immutable application command identity for a proposed reconciliation match. +-- This records reviewable candidate evidence only; it cannot approve, close, or +-- post a journal. + +CREATE TABLE accounting_core.reconciliation_match_command ( + reconciliation_match_command_id uuid PRIMARY KEY DEFAULT uuidv7(), + tenant_account_id uuid NOT NULL, + reconciliation_run_id uuid NOT NULL, + reconciliation_candidate_id uuid NOT NULL, + reconciliation_match_id uuid NOT NULL, + candidate_idempotency_key text NOT NULL + CHECK (btrim(candidate_idempotency_key) <> ''), + candidate_command_hash text NOT NULL + CHECK (candidate_command_hash ~ '^sha256:[0-9a-f]{64}$'), + source_payload_hash text NOT NULL + CHECK (source_payload_hash ~ '^sha256:[0-9a-f]{64}$'), + source_payload_reference text NOT NULL + CHECK (btrim(source_payload_reference) <> ''), + recorded_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY ( + tenant_account_id, + reconciliation_run_id, + reconciliation_candidate_id + ) REFERENCES accounting_core.reconciliation_candidate ( + tenant_account_id, + reconciliation_run_id, + reconciliation_candidate_id + ), + FOREIGN KEY ( + tenant_account_id, + reconciliation_run_id, + reconciliation_match_id + ) REFERENCES accounting_core.reconciliation_match ( + tenant_account_id, + reconciliation_run_id, + reconciliation_match_id + ), + UNIQUE (tenant_account_id, candidate_idempotency_key), + UNIQUE (tenant_account_id, reconciliation_run_id, reconciliation_match_id) +); + +CREATE INDEX reconciliation_match_command_run_index + ON accounting_core.reconciliation_match_command ( + tenant_account_id, + reconciliation_run_id, + recorded_at, + reconciliation_match_command_id + ); + +ALTER TABLE accounting_core.reconciliation_match_command ENABLE ROW LEVEL SECURITY; +ALTER TABLE accounting_core.reconciliation_match_command FORCE ROW LEVEL SECURITY; + +CREATE POLICY reconciliation_match_command_isolation + ON accounting_core.reconciliation_match_command + USING (tenant_account_id = accounting_core.current_tenant_account_id()) + WITH CHECK (tenant_account_id = accounting_core.current_tenant_account_id()); + +REVOKE ALL ON accounting_core.reconciliation_match_command FROM PUBLIC; + +CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_mutation() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION + 'recorded reconciliation match command evidence is immutable; create a new proposed match instead (reconciliation_match_command_immutable)' + USING ERRCODE = '23514'; +END; +$$; + +CREATE TRIGGER reconciliation_match_command_immutability_guard +BEFORE UPDATE OR DELETE +ON accounting_core.reconciliation_match_command +FOR EACH ROW EXECUTE FUNCTION accounting_core.reject_reconciliation_match_command_mutation(); + +COMMIT; From e38d0d1293df9a1af92985bd79be87cc044eb91a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:20:02 +0900 Subject: [PATCH 03/25] test: scope psycopg wheel hash contract --- tests/test_repository_contracts.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_repository_contracts.py b/tests/test_repository_contracts.py index 824c189b..1d46a58c 100644 --- a/tests/test_repository_contracts.py +++ b/tests/test_repository_contracts.py @@ -373,9 +373,15 @@ def test_quality_requirements_require_ci_coverage_wheels_and_packaging_backend( quality_requirements = (ROOT / "requirements-quality.txt").read_text( encoding="utf-8" ) + psycopg_binary_stanza = re.search( + r"(?ms)^psycopg-binary==3\.3\.4 \\\n" + r"(?: --hash=sha256:[0-9a-f]{64}(?: \\\n|\n))+", + quality_requirements, + ) + self.assertIsNotNone(psycopg_binary_stanza) self.assertIn( f"--hash=sha256:{PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH}", - quality_requirements, + psycopg_binary_stanza.group(0) if psycopg_binary_stanza else "", ) orphan_and_unpinned = validate_quality_requirements( From 75732b8b40a53fa19d6c74e4403bbb7127f3d54d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:22:17 +0900 Subject: [PATCH 04/25] fix: validate psycopg Python 3.14 hash --- scripts/validate_repository.py | 6 ++++++ tests/test_repository_contracts.py | 16 ++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index cac2c726..3ae72df4 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -418,6 +418,12 @@ def validate_quality_requirements(requirements_text: str) -> tuple[str, ...]: ) elif not package_hashes["psycopg-binary"]: errors.append("psycopg-binary must be hash locked") + elif PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH not in package_hashes[ + "psycopg-binary" + ]: + errors.append( + "psycopg-binary must pin the CPython 3.14 manylinux x86_64 wheel hash" + ) return tuple(errors) diff --git a/tests/test_repository_contracts.py b/tests/test_repository_contracts.py index 1d46a58c..f8c06932 100644 --- a/tests/test_repository_contracts.py +++ b/tests/test_repository_contracts.py @@ -367,7 +367,10 @@ def test_quality_requirements_require_ci_coverage_wheels_and_packaging_backend( ) self.assertEqual( universal_only, - ("coverage must pin the CPython 3.13 manylinux x86_64 wheel hash",), + ( + "coverage must pin the CPython 3.13 manylinux x86_64 wheel hash", + "psycopg-binary must pin the CPython 3.14 manylinux x86_64 wheel hash", + ), ) quality_requirements = (ROOT / "requirements-quality.txt").read_text( @@ -383,6 +386,14 @@ def test_quality_requirements_require_ci_coverage_wheels_and_packaging_backend( f"--hash=sha256:{PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH}", psycopg_binary_stanza.group(0) if psycopg_binary_stanza else "", ) + without_cp314 = quality_requirements.replace( + f" --hash=sha256:{PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH}\n", + "", + ) + self.assertIn( + "psycopg-binary must pin the CPython 3.14 manylinux x86_64 wheel hash", + validate_quality_requirements(without_cp314), + ) orphan_and_unpinned = validate_quality_requirements( f"--hash=sha256:{COVERAGE_UNIVERSAL_WHEEL_HASH}\n" @@ -420,7 +431,8 @@ def test_quality_requirements_require_ci_coverage_wheels_and_packaging_backend( "psycopg==3.3.4 --hash=sha256:" "b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a\n" "psycopg-binary==3.3.4 --hash=sha256:" - "c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3\n" + "c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3 " + f"--hash=sha256:{PSYCOPG_BINARY_CP314_MANYLINUX_X86_64_WHEEL_HASH}\n" "# comment and blank lines are ignored\n\n" ) self.assertEqual(inline_valid, ()) From 17340f96143453c9972f90e604c9d52d1cc3d8a2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:44:14 +0900 Subject: [PATCH 05/25] fix: validate reconciliation match sources --- CHANGELOG.md | 2 +- docs/DATA_MODEL.md | 2 +- docs/OPERABILITY.md | 2 +- ...9-reconciliation-match-command-evidence.md | 7 +- docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- .../reconciliation_match.py | 152 +++++++++++++++--- tests/test_reconciliation_match_api.py | 145 ++++++++++++++++- 7 files changed, 281 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00b91267..74a930b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [Unreleased] - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. -- Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts are required, and the evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. +- Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts must match the bound bank-entry and posted-journal source facts, and database source-conservation violations return stable validation errors. The evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. - Added migration `0018_bank_statement_balance_evidence.sql` and exact normalized balance facts to the immutable camt.053 registry. Opening and closing amounts, currency, direction, sequence, locator, and source hash now survive persistence as forced-RLS evidence for a later exact reconciliation bridge; no balance row grants posting, reversal, approval, close, or accounting-policy authority. ADR 0057 records the decision. - Added migration `0019_reconciliation_run_command_evidence.sql` and the tenant-scoped `POST /reconciliation-runs` / `GET /reconciliation-runs?reconciliation_run_id=` boundary. A run opens only as `evaluating` from one persisted statement and active bank-account assignment; the command binds the raw artifact payload hash (distinct from the normalized statement hash), exact retries replay immutable command/source evidence, and changed key evidence fails closed. Distinct keys may create separately auditable runs for later policy or cutoff evaluation. This slice does not match, approve, close, post, or select chart accounts. ADR 0058 records the decision. - Reconciliation-run bank, book, and knowledge cutoffs now require an explicit zero-offset UTC timezone (`Z` or `+00:00`); timezone-naive and non-UTC values fail before persistence or command hashing, so an ambiguous wall clock cannot become authoritative evidence. diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index e569e823..b5abe4d8 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -51,7 +51,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconc - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires the supplied amounts to equal the bound immutable bank-entry and posted-journal source facts in the run's accounting scope. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 7839eda5..164034de 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -45,7 +45,7 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; the public run API opens only `evaluating` scope and does not match, approve, close, or post journals. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run and quoted positive equal decimal amounts. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 98890f95..43432636 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -24,11 +24,14 @@ prevent cross-scope evidence and duplicate command or match identities. `accept_reconciliation_match` is the smallest durable command boundary: it accepts one exact 1:1 proposed match, requires quoted positive equal decimal -amounts, creates the candidate, proposed match, statement allocation, journal +amounts that equal the bound immutable bank-entry and posted-journal source +facts, creates the candidate, proposed match, statement allocation, journal allocation, and command evidence in one transaction, and grants no approval, close, chart-account, reversal, or posting authority. Exact retries return the stored document; reuse of the key with changed command or source evidence fails -closed. `POST /reconciliation-matches` exposes that command and +closed. Database source-conservation violations are translated to a stable +validation failure rather than leaking a driver error. `POST +/reconciliation-matches` exposes that command and `GET /reconciliation-matches?reconciliation_match_id=` reads the tenant-scoped document. diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 428935ee..6a3a3d97 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,7 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts matching the bound immutable bank entry and posted journal, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py index 9a5e4a05..f4812094 100644 --- a/src/accounting_information_platform/reconciliation_match.py +++ b/src/accounting_information_platform/reconciliation_match.py @@ -8,6 +8,8 @@ from typing import Mapping from uuid import UUID +from psycopg.errors import CheckViolation + from .core import ( AccountingValidationError, IdempotencyConflictError, @@ -15,7 +17,6 @@ _parse_amount, ) from .persistence import PostgresPostingLedger, _exact_amount_text -from .reconciliation_run import _parse_uuid def accept_reconciliation_match( @@ -23,7 +24,7 @@ def accept_reconciliation_match( ) -> dict[str, object]: """Persist one exact 1:1 proposed match for an evaluating reconciliation run.""" command = _require_command(payload, tenant_reference) - run_id = _parse_uuid( + run_id = _parse_match_uuid( str(command.get("reconciliation_run_id") or ""), "reconciliation_run_id", ) @@ -93,7 +94,7 @@ def accept_reconciliation_match( run = connection.execute( """ - SELECT run_status_code + SELECT run_status_code, accounting_book_id, currency_code FROM accounting_core.reconciliation_run WHERE tenant_account_id = %s AND reconciliation_run_id = %s @@ -110,6 +111,17 @@ def accept_reconciliation_match( "reconciliation matches can only be proposed on an evaluating run. " "Open a new evaluating reconciliation run, then retry the match." ) + _require_recorded_source_amounts( + connection, + tenant_id=tenant_id, + reconciliation_run_id=run_id, + accounting_book_id=run[1], + currency_code=run[2], + statement_reference=statement_reference, + journal_reference=journal_reference, + statement_amount=statement_amount, + journal_amount=journal_amount, + ) ledger._acquire_command_lock( connection, @@ -132,26 +144,32 @@ def accept_reconciliation_match( "Use the existing proposed match or a new source pair, then retry." ) - candidate_id = connection.execute( - """ - INSERT INTO accounting_core.reconciliation_candidate ( - tenant_account_id, reconciliation_run_id, - statement_entry_reference, journal_reference, - statement_amount, journal_amount, rule_code - ) - VALUES (%s, %s, %s, %s, %s, %s, %s) - RETURNING reconciliation_candidate_id - """, - ( - tenant_id, - run_id, - statement_reference, - journal_reference, - statement_amount, - journal_amount, - rule_code, - ), - ).fetchone()[0] + try: + candidate_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + run_id, + statement_reference, + journal_reference, + statement_amount, + journal_amount, + rule_code, + ), + ).fetchone()[0] + except CheckViolation as error: + raise AccountingValidationError( + "the proposed match conflicts with recorded source conservation evidence. " + "Refresh the source amounts and evaluating run, then retry the match." + ) from error match_id = connection.execute( """ INSERT INTO accounting_core.reconciliation_match ( @@ -217,7 +235,7 @@ def lookup_reconciliation_match( database_url: str, tenant_reference: str, reconciliation_match_id: str ) -> dict[str, object]: """Read one tenant-scoped proposed reconciliation match and its exact allocation.""" - match_id = _parse_uuid(reconciliation_match_id, "reconciliation_match_id") + match_id = _parse_match_uuid(reconciliation_match_id, "reconciliation_match_id") ledger = PostgresPostingLedger(database_url, tenant_reference) with ledger._session() as connection: tenant_id = ledger._require_tenant(connection) @@ -313,6 +331,92 @@ def _require_command(payload: object, tenant_reference: str) -> Mapping[str, obj return payload +def _parse_match_uuid(value: str, label: str) -> UUID: + """Parse a match command identifier with match-specific recovery guidance.""" + try: + return UUID(value) + except (ValueError, AttributeError) as error: + raise AccountingValidationError( + f"{label} must be a UUID. Supply a persisted {label}, then retry the match." + ) from error + + +def _require_recorded_source_amounts( + connection: object, + *, + tenant_id: UUID, + reconciliation_run_id: UUID, + accounting_book_id: UUID, + currency_code: str, + statement_reference: str, + journal_reference: str, + statement_amount: Decimal, + journal_amount: Decimal, +) -> None: + """Require command amounts to equal tenant-scoped immutable source facts.""" + statement_rows = connection.execute( + """ + SELECT entry.entry_amount, entry.entry_currency_code + FROM accounting_integration.bank_statement_entry AS entry + JOIN accounting_core.reconciliation_run_command AS run_command + ON run_command.tenant_account_id = entry.tenant_account_id + AND run_command.bank_statement_record_id = entry.bank_statement_record_id + JOIN accounting_core.reconciliation_run AS run_scope + ON run_scope.tenant_account_id = run_command.tenant_account_id + AND run_scope.reconciliation_run_id = run_command.reconciliation_run_id + WHERE run_command.tenant_account_id = %s + AND run_command.reconciliation_run_id = %s + AND (entry.source_entry_identity = %s OR entry.bank_statement_entry_id::text = %s) + AND run_scope.currency_code = entry.entry_currency_code + """, + (tenant_id, reconciliation_run_id, statement_reference, statement_reference), + ).fetchall() + if len(statement_rows) != 1: + raise AccountingValidationError( + "statement source evidence is not recorded exactly once for this reconciliation run. " + "Supply an entry reference from the bound bank statement, then retry the match." + ) + recorded_statement_amount = statement_rows[0][0] + if recorded_statement_amount != statement_amount: + raise AccountingValidationError( + "statement_amount does not match recorded statement source amount. " + "Supply the exact recorded statement amount, then retry the match." + ) + + journal_row = connection.execute( + """ + SELECT journal.transaction_currency_code, + COALESCE(SUM(line.debit_amount), 0), + COALESCE(SUM(line.credit_amount), 0) + FROM accounting_core.general_journal AS journal + LEFT JOIN accounting_core.journal_entry_line AS line + ON line.tenant_account_id = journal.tenant_account_id + AND line.general_journal_id = journal.general_journal_id + WHERE journal.tenant_account_id = %s + AND journal.accounting_book_id = %s + AND journal.journal_reference = %s + AND journal.journal_status_code = 'posted' + GROUP BY journal.general_journal_id, journal.transaction_currency_code + """, + (tenant_id, accounting_book_id, journal_reference), + ).fetchone() + if journal_row is None or journal_row[0] != currency_code: + raise AccountingValidationError( + "journal source evidence is not a posted journal in the reconciliation scope. " + "Supply a posted journal reference from the bound accounting book, then retry the match." + ) + if journal_row[1] != journal_row[2] or journal_row[1] <= 0: + raise AccountingValidationError( + "journal source evidence is not balanced and positive. " + "Supply a balanced posted journal, then retry the match." + ) + if journal_row[1] != journal_amount: + raise AccountingValidationError( + "journal_amount does not match recorded journal source amount. " + "Supply the exact recorded journal amount, then retry the match." + ) + + def _require_text(value: object, field_name: str) -> str: """Require one canonical non-empty command text field.""" if not isinstance(value, str) or not value or value.strip() != value: diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index f9823f58..41a1221a 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -5,6 +5,9 @@ import hashlib import unittest import uuid +from datetime import date +from decimal import Decimal +from unittest import mock import psycopg @@ -22,6 +25,9 @@ lookup_bank_statement_entries, lookup_reconciliation_match, ) +from accounting_information_platform.reconciliation_match import ( + _require_recorded_source_amounts, +) from tests import test_postgres_posting as posting @@ -106,11 +112,22 @@ def _open_run(self) -> tuple[str, str]: def _command(self) -> dict[str, object]: run_id, statement_reference = self._open_run() + journal = self.case.ledger.post( + self.case._two_line_proposal( + proposal_id=str(uuid.uuid4()), + idempotency_key=f"match-journal-{uuid.uuid4().hex}", + source_payload_hash="sha256:" + "9" * 64, + source_event_references=(f"urn:cwl:reconciliation:journal:{uuid.uuid4()}",), + transaction_date=date(2026, 8, 24), + accounting_date=date(2026, 8, 24), + ), + self.case.policy, + ) return { "tenant_reference": self.case.policy.tenant_reference, "reconciliation_run_id": run_id, "statement_entry_reference": statement_reference, - "journal_reference": "journal-match-fixture", + "journal_reference": journal.journal_reference, "statement_amount": "25000.00", "journal_amount": "25000.00", "rule_code": "provider_reference", @@ -182,6 +199,132 @@ def test_match_command_rejects_non_exact_or_unbalanced_amounts(self) -> None: changed, posting.DATABASE_URL, self.case.policy.tenant_reference ) + def test_match_command_requires_recorded_source_amounts(self) -> None: + """A proposed match cannot invent amounts or point at an absent journal.""" + command = self._command() + tenant = self.case.policy.tenant_reference + with self.assertRaisesRegex(AccountingValidationError, "does not match recorded"): + accept_reconciliation_match( + dict( + command, + statement_amount="24999.99", + journal_amount="24999.99", + candidate_idempotency_key=f"source-amount-{uuid.uuid4().hex}", + ), + posting.DATABASE_URL, + tenant, + ) + with self.assertRaisesRegex(AccountingValidationError, "journal source"): + accept_reconciliation_match( + dict( + command, + journal_reference="urn:cwl:accounting:general_journal:missing", + candidate_idempotency_key=f"missing-journal-{uuid.uuid4().hex}", + ), + posting.DATABASE_URL, + tenant, + ) + with self.assertRaisesRegex(AccountingValidationError, "not recorded exactly once"): + accept_reconciliation_match( + dict( + command, + statement_entry_reference="statement-entry-missing", + candidate_idempotency_key=f"missing-statement-{uuid.uuid4().hex}", + ), + posting.DATABASE_URL, + tenant, + ) + + def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> None: + """Defensive source checks reject impossible or mismatched journal evidence.""" + for journal_row, message in ( + (("KRW", Decimal("25000"), Decimal("24999")), "balanced and positive"), + (("KRW", Decimal("24999"), Decimal("24999")), "does not match recorded"), + ): + with self.subTest(message=message): + connection = mock.Mock() + statement_result = mock.Mock() + statement_result.fetchall.return_value = [(Decimal("25000"), "KRW")] + journal_result = mock.Mock() + journal_result.fetchone.return_value = journal_row + connection.execute.side_effect = [statement_result, journal_result] + with self.assertRaisesRegex(AccountingValidationError, message): + _require_recorded_source_amounts( + connection, + tenant_id=uuid.uuid4(), + reconciliation_run_id=uuid.uuid4(), + accounting_book_id=uuid.uuid4(), + currency_code="KRW", + statement_reference="statement-entry", + journal_reference="journal-reference", + statement_amount=Decimal("25000"), + journal_amount=Decimal("25000"), + ) + + def test_match_command_maps_source_conservation_guard_to_validation(self) -> None: + """A legacy cross-run amount conflict cannot escape as a raw database error.""" + command = self._command() + tenant = self.case.policy.tenant_reference + with psycopg.connect(posting.DATABASE_URL) as connection: + statement_id = connection.execute( + """ + SELECT bank_statement_record_id + FROM accounting_core.reconciliation_run_command + WHERE tenant_account_id = %s AND reconciliation_run_id = %s + """, + (self.case.tenant_id, command["reconciliation_run_id"]), + ).fetchone()[0] + second_run = accept_reconciliation_run( + { + "tenant_reference": tenant, + "bank_statement_record_id": statement_id, + "legal_entity_reference": self.case.policy.legal_entity_reference, + "accounting_book_reference": self.case.policy.accounting_book_reference, + "bank_cutoff_at": "2026-08-24T23:59:59Z", + "book_cutoff_at": "2026-08-24T23:59:59Z", + "matching_policy_version": "deterministic-v1", + "knowledge_cutoff_at": "2026-08-25T00:00:00Z", + "reconciliation_idempotency_key": f"run-conflict-{uuid.uuid4().hex}", + "source_payload_hash": "sha256:" + hashlib.sha256( + load_canonical_statement_fixture() + ).hexdigest(), + }, + posting.DATABASE_URL, + tenant, + ) + with psycopg.connect(posting.DATABASE_URL) as connection: + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, '24999.99', '24999.99', 'legacy-conflict') + """, + ( + self.case.tenant_id, + second_run["reconciliation_run_id"], + command["statement_entry_reference"], + command["journal_reference"], + ), + ) + with self.assertRaisesRegex(AccountingValidationError, "conservation evidence"): + accept_reconciliation_match(command, posting.DATABASE_URL, tenant) + + def test_match_command_uuid_errors_refer_to_match(self) -> None: + """Match endpoints use match-specific recovery guidance for UUID errors.""" + command = self._command() + tenant = self.case.policy.tenant_reference + with self.assertRaisesRegex(AccountingValidationError, "retry the match"): + accept_reconciliation_match( + dict(command, reconciliation_run_id="not-a-uuid"), + posting.DATABASE_URL, + tenant, + ) + with self.assertRaisesRegex(AccountingValidationError, "retry the match"): + lookup_reconciliation_match(posting.DATABASE_URL, tenant, "not-a-uuid") + def test_match_command_validation_and_run_lifecycle_fail_closed(self) -> None: """Malformed, missing-run, and non-evaluating commands write no evidence.""" command = self._command() From b73f57df33ee9f772aa8c84580f23de2ea5afe1d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 09:57:10 +0900 Subject: [PATCH 06/25] fix: classify reconciliation match conflicts --- CHANGELOG.md | 2 +- docs/OPERABILITY.md | 1 + .../0059-reconciliation-match-command-evidence.md | 4 +++- src/accounting_information_platform/http_api.py | 2 ++ tests/test_reconciliation_match_api.py | 15 +++++++++++++++ 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a930b7..aa5f9592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [Unreleased] - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. -- Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts must match the bound bank-entry and posted-journal source facts, and database source-conservation violations return stable validation errors. The evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. +- Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts must match the bound bank-entry and posted-journal source facts, and database source-conservation violations return stable validation errors. The HTTP boundary distinguishes malformed identifiers (400), absent sources (404), state conflicts (409), and source-content validation (422). The evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. - Added migration `0018_bank_statement_balance_evidence.sql` and exact normalized balance facts to the immutable camt.053 registry. Opening and closing amounts, currency, direction, sequence, locator, and source hash now survive persistence as forced-RLS evidence for a later exact reconciliation bridge; no balance row grants posting, reversal, approval, close, or accounting-policy authority. ADR 0057 records the decision. - Added migration `0019_reconciliation_run_command_evidence.sql` and the tenant-scoped `POST /reconciliation-runs` / `GET /reconciliation-runs?reconciliation_run_id=` boundary. A run opens only as `evaluating` from one persisted statement and active bank-account assignment; the command binds the raw artifact payload hash (distinct from the normalized statement hash), exact retries replay immutable command/source evidence, and changed key evidence fails closed. Distinct keys may create separately auditable runs for later policy or cutoff evaluation. This slice does not match, approve, close, post, or select chart accounts. ADR 0058 records the decision. - Reconciliation-run bank, book, and knowledge cutoffs now require an explicit zero-offset UTC timezone (`Z` or `+00:00`); timezone-naive and non-UTC values fail before persistence or command hashing, so an ambiguous wall clock cannot become authoritative evidence. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 164034de..792b3377 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -46,6 +46,7 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; the public run API opens only `evaluating` scope and does not match, approve, close, or post journals. Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 43432636..8912309a 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -33,7 +33,9 @@ closed. Database source-conservation violations are translated to a stable validation failure rather than leaking a driver error. `POST /reconciliation-matches` exposes that command and `GET /reconciliation-matches?reconciliation_match_id=` reads the tenant-scoped -document. +document. The HTTP boundary reports malformed identifiers as `400`, absent +source evidence as `404`, state conflicts as `409`, and source-content or +conservation validation failures as `422`. The command intentionally does not replace the existing pure split/aggregate allocation planner or the database approval workflow. It provides a bounded diff --git a/src/accounting_information_platform/http_api.py b/src/accounting_information_platform/http_api.py index 0440c7f6..1f90ee9b 100644 --- a/src/accounting_information_platform/http_api.py +++ b/src/accounting_information_platform/http_api.py @@ -1890,6 +1890,8 @@ def _reconciliation_match_status(error: AccountingValidationError) -> int: return 404 if "must be a UUID" in message or "is required" in message: return 400 + if "can only be proposed" in message or "already recorded for this run" in message: + return 409 return 422 diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index 41a1221a..ecb8712e 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -393,6 +393,20 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: "/reconciliation-matches", dict(command, source_payload_hash="sha256:" + "3" * 64), ) + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + connection.execute( + """ + UPDATE accounting_core.reconciliation_run + SET run_status_code = 'review_required' + WHERE reconciliation_run_id = %s + """, + (command["reconciliation_run_id"],), + ) + state_conflict_status, _state_conflict = self.case._http_json( + "POST", + "/reconciliation-matches", + dict(command, candidate_idempotency_key=f"http-state-{uuid.uuid4().hex}"), + ) wrong_status, _wrong = self.case._http_json( "POST", "/reconciliation-matches", @@ -432,6 +446,7 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: self.assertEqual(read_status, 200) self.assertEqual(read["reconciliation_match_id"], created["reconciliation_match_id"]) self.assertEqual(conflict_status, 409) + self.assertEqual(state_conflict_status, 409) self.assertEqual(wrong_status, 403) self.assertEqual(missing_header_status, 400) self.assertEqual(invalid_body_status, 400) From 06c02cc9037802287dce9b3e126f92750811385e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 10:11:40 +0900 Subject: [PATCH 07/25] fix: distinguish absent reconciliation journals --- docs/OPERABILITY.md | 1 - .../reconciliation_match.py | 18 ++++++++++++------ tests/test_reconciliation_match_api.py | 16 ++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 792b3377..0a5cfa56 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -45,7 +45,6 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; the public run API opens only `evaluating` scope and does not match, approve, close, or post journals. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py index f4812094..a6bcba03 100644 --- a/src/accounting_information_platform/reconciliation_match.py +++ b/src/accounting_information_platform/reconciliation_match.py @@ -385,7 +385,8 @@ def _require_recorded_source_amounts( journal_row = connection.execute( """ - SELECT journal.transaction_currency_code, + SELECT journal.journal_status_code, + journal.transaction_currency_code, COALESCE(SUM(line.debit_amount), 0), COALESCE(SUM(line.credit_amount), 0) FROM accounting_core.general_journal AS journal @@ -395,22 +396,27 @@ def _require_recorded_source_amounts( WHERE journal.tenant_account_id = %s AND journal.accounting_book_id = %s AND journal.journal_reference = %s - AND journal.journal_status_code = 'posted' - GROUP BY journal.general_journal_id, journal.transaction_currency_code + GROUP BY journal.general_journal_id, journal.journal_status_code, + journal.transaction_currency_code """, (tenant_id, accounting_book_id, journal_reference), ).fetchone() - if journal_row is None or journal_row[0] != currency_code: + if journal_row is None: + raise AccountingValidationError( + "journal source evidence is not recorded in the reconciliation scope. " + "Supply a recorded journal reference from the bound accounting book, then retry the match." + ) + if journal_row[0:2] != ("posted", currency_code): raise AccountingValidationError( "journal source evidence is not a posted journal in the reconciliation scope. " "Supply a posted journal reference from the bound accounting book, then retry the match." ) - if journal_row[1] != journal_row[2] or journal_row[1] <= 0: + if journal_row[2] != journal_row[3] or journal_row[2] <= 0: raise AccountingValidationError( "journal source evidence is not balanced and positive. " "Supply a balanced posted journal, then retry the match." ) - if journal_row[1] != journal_amount: + if journal_row[2] != journal_amount: raise AccountingValidationError( "journal_amount does not match recorded journal source amount. " "Supply the exact recorded journal amount, then retry the match." diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index ecb8712e..73eb9a3b 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -238,8 +238,10 @@ def test_match_command_requires_recorded_source_amounts(self) -> None: def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> None: """Defensive source checks reject impossible or mismatched journal evidence.""" for journal_row, message in ( - (("KRW", Decimal("25000"), Decimal("24999")), "balanced and positive"), - (("KRW", Decimal("24999"), Decimal("24999")), "does not match recorded"), + (("posted", "KRW", Decimal("25000"), Decimal("24999")), "balanced and positive"), + (("posted", "KRW", Decimal("24999"), Decimal("24999")), "does not match recorded"), + (("draft", "KRW", Decimal("25000"), Decimal("25000")), "not a posted journal"), + (("posted", "USD", Decimal("25000"), Decimal("25000")), "not a posted journal"), ): with self.subTest(message=message): connection = mock.Mock() @@ -393,6 +395,15 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: "/reconciliation-matches", dict(command, source_payload_hash="sha256:" + "3" * 64), ) + missing_journal_status, _missing_journal = self.case._http_json( + "POST", + "/reconciliation-matches", + dict( + command, + journal_reference="urn:cwl:accounting:general_journal:missing", + candidate_idempotency_key=f"http-missing-journal-{uuid.uuid4().hex}", + ), + ) with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: connection.execute( """ @@ -446,6 +457,7 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: self.assertEqual(read_status, 200) self.assertEqual(read["reconciliation_match_id"], created["reconciliation_match_id"]) self.assertEqual(conflict_status, 409) + self.assertEqual(missing_journal_status, 404) self.assertEqual(state_conflict_status, 409) self.assertEqual(wrong_status, 403) self.assertEqual(missing_header_status, 400) From 08e88d93846ec500536da7099698f1929333d006 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:03:31 -0700 Subject: [PATCH 08/25] test: reject cross-paired reconciliation command evidence --- ..._reconciliation_match_command_chain_red.py | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 tests/test_reconciliation_match_command_chain_red.py diff --git a/tests/test_reconciliation_match_command_chain_red.py b/tests/test_reconciliation_match_command_chain_red.py new file mode 100644 index 00000000..70d0b421 --- /dev/null +++ b/tests/test_reconciliation_match_command_chain_red.py @@ -0,0 +1,100 @@ +"""RED PostgreSQL contract for reconciliation command candidate/match lineage.""" + +from __future__ import annotations + +import unittest +import uuid + +import psycopg + +from tests.test_reconciliation_match_api import ReconciliationMatchApiTests +from tests import test_postgres_posting as posting + + +class ReconciliationMatchCommandChainTests(unittest.TestCase): + """Prove immutable command evidence names one real candidate-to-match chain.""" + + @classmethod + def setUpClass(cls) -> None: + posting.PostgresPostingTests.setUpClass() + + def setUp(self) -> None: + self.helper = ReconciliationMatchApiTests( + "test_proposed_match_is_persisted_and_replayed" + ) + self.helper.setUp() + self.addCleanup(self.helper.doCleanups) + self.addCleanup(self.helper.tearDown) + + def test_command_rejects_candidate_from_a_different_match(self) -> None: + """Cross-pair candidate/match provenance fails at the database boundary.""" + run_id, _ = self.helper._open_run() + tenant_id = self.helper.case.tenant_id + + with psycopg.connect(posting.DATABASE_URL) as connection: + candidate_ids: list[uuid.UUID] = [] + match_ids: list[uuid.UUID] = [] + for suffix in ("a", "b"): + candidate_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, + reconciliation_run_id, + statement_entry_reference, + journal_reference, + statement_amount, + journal_amount, + rule_code + ) + VALUES (%s, %s, %s, %s, '1.000000', '1.000000', 'chain-red') + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + run_id, + f"urn:cwl:statement:chain:{suffix}:{uuid.uuid4()}", + f"urn:cwl:journal:chain:{suffix}:{uuid.uuid4()}", + ), + ).fetchone()[0] + match_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, + reconciliation_run_id, + reconciliation_candidate_id, + match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (tenant_id, run_id, candidate_id), + ).fetchone()[0] + candidate_ids.append(candidate_id) + match_ids.append(match_id) + + with self.assertRaises(psycopg.errors.ForeignKeyViolation): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, + reconciliation_run_id, + reconciliation_candidate_id, + reconciliation_match_id, + candidate_idempotency_key, + candidate_command_hash, + source_payload_hash, + source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + tenant_id, + run_id, + candidate_ids[0], + match_ids[1], + f"chain-cross-pair-{uuid.uuid4().hex}", + "sha256:" + "a" * 64, + "sha256:" + "b" * 64, + "urn:cwl:object:cross-pair-provenance", + ), + ) From 3a42b1cb065dfa64c19ef514a9f6df892fd94152 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 29 Aug 2026 14:03:54 +0900 Subject: [PATCH 09/25] fix: bind match command evidence to candidate chain --- CHANGELOG.md | 1 + ..._reconciliation_match_command_evidence.sql | 15 +++- docs/DATA_MODEL.md | 1 + docs/ERD.md | 2 +- ...9-reconciliation-match-command-evidence.md | 4 +- docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- tests/test_reconciliation_match_api.py | 87 +++++++++++++++++++ 7 files changed, 107 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa5f9592..1483a05d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. - Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts must match the bound bank-entry and posted-journal source facts, and database source-conservation violations return stable validation errors. The HTTP boundary distinguishes malformed identifiers (400), absent sources (404), state conflicts (409), and source-content validation (422). The evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. - Added migration `0018_bank_statement_balance_evidence.sql` and exact normalized balance facts to the immutable camt.053 registry. Opening and closing amounts, currency, direction, sequence, locator, and source hash now survive persistence as forced-RLS evidence for a later exact reconciliation bridge; no balance row grants posting, reversal, approval, close, or accounting-policy authority. ADR 0057 records the decision. diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql index 0f564957..f509227e 100644 --- a/database/migrations/0020_reconciliation_match_command_evidence.sql +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -4,6 +4,15 @@ BEGIN; -- This records reviewable candidate evidence only; it cannot approve, close, or -- post a journal. +ALTER TABLE accounting_core.reconciliation_match + ADD CONSTRAINT reconciliation_match_chain_key + UNIQUE ( + tenant_account_id, + reconciliation_run_id, + reconciliation_match_id, + reconciliation_candidate_id + ); + CREATE TABLE accounting_core.reconciliation_match_command ( reconciliation_match_command_id uuid PRIMARY KEY DEFAULT uuidv7(), tenant_account_id uuid NOT NULL, @@ -31,11 +40,13 @@ CREATE TABLE accounting_core.reconciliation_match_command ( FOREIGN KEY ( tenant_account_id, reconciliation_run_id, - reconciliation_match_id + reconciliation_match_id, + reconciliation_candidate_id ) REFERENCES accounting_core.reconciliation_match ( tenant_account_id, reconciliation_run_id, - reconciliation_match_id + reconciliation_match_id, + reconciliation_candidate_id ), UNIQUE (tenant_account_id, candidate_idempotency_key), UNIQUE (tenant_account_id, reconciliation_run_id, reconciliation_match_id) diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index b5abe4d8..decec772 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -68,6 +68,7 @@ Financial-statement, cash-flow, changes-in-equity, aging, account-balance, ledge - Posted journals are never updated or deleted; finalized journal populations cannot be extended after receipt issuance. - Exact debit and credit amounts use PostgreSQL `numeric` and application `Decimal`; binary floating-point accounting amounts are rejected at input boundaries. - Command idempotency is tenant-scoped and tied to immutable source/command evidence so exact retries replay and changed evidence fails closed. +- `reconciliation_match_command` uses a candidate-inclusive composite foreign key to `reconciliation_match`, so its tenant, run, match, and candidate identifiers must name one persisted candidate-to-match chain rather than independently valid rows. ## Future extensions diff --git a/docs/ERD.md b/docs/ERD.md index 48dad56b..f4e5a6fc 100644 --- a/docs/ERD.md +++ b/docs/ERD.md @@ -79,7 +79,7 @@ erDiagram `bank_account_record` and `bank_account_assignment` map an opaque bank account onto one legal entity, book, and same-book cash chart account. The assignment composite foreign key requires that book to belong to the same legal entity. `bank_statement_record` and `bank_statement_entry` are append-only evidence. They retain `source_artifact_hash`, `normalized_payload_hash`, `ingestion_idempotency_key`, and `source_entry_hash` so a controller can prove which original artifact produced each entry without storing the raw XML in PostgreSQL. -`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_match_command` records the corresponding immutable proposed-match command identity and provenance over one exact 1:1 candidate/match/allocation chain; it is also evidence only. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. +`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_match_command` records the corresponding immutable proposed-match command identity and provenance over one exact 1:1 candidate/match/allocation chain; its candidate-inclusive composite foreign key makes the candidate and match one database-proven chain. It is also evidence only. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. `reconciliation_candidate` records a deterministic statement/journal candidate and its exact source amounts; after INSERT it is append-only. `reconciliation_match` records the reviewable disposition. `statement_match_allocation` and `journal_match_allocation` preserve exact many-to-many consumption and are append-only regardless of later match status. Database-owned conservation guards serialize by immutable source identity and reject cross-run source-amount conflicts or over-consumption. Only an `approved` match consumes active source capacity; changing that match to `rejected` or `superseded` releases active capacity without deleting or rewriting the historical candidate/allocation evidence. These reconciliation relations provide audit and operator-control evidence only: they do not post, reverse, close, approve accounting policy, or mutate authoritative journals. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 8912309a..377a2af0 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -20,7 +20,9 @@ forced-RLS `accounting_core.reconciliation_match_command` relation. It binds a tenant, evaluating reconciliation run, candidate, and match to a tenant-scoped candidate idempotency key, canonical command hash, source-payload hash, and immutable object-storage reference. Composite foreign keys and uniqueness rules -prevent cross-scope evidence and duplicate command or match identities. +prevent cross-scope evidence, duplicate command or match identities, and +candidate/match pair mixing: migration 0020 references the candidate-inclusive +unique key on `reconciliation_match` from the command row. `accept_reconciliation_match` is the smallest durable command boundary: it accepts one exact 1:1 proposed match, requires quoted positive equal decimal diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 6a3a3d97..14b74bd3 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,7 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts matching the bound immutable bank entry and posted journal, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts matching the bound immutable bank entry and posted journal, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index 73eb9a3b..be92ce9c 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -186,6 +186,93 @@ def test_match_command_evidence_is_immutable(self) -> None: (document["reconciliation_match_id"],), ) + def test_match_command_fk_proves_candidate_and_match_same_chain(self) -> None: + """Command evidence cannot combine a candidate with another candidate's match.""" + run_id, _ = self._open_run() + candidate_a_reference = f"candidate-a-{uuid.uuid4().hex}" + candidate_b_reference = f"candidate-b-{uuid.uuid4().hex}" + with psycopg.connect(posting.DATABASE_URL) as connection: + candidate_a = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, '1.00', '1.00', 'test-a') + RETURNING reconciliation_candidate_id + """, + ( + self.case.tenant_id, + run_id, + candidate_a_reference, + f"journal-a-{uuid.uuid4().hex}", + ), + ).fetchone()[0] + candidate_b = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, '1.00', '1.00', 'test-b') + RETURNING reconciliation_candidate_id + """, + ( + self.case.tenant_id, + run_id, + candidate_b_reference, + f"journal-b-{uuid.uuid4().hex}", + ), + ).fetchone()[0] + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (self.case.tenant_id, run_id, candidate_a), + ).fetchone()[0] + match_b = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (self.case.tenant_id, run_id, candidate_b), + ).fetchone()[0] + + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with self.assertRaises(psycopg.errors.ForeignKeyViolation): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + self.case.tenant_id, + run_id, + candidate_a, + match_b, + f"cross-chain-{uuid.uuid4().hex}", + "sha256:" + "1" * 64, + "sha256:" + "2" * 64, + "urn:cwl:object:cross-chain", + ), + ) + def test_match_command_rejects_non_exact_or_unbalanced_amounts(self) -> None: """The command rejects JSON numbers and non-conserving 1:1 evidence.""" command = self._command() From a418d45d19c93ed6b7df8fbdfb781efa03fa2e7b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:04:01 -0700 Subject: [PATCH 10/25] fix: bind match command to its candidate chain --- ...0_reconciliation_match_command_evidence.sql | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql index 0f564957..6e82c83c 100644 --- a/database/migrations/0020_reconciliation_match_command_evidence.sql +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -4,6 +4,18 @@ BEGIN; -- This records reviewable candidate evidence only; it cannot approve, close, or -- post a journal. +-- Command provenance must name the candidate actually referenced by the match. +-- The existing match primary key proves row identity, while this tenant/run +-- composite key gives downstream evidence a database-owned same-chain target. +ALTER TABLE accounting_core.reconciliation_match + ADD CONSTRAINT reconciliation_match_candidate_chain_unique + UNIQUE ( + tenant_account_id, + reconciliation_run_id, + reconciliation_match_id, + reconciliation_candidate_id + ); + CREATE TABLE accounting_core.reconciliation_match_command ( reconciliation_match_command_id uuid PRIMARY KEY DEFAULT uuidv7(), tenant_account_id uuid NOT NULL, @@ -31,11 +43,13 @@ CREATE TABLE accounting_core.reconciliation_match_command ( FOREIGN KEY ( tenant_account_id, reconciliation_run_id, - reconciliation_match_id + reconciliation_match_id, + reconciliation_candidate_id ) REFERENCES accounting_core.reconciliation_match ( tenant_account_id, reconciliation_run_id, - reconciliation_match_id + reconciliation_match_id, + reconciliation_candidate_id ), UNIQUE (tenant_account_id, candidate_idempotency_key), UNIQUE (tenant_account_id, reconciliation_run_id, reconciliation_match_id) From 0f7a9d46ff196bea04f8c3f75208ca9e6596bbb5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 28 Aug 2026 22:05:13 -0700 Subject: [PATCH 11/25] docs: record database-owned match command chain --- ...9-reconciliation-match-command-evidence.md | 29 ++++++++++++++----- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 8912309a..84ad5e45 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -13,14 +13,26 @@ persisted one candidate and its source evidence atomically. A caller that wrote those relations independently could leave a candidate without durable command identity or make a retry ambiguous. +Immutable command provenance must also prove one real candidate-to-match chain, +not merely prove that a candidate row and a match row both exist in the same +tenant and run. Independent foreign keys permit a cross-pair combination in +which command evidence names candidate A and match B even though match B points +to candidate C. Because command evidence is append-only, that mismatch must be +rejected by the database before it can become durable provenance. + ## Decision Migration `0020_reconciliation_match_command_evidence.sql` adds the immutable, forced-RLS `accounting_core.reconciliation_match_command` relation. It binds a tenant, evaluating reconciliation run, candidate, and match to a tenant-scoped candidate idempotency key, canonical command hash, source-payload hash, and -immutable object-storage reference. Composite foreign keys and uniqueness rules -prevent cross-scope evidence and duplicate command or match identities. +immutable object-storage reference. The candidate still has its own tenant/run +foreign key, while one database-owned composite foreign key binds +`(tenant_account_id, reconciliation_run_id, reconciliation_match_id, +reconciliation_candidate_id)` to the same composite identity on +`reconciliation_match`. The command therefore cannot combine a valid candidate +with a different valid match from the same run. Uniqueness rules additionally +prevent duplicate command or match identities. `accept_reconciliation_match` is the smallest durable command boundary: it accepts one exact 1:1 proposed match, requires quoted positive equal decimal @@ -46,11 +58,14 @@ authority boundaries. ## Consequences Proposed reconciliation evidence now has a durable retry identity and an -atomic candidate-to-allocation provenance chain. Legacy candidates or matches -without command evidence are not synthesized by this read boundary. A proposed -match remains non-authoritative: only the existing approval controls can record -a human decision, and any adjustment must re-enter the authoritative journal -command boundary. +atomic candidate-to-allocation provenance chain. A database write that attempts +to cross-pair a candidate with a match referencing another candidate fails at +the relational boundary, even when every identifier is otherwise valid in the +same tenant and run. Legacy candidates or matches without command evidence are +not synthesized by this read boundary. A proposed match remains +non-authoritative: only the existing approval controls can record a human +decision, and any adjustment must re-enter the authoritative journal command +boundary. ## References From 56381ee3b877a9f18d3a28de6ba508efb5f0e362 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 04:05:00 +0900 Subject: [PATCH 12/25] fix(reconciliation): enforce run evidence provenance --- CHANGELOG.md | 2 +- ...19_reconciliation_run_command_evidence.sql | 52 ++++++ docs/DATA_MODEL.md | 2 +- docs/OPERABILITY.md | 2 +- docs/TEST_STRATEGY.md | 2 +- ...058-reconciliation-run-command-evidence.md | 5 + docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- .../reconciliation_run.py | 12 ++ ...on_candidate_allocation_persistence_red.py | 43 ++++- ...conciliation_cross_run_conservation_red.py | 22 ++- ...iliation_database_control_hardening_red.py | 44 ++++- ...nciliation_match_allocation_balance_red.py | 42 ++++- tests/test_reconciliation_run_api.py | 163 +++++++++++++++++- 13 files changed, 373 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac0d7d15..ab66a68e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [Unreleased] - Added migration `0018_bank_statement_balance_evidence.sql` and exact normalized balance facts to the immutable camt.053 registry. Opening and closing amounts, currency, direction, effective date/time, sequence, locator, and source hash now survive persistence as forced-RLS evidence for a later exact reconciliation bridge; no balance row grants posting, reversal, approval, close, or accounting-policy authority. ADR 0057 records the decision. -- Added migration `0019_reconciliation_run_command_evidence.sql` and the tenant-scoped `POST /reconciliation-runs` / `GET /reconciliation-runs?reconciliation_run_id=` boundary. A run opens only as `evaluating` from one persisted statement and active bank-account assignment; the command binds the raw artifact payload hash (distinct from the normalized statement hash), exact retries replay immutable command/source evidence, and changed key evidence fails closed. Distinct keys may create separately auditable runs for later policy or cutoff evaluation. This slice does not match, approve, close, post, or select chart accounts. ADR 0058 records the decision. +- Added migration `0019_reconciliation_run_command_evidence.sql` and the tenant-scoped `POST /reconciliation-runs` / `GET /reconciliation-runs?reconciliation_run_id=` boundary. A run opens only as `evaluating` from one persisted statement and active bank-account assignment; the command binds the raw artifact payload hash (distinct from the normalized statement hash), exact retries replay immutable command/source evidence, and changed key evidence fails closed. Historical runs exclude statement, artifact, account, assignment, legal-entity, and accounting-book facts recorded after `knowledge_cutoff_at`; a deferred database provenance guard rejects orphan runs and commands whose statement belongs to another bank account. Distinct keys may create separately auditable runs for later policy or cutoff evaluation. This slice does not match, approve, close, post, or select chart accounts. ADR 0058 records the decision. - Reconciliation-run bank, book, and knowledge cutoffs now require an explicit zero-offset UTC timezone (`Z` or `+00:00`); timezone-naive and non-UTC values fail before persistence or command hashing, so an ambiguous wall clock cannot become authoritative evidence. - Exact reconciliation-run retries now resolve stored command evidence before live assignment validation, preserving replay after assignment rollover or closure while changed request evidence remains a conflict. - Added database-owned reconciliation approval snapshots in migration `0016_reconciliation_approval_evidence.sql` and the forward lock-order repair in `0017_reconciliation_approval_lock_order.sql`: command hashes and immutable object-storage references remain caller evidence, while PostgreSQL computes a versioned SHA-256 digest over the candidate and exact allocation rows. Approval, allocation, and terminal match transitions share a tenant/run/match advisory lock in parent-row-first order; late allocations, candidate retargeting, status-only decisions, stale snapshots, unbound legacy terminal rows, and row/advisory deadlocks fail closed, while supersession preserves terminal approval time. The migration upgrade check temporarily grants only its transaction's current migration user visibility over forced-RLS match rows and removes that policy before commit. This control evidence cannot post, reverse, close, or alter accounting policy. ADR 0055 records the decision. diff --git a/database/migrations/0019_reconciliation_run_command_evidence.sql b/database/migrations/0019_reconciliation_run_command_evidence.sql index 7300a6a2..3ab6d5a2 100644 --- a/database/migrations/0019_reconciliation_run_command_evidence.sql +++ b/database/migrations/0019_reconciliation_run_command_evidence.sql @@ -53,6 +53,58 @@ CREATE TRIGGER reconciliation_run_command_immutable_guard FOR EACH ROW EXECUTE FUNCTION accounting_core.reject_reconciliation_run_command_mutation(); +CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + command_count integer; +BEGIN + SELECT count(*) + INTO command_count + FROM accounting_core.reconciliation_run_command AS command + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id; + + IF command_count <> 1 THEN + RAISE EXCEPTION + 'reconciliation run must have exactly one command evidence row at commit (reconciliation_run_command_provenance)' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM accounting_core.reconciliation_run_command AS command + JOIN accounting_integration.bank_statement_record AS statement + ON statement.tenant_account_id = command.tenant_account_id + AND statement.bank_statement_record_id = command.bank_statement_record_id + JOIN accounting_core.reconciliation_run AS run + ON run.tenant_account_id = command.tenant_account_id + AND run.reconciliation_run_id = command.reconciliation_run_id + JOIN accounting_core.bank_account_assignment AS assignment + ON assignment.tenant_account_id = run.tenant_account_id + AND assignment.legal_entity_id = run.legal_entity_id + AND assignment.accounting_book_id = run.accounting_book_id + AND assignment.bank_account_assignment_id = run.bank_account_assignment_id + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id + AND statement.bank_account_record_id IS DISTINCT FROM assignment.bank_account_record_id + ) THEN + RAISE EXCEPTION + 'reconciliation run command bank account provenance does not match the run assignment (reconciliation_run_command_provenance)' + USING ERRCODE = '23514'; + END IF; + + RETURN NULL; +END; +$$; + +CREATE CONSTRAINT TRIGGER reconciliation_run_command_provenance_guard + AFTER INSERT ON accounting_core.reconciliation_run + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW + EXECUTE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance(); + ALTER TABLE accounting_core.reconciliation_run_command ENABLE ROW LEVEL SECURITY; ALTER TABLE accounting_core.reconciliation_run_command FORCE ROW LEVEL SECURITY; CREATE POLICY reconciliation_run_command_isolation diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 407ac18e..c370b198 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -50,7 +50,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0019_reconc - `statement_match_allocation`: append-only exact amount consumed from an immutable statement source reference by one reconciliation match. - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. -- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. +- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; a deferred database provenance guard requires exactly one command row and binds its statement to the run assignment's bank account. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 00921df8..3ce84255 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -42,7 +42,7 @@ Migration `0017_reconciliation_approval_lock_order.sql` is a forward repair for Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric amount, currency, credit/debit direction, typed effective date/time, sequence, locator, and source hash for every camt.053 balance. The effective date/time is distinct from statement period and system `recorded_at`; existing balance hashes remain on the statement row for compatibility. The numeric rows are immutable, forced-RLS evidence that a reconciliation bridge may read but that cannot post, reverse, or mutate a journal. -Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; the public run API opens only `evaluating` scope and does not match, approve, close, or post journals. +Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; new runs exclude source facts recorded after `knowledge_cutoff_at`, and a deferred database guard requires one command per run with statement-to-assignment bank-account provenance. The public run API opens only `evaluating` scope and does not match, approve, close, or post journals. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md index c284ee3f..23a1306c 100644 --- a/docs/TEST_STRATEGY.md +++ b/docs/TEST_STRATEGY.md @@ -32,7 +32,7 @@ Required regressions include: - direct update/delete of finalized journal, line, source-reference, reversal, receipt and proposal-source facts fails at the database boundary; - after an authoritative posting receipt exists, a late journal-line or source-reference insert into that finalized journal fails before it can extend the monetary/evidence population; - hard-closed periods reject later inserts; -- reconciliation-run command evidence is forced-RLS and immutable, and exact retries replay while changed command evidence conflicts; +- reconciliation-run command evidence is forced-RLS and immutable, exact retries replay while changed command evidence conflicts, historical runs exclude facts recorded after their knowledge cutoff, and deferred database provenance rejects orphan runs or cross-bank statement bindings; - a login that merely sets `accounting_core.journal_write_role` cannot insert into a soft-closed period; - a purpose-limited session login that is a member of `accounting_closing_writer` can exercise the supported soft-close exception path; - migration `0005` changes a pre-existing LOGIN `accounting_closing_writer` back to `NOLOGIN`; diff --git a/docs/adr/0058-reconciliation-run-command-evidence.md b/docs/adr/0058-reconciliation-run-command-evidence.md index 1515a932..abdbe97e 100644 --- a/docs/adr/0058-reconciliation-run-command-evidence.md +++ b/docs/adr/0058-reconciliation-run-command-evidence.md @@ -33,6 +33,11 @@ After the command lock, an existing idempotency key is resolved from its stored run evidence before live assignment validation, so an exact retry remains a replay even if that assignment later closes or overlaps; changed request fields still fail closed. +For a new run, every selected statement, artifact, bank account, assignment, +legal-entity, and accounting-book fact must have been recorded no later than +`knowledge_cutoff_at`. A deferred database trigger also requires exactly one +command row at commit and verifies that its statement belongs to the run's +assigned bank account. `GET /reconciliation-runs` returns the same tenant-scoped run document. Distinct idempotency keys may open distinct immutable runs for the same statement diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 999792a1..aca3870e 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -14,7 +14,7 @@ | PostgreSQL 18.6 test environment | The real regression environment uses PostgreSQL 18.6 to exercise the repository's PostgreSQL 18.4 compatibility pin. PostgreSQL's built-in `sha256(bytea)` and `encode(..., 'hex')` provide the database-owned approval snapshot digest. A row-level `BEFORE` trigger overwrites caller-supplied snapshot input, and a shared transaction-level advisory lock serializes approval, allocation, and terminal match transitions so a valid command cannot authorize a changed candidate/allocation population; migration 0017 applies the parent-row-first repair so concurrent approval and allocation cannot form a row/advisory deadlock. Immutable source-payload hash/reference provenance remains separate from that state digest. | `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, real PostgreSQL snapshot and lock-order regressions, ADR 0055, PostgreSQL binary-string, trigger, and transaction-isolation documentation | | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | -| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP reconciliation-run regressions, ADR 0058 | +| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and the deferred database guard rejects orphan runs and cross-bank statement provenance. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/src/accounting_information_platform/reconciliation_run.py b/src/accounting_information_platform/reconciliation_run.py index 35892c00..b3672acc 100644 --- a/src/accounting_information_platform/reconciliation_run.py +++ b/src/accounting_information_platform/reconciliation_run.py @@ -152,6 +152,12 @@ def accept_reconciliation_run( AND accounting_book.book_name = %s AND assignment.valid_from <= %s AND (assignment.valid_to IS NULL OR assignment.valid_to > %s) + AND statement.recorded_at <= %s + AND account.recorded_at <= %s + AND assignment.recorded_at <= %s + AND legal_entity.recorded_at <= %s + AND accounting_book.recorded_at <= %s + AND artifact.recorded_at <= %s """, ( tenant_id, @@ -160,6 +166,12 @@ def accept_reconciliation_run( accounting_book_reference, bank_cutoff_at, bank_cutoff_at, + knowledge_cutoff_at, + knowledge_cutoff_at, + knowledge_cutoff_at, + knowledge_cutoff_at, + knowledge_cutoff_at, + knowledge_cutoff_at, ), ).fetchall() if not binding_rows: diff --git a/tests/test_reconciliation_candidate_allocation_persistence_red.py b/tests/test_reconciliation_candidate_allocation_persistence_red.py index 068dd635..bfef495f 100644 --- a/tests/test_reconciliation_candidate_allocation_persistence_red.py +++ b/tests/test_reconciliation_candidate_allocation_persistence_red.py @@ -12,8 +12,11 @@ from tests import test_postgres_posting as posting from accounting_information_platform import ( + CAMT053_MESSAGE_DEFINITION, accept_bank_account_assignment, accept_bank_account_record, + accept_bank_statement_evidence, + load_canonical_statement_fixture, ) @@ -94,7 +97,7 @@ def setUp(self) -> None: "tenant_reference": self.case.policy.tenant_reference, "bank_account_reference": self.account_reference, "account_currency_code": "KRW", - "account_identifier": "acct-opaque-allocation-fixture", + "account_identifier": "acct-opaque-fixture-only", }, posting.DATABASE_URL, self.case.policy.tenant_reference, @@ -112,7 +115,21 @@ def setUp(self) -> None: posting.DATABASE_URL, self.case.policy.tenant_reference, ) - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + statement_payload = load_canonical_statement_fixture().replace( + b"Invoice 1001", f"Invoice {uuid.uuid4().hex[:8]}".encode(), 1 + ) + self.statement_record = accept_bank_statement_evidence( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": self.account_reference, + "message_definition_identifier": CAMT053_MESSAGE_DEFINITION, + "statement_payload": statement_payload.decode("utf-8"), + "ingestion_idempotency_key": f"statement-run-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + with psycopg.connect(posting.DATABASE_URL) as connection: assignments = connection.execute( """ SELECT a.tenant_account_id, a.legal_entity_id, a.accounting_book_id, @@ -131,7 +148,7 @@ def setUp(self) -> None: "bank_account_assignment_id": assignments[0][3], } self.run_reference = uuid.uuid4() - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: connection.execute( "SELECT set_config('app.tenant_account_id', %s, false)", (str(self.scope["tenant_account_id"]),), @@ -157,7 +174,25 @@ def setUp(self) -> None: VALID_FROM, ), ) - connection.execute("RESET app.tenant_account_id") + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, bank_statement_record_id, + reconciliation_idempotency_key, reconciliation_command_hash, + source_payload_hash, source_payload_reference + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + self.scope["tenant_account_id"], + self.run_reference, + self.statement_record["bank_statement_record_id"], + f"run-evidence-{uuid.uuid4().hex}", + "sha256:" + "c" * 64, + self.statement_record["source_artifact_hash"], + f"memory:{self.statement_record['source_artifact_hash']}", + ), + ) + connection.commit() def _insert_candidate( self, diff --git a/tests/test_reconciliation_cross_run_conservation_red.py b/tests/test_reconciliation_cross_run_conservation_red.py index 07be76b5..37401f94 100644 --- a/tests/test_reconciliation_cross_run_conservation_red.py +++ b/tests/test_reconciliation_cross_run_conservation_red.py @@ -26,7 +26,7 @@ def setUp(self) -> None: def _insert_run(self) -> uuid.UUID: run_reference = uuid.uuid4() - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: connection.execute( "SELECT set_config('app.tenant_account_id', %s, false)", (str(self.case.scope["tenant_account_id"]),), @@ -52,7 +52,25 @@ def _insert_run(self) -> uuid.UUID: allocation.VALID_FROM, ), ) - connection.execute("RESET app.tenant_account_id") + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, bank_statement_record_id, + reconciliation_idempotency_key, reconciliation_command_hash, + source_payload_hash, source_payload_reference + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + self.case.scope["tenant_account_id"], + run_reference, + self.case.statement_record["bank_statement_record_id"], + f"run-evidence-{uuid.uuid4().hex}", + "sha256:" + "c" * 64, + self.case.statement_record["source_artifact_hash"], + f"memory:{self.case.statement_record['source_artifact_hash']}", + ), + ) + connection.commit() return run_reference def _insert_candidate( diff --git a/tests/test_reconciliation_database_control_hardening_red.py b/tests/test_reconciliation_database_control_hardening_red.py index 7be7bc4f..dd6aa005 100644 --- a/tests/test_reconciliation_database_control_hardening_red.py +++ b/tests/test_reconciliation_database_control_hardening_red.py @@ -101,7 +101,7 @@ def test_match_cannot_retarget_candidate_after_allocations_exist(self) -> None: match_id, "stmt-retarget-a", "journal-retarget-a", "1000.00" ) - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: with self.assertRaisesRegex( psycopg.errors.CheckViolation, "reconciliation_match_identity_immutable", @@ -187,7 +187,7 @@ def test_superseded_match_cannot_reopen_or_rewrite_review_time(self) -> None: def test_conflicting_candidate_amount_cannot_enter_concurrently(self) -> None: """Two runs cannot concurrently establish different capacities for one source identity.""" second_run = uuid.uuid4() - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: connection.execute( """ INSERT INTO accounting_core.reconciliation_run ( @@ -209,6 +209,25 @@ def test_conflicting_candidate_amount_cannot_enter_concurrently(self) -> None: self.fixture.run_reference, ), ) + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, bank_statement_record_id, + reconciliation_idempotency_key, reconciliation_command_hash, + source_payload_hash, source_payload_reference + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + self.fixture.scope["tenant_account_id"], + second_run, + self.fixture.statement_record["bank_statement_record_id"], + f"run-evidence-{uuid.uuid4().hex}", + "sha256:" + "c" * 64, + self.fixture.statement_record["source_artifact_hash"], + f"memory:{self.fixture.statement_record['source_artifact_hash']}", + ), + ) + connection.commit() first_candidate = uuid.uuid4() second_candidate = uuid.uuid4() @@ -276,7 +295,7 @@ def test_statement_capacity_survives_assignment_rollover_for_same_bank_account(s ) self.fixture._approve_match(first_match) - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: assignment = connection.execute( """ SELECT bank_account_record_id, chart_account_id, valid_from @@ -342,6 +361,25 @@ def test_statement_capacity_survives_assignment_rollover_for_same_bank_account(s rollover_at, ), ) + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, bank_statement_record_id, + reconciliation_idempotency_key, reconciliation_command_hash, + source_payload_hash, source_payload_reference + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + self.fixture.scope["tenant_account_id"], + second_run, + self.fixture.statement_record["bank_statement_record_id"], + f"run-evidence-{uuid.uuid4().hex}", + "sha256:" + "c" * 64, + self.fixture.statement_record["source_artifact_hash"], + f"memory:{self.fixture.statement_record['source_artifact_hash']}", + ), + ) + connection.commit() second_candidate = connection.execute( """ INSERT INTO accounting_core.reconciliation_candidate ( diff --git a/tests/test_reconciliation_match_allocation_balance_red.py b/tests/test_reconciliation_match_allocation_balance_red.py index 3fb6ce69..63e529a2 100644 --- a/tests/test_reconciliation_match_allocation_balance_red.py +++ b/tests/test_reconciliation_match_allocation_balance_red.py @@ -10,8 +10,11 @@ import psycopg from accounting_information_platform import ( + CAMT053_MESSAGE_DEFINITION, accept_bank_account_assignment, accept_bank_account_record, + accept_bank_statement_evidence, + load_canonical_statement_fixture, ) from tests import test_postgres_posting as posting @@ -38,7 +41,7 @@ def setUp(self) -> None: "tenant_reference": self.case.policy.tenant_reference, "bank_account_reference": self.account_reference, "account_currency_code": "KRW", - "account_identifier": "acct-opaque-balance-fixture", + "account_identifier": "acct-opaque-fixture-only", }, posting.DATABASE_URL, self.case.policy.tenant_reference, @@ -56,7 +59,21 @@ def setUp(self) -> None: posting.DATABASE_URL, self.case.policy.tenant_reference, ) - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + statement_payload = load_canonical_statement_fixture().replace( + b"Invoice 1001", f"Invoice {uuid.uuid4().hex[:8]}".encode(), 1 + ) + self.statement_record = accept_bank_statement_evidence( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": self.account_reference, + "message_definition_identifier": CAMT053_MESSAGE_DEFINITION, + "statement_payload": statement_payload.decode("utf-8"), + "ingestion_idempotency_key": f"statement-balance-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + with psycopg.connect(posting.DATABASE_URL) as connection: assignment = connection.execute( """ SELECT a.tenant_account_id, a.legal_entity_id, a.accounting_book_id, @@ -77,7 +94,7 @@ def setUp(self) -> None: "bank_account_assignment_id": assignment[3], } self.run_reference = uuid.uuid4() - with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with psycopg.connect(posting.DATABASE_URL) as connection: connection.execute( """ INSERT INTO accounting_core.reconciliation_run ( @@ -99,6 +116,25 @@ def setUp(self) -> None: VALID_FROM, ), ) + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, bank_statement_record_id, + reconciliation_idempotency_key, reconciliation_command_hash, + source_payload_hash, source_payload_reference + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + self.scope["tenant_account_id"], + self.run_reference, + self.statement_record["bank_statement_record_id"], + f"run-evidence-{uuid.uuid4().hex}", + "sha256:" + "c" * 64, + self.statement_record["source_artifact_hash"], + f"memory:{self.statement_record['source_artifact_hash']}", + ), + ) + connection.commit() def _create_proposed_match(self) -> uuid.UUID: candidate_id = uuid.uuid4() diff --git a/tests/test_reconciliation_run_api.py b/tests/test_reconciliation_run_api.py index 1e81865b..b818a0a8 100644 --- a/tests/test_reconciliation_run_api.py +++ b/tests/test_reconciliation_run_api.py @@ -91,11 +91,31 @@ def _statement_and_command(self) -> tuple[dict[str, object], dict[str, object]]: "bank_cutoff_at": "2026-08-24T23:59:59Z", "book_cutoff_at": "2026-08-24T23:59:59Z", "matching_policy_version": "deterministic-v1", - "knowledge_cutoff_at": "2026-08-25T00:00:00Z", + "knowledge_cutoff_at": "2026-09-01T00:00:00Z", "reconciliation_idempotency_key": f"run-{uuid.uuid4().hex}", "source_payload_hash": source_payload_hash, } + def _assignment_scope(self, assignment_id: str | None = None) -> tuple[object, ...]: + """Return the tenant-scoped identifiers needed for direct SQL controls.""" + with psycopg.connect(posting.DATABASE_URL) as connection: + return connection.execute( + """ + SELECT assignment.tenant_account_id, + assignment.legal_entity_id, + assignment.accounting_book_id, + assignment.bank_account_assignment_id, + account.account_currency_code + FROM accounting_core.bank_account_assignment AS assignment + JOIN accounting_core.bank_account_record AS account + ON account.tenant_account_id = assignment.tenant_account_id + AND account.bank_account_record_id = assignment.bank_account_record_id + WHERE assignment.bank_account_assignment_id = COALESCE(%s::uuid, assignment.bank_account_assignment_id) + AND account.bank_account_reference = %s + """, + (assignment_id, self.account_reference), + ).fetchone() + def test_open_run_binds_statement_scope_and_replays(self) -> None: """An exact command opens one evaluating run and an exact retry replays it.""" statement, command = self._statement_and_command() @@ -150,6 +170,143 @@ def test_exact_retry_replays_after_assignment_closes(self) -> None: self.assertTrue(replay["replayed"]) self.assertEqual(first["reconciliation_run_id"], replay["reconciliation_run_id"]) + def test_statement_recorded_after_knowledge_cutoff_is_rejected(self) -> None: + """A historical run cannot include a statement learned after its cutoff.""" + _statement, command = self._statement_and_command() + historical = dict(command, knowledge_cutoff_at="2026-08-25T00:00:00Z") + with self.assertRaisesRegex(AccountingValidationError, "not bound"): + accept_reconciliation_run( + historical, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + + def test_assignment_recorded_after_knowledge_cutoff_is_rejected(self) -> None: + """A historical run cannot use an assignment learned after its cutoff.""" + _statement, command = self._statement_and_command() + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + connection.execute( + """ + UPDATE accounting_core.bank_account_assignment + SET recorded_at = '2026-09-02T00:00:00Z' + WHERE bank_account_assignment_id = ( + SELECT assignment.bank_account_assignment_id + FROM accounting_core.bank_account_assignment AS assignment + JOIN accounting_core.bank_account_record AS account + ON account.tenant_account_id = assignment.tenant_account_id + AND account.bank_account_record_id = assignment.bank_account_record_id + WHERE account.bank_account_reference = %s + ) + """, + (self.account_reference,), + ) + historical = dict(command, knowledge_cutoff_at="2026-09-01T00:00:00Z") + with self.assertRaisesRegex(AccountingValidationError, "not bound"): + accept_reconciliation_run( + historical, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + + def test_database_rejects_orphan_reconciliation_run_at_commit(self) -> None: + """A run cannot commit without exactly one immutable command-evidence row.""" + _statement, command = self._statement_and_command() + scope = self._assignment_scope() + assert scope is not None + with psycopg.connect(posting.DATABASE_URL) as connection: + with self.assertRaisesRegex(psycopg.Error, "exactly one command"): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run ( + tenant_account_id, legal_entity_id, accounting_book_id, + bank_account_assignment_id, currency_code, bank_cutoff_at, + book_cutoff_at, matching_policy_version, knowledge_cutoff_at, + run_status_code + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'evaluating') + """, + ( + scope[0], + scope[1], + scope[2], + scope[3], + scope[4], + command["bank_cutoff_at"], + command["book_cutoff_at"], + command["matching_policy_version"], + command["knowledge_cutoff_at"], + ), + ) + connection.commit() + + def test_database_rejects_command_for_different_bank_account_at_commit(self) -> None: + """A run command cannot bind evidence from another bank account.""" + _statement, command = self._statement_and_command() + second_account_reference = f"urn:cwl:bank_account:{uuid.uuid4().hex}" + accept_bank_account_record( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": second_account_reference, + "account_currency_code": "KRW", + "account_identifier": "acct-opaque-fixture-only", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + second_statement = accept_bank_statement_evidence( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": second_account_reference, + "message_definition_identifier": CAMT053_MESSAGE_DEFINITION, + "statement_payload": load_canonical_statement_fixture() + .replace(b"Invoice 1001", b"Invoice 1999", 1) + .decode("utf-8"), + "ingestion_idempotency_key": f"statement-run-second-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + artifact_store=self.store, + ) + scope = self._assignment_scope() + assert scope is not None + with psycopg.connect(posting.DATABASE_URL) as connection: + with self.assertRaisesRegex(psycopg.Error, "bank account provenance"): + connection.execute( + """ + WITH inserted_run AS ( + INSERT INTO accounting_core.reconciliation_run ( + tenant_account_id, legal_entity_id, accounting_book_id, + bank_account_assignment_id, currency_code, bank_cutoff_at, + book_cutoff_at, matching_policy_version, knowledge_cutoff_at, + run_status_code + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'evaluating') + RETURNING tenant_account_id, reconciliation_run_id + ) + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, + bank_statement_record_id, reconciliation_idempotency_key, + reconciliation_command_hash, source_payload_hash, + source_payload_reference + ) + SELECT tenant_account_id, reconciliation_run_id, %s, %s, %s, %s, %s + FROM inserted_run + """, + ( + scope[0], + scope[1], + scope[2], + scope[3], + scope[4], + command["bank_cutoff_at"], + command["book_cutoff_at"], + command["matching_policy_version"], + command["knowledge_cutoff_at"], + second_statement["bank_statement_record_id"], + f"direct-provenance-{uuid.uuid4().hex}", + "sha256:" + "1" * 64, + second_statement["source_artifact_hash"], + f"memory:{second_statement['source_artifact_hash']}", + ), + ) + connection.commit() + def test_wrong_source_hash_fails_before_run_persistence(self) -> None: """A run cannot claim a different immutable bank-statement source.""" _statement, command = self._statement_and_command() @@ -205,7 +362,7 @@ def test_statement_period_must_be_covered_by_bank_cutoff(self) -> None: before_start = dict( command, bank_cutoff_at="2026-08-22T23:59:59Z", - knowledge_cutoff_at="2026-08-25T00:00:00Z", + knowledge_cutoff_at="2026-09-01T00:00:00Z", ) with self.assertRaisesRegex(AccountingValidationError, "before the statement period"): accept_reconciliation_run( @@ -214,7 +371,7 @@ def test_statement_period_must_be_covered_by_bank_cutoff(self) -> None: before_end = dict( command, bank_cutoff_at="2026-08-24T12:00:00Z", - knowledge_cutoff_at="2026-08-25T00:00:00Z", + knowledge_cutoff_at="2026-09-01T00:00:00Z", ) with self.assertRaisesRegex( AccountingValidationError, "before the statement period end" From cb309f5e832e51608df70c1838bfdd43e90c3ec6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 04:33:13 +0900 Subject: [PATCH 13/25] fix(reconciliation): bind matches to source cash evidence --- CHANGELOG.md | 1 + ..._reconciliation_match_command_evidence.sql | 71 +++++ docs/DATA_MODEL.md | 3 +- docs/TEST_STRATEGY.md | 1 + ...9-reconciliation-match-command-evidence.md | 14 + docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- .../reconciliation_match.py | 89 ++++++- tests/test_reconciliation_match_api.py | 242 ++++++++++++++++-- 8 files changed, 392 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcf9a95e..0a54d672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. - Added the tenant-scoped `POST /reconciliation-matches` and `GET /reconciliation-matches?reconciliation_match_id=` command boundary with migration `0020_reconciliation_match_command_evidence.sql`. One exact 1:1 proposed match persists its candidate, statement/journal allocations, idempotency key, canonical command hash, and immutable source-payload provenance atomically; exact retries replay and changed evidence conflicts. Quoted positive equal decimal amounts must match the bound bank-entry and posted-journal source facts, and database source-conservation violations return stable validation errors. The HTTP boundary distinguishes malformed identifiers (400), absent sources (404), state conflicts (409), and source-content validation (422). The evidence cannot approve, close, select chart accounts, or post journals. ADR 0059 records the boundary. diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql index 6e82c83c..efd0476e 100644 --- a/database/migrations/0020_reconciliation_match_command_evidence.sql +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -89,4 +89,75 @@ BEFORE UPDATE OR DELETE ON accounting_core.reconciliation_match_command FOR EACH ROW EXECUTE FUNCTION accounting_core.reject_reconciliation_match_command_mutation(); +CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_match_command_allocations() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + statement_allocation_count bigint; + journal_allocation_count bigint; + statement_allocation_total numeric(30, 6); + journal_allocation_total numeric(30, 6); +BEGIN + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO statement_allocation_count, statement_allocation_total + FROM accounting_core.statement_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO journal_allocation_count, journal_allocation_total + FROM accounting_core.journal_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + IF statement_allocation_count <> 1 + OR journal_allocation_count <> 1 + OR statement_allocation_total <> journal_allocation_total THEN + RAISE EXCEPTION + 'reconciliation match command requires exactly one statement and one journal allocation with equal amounts (reconciliation_match_command_allocation)' + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +CREATE TRIGGER z_reconciliation_match_command_allocation_guard +AFTER INSERT +ON accounting_core.reconciliation_match_command +FOR EACH ROW EXECUTE FUNCTION accounting_core.enforce_reconciliation_match_command_allocations(); + +CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_allocation() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM accounting_core.reconciliation_match_command AS command + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id + AND command.reconciliation_match_id = NEW.reconciliation_match_id + ) THEN + RAISE EXCEPTION + 'reconciliation match command evidence freezes its allocation population; create a new proposed match instead (reconciliation_match_command_allocation_frozen)' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER z_reconciliation_match_command_allocation_frozen_guard +BEFORE INSERT +ON accounting_core.statement_match_allocation +FOR EACH ROW EXECUTE FUNCTION accounting_core.reject_reconciliation_match_command_allocation(); + +CREATE TRIGGER z_reconciliation_match_command_allocation_frozen_guard +BEFORE INSERT +ON accounting_core.journal_match_allocation +FOR EACH ROW EXECUTE FUNCTION accounting_core.reject_reconciliation_match_command_allocation(); + COMMIT; diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index caf7a2a0..cc8825f7 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -50,9 +50,8 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconc - `statement_match_allocation`: append-only exact amount consumed from an immutable statement source reference by one reconciliation match. - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. -- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires the supplied amounts to equal the bound immutable bank-entry and posted-journal source facts in the run's accounting scope. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; a deferred database provenance guard requires exactly one command row and binds its statement to the run assignment's bank account. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and from posted journals no later than the book cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard requires exactly one equal allocation on each side at command insert and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md index 23a1306c..b6f28f09 100644 --- a/docs/TEST_STRATEGY.md +++ b/docs/TEST_STRATEGY.md @@ -33,6 +33,7 @@ Required regressions include: - after an authoritative posting receipt exists, a late journal-line or source-reference insert into that finalized journal fails before it can extend the monetary/evidence population; - hard-closed periods reject later inserts; - reconciliation-run command evidence is forced-RLS and immutable, exact retries replay while changed command evidence conflicts, historical runs exclude facts recorded after their knowledge cutoff, and deferred database provenance rejects orphan runs or cross-bank statement bindings; +- reconciliation-match command evidence is forced-RLS and immutable, its run lock prevents a racing terminal transition, source entry/journal cutoffs and CRDT/DBIT cash-line direction are enforced, compound journals use the assigned cash line, and database triggers require one equal allocation per side while rejecting late allocations; - a login that merely sets `accounting_core.journal_write_role` cannot insert into a soft-closed period; - a purpose-limited session login that is a member of `accounting_closing_writer` can exercise the supported soft-close exception path; - migration `0005` changes a pre-existing LOGIN `accounting_closing_writer` back to `NOLOGIN`; diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 84ad5e45..62ef0355 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -34,6 +34,15 @@ reconciliation_candidate_id)` to the same composite identity on with a different valid match from the same run. Uniqueness rules additionally prevent duplicate command or match identities. +The command also binds source admission to the run snapshot: statement booking +and value timestamps must be no later than the bank cutoff, and journal +accounting date must be no later than the book cutoff. The journal amount is +read from the run assignment's cash chart account rather than the journal-wide +total; a `CRDT` statement requires that line on the debit side and a `DBIT` +statement requires it on the credit side. The run row is locked through the +candidate and allocation writes so a concurrent run transition cannot race the +match. + `accept_reconciliation_match` is the smallest durable command boundary: it accepts one exact 1:1 proposed match, requires quoted positive equal decimal amounts that equal the bound immutable bank-entry and posted-journal source @@ -49,6 +58,11 @@ document. The HTTP boundary reports malformed identifiers as `400`, absent source evidence as `404`, state conflicts as `409`, and source-content or conservation validation failures as `422`. +The database requires command insertion to observe exactly one statement and +one journal allocation with equal exact amounts, and rejects allocation rows +inserted after command evidence. These are evidence-integrity controls only; +they do not turn a proposed match into an approved accounting fact. + The command intentionally does not replace the existing pure split/aggregate allocation planner or the database approval workflow. It provides a bounded buyer-facing persistence seam for the common 1:1 case; many-to-many planning, diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 0f94904b..f9a05901 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,7 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and the deferred database guard rejects orphan runs and cross-bank statement provenance. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` requires an evaluating run, quoted positive equal decimal amounts matching the bound immutable bank entry and posted journal, tenant-bound source evidence, and a tenant-scoped idempotency key; exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP reconciliation-match regressions, ADR 0059 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run cutoffs, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards require exactly one equal allocation on each side at command insert and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py index a6bcba03..25d47b38 100644 --- a/src/accounting_information_platform/reconciliation_match.py +++ b/src/accounting_information_platform/reconciliation_match.py @@ -94,10 +94,12 @@ def accept_reconciliation_match( run = connection.execute( """ - SELECT run_status_code, accounting_book_id, currency_code + SELECT run_status_code, accounting_book_id, currency_code, + bank_account_assignment_id, bank_cutoff_at, book_cutoff_at FROM accounting_core.reconciliation_run WHERE tenant_account_id = %s AND reconciliation_run_id = %s + FOR UPDATE """, (tenant_id, run_id), ).fetchone() @@ -117,6 +119,9 @@ def accept_reconciliation_match( reconciliation_run_id=run_id, accounting_book_id=run[1], currency_code=run[2], + bank_account_assignment_id=run[3], + bank_cutoff_at=run[4], + book_cutoff_at=run[5], statement_reference=statement_reference, journal_reference=journal_reference, statement_amount=statement_amount, @@ -348,15 +353,19 @@ def _require_recorded_source_amounts( reconciliation_run_id: UUID, accounting_book_id: UUID, currency_code: str, + bank_account_assignment_id: UUID, + bank_cutoff_at: object, + book_cutoff_at: object, statement_reference: str, journal_reference: str, statement_amount: Decimal, journal_amount: Decimal, ) -> None: - """Require command amounts to equal tenant-scoped immutable source facts.""" + """Require exact source amounts, direction, assignment, and run cutoffs.""" statement_rows = connection.execute( """ - SELECT entry.entry_amount, entry.entry_currency_code + SELECT entry.entry_amount, entry.entry_currency_code, + entry.credit_debit_code FROM accounting_integration.bank_statement_entry AS entry JOIN accounting_core.reconciliation_run_command AS run_command ON run_command.tenant_account_id = entry.tenant_account_id @@ -368,8 +377,17 @@ def _require_recorded_source_amounts( AND run_command.reconciliation_run_id = %s AND (entry.source_entry_identity = %s OR entry.bank_statement_entry_id::text = %s) AND run_scope.currency_code = entry.entry_currency_code + AND (entry.booking_occurred_at IS NULL OR entry.booking_occurred_at <= %s) + AND (entry.value_occurred_at IS NULL OR entry.value_occurred_at <= %s) """, - (tenant_id, reconciliation_run_id, statement_reference, statement_reference), + ( + tenant_id, + reconciliation_run_id, + statement_reference, + statement_reference, + bank_cutoff_at, + bank_cutoff_at, + ), ).fetchall() if len(statement_rows) != 1: raise AccountingValidationError( @@ -382,24 +400,61 @@ def _require_recorded_source_amounts( "statement_amount does not match recorded statement source amount. " "Supply the exact recorded statement amount, then retry the match." ) + statement_direction = statement_rows[0][2] + if statement_direction not in {"CRDT", "DBIT"}: + raise AccountingValidationError( + "statement source evidence has an unsupported direction. " + "Supply a CRDT or DBIT statement entry, then retry the match." + ) journal_row = connection.execute( """ SELECT journal.journal_status_code, journal.transaction_currency_code, COALESCE(SUM(line.debit_amount), 0), - COALESCE(SUM(line.credit_amount), 0) + COALESCE(SUM(line.credit_amount), 0), + COALESCE( + SUM( + CASE + WHEN line.chart_account_id = assignment.chart_account_id + THEN line.debit_amount + ELSE 0 + END + ), + 0 + ), + COALESCE( + SUM( + CASE + WHEN line.chart_account_id = assignment.chart_account_id + THEN line.credit_amount + ELSE 0 + END + ), + 0 + ) FROM accounting_core.general_journal AS journal + JOIN accounting_core.bank_account_assignment AS assignment + ON assignment.tenant_account_id = journal.tenant_account_id + AND assignment.accounting_book_id = journal.accounting_book_id LEFT JOIN accounting_core.journal_entry_line AS line ON line.tenant_account_id = journal.tenant_account_id AND line.general_journal_id = journal.general_journal_id WHERE journal.tenant_account_id = %s AND journal.accounting_book_id = %s AND journal.journal_reference = %s + AND assignment.bank_account_assignment_id = %s + AND journal.accounting_date <= (%s::timestamptz AT TIME ZONE 'UTC')::date GROUP BY journal.general_journal_id, journal.journal_status_code, - journal.transaction_currency_code + journal.transaction_currency_code, assignment.chart_account_id """, - (tenant_id, accounting_book_id, journal_reference), + ( + tenant_id, + accounting_book_id, + journal_reference, + bank_account_assignment_id, + book_cutoff_at, + ), ).fetchone() if journal_row is None: raise AccountingValidationError( @@ -416,9 +471,25 @@ def _require_recorded_source_amounts( "journal source evidence is not balanced and positive. " "Supply a balanced posted journal, then retry the match." ) - if journal_row[2] != journal_amount: + cash_debit = journal_row[4] + cash_credit = journal_row[5] + expected_cash_debit = statement_direction == "CRDT" + if ( + expected_cash_debit + and cash_credit != 0 + ) or ( + not expected_cash_debit + and cash_debit != 0 + ): + expected_side = "debit" if expected_cash_debit else "credit" + raise AccountingValidationError( + "journal source evidence direction does not match the statement direction; " + f"the assigned cash line must carry the amount on the {expected_side} side. " + "Supply matching CRDT/DBIT source evidence, then retry the match." + ) + if (cash_debit if expected_cash_debit else cash_credit) != journal_amount: raise AccountingValidationError( - "journal_amount does not match recorded journal source amount. " + "journal_amount does not match recorded assigned cash line amount in the journal source. " "Supply the exact recorded journal amount, then retry the match." ) diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index 73eb9a3b..c4c276fe 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -5,7 +5,7 @@ import hashlib import unittest import uuid -from datetime import date +from datetime import date, datetime, timezone from decimal import Decimal from unittest import mock @@ -15,6 +15,7 @@ CAMT053_MESSAGE_DEFINITION, AccountingValidationError, IdempotencyConflictError, + JournalLineProposal, MemoryArtifactStore, accept_bank_account_assignment, accept_bank_account_record, @@ -69,8 +70,22 @@ def setUp(self) -> None: self.case.policy.tenant_reference, ) - def _open_run(self) -> tuple[str, str]: + def _open_run( + self, entry_index: int = 0, late_statement_entry: bool = False + ) -> tuple[str, str]: fixture = load_canonical_statement_fixture() + fixture = fixture.replace( + b"BANK-STMT-2026-08-24", f"BANK-STMT-{uuid.uuid4().hex[:12]}".encode(), 1 + ) + fixture = fixture.replace( + b"Invoice 1001", f"Invoice {uuid.uuid4().hex[:8]}".encode(), 1 + ) + if late_statement_entry: + fixture = fixture.replace( + b"2026-08-24T01:15:00+00:00", + b"2026-08-25T01:15:00+00:00", + 1, + ) statement = accept_bank_statement_evidence( { "tenant_reference": self.case.policy.tenant_reference, @@ -88,7 +103,7 @@ def _open_run(self) -> tuple[str, str]: self.case.policy.tenant_reference, str(statement["bank_statement_record_id"]), )["bank_statement_entries"] - entry = entries[0] + entry = entries[entry_index] statement_hash = "sha256:" + hashlib.sha256(fixture).hexdigest() run = accept_reconciliation_run( { @@ -99,7 +114,7 @@ def _open_run(self) -> tuple[str, str]: "bank_cutoff_at": "2026-08-24T23:59:59Z", "book_cutoff_at": "2026-08-24T23:59:59Z", "matching_policy_version": "deterministic-v1", - "knowledge_cutoff_at": "2026-08-25T00:00:00Z", + "knowledge_cutoff_at": "2026-09-01T00:00:00Z", "reconciliation_idempotency_key": f"run-match-{uuid.uuid4().hex}", "source_payload_hash": statement_hash, }, @@ -110,16 +125,50 @@ def _open_run(self) -> tuple[str, str]: entry["source_entry_identity"] or entry["bank_statement_entry_id"] ) - def _command(self) -> dict[str, object]: - run_id, statement_reference = self._open_run() + def _command( + self, + *, + entry_index: int = 0, + amount: str = "25000", + cash_direction: str = "debit", + accounting_date: date = date(2026, 8, 24), + extra_debit: str = "0", + late_statement_entry: bool = False, + ) -> dict[str, object]: + run_id, statement_reference = self._open_run(entry_index, late_statement_entry) + cash_debit, cash_credit = ( + (amount, "0") if cash_direction == "debit" else ("0", amount) + ) + if cash_direction == "debit" and Decimal(extra_debit) > 0: + lines = ( + JournalLineProposal(1, "cash_receipt", cash_debit, cash_credit), + JournalLineProposal(2, "accounts_receivable", extra_debit, "0"), + JournalLineProposal( + 3, + "usage_revenue", + "0", + str(Decimal(amount) + Decimal(extra_debit)), + ), + ) + elif cash_direction == "debit": + lines = ( + JournalLineProposal(1, "cash_receipt", cash_debit, cash_credit), + JournalLineProposal(2, "usage_revenue", "0", amount), + ) + else: + lines = ( + JournalLineProposal(1, "cash_receipt", cash_debit, cash_credit), + JournalLineProposal(2, "accounts_receivable", amount, "0"), + ) journal = self.case.ledger.post( self.case._two_line_proposal( proposal_id=str(uuid.uuid4()), idempotency_key=f"match-journal-{uuid.uuid4().hex}", source_payload_hash="sha256:" + "9" * 64, source_event_references=(f"urn:cwl:reconciliation:journal:{uuid.uuid4()}",), - transaction_date=date(2026, 8, 24), - accounting_date=date(2026, 8, 24), + transaction_date=accounting_date, + accounting_date=accounting_date, + lines=lines, ), self.case.policy, ) @@ -128,8 +177,8 @@ def _command(self) -> dict[str, object]: "reconciliation_run_id": run_id, "statement_entry_reference": statement_reference, "journal_reference": journal.journal_reference, - "statement_amount": "25000.00", - "journal_amount": "25000.00", + "statement_amount": f"{Decimal(amount):.2f}", + "journal_amount": f"{Decimal(amount):.2f}", "rule_code": "provider_reference", "candidate_idempotency_key": f"candidate-{uuid.uuid4().hex}", "source_payload_hash": "sha256:" + "1" * 64, @@ -235,18 +284,142 @@ def test_match_command_requires_recorded_source_amounts(self) -> None: tenant, ) + def test_match_uses_assigned_cash_line_for_compound_journal(self) -> None: + """A compound journal matches only the assigned cash line, not its total.""" + command = self._command(extra_debit="100") + document = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + self.assertEqual(document["allocated_amount"], "25000") + + def test_match_enforces_statement_and_cash_journal_direction(self) -> None: + """CRDT uses cash debit and DBIT uses cash credit for source matching.""" + with self.assertRaisesRegex(AccountingValidationError, "direction"): + accept_reconciliation_match( + self._command(cash_direction="credit"), + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + valid = accept_reconciliation_match( + self._command(entry_index=1, amount="10000", cash_direction="credit"), + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + self.assertEqual(valid["match_status_code"], "proposed") + + def test_match_rejects_journal_after_book_cutoff(self) -> None: + """A journal recorded after the run book cutoff is not matchable evidence.""" + with self.assertRaisesRegex(AccountingValidationError, "journal source"): + accept_reconciliation_match( + self._command(accounting_date=date(2026, 8, 25)), + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + + def test_match_rejects_statement_entry_after_bank_cutoff(self) -> None: + """A statement entry outside the run bank cutoff is not matchable evidence.""" + with self.assertRaisesRegex(AccountingValidationError, "not recorded exactly once"): + accept_reconciliation_match( + self._command(late_statement_entry=True), + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + + def test_match_command_requires_complete_allocation_evidence(self) -> None: + """Direct command evidence cannot omit its one-to-one allocation rows.""" + command = self._command() + tenant_id = self.case.tenant_id + with psycopg.connect(posting.DATABASE_URL) as connection: + candidate_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, %s, %s, 'direct-command') + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + command["reconciliation_run_id"], + command["statement_entry_reference"], + command["journal_reference"], + command["statement_amount"], + command["journal_amount"], + ), + ).fetchone()[0] + match_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (tenant_id, command["reconciliation_run_id"], candidate_id), + ).fetchone()[0] + with self.assertRaisesRegex(psycopg.errors.CheckViolation, "one statement"): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + tenant_id, + command["reconciliation_run_id"], + candidate_id, + match_id, + f"direct-command-{uuid.uuid4().hex}", + "sha256:" + "4" * 64, + command["source_payload_hash"], + command["source_payload_reference"], + ), + ) + + def test_match_command_rejects_late_allocation_after_command_evidence(self) -> None: + """Command evidence freezes its allocation population.""" + command = self._command() + document = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + with psycopg.connect(posting.DATABASE_URL) as connection: + with self.assertRaisesRegex(psycopg.errors.CheckViolation, "command evidence"): + connection.execute( + """ + INSERT INTO accounting_core.statement_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, statement_entry_reference, + allocated_amount + ) + VALUES (%s, %s, %s, %s, '1') + """, + ( + self.case.tenant_id, + command["reconciliation_run_id"], + document["reconciliation_match_id"], + command["statement_entry_reference"], + ), + ) + def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> None: """Defensive source checks reject impossible or mismatched journal evidence.""" for journal_row, message in ( - (("posted", "KRW", Decimal("25000"), Decimal("24999")), "balanced and positive"), - (("posted", "KRW", Decimal("24999"), Decimal("24999")), "does not match recorded"), - (("draft", "KRW", Decimal("25000"), Decimal("25000")), "not a posted journal"), - (("posted", "USD", Decimal("25000"), Decimal("25000")), "not a posted journal"), + (("posted", "KRW", Decimal("25000"), Decimal("24999"), Decimal("25000"), Decimal("0")), "balanced and positive"), + (("posted", "KRW", Decimal("24999"), Decimal("24999"), Decimal("24999"), Decimal("0")), "does not match recorded"), + (("draft", "KRW", Decimal("25000"), Decimal("25000"), Decimal("25000"), Decimal("0")), "not a posted journal"), + (("posted", "USD", Decimal("25000"), Decimal("25000"), Decimal("25000"), Decimal("0")), "not a posted journal"), ): with self.subTest(message=message): connection = mock.Mock() statement_result = mock.Mock() - statement_result.fetchall.return_value = [(Decimal("25000"), "KRW")] + statement_result.fetchall.return_value = [(Decimal("25000"), "KRW", "CRDT")] journal_result = mock.Mock() journal_result.fetchone.return_value = journal_row connection.execute.side_effect = [statement_result, journal_result] @@ -257,12 +430,37 @@ def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> reconciliation_run_id=uuid.uuid4(), accounting_book_id=uuid.uuid4(), currency_code="KRW", + bank_account_assignment_id=uuid.uuid4(), + bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), statement_reference="statement-entry", journal_reference="journal-reference", statement_amount=Decimal("25000"), journal_amount=Decimal("25000"), ) + def test_match_source_guard_rejects_unsupported_direction(self) -> None: + """The source guard remains defensive if a malformed row bypasses its DB check.""" + connection = mock.Mock() + statement_result = mock.Mock() + statement_result.fetchall.return_value = [(Decimal("25000"), "KRW", "OTHER")] + connection.execute.return_value = statement_result + with self.assertRaisesRegex(AccountingValidationError, "unsupported direction"): + _require_recorded_source_amounts( + connection, + tenant_id=uuid.uuid4(), + reconciliation_run_id=uuid.uuid4(), + accounting_book_id=uuid.uuid4(), + currency_code="KRW", + bank_account_assignment_id=uuid.uuid4(), + bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + statement_reference="statement-entry", + journal_reference="journal-reference", + statement_amount=Decimal("25000"), + journal_amount=Decimal("25000"), + ) + def test_match_command_maps_source_conservation_guard_to_validation(self) -> None: """A legacy cross-run amount conflict cannot escape as a raw database error.""" command = self._command() @@ -276,6 +474,14 @@ def test_match_command_maps_source_conservation_guard_to_validation(self) -> Non """, (self.case.tenant_id, command["reconciliation_run_id"]), ).fetchone()[0] + source_hash = connection.execute( + """ + SELECT source_artifact_hash + FROM accounting_integration.bank_statement_record + WHERE tenant_account_id = %s AND bank_statement_record_id = %s + """, + (self.case.tenant_id, statement_id), + ).fetchone()[0] second_run = accept_reconciliation_run( { "tenant_reference": tenant, @@ -285,11 +491,9 @@ def test_match_command_maps_source_conservation_guard_to_validation(self) -> Non "bank_cutoff_at": "2026-08-24T23:59:59Z", "book_cutoff_at": "2026-08-24T23:59:59Z", "matching_policy_version": "deterministic-v1", - "knowledge_cutoff_at": "2026-08-25T00:00:00Z", + "knowledge_cutoff_at": "2026-09-01T00:00:00Z", "reconciliation_idempotency_key": f"run-conflict-{uuid.uuid4().hex}", - "source_payload_hash": "sha256:" + hashlib.sha256( - load_canonical_statement_fixture() - ).hexdigest(), + "source_payload_hash": source_hash, }, posting.DATABASE_URL, tenant, From 10ead71a7b60e12911a686b9e57d0e9be62d4147 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 04:51:56 +0900 Subject: [PATCH 14/25] fix(reconciliation): close historical evidence gaps --- CHANGELOG.md | 1 + ...19_reconciliation_run_command_evidence.sql | 5 + ..._reconciliation_match_command_evidence.sql | 15 +- docs/DATA_MODEL.md | 4 +- docs/OPERABILITY.md | 4 +- docs/TEST_STRATEGY.md | 2 +- ...9-reconciliation-match-command-evidence.md | 6 + docs/doctoring/STANDARD_TRACEABILITY.md | 4 +- .../reconciliation_match.py | 11 +- tests/test_reconciliation_match_api.py | 149 ++++++++++++++++++ tests/test_reconciliation_run_api.py | 85 ++++++++++ 11 files changed, 274 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a54d672..58dbdfb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Closed additional match-evidence gaps: journal `posted_at` now respects the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, existing-run command inserts reapply bank-account provenance validation, and the psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. diff --git a/database/migrations/0019_reconciliation_run_command_evidence.sql b/database/migrations/0019_reconciliation_run_command_evidence.sql index 3ab6d5a2..c624e86a 100644 --- a/database/migrations/0019_reconciliation_run_command_evidence.sql +++ b/database/migrations/0019_reconciliation_run_command_evidence.sql @@ -105,6 +105,11 @@ CREATE CONSTRAINT TRIGGER reconciliation_run_command_provenance_guard FOR EACH ROW EXECUTE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance(); +CREATE TRIGGER reconciliation_run_command_provenance_insert_guard + AFTER INSERT ON accounting_core.reconciliation_run_command + FOR EACH ROW + EXECUTE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance(); + ALTER TABLE accounting_core.reconciliation_run_command ENABLE ROW LEVEL SECURITY; ALTER TABLE accounting_core.reconciliation_run_command FORCE ROW LEVEL SECURITY; CREATE POLICY reconciliation_run_command_isolation diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql index efd0476e..90f61bed 100644 --- a/database/migrations/0020_reconciliation_match_command_evidence.sql +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -98,7 +98,16 @@ DECLARE journal_allocation_count bigint; statement_allocation_total numeric(30, 6); journal_allocation_total numeric(30, 6); + candidate_statement_amount numeric(30, 6); + candidate_journal_amount numeric(30, 6); BEGIN + SELECT candidate.statement_amount, candidate.journal_amount + INTO candidate_statement_amount, candidate_journal_amount + FROM accounting_core.reconciliation_candidate AS candidate + WHERE candidate.tenant_account_id = NEW.tenant_account_id + AND candidate.reconciliation_run_id = NEW.reconciliation_run_id + AND candidate.reconciliation_candidate_id = NEW.reconciliation_candidate_id; + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) INTO statement_allocation_count, statement_allocation_total FROM accounting_core.statement_match_allocation AS allocation @@ -115,9 +124,11 @@ BEGIN IF statement_allocation_count <> 1 OR journal_allocation_count <> 1 - OR statement_allocation_total <> journal_allocation_total THEN + OR statement_allocation_total <> journal_allocation_total + OR statement_allocation_total <> candidate_statement_amount + OR journal_allocation_total <> candidate_journal_amount THEN RAISE EXCEPTION - 'reconciliation match command requires exactly one statement and one journal allocation with equal amounts (reconciliation_match_command_allocation)' + 'reconciliation match command requires exactly one statement and one journal allocation matching candidate amounts (reconciliation_match_command_allocation)' USING ERRCODE = '23514'; END IF; diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index cc8825f7..e31ef733 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -50,8 +50,8 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconc - `statement_match_allocation`: append-only exact amount consumed from an immutable statement source reference by one reconciliation match. - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. -- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; a deferred database provenance guard requires exactly one command row and binds its statement to the run assignment's bank account. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and from posted journals no later than the book cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard requires exactly one equal allocation on each side at command insert and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard requires exactly one equal allocation on each side matching the candidate amounts at command insert and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index b3765d4a..ff625bdf 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -43,9 +43,9 @@ Migration `0017_reconciliation_approval_lock_order.sql` is a forward repair for Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric amount, currency, credit/debit direction, typed effective date/time, sequence, locator, and source hash for every camt.053 balance. The effective date/time is distinct from statement period and system `recorded_at`; existing balance hashes remain on the statement row for compatibility. The numeric rows are immutable, forced-RLS evidence that a reconciliation bridge may read but that cannot post, reverse, or mutate a journal. -Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; new runs exclude source facts recorded after `knowledge_cutoff_at`, and a deferred database guard requires one command per run with statement-to-assignment bank-account provenance. The public run API opens only `evaluating` scope and does not match, approve, close, or post journals. +Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; new runs exclude source facts recorded after `knowledge_cutoff_at`, and both the deferred run guard and command INSERT guard require one command per run with statement-to-assignment bank-account provenance, including when a legacy run receives its command later. The public run API opens only `evaluating` scope and does not match, approve, close, or post journals. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` writes the candidate, proposed match, statement allocation, journal allocation, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only an `evaluating` run, quoted positive equal decimal amounts, a bound bank-statement entry, and a posted journal in the run's accounting book whose exact source amounts match the request. Database source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps respect the bank cutoff and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards require exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/TEST_STRATEGY.md b/docs/TEST_STRATEGY.md index b6f28f09..7558a599 100644 --- a/docs/TEST_STRATEGY.md +++ b/docs/TEST_STRATEGY.md @@ -32,7 +32,7 @@ Required regressions include: - direct update/delete of finalized journal, line, source-reference, reversal, receipt and proposal-source facts fails at the database boundary; - after an authoritative posting receipt exists, a late journal-line or source-reference insert into that finalized journal fails before it can extend the monetary/evidence population; - hard-closed periods reject later inserts; -- reconciliation-run command evidence is forced-RLS and immutable, exact retries replay while changed command evidence conflicts, historical runs exclude facts recorded after their knowledge cutoff, and deferred database provenance rejects orphan runs or cross-bank statement bindings; +- reconciliation-run command evidence is forced-RLS and immutable, exact retries replay while changed command evidence conflicts, historical runs exclude facts recorded after their knowledge cutoff, and both deferred run validation and command-insert validation reject orphan runs or cross-bank statement bindings; - reconciliation-match command evidence is forced-RLS and immutable, its run lock prevents a racing terminal transition, source entry/journal cutoffs and CRDT/DBIT cash-line direction are enforced, compound journals use the assigned cash line, and database triggers require one equal allocation per side while rejecting late allocations; - a login that merely sets `accounting_core.journal_write_role` cannot insert into a soft-closed period; - a purpose-limited session login that is a member of `accounting_closing_writer` can exercise the supported soft-close exception path; diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 62ef0355..48bf78ab 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -63,6 +63,12 @@ one journal allocation with equal exact amounts, and rejects allocation rows inserted after command evidence. These are evidence-integrity controls only; they do not turn a proposed match into an approved accounting fact. +Historical match admission also requires the posted journal fact to be known by +the run's `knowledge_cutoff_at`; a backdated accounting date alone is not +enough. The source database driver is loaded only inside the database command +path so dependency-free public imports remain available, while the runtime +database boundary still reports its existing fail-closed driver error. + The command intentionally does not replace the existing pure split/aggregate allocation planner or the database approval workflow. It provides a bounded buyer-facing persistence seam for the common 1:1 case; many-to-many planning, diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index f9a05901..521c3ae6 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -14,8 +14,8 @@ | PostgreSQL 18.6 test environment | The real regression environment uses PostgreSQL 18.6 to exercise the repository's PostgreSQL 18.4 compatibility pin. PostgreSQL's built-in `sha256(bytea)` and `encode(..., 'hex')` provide the database-owned approval snapshot digest. A row-level `BEFORE` trigger overwrites caller-supplied snapshot input, and a shared transaction-level advisory lock serializes approval, allocation, and terminal match transitions so a valid command cannot authorize a changed candidate/allocation population; migration 0017 applies the parent-row-first repair so concurrent approval and allocation cannot form a row/advisory deadlock. Immutable source-payload hash/reference provenance remains separate from that state digest. | `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, real PostgreSQL snapshot and lock-order regressions, ADR 0055, PostgreSQL binary-string, trigger, and transaction-isolation documentation | | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | -| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and the deferred database guard rejects orphan runs and cross-bank statement provenance. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run cutoffs, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards require exactly one equal allocation on each side at command insert and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | +| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run and knowledge cutoffs, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards require exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py index 25d47b38..6228c3cb 100644 --- a/src/accounting_information_platform/reconciliation_match.py +++ b/src/accounting_information_platform/reconciliation_match.py @@ -8,8 +8,6 @@ from typing import Mapping from uuid import UUID -from psycopg.errors import CheckViolation - from .core import ( AccountingValidationError, IdempotencyConflictError, @@ -95,7 +93,8 @@ def accept_reconciliation_match( run = connection.execute( """ SELECT run_status_code, accounting_book_id, currency_code, - bank_account_assignment_id, bank_cutoff_at, book_cutoff_at + bank_account_assignment_id, bank_cutoff_at, book_cutoff_at, + knowledge_cutoff_at FROM accounting_core.reconciliation_run WHERE tenant_account_id = %s AND reconciliation_run_id = %s @@ -122,6 +121,7 @@ def accept_reconciliation_match( bank_account_assignment_id=run[3], bank_cutoff_at=run[4], book_cutoff_at=run[5], + knowledge_cutoff_at=run[6], statement_reference=statement_reference, journal_reference=journal_reference, statement_amount=statement_amount, @@ -149,6 +149,8 @@ def accept_reconciliation_match( "Use the existing proposed match or a new source pair, then retry." ) + from psycopg.errors import CheckViolation + try: candidate_id = connection.execute( """ @@ -356,6 +358,7 @@ def _require_recorded_source_amounts( bank_account_assignment_id: UUID, bank_cutoff_at: object, book_cutoff_at: object, + knowledge_cutoff_at: object, statement_reference: str, journal_reference: str, statement_amount: Decimal, @@ -445,6 +448,7 @@ def _require_recorded_source_amounts( AND journal.journal_reference = %s AND assignment.bank_account_assignment_id = %s AND journal.accounting_date <= (%s::timestamptz AT TIME ZONE 'UTC')::date + AND journal.posted_at <= %s GROUP BY journal.general_journal_id, journal.journal_status_code, journal.transaction_currency_code, assignment.chart_account_id """, @@ -454,6 +458,7 @@ def _require_recorded_source_amounts( journal_reference, bank_account_assignment_id, book_cutoff_at, + knowledge_cutoff_at, ), ).fetchone() if journal_row is None: diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index c4c276fe..2638a03c 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -3,10 +3,14 @@ from __future__ import annotations import hashlib +import os +import subprocess +import sys import unittest import uuid from datetime import date, datetime, timezone from decimal import Decimal +from pathlib import Path from unittest import mock import psycopg @@ -70,6 +74,32 @@ def setUp(self) -> None: self.case.policy.tenant_reference, ) + def test_public_package_import_does_not_require_psycopg(self) -> None: + """Dependency-free public imports do not load the optional database driver.""" + package_root = Path(__file__).resolve().parents[1] + script = """ +import builtins + +real_import = builtins.__import__ + +def blocked_import(name, *args, **kwargs): + if name == "psycopg" or name.startswith("psycopg."): + raise ImportError("blocked for dependency-free import contract") + return real_import(name, *args, **kwargs) + +builtins.__import__ = blocked_import +import accounting_information_platform +""" + environment = dict(os.environ, PYTHONPATH=str(package_root / "src")) + completed = subprocess.run( + [sys.executable, "-c", script], + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + def _open_run( self, entry_index: int = 0, late_statement_entry: bool = False ) -> tuple[str, str]: @@ -408,6 +438,94 @@ def test_match_command_rejects_late_allocation_after_command_evidence(self) -> N ), ) + def test_match_command_rejects_allocation_amount_different_from_candidate(self) -> None: + """Command evidence cannot preserve allocations that disagree with its candidate.""" + command = self._command() + tenant_id = self.case.tenant_id + with psycopg.connect(posting.DATABASE_URL) as connection: + candidate_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, '24999', '24999', 'wrong-amount') + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + command["reconciliation_run_id"], + command["statement_entry_reference"], + command["journal_reference"], + ), + ).fetchone()[0] + match_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (tenant_id, command["reconciliation_run_id"], candidate_id), + ).fetchone()[0] + connection.execute( + """ + INSERT INTO accounting_core.statement_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, statement_entry_reference, + allocated_amount + ) + VALUES (%s, %s, %s, %s, '25000') + """, + ( + tenant_id, + command["reconciliation_run_id"], + match_id, + command["statement_entry_reference"], + ), + ) + connection.execute( + """ + INSERT INTO accounting_core.journal_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, journal_reference, + allocated_amount + ) + VALUES (%s, %s, %s, %s, '25000') + """, + ( + tenant_id, + command["reconciliation_run_id"], + match_id, + command["journal_reference"], + ), + ) + with self.assertRaisesRegex(psycopg.errors.CheckViolation, "candidate"): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + tenant_id, + command["reconciliation_run_id"], + candidate_id, + match_id, + f"wrong-amount-{uuid.uuid4().hex}", + "sha256:" + "5" * 64, + command["source_payload_hash"], + command["source_payload_reference"], + ), + ) + def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> None: """Defensive source checks reject impossible or mismatched journal evidence.""" for journal_row, message in ( @@ -433,6 +551,7 @@ def test_match_source_guard_rejects_unbalanced_or_wrong_journal_amounts(self) -> bank_account_assignment_id=uuid.uuid4(), bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + knowledge_cutoff_at=datetime(2026, 9, 1, tzinfo=timezone.utc), statement_reference="statement-entry", journal_reference="journal-reference", statement_amount=Decimal("25000"), @@ -455,11 +574,41 @@ def test_match_source_guard_rejects_unsupported_direction(self) -> None: bank_account_assignment_id=uuid.uuid4(), bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + knowledge_cutoff_at=datetime(2026, 9, 1, tzinfo=timezone.utc), + statement_reference="statement-entry", + journal_reference="journal-reference", + statement_amount=Decimal("25000"), + journal_amount=Decimal("25000"), + ) + + def test_match_source_guard_applies_journal_knowledge_cutoff(self) -> None: + """Historical runs cannot admit journals posted after their knowledge cutoff.""" + connection = mock.Mock() + statement_result = mock.Mock() + statement_result.fetchall.return_value = [(Decimal("25000"), "KRW", "CRDT")] + journal_result = mock.Mock() + journal_result.fetchone.return_value = None + connection.execute.side_effect = [statement_result, journal_result] + knowledge_cutoff = datetime(2026, 8, 25, tzinfo=timezone.utc) + with self.assertRaisesRegex(AccountingValidationError, "journal source"): + _require_recorded_source_amounts( + connection, + tenant_id=uuid.uuid4(), + reconciliation_run_id=uuid.uuid4(), + accounting_book_id=uuid.uuid4(), + currency_code="KRW", + bank_account_assignment_id=uuid.uuid4(), + bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + knowledge_cutoff_at=knowledge_cutoff, statement_reference="statement-entry", journal_reference="journal-reference", statement_amount=Decimal("25000"), journal_amount=Decimal("25000"), ) + journal_query, journal_parameters = connection.execute.call_args_list[1].args + self.assertIn("journal.posted_at <= %s", journal_query) + self.assertEqual(journal_parameters[-1], knowledge_cutoff) def test_match_command_maps_source_conservation_guard_to_validation(self) -> None: """A legacy cross-run amount conflict cannot escape as a raw database error.""" diff --git a/tests/test_reconciliation_run_api.py b/tests/test_reconciliation_run_api.py index b818a0a8..fe274656 100644 --- a/tests/test_reconciliation_run_api.py +++ b/tests/test_reconciliation_run_api.py @@ -307,6 +307,91 @@ def test_database_rejects_command_for_different_bank_account_at_commit(self) -> ) connection.commit() + def test_database_rejects_wrong_command_added_to_legacy_run(self) -> None: + """A command added to a pre-existing run still proves bank-account provenance.""" + _statement, command = self._statement_and_command() + second_account_reference = f"urn:cwl:bank_account:{uuid.uuid4().hex}" + accept_bank_account_record( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": second_account_reference, + "account_currency_code": "KRW", + "account_identifier": "acct-opaque-fixture-only", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + ) + second_statement = accept_bank_statement_evidence( + { + "tenant_reference": self.case.policy.tenant_reference, + "bank_account_reference": second_account_reference, + "message_definition_identifier": CAMT053_MESSAGE_DEFINITION, + "statement_payload": load_canonical_statement_fixture() + .replace(b"BANK-STMT-2026-08-24", b"BANK-STMT-LEGACY-2", 1) + .decode("utf-8"), + "ingestion_idempotency_key": f"statement-run-legacy-{uuid.uuid4().hex}", + }, + posting.DATABASE_URL, + self.case.policy.tenant_reference, + artifact_store=self.store, + ) + scope = self._assignment_scope() + assert scope is not None + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + connection.execute( + "ALTER TABLE accounting_core.reconciliation_run " + "DISABLE TRIGGER reconciliation_run_command_provenance_guard" + ) + run_id = connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run ( + tenant_account_id, legal_entity_id, accounting_book_id, + bank_account_assignment_id, currency_code, bank_cutoff_at, + book_cutoff_at, matching_policy_version, knowledge_cutoff_at, + run_status_code + ) + VALUES (%s, %s, %s, %s, 'KRW', %s, %s, %s, %s, 'evaluating') + RETURNING reconciliation_run_id + """, + ( + scope[0], + scope[1], + scope[2], + scope[3], + command["bank_cutoff_at"], + command["book_cutoff_at"], + command["matching_policy_version"], + command["knowledge_cutoff_at"], + ), + ).fetchone()[0] + connection.execute( + "ALTER TABLE accounting_core.reconciliation_run " + "ENABLE TRIGGER reconciliation_run_command_provenance_guard" + ) + with self.assertRaisesRegex(psycopg.Error, "bank account provenance"): + with psycopg.connect(posting.DATABASE_URL) as connection: + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_run_command ( + tenant_account_id, reconciliation_run_id, + bank_statement_record_id, reconciliation_idempotency_key, + reconciliation_command_hash, source_payload_hash, + source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + ( + scope[0], + run_id, + second_statement["bank_statement_record_id"], + f"legacy-provenance-{uuid.uuid4().hex}", + "sha256:" + "6" * 64, + second_statement["source_artifact_hash"], + f"memory:{second_statement['source_artifact_hash']}", + ), + ) + connection.commit() + def test_wrong_source_hash_fails_before_run_persistence(self) -> None: """A run cannot claim a different immutable bank-statement source.""" _statement, command = self._statement_and_command() From e7ef2e402375cf52dae7d5fa90443cb1e958f6df Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 05:15:41 +0900 Subject: [PATCH 15/25] fix(reconciliation): close concurrency and upgrade gaps --- CHANGELOG.md | 2 +- README.md | 2 +- ..._reconciliation_match_command_evidence.sql | 9 + ...iliation_run_command_provenance_repair.sql | 66 +++++++ docs/ARCHITECTURE.md | 1 + docs/DATA_MODEL.md | 4 +- docs/OPERABILITY.md | 7 +- ...058-reconciliation-run-command-evidence.md | 4 +- ...9-reconciliation-match-command-evidence.md | 16 +- docs/doctoring/STANDARD_TRACEABILITY.md | 4 +- docs/product-technical-gap-baseline.md | 3 +- scripts/validate_repository.py | 1 + .../persistence.py | 12 ++ .../reconciliation_match.py | 2 + ...st_foundation_install_manifest_contract.py | 30 +++ tests/test_reconciliation_match_api.py | 181 ++++++++++++++++++ ...tion_multi_match_documentation_contract.py | 1 + tests/test_reconciliation_run_api.py | 10 + 18 files changed, 342 insertions(+), 13 deletions(-) create mode 100644 database/migrations/0021_reconciliation_run_command_provenance_repair.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 58dbdfb6..7e6b8080 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Closed additional match-evidence gaps: journal `posted_at` now respects the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, existing-run command inserts reapply bank-account provenance validation, and the psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. +- Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. diff --git a/README.md b/README.md index 5eb20248..141fe942 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ factory/runner and provide the tenant-bound host boundary explicitly. `unittest` discovery also runs `tests/test_postgres_posting.py`, which needs a reachable PostgreSQL 18 instance and `ACCOUNTING_DATABASE_URL` (CI uses `postgresql://postgres:postgres@127.0.0.1:5432/accounting_test` and applies -the checked-in migration chain through `database/migrations/0020_reconciliation_match_command_evidence.sql`). Persistence is still +the checked-in migration chain through `database/migrations/0021_reconciliation_run_command_provenance_repair.sql`). Persistence is still local to this repository; it is not a Naruon or sibling checkout. Optional import smoke after the editable install above: diff --git a/database/migrations/0020_reconciliation_match_command_evidence.sql b/database/migrations/0020_reconciliation_match_command_evidence.sql index 90f61bed..c2838896 100644 --- a/database/migrations/0020_reconciliation_match_command_evidence.sql +++ b/database/migrations/0020_reconciliation_match_command_evidence.sql @@ -101,6 +101,15 @@ DECLARE candidate_statement_amount numeric(30, 6); candidate_journal_amount numeric(30, 6); BEGIN + -- Share the parent match lock with the allocation conservation trigger so + -- command insertion and a concurrent allocation cannot both commit. + PERFORM 1 + FROM accounting_core.reconciliation_match AS match + WHERE match.tenant_account_id = NEW.tenant_account_id + AND match.reconciliation_run_id = NEW.reconciliation_run_id + AND match.reconciliation_match_id = NEW.reconciliation_match_id + FOR UPDATE; + SELECT candidate.statement_amount, candidate.journal_amount INTO candidate_statement_amount, candidate_journal_amount FROM accounting_core.reconciliation_candidate AS candidate diff --git a/database/migrations/0021_reconciliation_run_command_provenance_repair.sql b/database/migrations/0021_reconciliation_run_command_provenance_repair.sql new file mode 100644 index 00000000..e401de19 --- /dev/null +++ b/database/migrations/0021_reconciliation_run_command_provenance_repair.sql @@ -0,0 +1,66 @@ +BEGIN; + +-- Apply the run-command provenance guards to installations that already +-- executed migration 0019 before the command-insert guard was added. +CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + command_count integer; +BEGIN + SELECT count(*) + INTO command_count + FROM accounting_core.reconciliation_run_command AS command + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id; + + IF command_count <> 1 THEN + RAISE EXCEPTION + 'reconciliation run must have exactly one command evidence row at commit (reconciliation_run_command_provenance)' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM accounting_core.reconciliation_run_command AS command + JOIN accounting_integration.bank_statement_record AS statement + ON statement.tenant_account_id = command.tenant_account_id + AND statement.bank_statement_record_id = command.bank_statement_record_id + JOIN accounting_core.reconciliation_run AS run + ON run.tenant_account_id = command.tenant_account_id + AND run.reconciliation_run_id = command.reconciliation_run_id + JOIN accounting_core.bank_account_assignment AS assignment + ON assignment.tenant_account_id = run.tenant_account_id + AND assignment.legal_entity_id = run.legal_entity_id + AND assignment.accounting_book_id = run.accounting_book_id + AND assignment.bank_account_assignment_id = run.bank_account_assignment_id + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id + AND statement.bank_account_record_id IS DISTINCT FROM assignment.bank_account_record_id + ) THEN + RAISE EXCEPTION + 'reconciliation run command bank account provenance does not match the run assignment (reconciliation_run_command_provenance)' + USING ERRCODE = '23514'; + END IF; + + RETURN NULL; +END; +$$; + +DROP TRIGGER IF EXISTS reconciliation_run_command_provenance_guard + ON accounting_core.reconciliation_run; +CREATE CONSTRAINT TRIGGER reconciliation_run_command_provenance_guard + AFTER INSERT ON accounting_core.reconciliation_run + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW + EXECUTE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance(); + +DROP TRIGGER IF EXISTS reconciliation_run_command_provenance_insert_guard + ON accounting_core.reconciliation_run_command; +CREATE TRIGGER reconciliation_run_command_provenance_insert_guard + AFTER INSERT ON accounting_core.reconciliation_run_command + FOR EACH ROW + EXECUTE FUNCTION accounting_core.enforce_reconciliation_run_command_provenance(); + +COMMIT; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a89cd3f2..353d3900 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -144,6 +144,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 18. `database/migrations/0018_bank_statement_balance_evidence.sql` — preserves exact numeric camt.053 balance facts, including typed effective date/time distinct from statement period and system recording time, as immutable, tenant-scoped evidence for reconciliation bridge reads. 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. 20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. +21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — reapplies run-command provenance guards to installations that already executed migration 0019 before its command-insert guard was added. ## Durable soft-close command evidence diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index e31ef733..aeb4ae06 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -40,7 +40,7 @@ The foundation ERD is maintained in [ERD.md](ERD.md). PostgreSQL migrations are ## Reconciliation control evidence -Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconciliation_match_command_evidence.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. +Migrations `0013_reconciliation_run_exception_evidence.sql` through `0021_reconciliation_run_command_provenance_repair.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. - `reconciliation_run`: immutable evaluated scope for one tenant, legal entity, accounting book, bank-account assignment, ISO currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Only the run status may progress; evaluated scope changes require a new run. - `reconciliation_exception`: explicit operator-owned exception with an exception code, next action, effective/system time, and open/resolved/superseded resolution status. @@ -51,7 +51,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0020_reconc - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard requires exactly one equal allocation on each side matching the candidate amounts at command insert and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index ff625bdf..1d32988b 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -2,7 +2,7 @@ ## Deployment preconditions -Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0020_reconciliation_match_command_evidence.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. +Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0021_reconciliation_run_command_provenance_repair.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. Required environment values are deployment-specific. At minimum, configure the accounting database URL and bind this AIS process to exactly one tenant reference. Secrets belong in an approved secret store; do not place database passwords, NTS credentials, bearer tokens or provider secrets in journal payloads, logs or outbox events. @@ -33,6 +33,7 @@ database/migrations/0017_reconciliation_approval_lock_order.sql database/migrations/0018_bank_statement_balance_evidence.sql database/migrations/0019_reconciliation_run_command_evidence.sql database/migrations/0020_reconciliation_match_command_evidence.sql +database/migrations/0021_reconciliation_run_command_provenance_repair.sql ``` Migration `0015_reconciliation_multi_match_conservation.sql` replaces the run-wide single-approved-match shortcut from `0014` with tenant/run-scoped match identity plus exact statement/journal allocation conservation. It permits multiple independently approved matches only when no authoritative source amount is over-consumed and grants no journal-posting authority. @@ -45,7 +46,9 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; new runs exclude source facts recorded after `knowledge_cutoff_at`, and both the deferred run guard and command INSERT guard require one command per run with statement-to-assignment bank-account provenance, including when a legacy run receives its command later. The public run API opens only `evaluating` scope and does not match, approve, close, or post journals. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps respect the bank cutoff and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards require exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. It recreates both the deferred run guard and immediate command-insert guard without changing immutable evidence rows; apply it after `0020` and before runtime traffic. + +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0058-reconciliation-run-command-evidence.md b/docs/adr/0058-reconciliation-run-command-evidence.md index abdbe97e..59a8a53f 100644 --- a/docs/adr/0058-reconciliation-run-command-evidence.md +++ b/docs/adr/0058-reconciliation-run-command-evidence.md @@ -37,7 +37,9 @@ For a new run, every selected statement, artifact, bank account, assignment, legal-entity, and accounting-book fact must have been recorded no later than `knowledge_cutoff_at`. A deferred database trigger also requires exactly one command row at commit and verifies that its statement belongs to the run's -assigned bank account. +assigned bank account. Migration `0021_reconciliation_run_command_provenance_repair.sql` +reapplies the deferred run trigger and adds the immediate command-insert guard +for installations that already applied migration 0019. `GET /reconciliation-runs` returns the same tenant-scoped run document. Distinct idempotency keys may open distinct immutable runs for the same statement diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 48bf78ab..1ae9cc17 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -35,13 +35,17 @@ with a different valid match from the same run. Uniqueness rules additionally prevent duplicate command or match identities. The command also binds source admission to the run snapshot: statement booking -and value timestamps must be no later than the bank cutoff, and journal -accounting date must be no later than the book cutoff. The journal amount is +and value timestamps must be no later than the bank cutoff, statement +`recorded_at` must be no later than the knowledge cutoff, and journal +accounting date plus `posted_at` must be no later than the book and knowledge +cutoffs. The journal amount is read from the run assignment's cash chart account rather than the journal-wide total; a `CRDT` statement requires that line on the debit side and a `DBIT` statement requires it on the credit side. The run row is locked through the candidate and allocation writes so a concurrent run transition cannot race the -match. +match. The command-evidence allocation guard also locks the parent match row +before counting allocations, sharing the allocation trigger's serialization +boundary. `accept_reconciliation_match` is the smallest durable command boundary: it accepts one exact 1:1 proposed match, requires quoted positive equal decimal @@ -63,6 +67,12 @@ one journal allocation with equal exact amounts, and rejects allocation rows inserted after command evidence. These are evidence-integrity controls only; they do not turn a proposed match into an approved accounting fact. +Migration `0021_reconciliation_run_command_provenance_repair.sql` is a +forward-only upgrade for installations that already applied migration 0019 +before the command-insert provenance guard existed. It recreates the deferred +run guard and immediate command-insert guard without changing existing +evidence rows. + Historical match admission also requires the posted journal fact to be known by the run's `knowledge_cutoff_at`; a backdated accounting date alone is not enough. The source database driver is loaded only inside the database command diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 521c3ae6..91673d8f 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -14,8 +14,8 @@ | PostgreSQL 18.6 test environment | The real regression environment uses PostgreSQL 18.6 to exercise the repository's PostgreSQL 18.4 compatibility pin. PostgreSQL's built-in `sha256(bytea)` and `encode(..., 'hex')` provide the database-owned approval snapshot digest. A row-level `BEFORE` trigger overwrites caller-supplied snapshot input, and a shared transaction-level advisory lock serializes approval, allocation, and terminal match transitions so a valid command cannot authorize a changed candidate/allocation population; migration 0017 applies the parent-row-first repair so concurrent approval and allocation cannot form a row/advisory deadlock. Immutable source-payload hash/reference provenance remains separate from that state digest. | `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, real PostgreSQL snapshot and lock-order regressions, ADR 0055, PostgreSQL binary-string, trigger, and transaction-isolation documentation | | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | -| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run and knowledge cutoffs, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards require exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | +| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, require exactly one equal allocation on each side matching candidate amounts at command insert, and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 9f1b22fd..3ae0401b 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -34,7 +34,8 @@ JSON contracts, and a durable outbox. The accounting posting foundation and the immutable `camt.053.001.14` bank-statement evidence registry are integrated protected `develop` facts. The current integration tree also includes database-owned candidate/match/allocation conservation, durable human approval snapshot controls, - and the proposed-match command boundary from migrations `0015` through `0020`; + the proposed-match command boundary from migrations `0015` through `0020`, and + the existing-installation provenance repair in migration `0021`; protected `develop` remains the release authority until this exact integration candidate passes its live gates. It does not transmit HomeTax/NTS filings, enforce purpose-bound application authorization, or provide a controller diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index 3ae72df4..8cbd6ca0 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -55,6 +55,7 @@ "database/migrations/0018_bank_statement_balance_evidence.sql", "database/migrations/0019_reconciliation_run_command_evidence.sql", "database/migrations/0020_reconciliation_match_command_evidence.sql", + "database/migrations/0021_reconciliation_run_command_provenance_repair.sql", "docs/PRD.md", "docs/TRD.md", "docs/ARCHITECTURE.md", diff --git a/src/accounting_information_platform/persistence.py b/src/accounting_information_platform/persistence.py index 23762d49..ab2d84a0 100644 --- a/src/accounting_information_platform/persistence.py +++ b/src/accounting_information_platform/persistence.py @@ -6349,6 +6349,15 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: f"{match_command_migration_path}. Restore " "database/migrations/0020_reconciliation_match_command_evidence.sql, then retry." ) + run_command_provenance_repair_migration_path = ( + migration_path.parent / "0021_reconciliation_run_command_provenance_repair.sql" + ) + if not run_command_provenance_repair_migration_path.is_file(): + raise AccountingValidationError( + "Reconciliation run-command provenance repair migration is missing at " + f"{run_command_provenance_repair_migration_path}. Restore " + "database/migrations/0021_reconciliation_run_command_provenance_repair.sql, then retry." + ) psycopg = _import_psycopg() try: with psycopg.connect( @@ -6390,6 +6399,9 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: connection.execute( match_command_migration_path.read_text(encoding="utf-8") ) + connection.execute( + run_command_provenance_repair_migration_path.read_text(encoding="utf-8") + ) except Exception as error: raise AccountingValidationError( "Foundation migration failed. Inspect the PostgreSQL error, restore a clean " diff --git a/src/accounting_information_platform/reconciliation_match.py b/src/accounting_information_platform/reconciliation_match.py index 6228c3cb..ee2d87e7 100644 --- a/src/accounting_information_platform/reconciliation_match.py +++ b/src/accounting_information_platform/reconciliation_match.py @@ -382,6 +382,7 @@ def _require_recorded_source_amounts( AND run_scope.currency_code = entry.entry_currency_code AND (entry.booking_occurred_at IS NULL OR entry.booking_occurred_at <= %s) AND (entry.value_occurred_at IS NULL OR entry.value_occurred_at <= %s) + AND entry.recorded_at <= %s """, ( tenant_id, @@ -390,6 +391,7 @@ def _require_recorded_source_amounts( statement_reference, bank_cutoff_at, bank_cutoff_at, + knowledge_cutoff_at, ), ).fetchall() if len(statement_rows) != 1: diff --git a/tests/test_foundation_install_manifest_contract.py b/tests/test_foundation_install_manifest_contract.py index c5849d0a..db45cfdd 100644 --- a/tests/test_foundation_install_manifest_contract.py +++ b/tests/test_foundation_install_manifest_contract.py @@ -159,6 +159,20 @@ def test_required_files_and_install_docs_include_match_command_evidence(self) -> self.assertIn(migration_twenty, text) self.assertLess(text.index(migration_nineteen), text.index(migration_twenty)) + def test_required_files_and_install_docs_include_run_provenance_repair(self) -> None: + """Existing installations receive the command provenance trigger repair.""" + migration_twenty = "database/migrations/0020_reconciliation_match_command_evidence.sql" + migration_twenty_one = ( + "database/migrations/0021_reconciliation_run_command_provenance_repair.sql" + ) + self.assertIn(migration_twenty_one, set(REQUIRED_FILES)) + for relative_path in ("docs/OPERABILITY.md", "docs/ARCHITECTURE.md"): + with self.subTest(relative_path=relative_path): + text = (ROOT / relative_path).read_text(encoding="utf-8") + self.assertIn(migration_twenty, text) + self.assertIn(migration_twenty_one, text) + self.assertLess(text.index(migration_twenty), text.index(migration_twenty_one)) + def test_install_fails_closed_when_approval_snapshot_migration_is_missing(self) -> None: """The canonical loader may not silently stop before database-owned approval evidence.""" original_is_file = Path.is_file @@ -239,6 +253,22 @@ def is_file(path: Path) -> bool: ROOT / "database/migrations/0001_accounting_foundation.sql", ) + def test_install_fails_closed_when_run_provenance_repair_is_missing(self) -> None: + """The canonical loader must apply the existing-installation trigger repair.""" + original_is_file = Path.is_file + + def is_file(path: Path) -> bool: + if path.name == "0021_reconciliation_run_command_provenance_repair.sql": + return False + return original_is_file(path) + + with patch.object(Path, "is_file", is_file): + with self.assertRaisesRegex(AccountingValidationError, "provenance repair"): + apply_foundation_migration( + "postgresql://unused", + ROOT / "database/migrations/0001_accounting_foundation.sql", + ) + def test_canonical_persistence_loader_fails_closed_when_conservation_is_missing(self) -> None: """Real PostgreSQL fixtures may not silently stop the authoritative chain at 0014.""" from accounting_information_platform.persistence import ( diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index 2638a03c..77503d85 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -6,6 +6,7 @@ import os import subprocess import sys +import threading import unittest import uuid from datetime import date, datetime, timezone @@ -438,6 +439,151 @@ def test_match_command_rejects_late_allocation_after_command_evidence(self) -> N ), ) + def test_match_command_serializes_concurrent_allocation_inserts(self) -> None: + """A concurrent allocation cannot commit around the command freeze.""" + command = self._command() + tenant_id = self.case.tenant_id + with psycopg.connect(posting.DATABASE_URL) as setup: + candidate_id = setup.execute( + """ + INSERT INTO accounting_core.reconciliation_candidate ( + tenant_account_id, reconciliation_run_id, + statement_entry_reference, journal_reference, + statement_amount, journal_amount, rule_code + ) + VALUES (%s, %s, %s, %s, '25000', '25000', 'concurrency') + RETURNING reconciliation_candidate_id + """, + ( + tenant_id, + command["reconciliation_run_id"], + command["statement_entry_reference"], + command["journal_reference"], + ), + ).fetchone()[0] + match_id = setup.execute( + """ + INSERT INTO accounting_core.reconciliation_match ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, match_status_code + ) + VALUES (%s, %s, %s, 'proposed') + RETURNING reconciliation_match_id + """, + (tenant_id, command["reconciliation_run_id"], candidate_id), + ).fetchone()[0] + setup.execute( + """ + INSERT INTO accounting_core.statement_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, statement_entry_reference, + allocated_amount + ) + VALUES (%s, %s, %s, %s, '25000') + """, + ( + tenant_id, + command["reconciliation_run_id"], + match_id, + command["statement_entry_reference"], + ), + ) + setup.execute( + """ + INSERT INTO accounting_core.journal_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, journal_reference, + allocated_amount + ) + VALUES (%s, %s, %s, %s, '25000') + """, + ( + tenant_id, + command["reconciliation_run_id"], + match_id, + command["journal_reference"], + ), + ) + + command_connection = psycopg.connect(posting.DATABASE_URL) + late_insert_started = threading.Event() + late_insert_finished = threading.Event() + late_insert_error: list[BaseException] = [] + + def insert_late_allocation() -> None: + try: + with psycopg.connect(posting.DATABASE_URL) as connection: + late_insert_started.set() + connection.execute( + """ + INSERT INTO accounting_core.statement_match_allocation ( + tenant_account_id, reconciliation_run_id, + reconciliation_match_id, statement_entry_reference, + allocated_amount + ) + VALUES (%s, %s, %s, 'concurrent-extra', '1') + """, + (tenant_id, command["reconciliation_run_id"], match_id), + ) + connection.commit() + except BaseException as error: # pragma: no cover - asserted below + late_insert_error.append(error) + finally: + late_insert_finished.set() + + try: + command_connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + tenant_id, + command["reconciliation_run_id"], + candidate_id, + match_id, + f"concurrent-command-{uuid.uuid4().hex}", + "sha256:" + "7" * 64, + command["source_payload_hash"], + command["source_payload_reference"], + ), + ) + thread = threading.Thread(target=insert_late_allocation) + thread.start() + self.assertTrue(late_insert_started.wait(2)) + self.assertFalse(late_insert_finished.wait(0.5)) + command_connection.commit() + thread.join(timeout=5) + self.assertFalse(thread.is_alive()) + self.assertEqual(len(late_insert_error), 1) + self.assertIsInstance(late_insert_error[0], psycopg.errors.CheckViolation) + finally: + command_connection.rollback() + command_connection.close() + + def test_match_command_allocation_guard_locks_match_before_counting(self) -> None: + """The command-side allocation check owns the same match lock as allocations.""" + migration = ( + Path(__file__).resolve().parents[1] + / "database/migrations/0020_reconciliation_match_command_evidence.sql" + ).read_text(encoding="utf-8") + function = migration.split( + "CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_match_command_allocations()", + 1, + )[1].split( + "CREATE TRIGGER z_reconciliation_match_command_allocation_guard", + 1, + )[0] + self.assertRegex( + function, + r"(?s)FROM accounting_core\.reconciliation_match.*?FOR UPDATE", + ) + def test_match_command_rejects_allocation_amount_different_from_candidate(self) -> None: """Command evidence cannot preserve allocations that disagree with its candidate.""" command = self._command() @@ -581,6 +727,41 @@ def test_match_source_guard_rejects_unsupported_direction(self) -> None: journal_amount=Decimal("25000"), ) + def test_match_source_guard_applies_statement_knowledge_cutoff(self) -> None: + """Historical matches exclude statement entries recorded after the snapshot.""" + connection = mock.Mock() + statement_result = mock.Mock() + statement_result.fetchall.return_value = [(Decimal("25000"), "KRW", "CRDT")] + journal_result = mock.Mock() + journal_result.fetchone.return_value = ( + "posted", + "KRW", + Decimal("25000"), + Decimal("25000"), + Decimal("25000"), + Decimal("0"), + ) + connection.execute.side_effect = [statement_result, journal_result] + knowledge_cutoff = datetime(2026, 8, 25, tzinfo=timezone.utc) + _require_recorded_source_amounts( + connection, + tenant_id=uuid.uuid4(), + reconciliation_run_id=uuid.uuid4(), + accounting_book_id=uuid.uuid4(), + currency_code="KRW", + bank_account_assignment_id=uuid.uuid4(), + bank_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + book_cutoff_at=datetime(2026, 8, 24, 23, 59, 59, tzinfo=timezone.utc), + knowledge_cutoff_at=knowledge_cutoff, + statement_reference="statement-entry", + journal_reference="journal-reference", + statement_amount=Decimal("25000"), + journal_amount=Decimal("25000"), + ) + statement_sql, statement_parameters = connection.execute.call_args_list[0].args + self.assertIn("entry.recorded_at <= %s", statement_sql) + self.assertEqual(statement_parameters[-1], knowledge_cutoff) + def test_match_source_guard_applies_journal_knowledge_cutoff(self) -> None: """Historical runs cannot admit journals posted after their knowledge cutoff.""" connection = mock.Mock() diff --git a/tests/test_reconciliation_multi_match_documentation_contract.py b/tests/test_reconciliation_multi_match_documentation_contract.py index 150fde9c..4e8fb3c2 100644 --- a/tests/test_reconciliation_multi_match_documentation_contract.py +++ b/tests/test_reconciliation_multi_match_documentation_contract.py @@ -42,6 +42,7 @@ def test_product_baseline_distinguishes_current_tree_from_remaining_close_packag """The product gap queue must not call delivered 0015-0017 controls future work.""" text = BASELINE.read_text(encoding="utf-8") self.assertIn("from migrations `0015` through `0020`", text) + self.assertIn("existing-installation provenance repair in migration `0021`", text) self.assertIn("close-package provenance remains open", text) self.assertIn("[delivered in current tree; migration 0015]", text) self.assertNotRegex( diff --git a/tests/test_reconciliation_run_api.py b/tests/test_reconciliation_run_api.py index fe274656..3d386796 100644 --- a/tests/test_reconciliation_run_api.py +++ b/tests/test_reconciliation_run_api.py @@ -6,6 +6,7 @@ import unittest import uuid from contextlib import nullcontext +from pathlib import Path from unittest import mock import psycopg @@ -337,7 +338,16 @@ def test_database_rejects_wrong_command_added_to_legacy_run(self) -> None: ) scope = self._assignment_scope() assert scope is not None + provenance_repair = ( + Path(__file__).resolve().parents[1] + / "database/migrations/0021_reconciliation_run_command_provenance_repair.sql" + ) with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + connection.execute( + "DROP TRIGGER IF EXISTS reconciliation_run_command_provenance_insert_guard " + "ON accounting_core.reconciliation_run_command" + ) + connection.execute(provenance_repair.read_text(encoding="utf-8")) connection.execute( "ALTER TABLE accounting_core.reconciliation_run " "DISABLE TRIGGER reconciliation_run_command_provenance_guard" From 270f0c7f0259b078978f919b6791f57f345ac0b8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 05:25:42 +0900 Subject: [PATCH 16/25] fix(reconciliation): validate legacy provenance on upgrade --- CHANGELOG.md | 2 +- ...iliation_run_command_provenance_repair.sql | 58 +++++++++++++++++++ docs/ARCHITECTURE.md | 2 +- docs/DATA_MODEL.md | 2 +- docs/OPERABILITY.md | 2 +- ...058-reconciliation-run-command-evidence.md | 6 +- docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- ...st_foundation_install_manifest_contract.py | 26 +++++++++ 8 files changed, 93 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6b8080..4f221d0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. +- Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations while failing closed on pre-existing cross-bank command evidence. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. - Extended the hash-locked PostgreSQL quality dependency set with the CPython 3.14 `psycopg-binary` wheel required by the central coverage runner; the repository contract now keeps that interpreter boundary executable. diff --git a/database/migrations/0021_reconciliation_run_command_provenance_repair.sql b/database/migrations/0021_reconciliation_run_command_provenance_repair.sql index e401de19..a7f3170b 100644 --- a/database/migrations/0021_reconciliation_run_command_provenance_repair.sql +++ b/database/migrations/0021_reconciliation_run_command_provenance_repair.sql @@ -48,6 +48,64 @@ BEGIN END; $$; +-- Forced RLS hides tenant rows from the migration session by default. Expose +-- only this immutable relation to the current migration user for the upgrade +-- preflight, then remove the policy before the transaction commits. +CREATE POLICY reconciliation_run_command_provenance_upgrade_visibility + ON accounting_core.reconciliation_run_command + FOR SELECT + TO current_user + USING (true); +CREATE POLICY reconciliation_run_command_provenance_run_upgrade_visibility + ON accounting_core.reconciliation_run + FOR SELECT + TO current_user + USING (true); +CREATE POLICY reconciliation_run_command_provenance_assignment_upgrade_visibility + ON accounting_core.bank_account_assignment + FOR SELECT + TO current_user + USING (true); +CREATE POLICY reconciliation_run_command_provenance_statement_upgrade_visibility + ON accounting_integration.bank_statement_record + FOR SELECT + TO current_user + USING (true); + +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM accounting_core.reconciliation_run_command AS command + JOIN accounting_integration.bank_statement_record AS statement + ON statement.tenant_account_id = command.tenant_account_id + AND statement.bank_statement_record_id = command.bank_statement_record_id + JOIN accounting_core.reconciliation_run AS run + ON run.tenant_account_id = command.tenant_account_id + AND run.reconciliation_run_id = command.reconciliation_run_id + JOIN accounting_core.bank_account_assignment AS assignment + ON assignment.tenant_account_id = run.tenant_account_id + AND assignment.legal_entity_id = run.legal_entity_id + AND assignment.accounting_book_id = run.accounting_book_id + AND assignment.bank_account_assignment_id = run.bank_account_assignment_id + WHERE statement.bank_account_record_id IS DISTINCT FROM assignment.bank_account_record_id + ) THEN + RAISE EXCEPTION + 'existing reconciliation run command provenance is invalid; repair the recorded evidence before applying this migration (reconciliation_run_command_provenance_upgrade)' + USING ERRCODE = '23514'; + END IF; +END; +$$; + +DROP POLICY reconciliation_run_command_provenance_upgrade_visibility + ON accounting_core.reconciliation_run_command; +DROP POLICY reconciliation_run_command_provenance_run_upgrade_visibility + ON accounting_core.reconciliation_run; +DROP POLICY reconciliation_run_command_provenance_assignment_upgrade_visibility + ON accounting_core.bank_account_assignment; +DROP POLICY reconciliation_run_command_provenance_statement_upgrade_visibility + ON accounting_integration.bank_statement_record; + DROP TRIGGER IF EXISTS reconciliation_run_command_provenance_guard ON accounting_core.reconciliation_run; CREATE CONSTRAINT TRIGGER reconciliation_run_command_provenance_guard diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 353d3900..64706fb4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -144,7 +144,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 18. `database/migrations/0018_bank_statement_balance_evidence.sql` — preserves exact numeric camt.053 balance facts, including typed effective date/time distinct from statement period and system recording time, as immutable, tenant-scoped evidence for reconciliation bridge reads. 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. 20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. -21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — reapplies run-command provenance guards to installations that already executed migration 0019 before its command-insert guard was added. +21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — preflights existing immutable command evidence and fails closed on cross-bank provenance before reapplying run-command guards to installations that already executed migration 0019 before its command-insert guard was added. ## Durable soft-close command evidence diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index aeb4ae06..4a9dcaba 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -50,7 +50,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0021_reconc - `statement_match_allocation`: append-only exact amount consumed from an immutable statement source reference by one reconciliation match. - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. -- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. +- `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Migration 0021 preflights existing command rows and fails closed on cross-bank provenance before reapplying those guards. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. - `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 1d32988b..58b93816 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -46,7 +46,7 @@ Migration `0018_bank_statement_balance_evidence.sql` preserves the exact numeric Migration `0019_reconciliation_run_command_evidence.sql` records the immutable command identity that opens a reconciliation run from one persisted bank statement and active bank-account assignment. The tenant-scoped idempotency key, command hash, source hash, and object-store reference are forced-RLS evidence; new runs exclude source facts recorded after `knowledge_cutoff_at`, and both the deferred run guard and command INSERT guard require one command per run with statement-to-assignment bank-account provenance, including when a legacy run receives its command later. The public run API opens only `evaluating` scope and does not match, approve, close, or post journals. -Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. It recreates both the deferred run guard and immediate command-insert guard without changing immutable evidence rows; apply it after `0020` and before runtime traffic. +Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. Its migration-only preflight scans existing immutable command evidence and fails closed on cross-bank provenance; it then recreates both the deferred run guard and immediate command-insert guard without changing evidence rows. Apply it after `0020` and before runtime traffic. Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. diff --git a/docs/adr/0058-reconciliation-run-command-evidence.md b/docs/adr/0058-reconciliation-run-command-evidence.md index 59a8a53f..de10a45b 100644 --- a/docs/adr/0058-reconciliation-run-command-evidence.md +++ b/docs/adr/0058-reconciliation-run-command-evidence.md @@ -38,8 +38,10 @@ legal-entity, and accounting-book fact must have been recorded no later than `knowledge_cutoff_at`. A deferred database trigger also requires exactly one command row at commit and verifies that its statement belongs to the run's assigned bank account. Migration `0021_reconciliation_run_command_provenance_repair.sql` -reapplies the deferred run trigger and adds the immediate command-insert guard -for installations that already applied migration 0019. +first uses migration-only visibility to scan existing immutable command rows and +fails closed on cross-bank provenance. It then reapplies the deferred run +trigger and adds the immediate command-insert guard for installations that +already applied migration 0019. `GET /reconciliation-runs` returns the same tenant-scoped run document. Distinct idempotency keys may open distinct immutable runs for the same statement diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 91673d8f..9bc72677 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -14,7 +14,7 @@ | PostgreSQL 18.6 test environment | The real regression environment uses PostgreSQL 18.6 to exercise the repository's PostgreSQL 18.4 compatibility pin. PostgreSQL's built-in `sha256(bytea)` and `encode(..., 'hex')` provide the database-owned approval snapshot digest. A row-level `BEFORE` trigger overwrites caller-supplied snapshot input, and a shared transaction-level advisory lock serializes approval, allocation, and terminal match transitions so a valid command cannot authorize a changed candidate/allocation population; migration 0017 applies the parent-row-first repair so concurrent approval and allocation cannot form a row/advisory deadlock. Immutable source-payload hash/reference provenance remains separate from that state digest. | `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, real PostgreSQL snapshot and lock-order regressions, ADR 0055, PostgreSQL binary-string, trigger, and transaction-isolation documentation | | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | -| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | +| Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 preflights existing command rows, fails closed on cross-bank provenance, and then reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | | Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, require exactly one equal allocation on each side matching candidate amounts at command insert, and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | diff --git a/tests/test_foundation_install_manifest_contract.py b/tests/test_foundation_install_manifest_contract.py index db45cfdd..e37dcfb7 100644 --- a/tests/test_foundation_install_manifest_contract.py +++ b/tests/test_foundation_install_manifest_contract.py @@ -173,6 +173,32 @@ def test_required_files_and_install_docs_include_run_provenance_repair(self) -> self.assertIn(migration_twenty_one, text) self.assertLess(text.index(migration_twenty), text.index(migration_twenty_one)) + def test_run_provenance_repair_rejects_existing_mismatches(self) -> None: + """The upgrade must fail closed on already stored cross-bank commands.""" + migration = ( + ROOT + / "database/migrations/0021_reconciliation_run_command_provenance_repair.sql" + ).read_text(encoding="utf-8") + self.assertIn( + "reconciliation_run_command_provenance_upgrade_visibility", + migration, + ) + for policy in ( + "reconciliation_run_command_provenance_run_upgrade_visibility", + "reconciliation_run_command_provenance_assignment_upgrade_visibility", + "reconciliation_run_command_provenance_statement_upgrade_visibility", + ): + with self.subTest(policy=policy): + self.assertIn(policy, migration) + self.assertIn( + "statement.bank_account_record_id IS DISTINCT FROM assignment.bank_account_record_id", + migration, + ) + self.assertIn( + "existing reconciliation run command provenance is invalid", + migration, + ) + def test_install_fails_closed_when_approval_snapshot_migration_is_missing(self) -> None: """The canonical loader may not silently stop before database-owned approval evidence.""" original_is_file = Path.is_file From d4581bfb833884e2a285525e10c496a30b300c17 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 05:32:54 +0900 Subject: [PATCH 17/25] docs: refresh product gap evidence date --- docs/product-technical-gap-baseline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 3ae0401b..ea531980 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -1,6 +1,6 @@ # Product and technical gap baseline -**Evidence refresh:** 2026-08-29 (Asia/Seoul) +**Evidence refresh:** 2026-08-30 (Asia/Seoul) This file is the durable buyer-visible gap queue for `accounting-information-platform`. It records authority, dependency order, acceptance evidence, and product gaps that From 20cb6b620173aaedecd0027d66308301f56f33b4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 08:55:05 +0900 Subject: [PATCH 18/25] fix(reconciliation): align amount precision domain --- CHANGELOG.md | 1 + .../0022_reconciliation_amount_precision.sql | 340 ++++++++++++++++++ docs/ARCHITECTURE.md | 1 + docs/DATA_MODEL.md | 3 +- docs/OPERABILITY.md | 5 +- ...9-reconciliation-match-command-evidence.md | 8 + docs/doctoring/STANDARD_TRACEABILITY.md | 1 + scripts/validate_repository.py | 1 + src/accounting_information_platform/core.py | 7 +- .../persistence.py | 12 + ...st_foundation_install_manifest_contract.py | 30 ++ tests/test_journal_proposal_ingest.py | 21 ++ ...ation_data_model_documentation_contract.py | 22 ++ 13 files changed, 449 insertions(+), 3 deletions(-) create mode 100644 database/migrations/0022_reconciliation_amount_precision.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f221d0a..4233a5b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Aligned reconciliation candidate and allocation amounts with the platform `numeric(38, 6)` monetary domain through migration `0022_reconciliation_amount_precision.sql`; exact trigger aggregates no longer overflow before conservation validation, and the shared parser rejects wider values before persistence. - Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations while failing closed on pre-existing cross-bank command evidence. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. diff --git a/database/migrations/0022_reconciliation_amount_precision.sql b/database/migrations/0022_reconciliation_amount_precision.sql new file mode 100644 index 00000000..9015daea --- /dev/null +++ b/database/migrations/0022_reconciliation_amount_precision.sql @@ -0,0 +1,340 @@ +BEGIN; + +-- Reconciliation monetary facts must use the same numeric(38, 6) domain as +-- journal lines and bank-statement evidence. The original 0014 tables used a +-- narrower legacy domain, which could reject valid source amounts. +ALTER TABLE accounting_core.reconciliation_candidate + ALTER COLUMN statement_amount TYPE numeric(38, 6), + ALTER COLUMN journal_amount TYPE numeric(38, 6); + +ALTER TABLE accounting_core.statement_match_allocation + ALTER COLUMN allocated_amount TYPE numeric(38, 6); + +ALTER TABLE accounting_core.journal_match_allocation + ALTER COLUMN allocated_amount TYPE numeric(38, 6); + +-- Keep aggregate variables unconstrained so an over-consumption comparison +-- remains an exact rejection instead of overflowing before the guard runs. +CREATE OR REPLACE FUNCTION accounting_core.reconciliation_match_approval_conservation_guard() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + source_row record; + source_capacity numeric; + consumed_amount numeric; + conservation_key text; + current_legal_entity_id uuid; + current_accounting_book_id uuid; + current_bank_account_record_id uuid; + current_currency_code text; + statement_allocation_count bigint; + journal_allocation_count bigint; + statement_allocation_total numeric; + journal_allocation_total numeric; +BEGIN + IF NEW.match_status_code <> 'approved' + OR (TG_OP = 'UPDATE' AND OLD.match_status_code = 'approved') THEN + RETURN NEW; + END IF; + + SELECT + run_scope.legal_entity_id, + run_scope.accounting_book_id, + bank_assignment.bank_account_record_id, + run_scope.currency_code + INTO + current_legal_entity_id, + current_accounting_book_id, + current_bank_account_record_id, + current_currency_code + FROM accounting_core.reconciliation_run AS run_scope + JOIN accounting_core.bank_account_assignment AS bank_assignment + ON bank_assignment.tenant_account_id = run_scope.tenant_account_id + AND bank_assignment.bank_account_assignment_id = run_scope.bank_account_assignment_id + WHERE run_scope.tenant_account_id = NEW.tenant_account_id + AND run_scope.reconciliation_run_id = NEW.reconciliation_run_id; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'match is outside the tenant reconciliation run (reconciliation_scope_mismatch)' + USING ERRCODE = '23514'; + END IF; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO statement_allocation_count, statement_allocation_total + FROM accounting_core.statement_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO journal_allocation_count, journal_allocation_total + FROM accounting_core.journal_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + IF statement_allocation_count = 0 + OR journal_allocation_count = 0 + OR statement_allocation_total <> journal_allocation_total THEN + RAISE EXCEPTION + 'approved reconciliation match requires non-empty equal statement and journal allocation totals (reconciliation_match_unbalanced)' + USING ERRCODE = '23514'; + END IF; + + FOR source_row IN + SELECT allocation.statement_entry_reference, + SUM(allocation.allocated_amount) AS allocation_amount + FROM accounting_core.statement_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id + GROUP BY allocation.statement_entry_reference + LOOP + conservation_key := concat_ws( + ':', + 'reconciliation-statement', + NEW.tenant_account_id::text, + current_legal_entity_id::text, + current_accounting_book_id::text, + current_bank_account_record_id::text, + current_currency_code, + source_row.statement_entry_reference + ); + PERFORM pg_advisory_xact_lock(hashtextextended(conservation_key, 0)); + + SELECT MAX(candidate.statement_amount) + INTO source_capacity + FROM accounting_core.reconciliation_candidate AS candidate + JOIN accounting_core.reconciliation_run AS candidate_run + ON candidate_run.tenant_account_id = candidate.tenant_account_id + AND candidate_run.reconciliation_run_id = candidate.reconciliation_run_id + JOIN accounting_core.bank_account_assignment AS candidate_assignment + ON candidate_assignment.tenant_account_id = candidate_run.tenant_account_id + AND candidate_assignment.bank_account_assignment_id = candidate_run.bank_account_assignment_id + WHERE candidate.tenant_account_id = NEW.tenant_account_id + AND candidate.statement_entry_reference = source_row.statement_entry_reference + AND candidate_run.legal_entity_id = current_legal_entity_id + AND candidate_run.accounting_book_id = current_accounting_book_id + AND candidate_assignment.bank_account_record_id = current_bank_account_record_id + AND candidate_run.currency_code = current_currency_code; + + SELECT COALESCE(SUM(allocation.allocated_amount), 0) + INTO consumed_amount + FROM accounting_core.statement_match_allocation AS allocation + JOIN accounting_core.reconciliation_match AS approved_match + ON approved_match.tenant_account_id = allocation.tenant_account_id + AND approved_match.reconciliation_run_id = allocation.reconciliation_run_id + AND approved_match.reconciliation_match_id = allocation.reconciliation_match_id + JOIN accounting_core.reconciliation_run AS consuming_run + ON consuming_run.tenant_account_id = allocation.tenant_account_id + AND consuming_run.reconciliation_run_id = allocation.reconciliation_run_id + JOIN accounting_core.bank_account_assignment AS consuming_assignment + ON consuming_assignment.tenant_account_id = consuming_run.tenant_account_id + AND consuming_assignment.bank_account_assignment_id = consuming_run.bank_account_assignment_id + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.statement_entry_reference = source_row.statement_entry_reference + AND approved_match.match_status_code = 'approved' + AND approved_match.reconciliation_match_id <> NEW.reconciliation_match_id + AND consuming_run.legal_entity_id = current_legal_entity_id + AND consuming_run.accounting_book_id = current_accounting_book_id + AND consuming_assignment.bank_account_record_id = current_bank_account_record_id + AND consuming_run.currency_code = current_currency_code; + + IF source_capacity IS NULL + OR consumed_amount + source_row.allocation_amount > source_capacity THEN + RAISE EXCEPTION + 'approving reconciliation match would over-consume statement source amount across active runs (reconciliation_allocation_overconsumed)' + USING ERRCODE = '23514'; + END IF; + END LOOP; + + FOR source_row IN + SELECT allocation.journal_reference, + SUM(allocation.allocated_amount) AS allocation_amount + FROM accounting_core.journal_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id + GROUP BY allocation.journal_reference + LOOP + conservation_key := concat_ws( + ':', + 'reconciliation-journal', + NEW.tenant_account_id::text, + current_legal_entity_id::text, + current_accounting_book_id::text, + current_currency_code, + source_row.journal_reference + ); + PERFORM pg_advisory_xact_lock(hashtextextended(conservation_key, 0)); + + SELECT MAX(candidate.journal_amount) + INTO source_capacity + FROM accounting_core.reconciliation_candidate AS candidate + JOIN accounting_core.reconciliation_run AS candidate_run + ON candidate_run.tenant_account_id = candidate.tenant_account_id + AND candidate_run.reconciliation_run_id = candidate.reconciliation_run_id + WHERE candidate.tenant_account_id = NEW.tenant_account_id + AND candidate.journal_reference = source_row.journal_reference + AND candidate_run.legal_entity_id = current_legal_entity_id + AND candidate_run.accounting_book_id = current_accounting_book_id + AND candidate_run.currency_code = current_currency_code; + + SELECT COALESCE(SUM(allocation.allocated_amount), 0) + INTO consumed_amount + FROM accounting_core.journal_match_allocation AS allocation + JOIN accounting_core.reconciliation_match AS approved_match + ON approved_match.tenant_account_id = allocation.tenant_account_id + AND approved_match.reconciliation_run_id = allocation.reconciliation_run_id + AND approved_match.reconciliation_match_id = allocation.reconciliation_match_id + JOIN accounting_core.reconciliation_run AS consuming_run + ON consuming_run.tenant_account_id = allocation.tenant_account_id + AND consuming_run.reconciliation_run_id = allocation.reconciliation_run_id + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.journal_reference = source_row.journal_reference + AND approved_match.match_status_code = 'approved' + AND approved_match.reconciliation_match_id <> NEW.reconciliation_match_id + AND consuming_run.legal_entity_id = current_legal_entity_id + AND consuming_run.accounting_book_id = current_accounting_book_id + AND consuming_run.currency_code = current_currency_code; + + IF source_capacity IS NULL + OR consumed_amount + source_row.allocation_amount > source_capacity THEN + RAISE EXCEPTION + 'approving reconciliation match would over-consume journal source amount across active runs (reconciliation_allocation_overconsumed)' + USING ERRCODE = '23514'; + END IF; + END LOOP; + + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION accounting_core.reconciliation_approval_insert_guard() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + current_status text; + statement_allocation_count bigint; + journal_allocation_count bigint; + statement_allocation_total numeric; + journal_allocation_total numeric; +BEGIN + SELECT match.match_status_code + INTO current_status + FROM accounting_core.reconciliation_match AS match + WHERE match.tenant_account_id = NEW.tenant_account_id + AND match.reconciliation_run_id = NEW.reconciliation_run_id + AND match.reconciliation_match_id = NEW.reconciliation_match_id + FOR UPDATE OF match; + + IF NOT FOUND OR current_status <> 'proposed' THEN + RAISE EXCEPTION + 'reconciliation approval evidence requires a proposed match in the same tenant/run scope (reconciliation_approval_scope_mismatch)' + USING ERRCODE = '23514'; + END IF; + + PERFORM accounting_core.reconciliation_match_snapshot_lock( + NEW.tenant_account_id, + NEW.reconciliation_run_id, + NEW.reconciliation_match_id + ); + + IF NEW.approval_decision_code = 'approved' THEN + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO statement_allocation_count, statement_allocation_total + FROM accounting_core.statement_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO journal_allocation_count, journal_allocation_total + FROM accounting_core.journal_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + IF statement_allocation_count = 0 + OR journal_allocation_count = 0 + OR statement_allocation_total <> journal_allocation_total THEN + RAISE EXCEPTION + 'approved reconciliation evidence requires non-empty equal statement and journal allocation totals; add or correct allocations before recording approval evidence (reconciliation_match_unbalanced)' + USING ERRCODE = '23514'; + END IF; + END IF; + + NEW.reconciliation_snapshot_version := 1; + NEW.reconciliation_snapshot_hash := + accounting_core.reconciliation_match_snapshot_hash( + NEW.tenant_account_id, + NEW.reconciliation_run_id, + NEW.reconciliation_match_id + ); + IF NEW.reconciliation_snapshot_hash IS NULL THEN + RAISE EXCEPTION + 'reconciliation approval evidence has no reviewable candidate snapshot (reconciliation_snapshot_missing)' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END; +$$; + +CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_match_command_allocations() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + statement_allocation_count bigint; + journal_allocation_count bigint; + statement_allocation_total numeric; + journal_allocation_total numeric; + candidate_statement_amount numeric; + candidate_journal_amount numeric; +BEGIN + PERFORM 1 + FROM accounting_core.reconciliation_match AS match + WHERE match.tenant_account_id = NEW.tenant_account_id + AND match.reconciliation_run_id = NEW.reconciliation_run_id + AND match.reconciliation_match_id = NEW.reconciliation_match_id + FOR UPDATE; + + SELECT candidate.statement_amount, candidate.journal_amount + INTO candidate_statement_amount, candidate_journal_amount + FROM accounting_core.reconciliation_candidate AS candidate + WHERE candidate.tenant_account_id = NEW.tenant_account_id + AND candidate.reconciliation_run_id = NEW.reconciliation_run_id + AND candidate.reconciliation_candidate_id = NEW.reconciliation_candidate_id; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO statement_allocation_count, statement_allocation_total + FROM accounting_core.statement_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + SELECT COUNT(*), COALESCE(SUM(allocation.allocated_amount), 0) + INTO journal_allocation_count, journal_allocation_total + FROM accounting_core.journal_match_allocation AS allocation + WHERE allocation.tenant_account_id = NEW.tenant_account_id + AND allocation.reconciliation_run_id = NEW.reconciliation_run_id + AND allocation.reconciliation_match_id = NEW.reconciliation_match_id; + + IF statement_allocation_count <> 1 + OR journal_allocation_count <> 1 + OR statement_allocation_total <> journal_allocation_total + OR statement_allocation_total <> candidate_statement_amount + OR journal_allocation_total <> candidate_journal_amount THEN + RAISE EXCEPTION + 'reconciliation match command requires exactly one statement and one journal allocation matching candidate amounts (reconciliation_match_command_allocation)' + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$$; + +COMMIT; diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 64706fb4..125e23d2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -145,6 +145,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. 20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. 21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — preflights existing immutable command evidence and fails closed on cross-bank provenance before reapplying run-command guards to installations that already executed migration 0019 before its command-insert guard was added. +22. `database/migrations/0022_reconciliation_amount_precision.sql` — widens reconciliation candidate/allocation monetary facts to the platform `numeric(38, 6)` domain and keeps exact conservation aggregates from overflowing before validation. ## Durable soft-close command evidence diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 4a9dcaba..16bcd59c 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -40,7 +40,7 @@ The foundation ERD is maintained in [ERD.md](ERD.md). PostgreSQL migrations are ## Reconciliation control evidence -Migrations `0013_reconciliation_run_exception_evidence.sql` through `0021_reconciliation_run_command_provenance_repair.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. +Migrations `0013_reconciliation_run_exception_evidence.sql` through `0022_reconciliation_amount_precision.sql` persist deterministic reconciliation as accounting control evidence without granting posting, reversal, close, or accounting-policy authority. - `reconciliation_run`: immutable evaluated scope for one tenant, legal entity, accounting book, bank-account assignment, ISO currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Only the run status may progress; evaluated scope changes require a new run. - `reconciliation_exception`: explicit operator-owned exception with an exception code, next action, effective/system time, and open/resolved/superseded resolution status. @@ -52,6 +52,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0021_reconc - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Migration 0021 preflights existing command rows and fails closed on cross-bank provenance before reapplying those guards. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. - `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- Reconciliation candidate and allocation amounts use PostgreSQL `numeric(38, 6)`, matching journal and bank-statement evidence. Migration `0022_reconciliation_amount_precision.sql` upgrades historical `numeric(30, 6)` columns and keeps exact trigger aggregates from overflowing before conservation validation. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 58b93816..b2ae4d58 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -2,7 +2,7 @@ ## Deployment preconditions -Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0021_reconciliation_run_command_provenance_repair.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. +Use PostgreSQL 18 and keep the migration owner, application runtime login and administrative / break-glass identities separate. Apply migrations in numeric order through `0022_reconciliation_amount_precision.sql` before starting the service. Do not run the application with a table-owner, superuser or `BYPASSRLS` login. Required environment values are deployment-specific. At minimum, configure the accounting database URL and bind this AIS process to exactly one tenant reference. Secrets belong in an approved secret store; do not place database passwords, NTS credentials, bearer tokens or provider secrets in journal payloads, logs or outbox events. @@ -34,6 +34,7 @@ database/migrations/0018_bank_statement_balance_evidence.sql database/migrations/0019_reconciliation_run_command_evidence.sql database/migrations/0020_reconciliation_match_command_evidence.sql database/migrations/0021_reconciliation_run_command_provenance_repair.sql +database/migrations/0022_reconciliation_amount_precision.sql ``` Migration `0015_reconciliation_multi_match_conservation.sql` replaces the run-wide single-approved-match shortcut from `0014` with tenant/run-scoped match identity plus exact statement/journal allocation conservation. It permits multiple independently approved matches only when no authoritative source amount is over-consumed and grants no journal-posting authority. @@ -48,6 +49,8 @@ Migration `0019_reconciliation_run_command_evidence.sql` records the immutable c Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. Its migration-only preflight scans existing immutable command evidence and fails closed on cross-bank provenance; it then recreates both the deferred run guard and immediate command-insert guard without changing evidence rows. Apply it after `0020` and before runtime traffic. +Migration `0022_reconciliation_amount_precision.sql` widens reconciliation candidate and allocation amounts from the historical `numeric(30, 6)` domain to the platform-wide `numeric(38, 6)` domain. It also replaces the conservation and command-allocation trigger functions with unconstrained exact aggregate variables, so valid boundary-sized amounts are compared before rejection rather than overflowing during `SUM`. Apply it after `0021`; the migration changes no evidence values and runtime parsing rejects amounts outside the same domain before persistence. + Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 1ae9cc17..3f0db0da 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -73,6 +73,14 @@ before the command-insert provenance guard existed. It recreates the deferred run guard and immediate command-insert guard without changing existing evidence rows. +Migration `0022_reconciliation_amount_precision.sql` widens the candidate and +allocation amount columns from the historical `numeric(30, 6)` definition to +the platform-wide `numeric(38, 6)` definition. It also uses unconstrained +PostgreSQL numeric variables in the conservation and command-allocation +triggers, preventing aggregate overflow from masking an exact conservation +rejection. The shared amount parser rejects values outside that same storage +domain before a command reaches the database. + Historical match admission also requires the posted journal fact to be known by the run's `knowledge_cutoff_at`; a backdated accounting date alone is not enough. The source database driver is loaded only inside the database command diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 9bc72677..123665d4 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -16,6 +16,7 @@ | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 preflights existing command rows, fails closed on cross-bank provenance, and then reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | | Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, require exactly one equal allocation on each side matching candidate amounts at command insert, and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | +| Reconciliation amount precision / PostgreSQL numeric domain | Candidate and statement/journal allocation amounts share the platform `numeric(38, 6)` domain used by authoritative journal and bank-statement facts. Migration 0022 widens historical reconciliation columns and uses unconstrained exact aggregate variables in conservation guards; the shared `_parse_amount` boundary rejects values wider than that storage domain before persistence. This preserves exact rejection semantics without rounding or aggregate overflow. | `0022_reconciliation_amount_precision.sql`, `_parse_amount`, precision boundary regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | | CloudEvents 1.0.2 | Commit authoritative events through a transactional outbox and replay by event identity | Outbox table and architecture | diff --git a/scripts/validate_repository.py b/scripts/validate_repository.py index 8cbd6ca0..f9a6dced 100644 --- a/scripts/validate_repository.py +++ b/scripts/validate_repository.py @@ -56,6 +56,7 @@ "database/migrations/0019_reconciliation_run_command_evidence.sql", "database/migrations/0020_reconciliation_match_command_evidence.sql", "database/migrations/0021_reconciliation_run_command_provenance_repair.sql", + "database/migrations/0022_reconciliation_amount_precision.sql", "docs/PRD.md", "docs/TRD.md", "docs/ARCHITECTURE.md", diff --git a/src/accounting_information_platform/core.py b/src/accounting_information_platform/core.py index 262bc403..40c37866 100644 --- a/src/accounting_information_platform/core.py +++ b/src/accounting_information_platform/core.py @@ -22,6 +22,7 @@ _CODE_PATTERN = re.compile(r"^[a-z][a-z0-9_]{1,63}$") _CURRENCY_PATTERN = re.compile(r"^[A-Z]{3}$") _DECIMAL_PATTERN = re.compile(r"^(0|[1-9][0-9]*)(\.[0-9]{1,6})?$") +_MAX_STORAGE_AMOUNT = Decimal("9" * 32 + ".999999") _HASH_PATTERN = re.compile(r"^sha256:[0-9a-f]{64}$") _PROPOSAL_ID_PATTERN = re.compile( r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" @@ -703,7 +704,7 @@ def _reversal_command_hash( def _parse_amount(value: Decimal | str) -> Decimal: - """Parse a canonical non-negative decimal with at most six fractional digits.""" + """Parse a canonical amount within the PostgreSQL numeric(38, 6) domain.""" text = str(value) if _DECIMAL_PATTERN.fullmatch(text) is None: raise AccountingValidationError( @@ -716,6 +717,10 @@ def _parse_amount(value: Decimal | str) -> Decimal: raise AccountingValidationError( "amount is not a valid decimal. Supply a canonical non-negative decimal string, then retry ingest." ) from error + if amount > _MAX_STORAGE_AMOUNT: + raise AccountingValidationError( + "amount exceeds the supported monetary precision. Supply an amount within numeric(38, 6), then retry ingest." + ) return amount diff --git a/src/accounting_information_platform/persistence.py b/src/accounting_information_platform/persistence.py index ab2d84a0..cc24ca29 100644 --- a/src/accounting_information_platform/persistence.py +++ b/src/accounting_information_platform/persistence.py @@ -6358,6 +6358,15 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: f"{run_command_provenance_repair_migration_path}. Restore " "database/migrations/0021_reconciliation_run_command_provenance_repair.sql, then retry." ) + amount_precision_migration_path = ( + migration_path.parent / "0022_reconciliation_amount_precision.sql" + ) + if not amount_precision_migration_path.is_file(): + raise AccountingValidationError( + "Reconciliation amount-precision migration is missing at " + f"{amount_precision_migration_path}. Restore " + "database/migrations/0022_reconciliation_amount_precision.sql, then retry." + ) psycopg = _import_psycopg() try: with psycopg.connect( @@ -6402,6 +6411,9 @@ def apply_foundation_migration(database_url: str, migration_path: Path) -> None: connection.execute( run_command_provenance_repair_migration_path.read_text(encoding="utf-8") ) + connection.execute( + amount_precision_migration_path.read_text(encoding="utf-8") + ) except Exception as error: raise AccountingValidationError( "Foundation migration failed. Inspect the PostgreSQL error, restore a clean " diff --git a/tests/test_foundation_install_manifest_contract.py b/tests/test_foundation_install_manifest_contract.py index e37dcfb7..61facc50 100644 --- a/tests/test_foundation_install_manifest_contract.py +++ b/tests/test_foundation_install_manifest_contract.py @@ -173,6 +173,20 @@ def test_required_files_and_install_docs_include_run_provenance_repair(self) -> self.assertIn(migration_twenty_one, text) self.assertLess(text.index(migration_twenty), text.index(migration_twenty_one)) + def test_required_files_and_install_docs_include_amount_precision_repair(self) -> None: + """Existing installations receive the reconciliation precision repair.""" + migration_twenty_one = ( + "database/migrations/0021_reconciliation_run_command_provenance_repair.sql" + ) + migration_twenty_two = "database/migrations/0022_reconciliation_amount_precision.sql" + self.assertIn(migration_twenty_two, set(REQUIRED_FILES)) + for relative_path in ("docs/OPERABILITY.md", "docs/ARCHITECTURE.md"): + with self.subTest(relative_path=relative_path): + text = (ROOT / relative_path).read_text(encoding="utf-8") + self.assertIn(migration_twenty_one, text) + self.assertIn(migration_twenty_two, text) + self.assertLess(text.index(migration_twenty_one), text.index(migration_twenty_two)) + def test_run_provenance_repair_rejects_existing_mismatches(self) -> None: """The upgrade must fail closed on already stored cross-bank commands.""" migration = ( @@ -295,6 +309,22 @@ def is_file(path: Path) -> bool: ROOT / "database/migrations/0001_accounting_foundation.sql", ) + def test_install_fails_closed_when_amount_precision_migration_is_missing(self) -> None: + """The canonical loader must apply the reconciliation precision repair.""" + original_is_file = Path.is_file + + def is_file(path: Path) -> bool: + if path.name == "0022_reconciliation_amount_precision.sql": + return False + return original_is_file(path) + + with patch.object(Path, "is_file", is_file): + with self.assertRaisesRegex(AccountingValidationError, "amount-precision"): + apply_foundation_migration( + "postgresql://unused", + ROOT / "database/migrations/0001_accounting_foundation.sql", + ) + def test_canonical_persistence_loader_fails_closed_when_conservation_is_missing(self) -> None: """Real PostgreSQL fixtures may not silently stop the authoritative chain at 0014.""" from accounting_information_platform.persistence import ( diff --git a/tests/test_journal_proposal_ingest.py b/tests/test_journal_proposal_ingest.py index ac4d9741..4239fc02 100644 --- a/tests/test_journal_proposal_ingest.py +++ b/tests/test_journal_proposal_ingest.py @@ -209,6 +209,27 @@ def test_more_than_six_fractional_digits_fail_closed_without_rounding(self) -> N self.assertEqual(two.lines[0].debit_amount, Decimal("25000.50")) self.assertEqual(two.lines[1].credit_amount, Decimal("25000.50")) + def test_more_than_numeric_precision_fails_closed_without_rounding(self) -> None: + """An amount wider than numeric(38, 6) cannot reach persistence.""" + too_wide = self._billing_proposal() + too_wide_amount = "1" + "0" * 32 + too_wide["lines"][0]["debit_amount"] = too_wide_amount + too_wide["lines"][1]["credit_amount"] = too_wide_amount + + with self.assertRaisesRegex( + AccountingValidationError, "supported monetary precision" + ): + ingest_journal_proposal(too_wide) + + maximum = self._billing_proposal() + maximum_amount = "9" * 32 + ".999999" + maximum["lines"][0]["debit_amount"] = maximum_amount + maximum["lines"][1]["credit_amount"] = maximum_amount + self.assertEqual( + ingest_journal_proposal(maximum).lines[0].debit_amount, + Decimal(maximum_amount), + ) + def test_proposal_contract_version_must_be_a_non_bool_int(self) -> None: """Bool and non-int contract versions fail closed before posting.""" bool_version = self._billing_proposal(proposal_contract_version=True) diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index 6079d56c..b3671ec8 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -46,6 +46,28 @@ def test_erd_shows_reconciliation_evidence_chain(self) -> None: self.assertIn("active", text.lower()) self.assertIn("approved", text.lower()) + def test_reconciliation_amounts_use_platform_numeric_precision(self) -> None: + """Forward migration widens reconciliation amounts to the platform precision contract.""" + migration_path = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ) + if not migration_path.is_file(): + self.fail("reconciliation amount precision migration is missing") + migration = migration_path.read_text(encoding="utf-8") + self.assertNotIn("numeric(30, 6)", migration) + for table_name, column_name in ( + ("reconciliation_candidate", "statement_amount"), + ("reconciliation_candidate", "journal_amount"), + ("statement_match_allocation", "allocated_amount"), + ("journal_match_allocation", "allocated_amount"), + ): + with self.subTest(table_name=table_name, column_name=column_name): + self.assertIn( + f"ALTER COLUMN {column_name} TYPE numeric(38, 6)", + migration, + ) + if __name__ == "__main__": unittest.main() From b740bccba0d42589e5ed0940352333f4f47f6e2c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:00:39 +0900 Subject: [PATCH 19/25] fix(reconciliation): rebuild dependent amount trigger --- .../0022_reconciliation_amount_precision.sql | 16 +++++++++++++ ...ation_data_model_documentation_contract.py | 23 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/database/migrations/0022_reconciliation_amount_precision.sql b/database/migrations/0022_reconciliation_amount_precision.sql index 9015daea..d187bea4 100644 --- a/database/migrations/0022_reconciliation_amount_precision.sql +++ b/database/migrations/0022_reconciliation_amount_precision.sql @@ -3,6 +3,11 @@ BEGIN; -- Reconciliation monetary facts must use the same numeric(38, 6) domain as -- journal lines and bank-statement evidence. The original 0014 tables used a -- narrower legacy domain, which could reject valid source amounts. +-- PostgreSQL does not permit changing a column type while an UPDATE OF trigger +-- names that column, so the existing registration is rebuilt below. +DROP TRIGGER reconciliation_candidate_capacity_guard + ON accounting_core.reconciliation_candidate; + ALTER TABLE accounting_core.reconciliation_candidate ALTER COLUMN statement_amount TYPE numeric(38, 6), ALTER COLUMN journal_amount TYPE numeric(38, 6); @@ -337,4 +342,15 @@ BEGIN END; $$; +CREATE TRIGGER reconciliation_candidate_capacity_guard +BEFORE INSERT OR UPDATE OF + tenant_account_id, + reconciliation_run_id, + statement_entry_reference, + journal_reference, + statement_amount, + journal_amount +ON accounting_core.reconciliation_candidate +FOR EACH ROW EXECUTE FUNCTION accounting_core.reconciliation_candidate_capacity_guard(); + COMMIT; diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index b3671ec8..2ee70ff1 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -68,6 +68,29 @@ def test_reconciliation_amounts_use_platform_numeric_precision(self) -> None: migration, ) + def test_precision_migration_rebuilds_column_dependent_candidate_trigger(self) -> None: + """The type repair must preserve the candidate UPDATE OF trigger registration.""" + migration = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ).read_text(encoding="utf-8") + self.assertIn( + "DROP TRIGGER reconciliation_candidate_capacity_guard", + migration, + ) + self.assertIn( + "CREATE TRIGGER reconciliation_candidate_capacity_guard", + migration, + ) + self.assertLess( + migration.index("DROP TRIGGER reconciliation_candidate_capacity_guard"), + migration.index("ALTER TABLE accounting_core.reconciliation_candidate"), + ) + self.assertGreater( + migration.index("CREATE TRIGGER reconciliation_candidate_capacity_guard"), + migration.index("ALTER COLUMN journal_amount TYPE numeric(38, 6)"), + ) + if __name__ == "__main__": unittest.main() From 134ecb528cee30ba840dabf6d99d1a2bb3539779 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:02:55 +0900 Subject: [PATCH 20/25] docs: update migration chain guidance --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 141fe942..fc25488c 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ factory/runner and provide the tenant-bound host boundary explicitly. `unittest` discovery also runs `tests/test_postgres_posting.py`, which needs a reachable PostgreSQL 18 instance and `ACCOUNTING_DATABASE_URL` (CI uses `postgresql://postgres:postgres@127.0.0.1:5432/accounting_test` and applies -the checked-in migration chain through `database/migrations/0021_reconciliation_run_command_provenance_repair.sql`). Persistence is still +the checked-in migration chain through `database/migrations/0022_reconciliation_amount_precision.sql`). Persistence is still local to this repository; it is not a Naruon or sibling checkout. Optional import smoke after the editable install above: From 98ba7dfbc704b3e0c53d12bf8048ada8cf0f5bce Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:18:01 +0900 Subject: [PATCH 21/25] fix(reconciliation): close command evidence races --- CHANGELOG.md | 2 +- .../0022_reconciliation_amount_precision.sql | 54 ++++++++++++++++++- docs/ARCHITECTURE.md | 2 +- docs/DATA_MODEL.md | 2 +- docs/OPERABILITY.md | 4 +- ...9-reconciliation-match-command-evidence.md | 16 ++++-- docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- ...on_candidate_allocation_persistence_red.py | 36 +++++++++++++ ...ation_data_model_documentation_contract.py | 36 +++++++++++++ 9 files changed, 141 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4233a5b0..191a9438 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Aligned reconciliation candidate and allocation amounts with the platform `numeric(38, 6)` monetary domain through migration `0022_reconciliation_amount_precision.sql`; exact trigger aggregates no longer overflow before conservation validation, and the shared parser rejects wider values before persistence. +- Aligned reconciliation candidate and allocation amounts with the platform `numeric(38, 6)` monetary domain through migration `0022_reconciliation_amount_precision.sql`; exact trigger aggregates no longer overflow before conservation validation, command evidence is admitted only for still-proposed matches, and allocation-freeze checks re-read command evidence after the parent-match lock. The shared parser rejects wider values before persistence. - Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations while failing closed on pre-existing cross-bank command evidence. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. diff --git a/database/migrations/0022_reconciliation_amount_precision.sql b/database/migrations/0022_reconciliation_amount_precision.sql index d187bea4..c0223dde 100644 --- a/database/migrations/0022_reconciliation_amount_precision.sql +++ b/database/migrations/0022_reconciliation_amount_precision.sql @@ -293,6 +293,7 @@ RETURNS trigger LANGUAGE plpgsql AS $$ DECLARE + current_match_status text; statement_allocation_count bigint; journal_allocation_count bigint; statement_allocation_total numeric; @@ -300,12 +301,25 @@ DECLARE candidate_statement_amount numeric; candidate_journal_amount numeric; BEGIN - PERFORM 1 + SELECT match.match_status_code + INTO current_match_status FROM accounting_core.reconciliation_match AS match WHERE match.tenant_account_id = NEW.tenant_account_id AND match.reconciliation_run_id = NEW.reconciliation_run_id AND match.reconciliation_match_id = NEW.reconciliation_match_id - FOR UPDATE; + FOR UPDATE OF match; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'reconciliation match command requires a match in the same tenant/run scope (reconciliation_match_command_scope_mismatch)' + USING ERRCODE = '23514'; + END IF; + + IF current_match_status <> 'proposed' THEN + RAISE EXCEPTION + 'reconciliation match command evidence requires a proposed match; terminal matches already have review provenance (reconciliation_match_command_status)' + USING ERRCODE = '23514'; + END IF; SELECT candidate.statement_amount, candidate.journal_amount INTO candidate_statement_amount, candidate_journal_amount @@ -342,6 +356,42 @@ BEGIN END; $$; +-- Recheck command evidence after acquiring the match lock. A later statement +-- snapshot prevents an allocation from observing a stale pre-command view +-- after it waited for a concurrent command transaction to finish. +CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_allocation() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + PERFORM 1 + FROM accounting_core.reconciliation_match AS match + WHERE match.tenant_account_id = NEW.tenant_account_id + AND match.reconciliation_run_id = NEW.reconciliation_run_id + AND match.reconciliation_match_id = NEW.reconciliation_match_id + FOR UPDATE OF match; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'reconciliation match command allocation requires a match in the same tenant/run scope (reconciliation_match_command_scope_mismatch)' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + SELECT 1 + FROM accounting_core.reconciliation_match_command AS command + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id + AND command.reconciliation_match_id = NEW.reconciliation_match_id + ) THEN + RAISE EXCEPTION + 'reconciliation match command evidence freezes its allocation population; create a new proposed match instead (reconciliation_match_command_allocation_frozen)' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END; +$$; + CREATE TRIGGER reconciliation_candidate_capacity_guard BEFORE INSERT OR UPDATE OF tenant_account_id, diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 125e23d2..64ec83c7 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -145,7 +145,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. 20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. 21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — preflights existing immutable command evidence and fails closed on cross-bank provenance before reapplying run-command guards to installations that already executed migration 0019 before its command-insert guard was added. -22. `database/migrations/0022_reconciliation_amount_precision.sql` — widens reconciliation candidate/allocation monetary facts to the platform `numeric(38, 6)` domain and keeps exact conservation aggregates from overflowing before validation. +22. `database/migrations/0022_reconciliation_amount_precision.sql` — widens reconciliation candidate/allocation monetary facts to the platform `numeric(38, 6)` domain, keeps exact conservation aggregates from overflowing before validation, admits command provenance only for proposed matches, and rechecks allocation-freeze evidence after locking the parent match. ## Durable soft-close command evidence diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 16bcd59c..193e8ffd 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -51,7 +51,7 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0022_reconc - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Migration 0021 preflights existing command rows and fails closed on cross-bank provenance before reapplying those guards. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, admits provenance only while the match is `proposed`, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts by rechecking command evidence after that parent lock. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. - Reconciliation candidate and allocation amounts use PostgreSQL `numeric(38, 6)`, matching journal and bank-statement evidence. Migration `0022_reconciliation_amount_precision.sql` upgrades historical `numeric(30, 6)` columns and keeps exact trigger aggregates from overflowing before conservation validation. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index b2ae4d58..96598259 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -49,9 +49,9 @@ Migration `0019_reconciliation_run_command_evidence.sql` records the immutable c Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. Its migration-only preflight scans existing immutable command evidence and fails closed on cross-bank provenance; it then recreates both the deferred run guard and immediate command-insert guard without changing evidence rows. Apply it after `0020` and before runtime traffic. -Migration `0022_reconciliation_amount_precision.sql` widens reconciliation candidate and allocation amounts from the historical `numeric(30, 6)` domain to the platform-wide `numeric(38, 6)` domain. It also replaces the conservation and command-allocation trigger functions with unconstrained exact aggregate variables, so valid boundary-sized amounts are compared before rejection rather than overflowing during `SUM`. Apply it after `0021`; the migration changes no evidence values and runtime parsing rejects amounts outside the same domain before persistence. +Migration `0022_reconciliation_amount_precision.sql` widens reconciliation candidate and allocation amounts from the historical `numeric(30, 6)` domain to the platform-wide `numeric(38, 6)` domain. It also replaces the conservation and command-allocation trigger functions with unconstrained exact aggregate variables, so valid boundary-sized amounts are compared before rejection rather than overflowing during `SUM`. Command provenance is accepted only for a still-proposed match, and allocation-freeze checks acquire the parent match lock before taking their command-evidence snapshot. Apply it after `0021`; the migration changes no evidence values and runtime parsing rejects amounts outside the same domain before persistence. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Migration `0022` preserves that boundary for upgrades by refusing command provenance on terminal matches and rechecking the command-freeze population after the parent lock. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 3f0db0da..518d34e4 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -63,9 +63,13 @@ source evidence as `404`, state conflicts as `409`, and source-content or conservation validation failures as `422`. The database requires command insertion to observe exactly one statement and -one journal allocation with equal exact amounts, and rejects allocation rows -inserted after command evidence. These are evidence-integrity controls only; -they do not turn a proposed match into an approved accounting fact. +one journal allocation with equal exact amounts while the match is still +`proposed`, and rejects allocation rows inserted after command evidence. The +allocation guard locks the parent match before its command-evidence query, so +a transaction that waited on a concurrent command cannot use a stale snapshot +to extend the frozen allocation population. These are evidence-integrity +controls only; they do not turn a proposed match into an approved accounting +fact. Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-only upgrade for installations that already applied migration 0019 @@ -78,8 +82,10 @@ allocation amount columns from the historical `numeric(30, 6)` definition to the platform-wide `numeric(38, 6)` definition. It also uses unconstrained PostgreSQL numeric variables in the conservation and command-allocation triggers, preventing aggregate overflow from masking an exact conservation -rejection. The shared amount parser rejects values outside that same storage -domain before a command reaches the database. +rejection. Its forward-recreated command guard rejects provenance for terminal +matches, and its allocation-freeze guard takes the parent match lock before +checking command evidence. The shared amount parser rejects values outside +that same storage domain before a command reaches the database. Historical match admission also requires the posted journal fact to be known by the run's `knowledge_cutoff_at`; a backdated accounting date alone is not diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index 123665d4..a65ac6db 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,7 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 preflights existing command rows, fails closed on cross-bank provenance, and then reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, require exactly one equal allocation on each side matching candidate amounts at command insert, and reject later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, admit command provenance only for `proposed`, require exactly one equal allocation on each side matching candidate amounts at command insert, and recheck command evidence after that lock before rejecting later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `0022_reconciliation_amount_precision.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | Reconciliation amount precision / PostgreSQL numeric domain | Candidate and statement/journal allocation amounts share the platform `numeric(38, 6)` domain used by authoritative journal and bank-statement facts. Migration 0022 widens historical reconciliation columns and uses unconstrained exact aggregate variables in conservation guards; the shared `_parse_amount` boundary rejects values wider than that storage domain before persistence. This preserves exact rejection semantics without rounding or aggregate overflow. | `0022_reconciliation_amount_precision.sql`, `_parse_amount`, precision boundary regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | diff --git a/tests/test_reconciliation_candidate_allocation_persistence_red.py b/tests/test_reconciliation_candidate_allocation_persistence_red.py index bfef495f..97f679d7 100644 --- a/tests/test_reconciliation_candidate_allocation_persistence_red.py +++ b/tests/test_reconciliation_candidate_allocation_persistence_red.py @@ -262,6 +262,42 @@ def _approve_match(self, match_id: uuid.UUID) -> None: (self.scope["tenant_account_id"], self.run_reference, match_id), ) + def test_command_provenance_requires_a_proposed_match(self) -> None: + """Terminal matches cannot be relabeled as command-created proposals.""" + candidate_id = self._insert_candidate("terminal-statement", "terminal-journal") + match_id = self._insert_match(candidate_id) + self._insert_allocations( + match_id, + "terminal-statement", + "terminal-journal", + "1000.00", + ) + self._approve_match(match_id) + + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with self.assertRaisesRegex(psycopg.errors.CheckViolation, "proposed"): + connection.execute( + """ + INSERT INTO accounting_core.reconciliation_match_command ( + tenant_account_id, reconciliation_run_id, + reconciliation_candidate_id, reconciliation_match_id, + candidate_idempotency_key, candidate_command_hash, + source_payload_hash, source_payload_reference + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + self.scope["tenant_account_id"], + self.run_reference, + candidate_id, + match_id, + f"terminal-command-{uuid.uuid4().hex}", + "sha256:" + "1" * 64, + "sha256:" + "2" * 64, + "urn:cwl:object:terminal-command", + ), + ) + def _record_approval( self, match_id: uuid.UUID, diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index 2ee70ff1..bb86dbbb 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -91,6 +91,42 @@ def test_precision_migration_rebuilds_column_dependent_candidate_trigger(self) - migration.index("ALTER COLUMN journal_amount TYPE numeric(38, 6)"), ) + def test_precision_migration_rechecks_command_freeze_after_parent_lock(self) -> None: + """Allocation freeze evidence must use a snapshot taken after the match lock.""" + migration = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ).read_text(encoding="utf-8") + marker = ( + "CREATE OR REPLACE FUNCTION " + "accounting_core.reject_reconciliation_match_command_allocation()" + ) + self.assertIn(marker, migration) + function = migration.split(marker, 1)[1].split( + "CREATE TRIGGER reconciliation_candidate_capacity_guard", 1 + )[0] + self.assertRegex( + function, + r"(?s)FROM accounting_core\.reconciliation_match.*?FOR UPDATE.*?IF EXISTS", + ) + + def test_precision_migration_rejects_non_proposed_command_matches(self) -> None: + """Command provenance can only be recorded for a still-proposed match.""" + migration = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ).read_text(encoding="utf-8") + function = migration.split( + "CREATE OR REPLACE FUNCTION accounting_core.enforce_reconciliation_match_command_allocations()", + 1, + )[1].split( + "CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_allocation()", + 1, + )[0] + self.assertIn("current_match_status", function) + self.assertIn("match_status_code", function) + self.assertIn("reconciliation_match_command_status", function) + if __name__ == "__main__": unittest.main() From 39d134d346dde1f080784342ef2d8c336334ad9a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:40:56 +0900 Subject: [PATCH 22/25] fix(reconciliation): persist command freeze marker --- CHANGELOG.md | 2 +- .../0022_reconciliation_amount_precision.sql | 62 +++++++++++++++---- docs/DATA_MODEL.md | 4 +- docs/OPERABILITY.md | 4 +- ...9-reconciliation-match-command-evidence.md | 17 ++--- docs/doctoring/STANDARD_TRACEABILITY.md | 2 +- ...ation_data_model_documentation_contract.py | 25 +++++++- 7 files changed, 89 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 191a9438..5a7cffeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## [Unreleased] -- Aligned reconciliation candidate and allocation amounts with the platform `numeric(38, 6)` monetary domain through migration `0022_reconciliation_amount_precision.sql`; exact trigger aggregates no longer overflow before conservation validation, command evidence is admitted only for still-proposed matches, and allocation-freeze checks re-read command evidence after the parent-match lock. The shared parser rejects wider values before persistence. +- Aligned reconciliation candidate and allocation amounts with the platform `numeric(38, 6)` monetary domain through migration `0022_reconciliation_amount_precision.sql`; exact trigger aggregates no longer overflow before conservation validation, command evidence is admitted only for still-proposed matches, and allocation-freeze checks read a durable command-evidence marker from the locked parent match. Existing command rows backfill that marker, and the shared parser rejects wider values before persistence. - Closed additional match-evidence gaps: journal `posted_at` and statement `recorded_at` now respect the run knowledge cutoff, direct command evidence must match candidate amounts as well as allocation totals, command-side allocation checks serialize on the match row, and migration `0021_reconciliation_run_command_provenance_repair.sql` reapplies provenance guards to existing installations while failing closed on pre-existing cross-bank command evidence. The psycopg exception import is deferred until the database command path. Dependency-free public imports remain usable; the match boundary remains review-only. - Hardened proposed-match source admission: the run row is locked through match persistence; statement booking/value timestamps and journal accounting dates respect the run cutoffs; CRDT/DBIT must agree with the debit/credit side of the assigned cash chart line; and compound journals match that cash line rather than the journal-wide total. Migration `0020_reconciliation_match_command_evidence.sql` now requires exactly one equal statement/journal allocation at command insert and freezes allocations after command evidence. These controls remain review evidence only and cannot approve, close, or post. - Hardened migration `0020_reconciliation_match_command_evidence.sql` so immutable match-command evidence cannot mix an independently valid candidate with another candidate's match; the candidate-inclusive composite foreign key now proves the persisted chain. diff --git a/database/migrations/0022_reconciliation_amount_precision.sql b/database/migrations/0022_reconciliation_amount_precision.sql index c0223dde..49e2f1e4 100644 --- a/database/migrations/0022_reconciliation_amount_precision.sql +++ b/database/migrations/0022_reconciliation_amount_precision.sql @@ -18,6 +18,37 @@ ALTER TABLE accounting_core.statement_match_allocation ALTER TABLE accounting_core.journal_match_allocation ALTER COLUMN allocated_amount TYPE numeric(38, 6); +-- A command transaction records its freeze on the locked match tuple. Unlike +-- a command-table lookup, the row lock makes a waiting allocation recheck the +-- committed tuple version after the command transaction finishes. +ALTER TABLE accounting_core.reconciliation_match + ADD COLUMN command_evidence_recorded_at timestamptz; + +CREATE POLICY reconciliation_match_command_marker_match_upgrade_visibility + ON accounting_core.reconciliation_match + FOR UPDATE + TO current_user + USING (true) + WITH CHECK (true); +CREATE POLICY reconciliation_match_command_marker_source_upgrade_visibility + ON accounting_core.reconciliation_match_command + FOR SELECT + TO current_user + USING (true); + +UPDATE accounting_core.reconciliation_match AS match +SET command_evidence_recorded_at = command.recorded_at +FROM accounting_core.reconciliation_match_command AS command +WHERE command.tenant_account_id = match.tenant_account_id + AND command.reconciliation_run_id = match.reconciliation_run_id + AND command.reconciliation_match_id = match.reconciliation_match_id + AND match.command_evidence_recorded_at IS NULL; + +DROP POLICY reconciliation_match_command_marker_match_upgrade_visibility + ON accounting_core.reconciliation_match; +DROP POLICY reconciliation_match_command_marker_source_upgrade_visibility + ON accounting_core.reconciliation_match_command; + -- Keep aggregate variables unconstrained so an over-consumption comparison -- remains an exact rejection instead of overflowing before the guard runs. CREATE OR REPLACE FUNCTION accounting_core.reconciliation_match_approval_conservation_guard() @@ -321,6 +352,19 @@ BEGIN USING ERRCODE = '23514'; END IF; + UPDATE accounting_core.reconciliation_match AS match + SET command_evidence_recorded_at = NEW.recorded_at + WHERE match.tenant_account_id = NEW.tenant_account_id + AND match.reconciliation_run_id = NEW.reconciliation_run_id + AND match.reconciliation_match_id = NEW.reconciliation_match_id + AND match.command_evidence_recorded_at IS NULL; + + IF NOT FOUND THEN + RAISE EXCEPTION + 'reconciliation match command evidence is already recorded; create a new proposed match instead (reconciliation_match_command_duplicate)' + USING ERRCODE = '23514'; + END IF; + SELECT candidate.statement_amount, candidate.journal_amount INTO candidate_statement_amount, candidate_journal_amount FROM accounting_core.reconciliation_candidate AS candidate @@ -356,15 +400,17 @@ BEGIN END; $$; --- Recheck command evidence after acquiring the match lock. A later statement --- snapshot prevents an allocation from observing a stale pre-command view --- after it waited for a concurrent command transaction to finish. +-- Recheck the durable parent marker after acquiring the match lock. The +-- locked tuple is the synchronization point for a command and allocation. CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_allocation() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE + command_evidence_recorded_at timestamptz; BEGIN - PERFORM 1 + SELECT match.command_evidence_recorded_at + INTO command_evidence_recorded_at FROM accounting_core.reconciliation_match AS match WHERE match.tenant_account_id = NEW.tenant_account_id AND match.reconciliation_run_id = NEW.reconciliation_run_id @@ -377,13 +423,7 @@ BEGIN USING ERRCODE = '23514'; END IF; - IF EXISTS ( - SELECT 1 - FROM accounting_core.reconciliation_match_command AS command - WHERE command.tenant_account_id = NEW.tenant_account_id - AND command.reconciliation_run_id = NEW.reconciliation_run_id - AND command.reconciliation_match_id = NEW.reconciliation_match_id - ) THEN + IF command_evidence_recorded_at IS NOT NULL THEN RAISE EXCEPTION 'reconciliation match command evidence freezes its allocation population; create a new proposed match instead (reconciliation_match_command_allocation_frozen)' USING ERRCODE = '23514'; diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 193e8ffd..233724e7 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -51,8 +51,8 @@ Migrations `0013_reconciliation_run_exception_evidence.sql` through `0022_reconc - `journal_match_allocation`: append-only exact amount consumed from an immutable journal source reference by one reconciliation match. - `reconciliation_approval`: one immutable tenant/run/match-scoped human decision with command identity, immutable object-storage source-payload hash/reference, approver, purpose, decision, and effective/system times. PostgreSQL owns its version-1 snapshot hash over the candidate and allocation rows; a caller-supplied snapshot value is ignored. - `reconciliation_run_command`: one immutable tenant/run/statement command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and the evidence recorded when an `evaluating` run opens. New runs select only source facts recorded no later than their knowledge cutoff; deferred run validation and command-insert validation require exactly one command row and bind its statement to the run assignment's bank account, including for runs that predate the command row. Migration 0021 preflights existing command rows and fails closed on cross-bank provenance before reapplying those guards. Exact retries replay the run; changed command evidence under the same key fails closed. It does not select a chart account or authorize matching, approval, close, or posting. -- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, admits provenance only while the match is `proposed`, requires exactly one equal allocation on each side matching the candidate amounts at command insert, and freezes later allocation inserts by rechecking command evidence after that parent lock. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. -- Reconciliation candidate and allocation amounts use PostgreSQL `numeric(38, 6)`, matching journal and bank-statement evidence. Migration `0022_reconciliation_amount_precision.sql` upgrades historical `numeric(30, 6)` columns and keeps exact trigger aggregates from overflowing before conservation validation. +- `reconciliation_match_command`: one immutable tenant/run/candidate/match command identity with tenant-scoped idempotency key, canonical command hash, immutable source-payload hash/reference, and one exact 1:1 statement/journal allocation chain. Admission requires exact amounts from bank entries no later than the bank cutoff and knowledge cutoff and from posted journals no later than the book cutoff and knowledge cutoff; the journal amount comes from the run assignment's cash chart line, with CRDT mapped to cash debit and DBIT to cash credit. A database guard locks the parent match, admits provenance only while the match is `proposed`, requires exactly one equal allocation on each side matching the candidate amounts at command insert, records a durable `command_evidence_recorded_at` marker on the parent match, and freezes later allocation inserts by reading that locked parent marker. Exact retries replay the proposed match; changed evidence under the same key fails closed. It is review evidence only and does not approve, select a chart account, close, or post. +- Reconciliation candidate and allocation amounts use PostgreSQL `numeric(38, 6)`, matching journal and bank-statement evidence. Migration `0022_reconciliation_amount_precision.sql` upgrades historical `numeric(30, 6)` columns, keeps exact trigger aggregates from overflowing before conservation validation, and backfills the durable match-level command-evidence marker for existing commands. - `ReconciliationClosePackage` is a read-only schema-versioned manifest over those rows, not another database authority. It carries every reviewed match identity with an approved decision, exact tenant/run scope, PostgreSQL snapshot digest, durable approval-evidence reference, immutable run cutoff, and source-population references; incomplete, rejected, or unrelated approval evidence fails closed before export. Approved allocations are conserved by immutable source identity across active reconciliation runs in the same accounting/bank scope. Only matches whose current `match_status_code` is `approved` consume active capacity; `rejected` or `superseded` matches release capacity while their candidate and allocation rows remain durable historical evidence. Cross-run source-amount conflicts and over-consumption fail closed under database-owned guards and transaction-scoped advisory serialization. Approval and allocation transitions share a match-level advisory lock, and allocations plus candidate identity are frozen once approval evidence exists, so the durable decision cannot authorize a changed proposed state. Migration 0016 refuses to install over existing non-proposed matches that lack durable approval evidence; terminal approval timestamps remain immutable through explicit supersession. Reconciliation evidence therefore records and explains matching decisions but cannot itself post, reverse, close, or mutate authoritative journal facts. diff --git a/docs/OPERABILITY.md b/docs/OPERABILITY.md index 96598259..fa7c49ad 100644 --- a/docs/OPERABILITY.md +++ b/docs/OPERABILITY.md @@ -49,9 +49,9 @@ Migration `0019_reconciliation_run_command_evidence.sql` records the immutable c Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-compatible upgrade for installations that already applied `0019` before the command-insert provenance guard existed. Its migration-only preflight scans existing immutable command evidence and fails closed on cross-bank provenance; it then recreates both the deferred run guard and immediate command-insert guard without changing evidence rows. Apply it after `0020` and before runtime traffic. -Migration `0022_reconciliation_amount_precision.sql` widens reconciliation candidate and allocation amounts from the historical `numeric(30, 6)` domain to the platform-wide `numeric(38, 6)` domain. It also replaces the conservation and command-allocation trigger functions with unconstrained exact aggregate variables, so valid boundary-sized amounts are compared before rejection rather than overflowing during `SUM`. Command provenance is accepted only for a still-proposed match, and allocation-freeze checks acquire the parent match lock before taking their command-evidence snapshot. Apply it after `0021`; the migration changes no evidence values and runtime parsing rejects amounts outside the same domain before persistence. +Migration `0022_reconciliation_amount_precision.sql` widens reconciliation candidate and allocation amounts from the historical `numeric(30, 6)` domain to the platform-wide `numeric(38, 6)` domain. It also replaces the conservation and command-allocation trigger functions with unconstrained exact aggregate variables, so valid boundary-sized amounts are compared before rejection rather than overflowing during `SUM`. Command provenance is accepted only for a still-proposed match; the command records a durable `command_evidence_recorded_at` marker on the locked parent, and allocation-freeze checks read that marker after acquiring the same parent lock. Existing command rows backfill the marker. Apply it after `0021`; the migration changes no monetary evidence values and runtime parsing rejects amounts outside the same domain before persistence. -Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Migration `0022` preserves that boundary for upgrades by refusing command provenance on terminal matches and rechecking the command-freeze population after the parent lock. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. +Migration `0020_reconciliation_match_command_evidence.sql` records the immutable command identity for one exact 1:1 proposed match. `POST /reconciliation-matches` locks the evaluating run and writes the candidate, proposed match, statement/journal allocations, idempotency key, canonical command hash, and source-payload provenance in one transaction; exact retries replay and changed evidence conflicts. The route accepts only a bound bank-statement entry whose booking/value timestamps and `recorded_at` respect the bank and knowledge cutoffs, and a posted journal whose accounting date and `posted_at` respect the book and knowledge cutoffs; its amount must be on the run assignment's cash chart line, with CRDT requiring cash debit and DBIT requiring cash credit. Database guards lock the parent match while requiring exactly one equal allocation on each side matching candidate amounts at command insert and reject later allocations. Migration `0022` preserves that boundary for upgrades by refusing command provenance on terminal matches, backfilling a durable parent-row freeze marker, and reading that marker after the parent lock. Source-conservation violations fail as stable HTTP 422 validation errors rather than raw driver failures. Malformed identifiers are 400, absent source evidence is 404, state conflicts are 409, and source-content validation is 422. `GET /reconciliation-matches` reads the tenant-scoped evidence. This command does not select chart accounts, approve a match, close a period, or post a journal. Migration `0007_runtime_tenant_binding.sql` replaces caller-selected tenant authority with owner-controlled runtime-login binding. Migration `0008_fiscal_period_open_command.sql` adds forced-RLS, append-only command evidence so fiscal-period-open retries are bound to the original tenant key and source hash. Both must be installed before runtime database privileges are treated as production-ready. diff --git a/docs/adr/0059-reconciliation-match-command-evidence.md b/docs/adr/0059-reconciliation-match-command-evidence.md index 518d34e4..733cedfd 100644 --- a/docs/adr/0059-reconciliation-match-command-evidence.md +++ b/docs/adr/0059-reconciliation-match-command-evidence.md @@ -64,12 +64,12 @@ conservation validation failures as `422`. The database requires command insertion to observe exactly one statement and one journal allocation with equal exact amounts while the match is still -`proposed`, and rejects allocation rows inserted after command evidence. The -allocation guard locks the parent match before its command-evidence query, so -a transaction that waited on a concurrent command cannot use a stale snapshot -to extend the frozen allocation population. These are evidence-integrity -controls only; they do not turn a proposed match into an approved accounting -fact. +`proposed`, and records `command_evidence_recorded_at` on the locked parent +match. Allocation rows inserted after command evidence read that durable +parent marker after acquiring the same row lock, so a transaction that waited +on a concurrent command cannot use a stale command-table snapshot to extend +the frozen allocation population. These are evidence-integrity controls only; +they do not turn a proposed match into an approved accounting fact. Migration `0021_reconciliation_run_command_provenance_repair.sql` is a forward-only upgrade for installations that already applied migration 0019 @@ -84,8 +84,9 @@ PostgreSQL numeric variables in the conservation and command-allocation triggers, preventing aggregate overflow from masking an exact conservation rejection. Its forward-recreated command guard rejects provenance for terminal matches, and its allocation-freeze guard takes the parent match lock before -checking command evidence. The shared amount parser rejects values outside -that same storage domain before a command reaches the database. +checking the durable parent-row command-evidence marker. Existing command rows +backfill that marker. The shared amount parser rejects values outside that same +storage domain before a command reaches the database. Historical match admission also requires the posted journal fact to be known by the run's `knowledge_cutoff_at`; a backdated accounting date alone is not diff --git a/docs/doctoring/STANDARD_TRACEABILITY.md b/docs/doctoring/STANDARD_TRACEABILITY.md index a65ac6db..92420ef7 100644 --- a/docs/doctoring/STANDARD_TRACEABILITY.md +++ b/docs/doctoring/STANDARD_TRACEABILITY.md @@ -15,7 +15,7 @@ | PostgreSQL source-conservation controls | Migration 0015 serializes candidate amount admission on stable statement (`bank_account_record_id`) and journal source identities before conflict reads, and uses the same stable statement identity across effective-dated assignment rollover during approval capacity checks. Allocation evidence freezes the candidate identity before approval; `superseded` is terminal; and the migration-0016 legacy-row check temporarily grants only the current migration user visibility over forced-RLS match rows before dropping that policy in the same transaction. Migration 0017 keeps approval/allocation lock acquisition parent-row-first. | `0015_reconciliation_multi_match_conservation.sql`, `0016_reconciliation_approval_evidence.sql`, `0017_reconciliation_approval_lock_order.sql`, hardening RED/GREEN regressions, ADR 0054, ADR 0055 | | ISO 20022 balance evidence / PostgreSQL numeric controls | The camt.053 adapter retains every `Bal` as an immutable normalized fact with exact decimal amount, currency, CRDT/DBIT direction, typed effective date/time distinct from statement period and system `recorded_at`, source locator, and source hash. Migration 0018 stores those values in a forced-RLS relational table; opening and closing balance hashes remain compatibility fields, and a reconciliation bridge must fail closed when numeric balance evidence is absent rather than infer it. | `0018_bank_statement_balance_evidence.sql`, bank-statement parser and persistence regressions, ADR 0057, ISO 20022 references above, PostgreSQL constraints and row-level-security references | | Reconciliation run command evidence / PostgreSQL idempotency controls | Migration 0019 records one immutable tenant/run/statement command identity, canonical command hash, raw bank-statement artifact payload hash/reference distinct from the normalized statement hash, and forced-RLS evidence for opening an `evaluating` run. The public API requires the active statement assignment and exact raw artifact source hash, excludes selected source facts recorded after the knowledge cutoff, requires explicit zero-offset UTC cutoffs before persistence or command hashing, resolves existing command evidence before live assignment validation so exact retries survive assignment rollover or closure, permits separately auditable runs under distinct keys, rejects changed evidence, and deferred run plus command-insert database guards reject orphan runs and cross-bank statement provenance, including legacy runs receiving a command later. Migration 0021 preflights existing command rows, fails closed on cross-bank provenance, and then reapplies both guards for installations that already applied 0019 before the command-insert guard was added. The boundary grants no matching, approval, close, posting, or chart-account authority. | `0019_reconciliation_run_command_evidence.sql`, `0021_reconciliation_run_command_provenance_repair.sql`, `accept_reconciliation_run`, `lookup_reconciliation_run`, HTTP and direct-SQL reconciliation-run regressions, ADR 0058 | -| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, admit command provenance only for `proposed`, require exactly one equal allocation on each side matching candidate amounts at command insert, and recheck command evidence after that lock before rejecting later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `0022_reconciliation_amount_precision.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | +| Reconciliation match command evidence / exact 1:1 persistence | Migration 0020 records one immutable tenant/run/candidate/match command identity, canonical command hash, source-payload hash/reference, and forced-RLS provenance over the candidate, proposed match, and exact statement/journal allocations. Its candidate-inclusive composite foreign key proves that command candidate and match identifiers are one persisted chain. `accept_reconciliation_match` locks the evaluating run, requires exact equal Decimal amounts, excludes bank and book facts beyond their run, bank and book cutoffs, and knowledge cutoff, and binds CRDT/DBIT to the debit/credit side of the assigned cash chart line rather than a journal-wide total; compound journals therefore retain their exact cash evidence. Database guards lock the parent match, admit command provenance only for `proposed`, require exactly one equal allocation on each side matching candidate amounts at command insert, record a durable parent-row command-evidence marker, and read that marker after the same parent lock before rejecting later allocations. Exact retries replay while changed evidence conflicts, and database source-conservation violations become stable validation failures. `POST`/`GET /reconciliation-matches` expose reviewable proposed evidence only; the command cannot select chart accounts, approve, close, reverse, or post a journal. | `0020_reconciliation_match_command_evidence.sql`, `0022_reconciliation_amount_precision.sql`, `accept_reconciliation_match`, `lookup_reconciliation_match`, HTTP and direct-SQL reconciliation-match regressions, ADR 0059 | | Reconciliation amount precision / PostgreSQL numeric domain | Candidate and statement/journal allocation amounts share the platform `numeric(38, 6)` domain used by authoritative journal and bank-statement facts. Migration 0022 widens historical reconciliation columns and uses unconstrained exact aggregate variables in conservation guards; the shared `_parse_amount` boundary rejects values wider than that storage domain before persistence. This preserves exact rejection semantics without rounding or aggregate overflow. | `0022_reconciliation_amount_precision.sql`, `_parse_amount`, precision boundary regressions, ADR 0059 | | RFC 9112 | The standalone HTTP/1.1 command boundary deliberately does not implement transfer coding: any request carrying `Transfer-Encoding` fails closed with HTTP 400 and connection close rather than being combined with a `Content-Length` interpretation. A valid `Content-Length` is an exact octet contract; premature EOF/short reads are incomplete messages, fail with HTTP 400, and close the connection before JSON/domain processing. This prevents ambiguous message boundaries from becoming request-smuggling or valid-prefix acceptance paths | `JournalProposalHandler._read_body`, HTTP request-boundary RED/GREEN regressions, RFC 9112 §§6.2–6.3 and §8 | | RFC 9562 | New persistence identifiers use UUIDv7 | Initial migration | diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index bb86dbbb..2de304f1 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -92,7 +92,7 @@ def test_precision_migration_rebuilds_column_dependent_candidate_trigger(self) - ) def test_precision_migration_rechecks_command_freeze_after_parent_lock(self) -> None: - """Allocation freeze evidence must use a snapshot taken after the match lock.""" + """Allocation freeze evidence must use the locked parent marker.""" migration = ( ROOT / "database/migrations/0022_reconciliation_amount_precision.sql" @@ -107,7 +107,8 @@ def test_precision_migration_rechecks_command_freeze_after_parent_lock(self) -> )[0] self.assertRegex( function, - r"(?s)FROM accounting_core\.reconciliation_match.*?FOR UPDATE.*?IF EXISTS", + r"(?s)FROM accounting_core\.reconciliation_match.*?FOR UPDATE.*?" + r"command_evidence_recorded_at IS NOT NULL", ) def test_precision_migration_rejects_non_proposed_command_matches(self) -> None: @@ -127,6 +128,26 @@ def test_precision_migration_rejects_non_proposed_command_matches(self) -> None: self.assertIn("match_status_code", function) self.assertIn("reconciliation_match_command_status", function) + def test_precision_migration_persists_command_freeze_on_parent_match(self) -> None: + """Allocation guards must read a durable freeze marker from the locked parent.""" + migration = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ).read_text(encoding="utf-8") + self.assertIn( + "ADD COLUMN command_evidence_recorded_at timestamptz", + migration, + ) + self.assertRegex( + migration, + r"(?s)UPDATE accounting_core\.reconciliation_match.*?SET command_evidence_recorded_at", + ) + function = migration.split( + "CREATE OR REPLACE FUNCTION accounting_core.reject_reconciliation_match_command_allocation()", + 1, + )[1].split("CREATE TRIGGER reconciliation_candidate_capacity_guard", 1)[0] + self.assertIn("command_evidence_recorded_at IS NOT NULL", function) + if __name__ == "__main__": unittest.main() From 37c7aeff474c6f7f13d1ee5821e3c0b2672f1ed1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:45:36 +0900 Subject: [PATCH 23/25] docs(reconciliation): describe durable freeze marker --- docs/ARCHITECTURE.md | 2 +- docs/ERD.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 64ec83c7..c1f1f3f3 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -145,7 +145,7 @@ Shared fiscal-calendar dates do not collapse independent accounting books into o 19. `database/migrations/0019_reconciliation_run_command_evidence.sql` — records immutable tenant-scoped run-command idempotency, source hash/reference, and the statement bound to an evaluating reconciliation scope. 20. `database/migrations/0020_reconciliation_match_command_evidence.sql` — records immutable tenant-scoped exact 1:1 proposed-match command identity, source hash/reference, and the candidate/match allocation chain. 21. `database/migrations/0021_reconciliation_run_command_provenance_repair.sql` — preflights existing immutable command evidence and fails closed on cross-bank provenance before reapplying run-command guards to installations that already executed migration 0019 before its command-insert guard was added. -22. `database/migrations/0022_reconciliation_amount_precision.sql` — widens reconciliation candidate/allocation monetary facts to the platform `numeric(38, 6)` domain, keeps exact conservation aggregates from overflowing before validation, admits command provenance only for proposed matches, and rechecks allocation-freeze evidence after locking the parent match. +22. `database/migrations/0022_reconciliation_amount_precision.sql` — widens reconciliation candidate/allocation monetary facts to the platform `numeric(38, 6)` domain, keeps exact conservation aggregates from overflowing before validation, admits command provenance only for proposed matches, records/backfills a durable parent-match command-evidence marker, and rechecks allocation-freeze evidence from that locked parent row. ## Durable soft-close command evidence diff --git a/docs/ERD.md b/docs/ERD.md index 0073510f..a2e1908e 100644 --- a/docs/ERD.md +++ b/docs/ERD.md @@ -80,7 +80,7 @@ erDiagram `bank_account_record` and `bank_account_assignment` map an opaque bank account onto one legal entity, book, and same-book cash chart account. The assignment composite foreign key requires that book to belong to the same legal entity. `bank_statement_record`, `bank_statement_balance`, and `bank_statement_entry` are append-only evidence. Balance rows retain exact amount, currency, CRDT/DBIT direction, typed `balance_effective_at`, source locator, and source hash; statement and entry rows retain `source_artifact_hash`, `normalized_payload_hash`, `ingestion_idempotency_key`, and `source_entry_hash` so a controller can prove which original artifact produced each fact without storing the raw XML in PostgreSQL. None of these evidence rows can post, reverse, approve, close, or mutate a journal. -`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_match_command` records the corresponding immutable proposed-match command identity and provenance over one exact 1:1 candidate/match/allocation chain; its candidate-inclusive composite foreign key makes the candidate and match one database-proven chain. It is also evidence only. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. +`reconciliation_run` binds one evaluated reconciliation to tenant, legal entity, accounting book, bank-account assignment, currency, bank/book cutoffs, matching-policy version, and knowledge cutoff. Its evaluated scope is immutable. `reconciliation_run_command` records the command hash, tenant-scoped idempotency key, source hash/reference, and exact statement that opened the scope; it is immutable evidence, not a matching or posting authority. `reconciliation_match_command` records the corresponding immutable proposed-match command identity and provenance over one exact 1:1 candidate/match/allocation chain; its candidate-inclusive composite foreign key makes the candidate and match one database-proven chain. Its command evidence also records a durable parent-match freeze marker so concurrent allocation checks use the locked tuple rather than a stale command-table snapshot. It is also evidence only. `reconciliation_exception` and `reconciliation_evidence` retain explicit exception ownership, next action, effective/system time, evidence references, and optional hashes rather than hiding unresolved items in derived status text. `reconciliation_candidate` records a deterministic statement/journal candidate and its exact source amounts; after INSERT it is append-only. `reconciliation_match` records the reviewable disposition. `statement_match_allocation` and `journal_match_allocation` preserve exact many-to-many consumption and are append-only regardless of later match status. Database-owned conservation guards serialize by immutable source identity and reject cross-run source-amount conflicts or over-consumption. Only an `approved` match consumes active source capacity; changing that match to `rejected` or `superseded` releases active capacity without deleting or rewriting the historical candidate/allocation evidence. These reconciliation relations provide audit and operator-control evidence only: they do not post, reverse, close, approve accounting policy, or mutate authoritative journals. From 03dc547155102a02924becb43089c9837c96d2ae Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 09:55:57 +0900 Subject: [PATCH 24/25] fix(reconciliation): protect command freeze marker --- .../0022_reconciliation_amount_precision.sql | 42 +++++++++++++++++++ ...ation_data_model_documentation_contract.py | 18 ++++++++ tests/test_reconciliation_match_api.py | 30 +++++++++++++ 3 files changed, 90 insertions(+) diff --git a/database/migrations/0022_reconciliation_amount_precision.sql b/database/migrations/0022_reconciliation_amount_precision.sql index 49e2f1e4..f5862418 100644 --- a/database/migrations/0022_reconciliation_amount_precision.sql +++ b/database/migrations/0022_reconciliation_amount_precision.sql @@ -49,6 +49,48 @@ DROP POLICY reconciliation_match_command_marker_match_upgrade_visibility DROP POLICY reconciliation_match_command_marker_source_upgrade_visibility ON accounting_core.reconciliation_match_command; +CREATE OR REPLACE FUNCTION accounting_core.reconciliation_match_command_marker_immutability() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + command_recorded_at timestamptz; +BEGIN + IF NEW.command_evidence_recorded_at IS NOT DISTINCT FROM OLD.command_evidence_recorded_at THEN + RETURN NEW; + END IF; + + IF OLD.command_evidence_recorded_at IS NOT NULL THEN + RAISE EXCEPTION + 'reconciliation match command evidence marker is immutable (reconciliation_match_command_marker_immutable)' + USING ERRCODE = '23514'; + END IF; + + IF NEW.command_evidence_recorded_at IS NULL THEN + RETURN NEW; + END IF; + + SELECT command.recorded_at + INTO command_recorded_at + FROM accounting_core.reconciliation_match_command AS command + WHERE command.tenant_account_id = NEW.tenant_account_id + AND command.reconciliation_run_id = NEW.reconciliation_run_id + AND command.reconciliation_match_id = NEW.reconciliation_match_id; + + IF NOT FOUND OR command_recorded_at IS DISTINCT FROM NEW.command_evidence_recorded_at THEN + RAISE EXCEPTION + 'reconciliation match command evidence marker must be written by command evidence (reconciliation_match_command_marker_immutable)' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER reconciliation_match_command_marker_immutability_guard +BEFORE UPDATE OF command_evidence_recorded_at +ON accounting_core.reconciliation_match +FOR EACH ROW EXECUTE FUNCTION accounting_core.reconciliation_match_command_marker_immutability(); + -- Keep aggregate variables unconstrained so an over-consumption comparison -- remains an exact rejection instead of overflowing before the guard runs. CREATE OR REPLACE FUNCTION accounting_core.reconciliation_match_approval_conservation_guard() diff --git a/tests/test_reconciliation_data_model_documentation_contract.py b/tests/test_reconciliation_data_model_documentation_contract.py index 2de304f1..28f93406 100644 --- a/tests/test_reconciliation_data_model_documentation_contract.py +++ b/tests/test_reconciliation_data_model_documentation_contract.py @@ -148,6 +148,24 @@ def test_precision_migration_persists_command_freeze_on_parent_match(self) -> No )[1].split("CREATE TRIGGER reconciliation_candidate_capacity_guard", 1)[0] self.assertIn("command_evidence_recorded_at IS NOT NULL", function) + def test_precision_migration_makes_command_freeze_marker_immutable(self) -> None: + """Direct callers cannot clear or rewrite a command freeze marker.""" + migration = ( + ROOT + / "database/migrations/0022_reconciliation_amount_precision.sql" + ).read_text(encoding="utf-8") + self.assertIn( + "CREATE OR REPLACE FUNCTION " + "accounting_core.reconciliation_match_command_marker_immutability()", + migration, + ) + self.assertIn( + "BEFORE UPDATE OF command_evidence_recorded_at", + migration, + ) + self.assertIn("OLD.command_evidence_recorded_at IS NOT NULL", migration) + self.assertIn("reconciliation_match_command_marker_immutable", migration) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index 77503d85..ad2e15da 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -439,6 +439,36 @@ def test_match_command_rejects_late_allocation_after_command_evidence(self) -> N ), ) + def test_match_command_freeze_marker_cannot_be_cleared_or_rewritten(self) -> None: + """The durable parent freeze survives direct marker tampering attempts.""" + command = self._command() + document = accept_reconciliation_match( + command, posting.DATABASE_URL, self.case.policy.tenant_reference + ) + match_id = document["reconciliation_match_id"] + update = """ + UPDATE accounting_core.reconciliation_match + SET command_evidence_recorded_at = %s + WHERE tenant_account_id = %s + AND reconciliation_run_id = %s + AND reconciliation_match_id = %s + """ + for replacement in (None, datetime.now(timezone.utc)): + with psycopg.connect(posting.DATABASE_URL, autocommit=True) as connection: + with self.assertRaisesRegex( + psycopg.errors.CheckViolation, + "command.*marker.*immutable", + ): + connection.execute( + update, + ( + replacement, + self.case.tenant_id, + command["reconciliation_run_id"], + match_id, + ), + ) + def test_match_command_serializes_concurrent_allocation_inserts(self) -> None: """A concurrent allocation cannot commit around the command freeze.""" command = self._command() From e813c8f251ac2f3b0728b7932cd9e95be0f4a2ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 30 Aug 2026 10:32:48 +0900 Subject: [PATCH 25/25] fix(http): advertise reconciliation match lookup --- src/accounting_information_platform/http_api.py | 4 +++- tests/test_reconciliation_match_api.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/accounting_information_platform/http_api.py b/src/accounting_information_platform/http_api.py index 1f90ee9b..6595dc46 100644 --- a/src/accounting_information_platform/http_api.py +++ b/src/accounting_information_platform/http_api.py @@ -252,7 +252,9 @@ def do_GET(self) -> None: "GET /bank-statement-entries, GET /receivable-agings, " "GET /payable-agings, GET /period-close-packages, GET /journals, " "GET /journal-reversals, GET /period-closes, GET /fiscal-periods, " - "GET /reconciliation-runs?reconciliation_run_id=, GET /outbox-events?event_type_code=, " + "GET /reconciliation-runs?reconciliation_run_id=, " + "GET /reconciliation-matches?reconciliation_match_id=, " + "GET /outbox-events?event_type_code=, " "or GET /audit-events, then retry.", ) diff --git a/tests/test_reconciliation_match_api.py b/tests/test_reconciliation_match_api.py index ad2e15da..b397bd80 100644 --- a/tests/test_reconciliation_match_api.py +++ b/tests/test_reconciliation_match_api.py @@ -1017,6 +1017,9 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: f"/reconciliation-matches?reconciliation_match_id={uuid.uuid4()}", None, ) + unknown_status, unknown = self.case._http_json( + "GET", "/unsupported-reconciliation-resource", None + ) self.assertEqual(status, 200) self.assertEqual(read_status, 200) self.assertEqual(read["reconciliation_match_id"], created["reconciliation_match_id"]) @@ -1031,6 +1034,11 @@ def test_http_routes_persist_and_read_the_proposed_match(self) -> None: self.assertEqual(missing_get_header_status, 400) self.assertEqual(invalid_id_status, 400) self.assertEqual(missing_status, 404) + self.assertEqual(unknown_status, 404) + self.assertIn( + "GET /reconciliation-matches?reconciliation_match_id=", + unknown["error_message"], + ) if __name__ == "__main__":