diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e71e0b9d..8b5a6866 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,7 @@ jobs: - name: Run tests (Windows) if: runner.os == 'Windows' - run: .\\mvnw.cmd -B test + run: .\mvnw.cmd -B test - name: Report uncovered JaCoCo branches if: failure() @@ -60,7 +60,9 @@ jobs: coverage_targets = { "com/xtrmetl/etl/job/EtlJobService": "EtlJobService.java", + "com/xtrmetl/etl/job/EtlJobReplayService": "EtlJobReplayService.java", "com/xtrmetl/etl/controller/EtlJobController": "EtlJobController.java", + "com/xtrmetl/etl/controller/EtlJobReplayController": "EtlJobReplayController.java", "com/xtrmetl/etl/service/Sha256Digest": "Sha256Digest.java", } reports = sorted(Path(".").glob("**/target/site/jacoco/jacoco.xml")) @@ -135,4 +137,4 @@ jobs: - name: Run tests (Windows) if: runner.os == 'Windows' - run: .\\mvnw.cmd -B test + run: .\mvnw.cmd -B test diff --git a/.github/workflows/postgresql-migration-integration.yml b/.github/workflows/postgresql-migration-integration.yml new file mode 100644 index 00000000..867b5ca1 --- /dev/null +++ b/.github/workflows/postgresql-migration-integration.yml @@ -0,0 +1,56 @@ +name: PostgreSQL Migration Integration + +on: + pull_request: + branches: + - develop + paths: + - "etl-service/src/main/resources/db/migration/**" + - "etl-service/src/test/postgresql/**" + - "scripts/verify-postgresql-migrations.sh" + - ".github/workflows/postgresql-migration-integration.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: postgresql-migration-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + replay_lineage_migration: + name: replay-lineage-migration + if: github.event_name != 'workflow_dispatch' || github.ref_name == github.event.repository.default_branch + runs-on: ubuntu-latest + timeout-minutes: 15 + services: + postgresql_database: + image: postgres:18-alpine + env: + POSTGRES_DB: mightyetl_replay_test + POSTGRES_USER: mightyetl_test + POSTGRES_PASSWORD: mightyetl_test_password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready --username=mightyetl_test --dbname=mightyetl_replay_test" + --health-interval 5s + --health-timeout 5s + --health-retries 12 + env: + PGHOST: 127.0.0.1 + PGPORT: "5432" + PGDATABASE: mightyetl_replay_test + PGUSER: mightyetl_test + PGPASSWORD: mightyetl_test_password + steps: + - name: Check out exact pull-request head or protected default branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 + with: + repository: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event.repository.default_branch }} + persist-credentials: false + - name: Verify versioned migrations and immutable replay lineage + shell: bash + run: bash scripts/verify-postgresql-migrations.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 44da5d3b..2f0ac24c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Authenticated operators can now create an ordinary pending durable job from an immutable failed or cancelled source only after resupplying a byte-identical bounded JSON payload; the terminal source remains unchanged and succeeded sources remain non-replayable. +- Replay lineage now remains database-authoritative across every writer: PostgreSQL validates the terminal immediate source, first root, exact source/root/generation continuity, generation-one identity, and one-step generation succession, while immutable lineage fields prevent post-insert reparenting. +- Once a terminal job is referenced as a replay source or root, PostgreSQL row-lock serialization now freezes its status, request evidence, attempt/failure state, cancellation evidence, and lifecycle timestamps so descendants cannot silently acquire different historical meaning. +- PostgreSQL migration verification now reconstructs and validates the exact replay-index column, predicate, and one-column nonunique shape in addition to readiness and validity, so a same-named but incorrectly defined index fails closed. - Authenticated operators can now perform owner-scoped durable-job cancellation for `PENDING` and `RUNNING` work through an idempotent action that commits terminal `CANCELLED`, clears payload and lease state, stores only `cancellation_key_hash` plus a fixed code and timestamp, and returns stable RFC 9457 conflicts when success or failure already won. - Cancellation-first races now make the former exact lease stale and roll back transactional target and response-ledger effects; success-first races remain `SUCCEEDED`, while same-key cancellation replays and different-key reuse fails closed. - Owner-scoped durable-job status responses now emit deterministic weak SHA-256 `ETag` validators; ordinary and wildcard `If-None-Match` requests return an empty RFC 9110 `304 Not Modified` response only after authenticated owner-safe lookup, while `Cache-Control: no-store` remains unchanged. @@ -37,9 +41,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Transactional migration `V7__add_etl_job_replay_lineage.sql`, owner-scoped replay admission, the named `etl_job_owner_identity_unique` support key, composite owner-scoped foreign keys for immediate-source and root lineage, `validate_etl_job_replay_lineage()` plus `etl_job_replay_lineage_guard_trigger`, and immutable `replay_source_job_record_id`, `replay_root_job_record_id`, and `replay_generation_count` evidence with concurrency, tenant-integrity, exact-continuity, worker-compatibility, rollout, incident, and rollback tests and documentation. +- PostgreSQL 18 rehearsal now executes valid first- and second-generation replay chains and rejects nonterminal sources, different generation-one roots, derived roots, skipped generations, cross-owner references, lineage mutation, referenced root/source evidence mutation, and protected source/root deletion before transactionally rehearsing trigger, function, constraint, and column rollback. - Transactional migration `V6__add_etl_job_cancellation.sql`, owner-safe cancellation API and replay model, exact lease-invalidation and cancellation-versus-success integration tests, plus rollout, incident, connector-limitation, and rollback evidence in `docs/operations/durable-job-cancellation.md`. - Deterministic ordinary, wildcard, changed-state, changed-failure-code, null-versus-empty, and unrelated-response conditional polling tests, complete controller Javadoc, privacy and rollback guidance, and APA 7th standards evidence in `docs/etl/durable-job-polling.md`. -- Controller-scoped polling advice, deterministic active/terminal lifecycle tests, disabled-worker fail-closed behavior, sub-second rounding coverage, rollback guidance, and APA 7th standards evidence in `docs/etl/durable-job-polling.md`. +- Controller-scoped polling advice, deterministic active/terminal lifecycle tests, disabled-worker fail-closed behavior, sub-second rounding coverage, rollback guidance, and APA 7th RFC 9110 evidence in `docs/etl/durable-job-polling.md`. - Owner-scoped durable job list models and HTTP contract, strict cursor and page-limit validation, one-extra-row next-page detection, the descriptive `etl_job_owner_pagination_index`, deterministic tenant-isolation and equal-timestamp tests, migration rollback guidance, and APA 7th standards evidence in `docs/etl/durable-job-intake.md`. - A production rollout and invalid-index recovery runbook for the nonblocking durable-job claim index: `docs/operations/durable-job-claim-index-rollout.md`. - PostgreSQL `FOR UPDATE SKIP LOCKED` durable-job claiming, per-process and per-claim lease fencing, expiry reclaim, bounded attempts, exact-live-lease transitions, terminal payload clearing, stable failure codes, and finite-cardinality worker metrics. @@ -77,6 +83,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Replay uses owner-scoped source selection, byte-exact digest verification, a versioned principal-scoped key domain, immutable relational lineage, composite owner-scoped foreign keys, and a database trigger that independently rejects cross-tenant references, nonterminal sources, false roots, generation discontinuities, lineage mutation, and mutation of referenced terminal evidence without retaining raw principals or replay keys; this evidence does not prove external connector safety, so connector-native idempotency, transaction participation, or compensation remains required. - Cancellation stores only a principal-scoped SHA-256 replay identity and a fixed machine code, exposes no raw principal, key, hash, payload, lease, SQL, exception, or target detail, and requires an owner-matched conditional database update before reporting success. - Conditional status validators are SHA-256 digests of only the complete owner-authorized operator-safe representation; payloads, raw principals, idempotency keys, internal hashes, leases, SQL, and exception text remain excluded, and wildcard evaluation occurs only after owner-safe lookup. - Polling advice exposes only a bounded delay integer and is omitted when local execution is disabled or terminal; it never contains job, lease, principal, key, hash, payload, SQL, exception, target, or queue-depth data. @@ -164,7 +171,7 @@ Through code analysis, identified the platform as: - Microservices-based architecture using Spring Cloud - Real-time Change Data Capture using Debezium - Event streaming via Apache Kafka -- Service discovery with Netflix Eureka +- Service discovery and registration - Distributed tracing with Zipkin #### Key Components Documented @@ -283,5 +290,5 @@ This changelog will be updated: --- **Changelog Version**: 1.0 -**Last Updated**: 2026-08-06 +**Last Updated**: 2026-08-07 **Maintained By**: Development Team diff --git a/docs/adr/2026-08-07-immutable-durable-job-replay.md b/docs/adr/2026-08-07-immutable-durable-job-replay.md new file mode 100644 index 00000000..225c2ab6 --- /dev/null +++ b/docs/adr/2026-08-07-immutable-durable-job-replay.md @@ -0,0 +1,203 @@ +# ADR: Immutable durable-job replay lineage + +- **Status:** Proposed while the replay pull request is stacked; Accepted only after direct-`develop` gates and merge +- **Date:** 2026-08-07 +- **Decision owners:** mightyETL maintainers +- **Scope:** `etl-service` durable-job admission, persistence, operator API, provenance, and PostgreSQL rollout + +## Context + +Durable jobs deliberately clear `request_payload` after success, failure, or cancellation. Operators nevertheless need a controlled way to retry failed or cancelled work without weakening source immutability, owner isolation, idempotency, lease fencing, or auditability. + +Rewinding a terminal row to `PENDING` would erase the original terminal fact, mix attempt histories, invalidate conditional status validators, and make concurrent cancellation or success reasoning substantially harder. Retaining terminal payloads solely for replay would expand sensitive-data retention. Treating semantically equivalent JSON as the same work would also allow hidden payload changes under a replay label. + +A structural schema with nullable source, root, and generation fields is not sufficient on its own. Composite foreign keys can prove same-owner existence, and check constraints can prove field completeness and bounds, but they cannot prove that a source is terminal, the replay row preserves the exact source request digest, a root is the first row in the chain, a generation increments exactly once, or an existing replay has never been reparented. Those properties must remain true even for maintenance scripts, data imports, and future writers that do not execute the Java service. + +Cross-row validity also creates a temporal requirement. Once a descendant commits a reference to terminal source or root evidence, a later direct writer must not change the referenced status, digest, payload state, attempt/failure state, cancellation evidence, or lifecycle timestamps. Otherwise the descendant's historical meaning can change after admission even though its lineage identifiers remain untouched. + +The trigger must preserve that temporal boundary without introducing a deadlock. Child insertion needs source/root row locks to bind exact evidence. A parent update already owns the parent row lock before a `BEFORE UPDATE` trigger runs, so locking child rows and then returning to ancestors would invert lock order. The parent mutation path therefore needs an indexed descendant existence lookup without child row locks. + +The source/root foreign keys and descendant lookup also need bounded access paths as history grows. Building those indexes inside the transactional lineage migration would block production writers, while grouping multiple `CREATE INDEX CONCURRENTLY` statements into one nontransactional Flyway migration would make partial failure recovery ambiguous. + +## Decision + +Replay creates a **new** durable job. The terminal source remains unchanged. + +The authenticated owner submits the source identifier, a replay-specific `Idempotency-Key`, and the complete bounded JSON text. Admission parses the payload through the ordinary durable-intake boundary and requires its SHA-256 digest to equal the immutable source `request_digest`. Only `FAILED` and `CANCELLED` sources are eligible. `SUCCEEDED` remains excluded because payload equality is not evidence that committed target effects may be repeated safely. + +A derived row stores: + +- `replay_source_job_record_id`: immediate source; +- `replay_root_job_record_id`: immutable first root of the replay chain; +- `replay_generation_count`: bounded positive generation. + +Source and root references are composite owner-scoped foreign keys to `(job_record_id, principal_scope_hash)` and use `ON DELETE RESTRICT`. A root row has all three lineage fields null; a replay row has all three non-null. Generation cannot exceed the supported bound. + +V7 creates the PL/pgSQL function `validate_etl_job_replay_lineage()` and the row-level `etl_job_replay_lineage_guard_trigger`. The trigger is the database authority for exact request and lineage continuity: + +- replay rows are inserted only as `PENDING`, attempt zero, with a retained payload; +- immediate source and root belong to the same principal namespace as the new row; +- the immediate source is `FAILED` or `CANCELLED`; +- the replay row `request_digest` equals the immediate source `request_digest`; +- the root is terminal and has no lineage fields; +- generation one uses the same source and root; +- later generations retain that root and equal the immediate source generation plus one; +- lineage columns are immutable after insertion; and +- after any descendant references a row as immediate source or root, the terminal status, request evidence, attempt/failure state, cancellation evidence, and lifecycle timestamps are immutable. + +The trigger uses one directional lock protocol: + +- an INSERT locks source and root rows with `FOR UPDATE`, validates exact evidence, and then commits or rejects; +- an UPDATE already owns its parent row lock, performs an indexed descendant existence read without locking child rows, rejects protected evidence changes when a descendant exists, and returns before the INSERT-only source/root locking path. + +If a parent update commits first, a later child validates and binds the resulting evidence. If child insertion locks and references the parent first, a later conflicting parent update waits, observes the committed descendant, and fails closed. Ordinary lifecycle updates remain available until the row becomes referenced historical evidence. + +V8 and V9 provide the online lookup boundary: + +- `V8__add_etl_job_replay_source_lookup_index.sql` creates the partial `etl_job_replay_source_lookup_index` concurrently; +- `V9__add_etl_job_replay_root_lookup_index.sql` creates the partial `etl_job_replay_root_lookup_index` concurrently; +- each migration owns exactly one `CREATE INDEX CONCURRENTLY` statement; +- each companion `.sql.conf` sets `executeInTransaction=false`; +- migration verification requires both indexes to be ready and valid. + +A failed concurrent build is repaired by inspecting catalog state, removing only the invalid artifact with `DROP INDEX CONCURRENTLY`, repairing the exact Flyway migration record through the approved deployment process, and rerunning without editing an applied migration. + +The Java service independently verifies source digest equality and the inherited root before insertion. This is defense in depth; it does not replace the trigger. The new row enters the ordinary `PENDING` lifecycle and uses the existing worker, PostgreSQL claim, exact lease, success, failure, polling, cancellation, and ETag contracts. + +```mermaid +sequenceDiagram + participant O as Authenticated owner + participant A as Replay API + participant P as PostgreSQL transaction + participant T as Lineage trigger + participant W as Ordinary worker + + O->>A: source id + exact payload + replay key + A->>A: bounded JSON validation and SHA-256 + A->>P: owner-matched terminal source lock + P->>P: verify FAILED/CANCELLED and digest equality + P->>P: verify replay-key identity and inherited root + P->>T: insert new PENDING row with source/root/generation + T->>T: FOR UPDATE source/root and verify exact evidence + T-->>P: accept or reject before persistence + P-->>A: commit derived job + A-->>O: 202 + Location + Idempotency-Replayed + W->>P: ordinary lease-fenced claim + P->>T: later mutation of referenced terminal evidence + T->>T: indexed child-existence read without child lock + T-->>P: reject protected evidence mutation +``` + +## Replay identity + +The stored replay identity uses a versioned replay-specific domain and the principal namespace. It is intentionally distinct from ordinary submission and cancellation domains. Raw principals and raw replay keys are not retained. The domain string is persistence compatibility behavior and cannot change without a migration or dual-read period. + +NIST SP 800-185 motivates explicit domain separation, but the SHA-256 construction does not claim cSHAKE, KMAC, TupleHash, or ParallelHash conformance. + +## HTTP behavior + +- first committed replay: `202 Accepted`, `Location`, `Cache-Control: no-store`, `Idempotency-Replayed: false`; +- committed same-intent retry: same derived job, current lifecycle state, `Idempotency-Replayed: true`; +- absent or foreign source: indistinguishable `404 etl_job_not_found`; +- active source: stable `409` problem; +- succeeded source: stable `409` problem; +- byte-different payload: stable `422` problem; +- conflicting replay-key reuse: stable `422` problem; +- unresolved concurrent admission: stable retryable `409` problem; +- generation exhaustion: stable `409` problem. + +All covered request failures use fixed RFC 9457 metadata and exclude exception messages, SQL, hashes, identifiers, payloads, and target details. A database-trigger rejection is an internal integrity incident; raw PL/pgSQL text must not enter the client response or ordinary telemetry. + +## Connector boundary + +The replay transaction can prove source ownership, payload fidelity, replay identity, exact lineage, referenced evidence immutability, and durable admission. It cannot prove that a remote warehouse, file system, API, or broker will suppress duplicate effects. Replay is enabled for a connector only when target effects participate in the mightyETL transaction or the connector provides independently tested idempotency or compensation. + +## Alternatives rejected + +### Rewind the terminal row + +Rejected because it destroys terminal history, combines multiple execution episodes into one identity, complicates ETag semantics, and weakens race reasoning. + +### Retain terminal payloads indefinitely + +Rejected because replay does not justify expanding sensitive payload retention. The operator must recover the exact payload from an approved upstream or encrypted audit source. + +### Accept semantic JSON equivalence + +Rejected because normalization can obscure a changed request and creates a second canonicalization contract. Replay fidelity is byte-exact, matching durable submission identity. + +### Permit succeeded-source replay + +Rejected in the initial slice because a succeeded job may already have committed irreversible external effects. + +### Rely only on application validation + +Rejected because maintenance scripts, migrations, import processes, or future services can write directly to the table. Relational lineage must remain valid independently of one application binary. + +### Use only foreign keys and check constraints + +Rejected because those constraints cannot express exact digest equality, generation succession, immutable cross-row root identity, or the transition from mutable terminal state to referenced immutable evidence. A row-level trigger is required for those cross-row and temporal invariants. + +### Lock descendants during parent mutation + +Rejected because child insertion already locks ancestors. A parent trigger that locks child rows and then reaches ancestors creates child-to-ancestor lock inversion and can deadlock replay admission against lifecycle maintenance. + +### Build both indexes in one nontransactional migration + +Rejected because a second-index failure could leave a valid first index and a failed Flyway version with no one-artifact repair boundary. Separate V8/V9 migrations preserve auditable failure and rollback semantics. + +### Leave referenced evidence mutable + +Rejected because a descendant would preserve the same source/root identifiers while the status, digest, terminal payload state, failure or cancellation evidence, or timestamps behind those identifiers changed. Audit and provenance exports would then describe a moving historical fact. + +## Consequences + +### Positive + +- terminal sources remain immutable once replayed; +- each execution episode has a distinct opaque job identity; +- lineage supports incident analysis and future PROV-compatible export; +- ordinary worker and cancellation machinery is reused; +- payload retention does not increase; +- concurrent retries have one database-owned outcome; +- database maintenance and import paths cannot create digest-divergent, discontinuous, or reparented lineage; +- cross-owner, nonterminal, derived-root, skipped-generation, lineage-mutation, and referenced-evidence-mutation attempts fail closed; +- source/root lookups remain indexed as durable history grows; +- concurrent index failure has one migration and one artifact to repair. + +### Costs + +- operators must possess the exact original payload bytes; +- self-referencing lineage constrains retention and deletion order; +- every connector needs an explicit replay-safety classification; +- migrations and generation bounds require real PostgreSQL verification; +- the trigger adds same-transaction ancestor locks to replay insertion and indexed descendant reads to referenced-evidence updates; +- a terminal row cannot receive later maintenance edits after it becomes lineage evidence without a separately reviewed migration strategy; +- V8/V9 are nontransactional and require explicit ready/valid inspection and invalid-index recovery; +- trigger, function, and index lifecycle must be included in downgrade and disaster-recovery rehearsals. + +## Verification + +Acceptance requires exact-head tests for source immutability, owner isolation, exact payload matching, same-key replay, conflicting-key reuse, concurrent admission, lineage inheritance, generation exhaustion, ordinary worker behavior, cancellation compatibility, RFC 9457 responses, privacy exclusions, lock-order safety, and zero-missed configured production coverage. + +A direct-`develop` GitHub Actions gate applies every versioned migration to PostgreSQL 18, verifies replay and cancellation columns, verifies both lineage foreign keys use `ON DELETE RESTRICT`, verifies trigger and function presence, verifies both replay lookup indexes are ready and valid, executes valid generation-one and generation-two inserts, rejects nonterminal sources, request-digest mismatch, different generation-one roots, derived roots, skipped generations, cross-owner references, lineage mutation, referenced root-digest mutation, and referenced immediate-source failure-evidence mutation, protects source/root deletion, rehearses ordered rollback, and creates a non-empty schema-only dump. SAST, security, dependency, SBOM, review-thread, and non-author exact-head approval gates remain mandatory. + +## References — APA 7th edition + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://www.rfc-editor.org/rfc/rfc9110 + +National Institute of Standards and Technology. (2016). *SHA-3 derived functions: cSHAKE, KMAC, TupleHash, and ParallelHash* (NIST Special Publication 800-185). U.S. Department of Commerce. https://doi.org/10.6028/NIST.SP.800-185 + +Nottingham, M., Wilde, E., & Dalal, S. (2023). *Problem details for HTTP APIs* (RFC 9457). RFC Editor. https://www.rfc-editor.org/rfc/rfc9457 + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Constraints*. https://www.postgresql.org/docs/18/ddl-constraints.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE INDEX*. https://www.postgresql.org/docs/18/sql-createindex.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE TRIGGER*. https://www.postgresql.org/docs/18/sql-createtrigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: INSERT*. https://www.postgresql.org/docs/18/sql-insert.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: PL/pgSQL trigger functions*. https://www.postgresql.org/docs/18/plpgsql-trigger.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. https://www.w3.org/TR/prov-o/ diff --git a/docs/doctoring/durable-job-replay-key-domain-separation.md b/docs/doctoring/durable-job-replay-key-domain-separation.md new file mode 100644 index 00000000..17579fc8 --- /dev/null +++ b/docs/doctoring/durable-job-replay-key-domain-separation.md @@ -0,0 +1,83 @@ +# Durable-job replay-key domain separation + +## Decision + +Replay-created jobs store one versioned principal-scoped replay identity in +`submission_key_hash`: + +```text +SHA-256( + "mightyetl:durable-job-replay:v1:" + || principal_scope_hash + || ":" + || normalized_replay_key +) +``` + +The same identity feeds a separately versioned transaction-lock input. Replay identity is isolated +from ordinary submission-key hashing and from another authenticated principal. It proves only +idempotent admission; every source lookup and created-job query independently binds the owner hash. + +## Threat addressed + +Using the ordinary raw-key digest would allow one client key to collide across the submit and replay +APIs. It would also make a replay key's stored equality directly comparable between principal +namespaces. The versioned operation domain and owner hash prevent those cross-protocol and +cross-tenant equality channels. + +Within one principal namespace, the same replay key intentionally identifies only one replay intent. +An existing row with another immediate source or request digest returns +`etl_job_replay_key_reused` rather than creating a second row. + +## Compatibility boundary + +These exact strings are persisted behavior: + +```text +mightyetl:durable-job-replay:v1: +mightyetl:durable-job-replay-lock:v1: +``` + +Changing the replay domain would make existing requests stop replaying their created jobs. Changing +the lock domain could let old and new binaries concurrently use different locks for the same +identity. Either change requires an explicit migration and mixed-version deployment analysis. + +The implementation does not claim cSHAKE, KMAC, or TupleHash conformance. It uses the existing +SHA-256 utility with fixed-width principal hash, explicit separators, and one final bounded key. +NIST SP 800-185 is methodological evidence for customization and domain separation, not an +implementation-conformance claim. + +## Test evidence + +Service integration requires: + +- quoted and legacy-raw representations of one key replay the same created job; +- one replay key with another source or payload fails closed; +- an ordinary submission key cannot accidentally identify a replay-created row because the replay + domain changes the digest input; +- an unavailable transaction lock returns `etl_job_replay_in_progress` before table access. + +The table's `etl_job_submission_scope_unique` constraint remains a second integrity boundary after +the transaction lock. + +## Privacy + +Raw replay keys and replay hashes are excluded from HTTP bodies, headers, RFC 9457 problems, ordinary +logs, metrics, status and list resources, lineage exports, and worker leases. The stored hash remains +pseudonymous internal security data and must not be published merely because it is one-way. + +## Rollback + +Keep both versioned derivations available while replay-created rows can receive retries or while old +and new binaries overlap. Do not derive candidate hashes from logged user input during diagnosis. +Never rewrite a replay-created `submission_key_hash` without preserving unique-constraint and exact +idempotency evidence. + +## References — APA 7th + +Kelsey, J., Chang, S., & Perlner, R. (2016). *SHA-3 derived functions: cSHAKE, KMAC, TupleHash, and +ParallelHash* (NIST Special Publication 800-185). National Institute of Standards and Technology. +https://doi.org/10.6028/NIST.SP.800-185 + +National Institute of Standards and Technology. (2025, March 12). *Decision to update FIPS 202 and +revise SP 800-185*. https://csrc.nist.gov/news/2025/decision-to-update-fips-202-and-revise-sp-800-185 diff --git a/docs/doctoring/durable-job-replay-standards-evidence.md b/docs/doctoring/durable-job-replay-standards-evidence.md new file mode 100644 index 00000000..47b3bc88 --- /dev/null +++ b/docs/doctoring/durable-job-replay-standards-evidence.md @@ -0,0 +1,109 @@ +# Durable job replay standards evidence + +## Decision + +mightyETL models replay as creation of a new durable job derived from an immutable terminal source. The source is never returned to `PENDING`. Only owner-scoped `FAILED` and `CANCELLED` sources are eligible, and the operator must resupply byte-identical bounded JSON whose SHA-256 digest equals the source `request_digest`. + +The accepted replay response uses `202 Accepted` with a monitor URI because durable admission does not assert completion. Deterministic failures use RFC 9457 problem details. PostgreSQL owns the replay transaction, uniqueness, owner-scoped foreign keys, immediate-source digest equality, exact source/root/generation continuity, lineage-column immutability, referenced-evidence immutability, and source-row serialization. Replay lineage is compatible with PROV-O derivation semantics, while the relational database remains authoritative. + +## Normative mapping + +| Product contract | Primary authority | Application | +|---|---|---| +| Noncommittal durable admission | RFC 9110, section 15.3.3 | `202 Accepted`, `Location`, and no claim of execution completion | +| Stable machine-readable failures | RFC 9457 | Fixed problem type, title, status, detail, and `error_code` without exception text | +| Atomic new-row creation and conflict handling | PostgreSQL 18 `INSERT` and transaction documentation | One transaction validates source ownership, digest, replay identity, and lineage before inserting one new job | +| Same-owner source and root existence | PostgreSQL 18 constraints | Composite foreign keys bind source and root to the new row's `principal_scope_hash`, and `ON DELETE RESTRICT` protects retained history | +| Exact request and lineage transition | PostgreSQL 18 `CREATE TRIGGER` and PL/pgSQL trigger functions | A row-level `BEFORE INSERT OR UPDATE OF` trigger validates terminal source, immediate-source digest equality, first root, exact generation successor, initial pending lifecycle, and lineage-column immutability | +| Online descendant and foreign-key lookup | PostgreSQL 18 `CREATE INDEX`, `pg_index`, and system-information functions | Separate partial source/root indexes are built with one `CREATE INDEX CONCURRENTLY` per nontransactional Flyway migration and must match their exact ready, valid, nonunique, one-column partial-index definitions | +| Derivation lineage | W3C PROV-O | New job is derived from the immediate source and preserves an immutable first-root/generation chain | +| Domain separation rationale | NIST SP 800-185 | Replay-key hashing uses a versioned replay-specific domain; the SHA-256 construction does not claim cSHAKE or TupleHash conformance | + +## Why constraints and a trigger are both required + +The composite owner-scoped foreign keys prove that the named source and root exist in the same tenant namespace. The complete-lineage check proves that lineage fields are either all null or all present, bounds generation, and rejects direct self-reference. Those declarative rules cannot express all cross-row temporal invariants: + +- source must already be `FAILED` or `CANCELLED`; +- the replay row `request_digest` must equal the immediate source `request_digest`; +- the root must be the first job, with every lineage field null; +- generation one must use the same row as source and root; +- every later generation must inherit that root and equal the immediate source generation plus one; +- an existing replay row must never be reparented; and +- terminal evidence must become immutable after any descendant references the row. + +PostgreSQL `CREATE TRIGGER` permits a row-level trigger to run before selected insert or update events, and PL/pgSQL trigger functions receive `NEW`, `OLD`, `TG_OP`, and related context. mightyETL uses that database mechanism to reject invalid writes before persistence. The service repeats source-digest and inherited-root checks as defense in depth, but a maintenance script or import path cannot bypass the relational authority merely by omitting Java validation. + +The trigger raises the fixed SQLSTATE class `23514` without embedding principal values, job identifiers, payloads, hashes, SQL, or exception causes. Application and operator logs must classify the failure with a finite internal integrity code rather than copying raw database text. + +## Concurrency and lock-order evidence + +Child insertion needs an exact source/root snapshot. The INSERT trigger therefore locks the immediate source and root with PostgreSQL `FOR UPDATE` before validating status, digest, root identity, and generation continuity. + +A parent UPDATE already owns the parent row lock before the `BEFORE UPDATE` trigger executes. The immutable-evidence path performs an indexed descendant existence lookup without taking child row locks, rejects the mutation when a descendant exists, and returns before the INSERT-only source/root lock path. This establishes one lock direction: + +```text +child insertion → source/root row locks +parent mutation → existing parent row lock + descendant existence read +``` + +A parent update that commits first defines the evidence a later child validates. A child that obtains the source/root lock first causes a later conflicting parent update to wait; after the child commits, the parent sees the descendant and fails closed. The parent never locks a child and then reaches back to an ancestor, avoiding child-to-ancestor lock inversion. + +## Why online indexes are separate migrations + +The self-referencing foreign keys and immutable-evidence lookup need indexes beginning with `replay_source_job_record_id` and `replay_root_job_record_id`. Building them with ordinary `CREATE INDEX` would block production writes. PostgreSQL `CREATE INDEX CONCURRENTLY` preserves table availability but cannot run inside a transaction block and can leave an invalid index after cancellation or failure. + +mightyETL therefore uses two migrations: + +- `V8__add_etl_job_replay_source_lookup_index.sql` owns only `etl_job_replay_source_lookup_index`; +- `V9__add_etl_job_replay_root_lookup_index.sql` owns only `etl_job_replay_root_lookup_index`. + +Each companion `.sql.conf` sets `executeInTransaction=false`. One index per nontransactional migration makes failure, Flyway repair, and rollback independently auditable. PostgreSQL migration verification requires both `pg_index.indisready` and `pg_index.indisvalid`. + +Ready and valid flags alone do not prove that a same-named index has the required definition. `pg_index.indnkeyatts` and `pg_index.indnatts` establish that the contract has exactly one key attribute and no included attributes, while `indisunique` proves the expected nonunique shape. `pg_get_indexdef` reconstructs each indexed column and `pg_get_expr` reconstructs each stored partial predicate. The verifier therefore rejects a same-named index unless it targets the exact source or root column and has the matching `IS NOT NULL` predicate. An interrupted or definition-mismatched build is not accepted as passing evidence; operators remove only the failed artifact with `DROP INDEX CONCURRENTLY`, repair the exact Flyway migration record through the approved process, and rerun without editing an applied migration. + +## Security and privacy boundary + +The HTTP response and ordinary telemetry exclude raw principals, replay keys, payloads, request digests, internal hashes, source/root identifiers, lease identifiers, SQL, target identities, and exception messages. Foreign-owned and absent source identifiers remain indistinguishable. `SUCCEEDED` is excluded because repeating a committed target effect is not safe merely because the original request bytes are known. + +Connector replay is enabled only when target effects participate in the mightyETL transaction or the connector supplies independently tested idempotency or compensation. Payload equality is evidence of replay fidelity, not evidence that an external system will suppress duplicate effects. + +## Verification obligations + +- real PostgreSQL 18 migration rehearsal for trigger and function presence, composite self-referencing foreign keys, and `ON DELETE RESTRICT`; +- both replay lookup indexes present, ready, valid, nonunique, one-column, and bound to the exact source/root columns and `IS NOT NULL` predicates after separate V8/V9 concurrent builds; +- exact-payload acceptance and immediate-source digest mismatch rejection at both service and database boundaries; +- owner-safe missing/foreign behavior; +- same-key replay, key reuse conflict, and concurrent admission tests; +- source immutability and source/root/generation lineage tests; +- database rejection of nonterminal sources, generation-one root divergence, derived roots, skipped generations, cross-owner references, lineage mutation, and referenced-evidence mutation; +- lock-order regression proving the parent UPDATE path does not lock descendants before ancestor validation; +- service defense-in-depth rejection when an inherited root is itself a replay row; +- generation-bound rejection; +- ordinary worker claim, lease fencing, cancellation, polling, and ETag compatibility; +- transactional V7 rollback rehearsal plus operational V8/V9 concurrent-index rollback and invalid-index recovery evidence; +- configured production instruction, line, method, and branch coverage with zero misses; +- direct-base CI, dependency, SBOM, SAST, security, review-thread, and independent-approval gates before merge. + +## References — APA 7th edition + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://www.rfc-editor.org/rfc/rfc9110 + +National Institute of Standards and Technology. (2016). *SHA-3 derived functions: cSHAKE, KMAC, TupleHash, and ParallelHash* (NIST Special Publication 800-185). U.S. Department of Commerce. https://doi.org/10.6028/NIST.SP.800-185 + +Nottingham, M., Wilde, E., & Dalal, S. (2023). *Problem details for HTTP APIs* (RFC 9457). RFC Editor. https://www.rfc-editor.org/rfc/rfc9457 + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Constraints*. https://www.postgresql.org/docs/18/ddl-constraints.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE INDEX*. https://www.postgresql.org/docs/18/sql-createindex.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE TRIGGER*. https://www.postgresql.org/docs/18/sql-createtrigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: INSERT*. https://www.postgresql.org/docs/18/sql-insert.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: pg_index*. https://www.postgresql.org/docs/18/catalog-pg-index.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: PL/pgSQL trigger functions*. https://www.postgresql.org/docs/18/plpgsql-trigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: System information functions and operators*. https://www.postgresql.org/docs/18/functions-info.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. https://www.w3.org/TR/prov-o/ diff --git a/docs/operations/durable-job-replay.md b/docs/operations/durable-job-replay.md new file mode 100644 index 00000000..6bbfe716 --- /dev/null +++ b/docs/operations/durable-job-replay.md @@ -0,0 +1,345 @@ +# Durable ETL job replay + +## Purpose + +`POST /api/etl/jobs/{source_job_record_id}/replays` creates a new ordinary durable job from an +immutable failed or cancelled source. The operator must resupply the complete bounded JSON payload. +mightyETL validates that payload through the ordinary intake contract and requires its SHA-256 digest +to equal the terminal source `request_digest` before any new row is inserted. + +Replay never changes a terminal source back to `PENDING`. + +## HTTP contract + +```http +POST /api/etl/jobs/cf4f083f-8c90-4f34-a8b6-b53761de44ef/replays HTTP/1.1 +Authorization: Basic +Idempotency-Key: "1e05bdca-447c-4ad3-882c-e33963ce517c" +Content-Type: application/json + +[{"id":"record_alpha"}] +``` + +A first replay returns: + +```http +HTTP/1.1 202 Accepted +Location: /api/etl/jobs/86e4d474-dabf-4d6a-9de4-4e8230589363 +Cache-Control: no-store +Idempotency-Replayed: false +Content-Type: application/json +``` + +The representation is the existing accepted-job model. A later identical retry returns the same new +job and its current lifecycle state with `Idempotency-Replayed: true`. `202 Accepted` is +noncommittal: it proves durable admission of the new job, not completion of its ETL effects. + +Malformed, missing, and foreign-owned source identifiers share `404 etl_job_not_found`. Active +sources return `409 etl_job_replay_source_active`; succeeded sources return +`409 etl_job_replay_source_succeeded`. A mismatching payload returns +`422 etl_job_replay_payload_mismatch`. Reusing one replay key with another source or payload returns +`422 etl_job_replay_key_reused`. + +## Source immutability + +The source row is selected under the authenticated principal and a row lock, but never updated. Its +status, failure or cancellation evidence, timestamps, digest, terminal null payload, and lineage +remain unchanged. Only `FAILED` and `CANCELLED` are eligible. + +The client must resupply exact JSON text because terminal jobs deliberately clear `request_payload`. +Whitespace or field-order changes produce another digest even when a parser would consider the JSON +semantically equivalent. This byte-exact rule mirrors durable submission and prevents operators from +silently changing the work while claiming to replay it. + +## Replay identity and concurrency + +The normalized key is stored only through this versioned principal-scoped identity: + +```text +SHA-256( + "mightyetl:durable-job-replay:v1:" + || principal_scope_hash + || ":" + || normalized_replay_key +) +``` + +The value occupies the replay-created row's existing `submission_key_hash` field. It cannot collide +with ordinary raw-key submission identities unless SHA-256 itself collides. The exact domain string +is persisted compatibility behavior and cannot change without a migration. + +A PostgreSQL transaction-level try-lock serializes one replay key within a principal namespace. A +concurrent request that cannot acquire it receives `409 etl_job_replay_in_progress`; retry after the +first transaction commits returns the created job. The existing +`etl_job_submission_scope_unique` constraint remains a second integrity boundary. + +## Immutable owner-scoped lineage + +`V7__add_etl_job_replay_lineage.sql` adds: + +```text +replay_source_job_record_id +replay_root_job_record_id +replay_generation_count +``` + +Root jobs have all fields null. Replay rows have all fields non-null, references different from their +own job identifier, and generation 1 through 100. Both self-referencing foreign keys use +`ON DELETE RESTRICT`; deleting a source or root cannot silently cascade through audit history. + +The migration also adds the named support key: + +```text +etl_job_owner_identity_unique +UNIQUE (job_record_id, principal_scope_hash) +``` + +Both lineage relationships are composite owner-scoped foreign keys. They reference +`(job_record_id, principal_scope_hash)` rather than only the opaque job identifier. The database +therefore rejects cross-owner lineage even if application code, a maintenance script, or a future +import path attempts to pair one tenant's new job with another tenant's source or root. Application +owner predicates remain mandatory, but they are no longer the only tenant-integrity boundary. + +Declarative foreign keys and checks do not establish that the selected source is terminal, that the +root is the first row in the lineage, that the replay request digest equals the immediate source +digest, or that generation advances exactly once. V7 therefore creates: + +```text +validate_etl_job_replay_lineage() +etl_job_replay_lineage_guard_trigger +``` + +The `BEFORE INSERT OR UPDATE OF` database trigger enforces these additional invariants: + +- replay-created rows start as `PENDING`, with attempt zero and a retained payload; +- the exact immediate source is same-owner and `FAILED` or `CANCELLED`; +- the replay row's `request_digest` exactly equals the immediate source `request_digest`; +- the exact root is same-owner, terminal, and has all lineage fields null; +- generation 1 uses the same row as immediate source and root; +- every later generation equals the source generation plus one and inherits the same first root; +- lineage fields are immutable after insertion; and +- after a descendant references a row as immediate source or root, its status, request evidence, + attempt/failure state, cancellation evidence, and lifecycle timestamps are immutable. + +Child insertion locks the immediate source and root with PostgreSQL `FOR UPDATE` while validating +lineage. A parent update already owns the parent row lock before the trigger executes; if the update +would change immutable replay evidence, the trigger performs a bounded descendant existence lookup +without taking child locks. This preserves one lock direction: a child waits for its source/root, +while a parent never waits on a child and then reaches back to the ancestor. The result is a +fail-closed serialization boundary without the child-to-ancestor lock inversion that could otherwise +deadlock concurrent replay admission and lifecycle maintenance. + +The service performs the same root-identity and request-digest checks before insertion as defense in +depth. The database trigger remains authoritative for maintenance scripts, data imports, and other +writers that do not execute Java service code. + +```mermaid +flowchart LR + R0[Root terminal job
generation null] -->|same-owner replay| R1[Replay job
generation 1] + R1 -->|later terminal + same-owner replay| R2[Replay job
generation 2] + R0 -. owner-scoped first-root reference .-> R2 +``` + +A replay of a root uses the source as root and generation 1. A replay of a replay inherits the first +root and increments the immediate source generation. Generation 100 returns +`409 etl_job_replay_generation_exhausted` instead of creating generation 101. + +## Replay lookup indexes + +`V8__add_etl_job_replay_source_lookup_index.sql` and +`V9__add_etl_job_replay_root_lookup_index.sql` build partial indexes whose leading columns are the +immediate-source and first-root identifiers. They support composite foreign-key enforcement and the +bounded descendant existence lookup used by immutable-evidence updates. + +Each migration owns exactly one `CREATE INDEX CONCURRENTLY` statement so ordinary durable-job +inserts, lifecycle updates, and deletes remain available during the build. PostgreSQL prohibits +concurrent index creation inside a transaction block, so the companion `.sql.conf` files set +`executeInTransaction=false`; application configuration also sets +`spring.flyway.postgresql.transactional-lock=false` so Flyway does not wrap these migrations in a +PostgreSQL transactional advisory lock. + +Migration verification requires both indexes to be `indisready` and `indisvalid`, then reconstructs +the exact indexed column, one-key/one-attribute nonunique shape, and `IS NOT NULL` partial predicate +from PostgreSQL catalogs. A same-named ready and valid index with a different column, included +attribute, uniqueness contract, or predicate fails closed instead of being mistaken for the required +lineage-support index. + +## Worker behavior + +The new row is an ordinary `PENDING` job with the verified payload. The existing PostgreSQL claim, +lease fencing, attempts, retry, success, failure, cancellation, pagination, `Retry-After`, and ETag +contracts apply unchanged. No replay-specific worker or scheduler exists. + +Ordinary lifecycle updates may change status, payload, lease, attempts, failure, or cancellation +fields while retaining the exact lineage only until that row becomes historical evidence for a +subsequent replay. After any descendant references the row as source or root, the terminal evidence +covered by V7 is frozen. Any writer that attempts to reparent a replay row, alter its generation, or +change referenced terminal evidence receives a database constraint failure and must be treated as an +integrity incident. + +## Provenance export + +The relational rows are authoritative. A future owner-authorized JSON-LD export may represent: + +```text +source job → prov:Entity +replay action → prov:Activity +new job → prov:Entity +new job → prov:wasDerivedFrom → source job +replay action → prov:used → source job +new job → prov:wasGeneratedBy → replay action +``` + +PROV export never grants authority and never substitutes for owner predicates, database constraints, +trigger enforcement, or replay-key idempotency. + +## Rollout + +1. Rehearse V7 on a representative PostgreSQL 18 copy and inspect existing row count, table lock + duration, foreign-key validation time, and trigger creation. +2. Rehearse `V8__add_etl_job_replay_source_lookup_index.sql` and + `V9__add_etl_job_replay_root_lookup_index.sql` independently and record concurrent build duration, + disk growth, lock waits, and whether each resulting index is ready and valid. +3. Verify exact-head cross-platform CI, full reactor tests, zero-missed configured coverage, + dependency review, SBOM, SAST, security scan, review threads, and independent approval. +4. Apply V7 before serving the replay route, then apply V8 and V9 before treating replay migrations + as operationally ready. +5. Verify that exactly one `validate_etl_job_replay_lineage` function and one + `etl_job_replay_lineage_guard_trigger` exist on `etl_job_records`, and that both replay lookup + indexes are ready and valid with the exact column, one-column nonunique shape, and partial + predicate recorded in the migration contract. +6. Smoke-test a disposable failed source, exact payload acceptance, same-key retry, and key conflict. +7. Confirm the source is unchanged and the new row has source/root/generation lineage. +8. In an isolated migration rehearsal, confirm PostgreSQL rejects: + - nonterminal replay sources; + - replay rows whose request digest differs from their immediate source; + - generation-one rows whose source differs from root; + - derived replay rows used as root; + - skipped generations; + - source or root references from another `principal_scope_hash`; + - post-insert lineage mutation; + - request-digest mutation on a referenced lineage root; and + - failure-evidence mutation on a referenced immediate source. +9. Confirm both source and root deletion remain protected by `ON DELETE RESTRICT`. +10. Claim the new pending row through the ordinary worker and verify no replay-only execution path. +11. Monitor replay acceptance, in-progress conflicts, payload mismatches, generation exhaustion, + database lock waits, trigger rejections, concurrent-index failures, and failed foreign-key + deletion or tenant-boundary attempts using fixed-cardinality signals. + +Logs and metric labels must not contain payloads, raw principals, raw keys, hashes, source/new job +identifiers, lineage identifiers, SQL, exception messages, or target identities. + +## Incident response + +### Payload mismatch + +Recover the exact source payload from the approved upstream evidence or encrypted audit archive. Do +not change the source digest, bypass verification, or reconstruct payload text from an operator's +memory. If the exact payload is unavailable, the job is not replayable through this endpoint. + +### Replay key conflict + +Read the already-created replay job associated with the operator's prior request. A key is one +principal-scoped replay intent and cannot be reused for another source or payload. Use a new key only +for a deliberately separate replay. + +### Concurrent replay-index migration failure + +A cancelled or interrupted `CREATE INDEX CONCURRENTLY` can leave an invalid index behind. Stop the +migration rollout and preserve the exact application SHA, Flyway schema history, PostgreSQL logs, and +sanitized index metadata. Do not mark the migration successful while either replay lookup index is +missing, not ready, invalid, or differently defined from its exact indexed-column, attribute-count, +nonunique, and partial-predicate contract. + +Confirm no active migration process is still using the affected index, then remove only the invalid +artifact outside an explicit transaction: + +```sql +DROP INDEX CONCURRENTLY etl_job_replay_source_lookup_index; +DROP INDEX CONCURRENTLY etl_job_replay_root_lookup_index; +``` + +Drop only the index that actually failed; the two commands are shown together as the complete replay +index inventory. After the invalid index is removed, repair the failed Flyway migration record using +the approved deployment procedure and rerun that exact migration. Its companion configuration must +still contain `executeInTransaction=false`. Do not edit an applied migration, create an untracked +replacement index, or bypass exact catalog-definition verification. + +### Trigger, referenced evidence, or lineage integrity rejection + +Stop replay admission for the affected deployment. Preserve the deployed SHA, Flyway history, +transaction boundary, fixed error classification, and sanitized database evidence. Do not retry by +disabling the trigger or rewriting the source, root, generation, request digest, status, failure or +cancellation evidence, or timestamps. Determine whether the attempted write came from a stale binary, +maintenance script, import path, migration defect, or unauthorized writer. Treat cross-owner attempts +as tenant-isolation incidents even when PostgreSQL rejected them. + +### Broken lineage or missing root + +Stop replay admission. Preserve affected rows, deployed SHA, Flyway history, and backup evidence. +Do not null lineage fields to make constraints pass. Repair requires a reviewed migration based on +verified source/root ownership and generation. Never update lineage columns in place merely to pass +the trigger. + +## Rollback + +Stop serving replay admission before rolling application binaries back. Older binaries ignore lineage +columns, but deletion or retention tooling might not understand the new `ON DELETE RESTRICT` +relationships or trigger. + +V8 and V9 are nontransactional by design. If a rollback requires removing their indexes, perform +`DROP INDEX CONCURRENTLY etl_job_replay_root_lookup_index` and +`DROP INDEX CONCURRENTLY etl_job_replay_source_lookup_index` outside an explicit transaction before +rolling back V7. Preserve Flyway history and record the operational reason; do not pretend a partial +concurrent build was atomic. + +Do not drop V7 while replay rows exist. Archive or remove replay lineages from leaf to root under an +approved retention policy, preserving external audit evidence. Then a separately reviewed migration +may remove the trigger, function, composite foreign keys, `etl_job_owner_identity_unique`, and +lineage columns. Never edit the applied V7 file or mutate terminal sources back to pending. + +A controlled V7 rollback rehearsal must drop the trigger before its function and remove dependent +constraints before columns, all inside a transaction that is rolled back. After rollback, verify all +three columns, four named constraints, the trigger, and the function are restored. + +The replay-key domain must remain readable while any replay-created row can receive an idempotent +retry. A domain change requires a versioned migration or dual-read period, not a silent constant edit. + +## Connector limitation + +Matching the original payload does not prove that replaying a connector is externally safe. Enable +replay for a connector only when its target effects participate in the mightyETL transaction or the +connector provides independently tested idempotency or compensation. Succeeded jobs remain excluded +from this first slice. + +## References — APA 7th + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. +https://www.rfc-editor.org/rfc/rfc9110 + +Nottingham, M., Wilde, E., & Dalal, S. (2023). *Problem details for HTTP APIs* (RFC 9457). RFC Editor. +https://www.rfc-editor.org/rfc/rfc9457 + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Constraints*. +https://www.postgresql.org/docs/18/ddl-constraints.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE INDEX*. +https://www.postgresql.org/docs/18/sql-createindex.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE TRIGGER*. +https://www.postgresql.org/docs/18/sql-createtrigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: INSERT*. +https://www.postgresql.org/docs/18/sql-insert.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: pg_index*. +https://www.postgresql.org/docs/18/catalog-pg-index.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: PL/pgSQL trigger functions*. +https://www.postgresql.org/docs/18/plpgsql-trigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: System information functions and operators*. +https://www.postgresql.org/docs/18/functions-info.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. +https://www.w3.org/TR/prov-o/ diff --git a/docs/superpowers/plans/2026-08-06-durable-job-replay.md b/docs/superpowers/plans/2026-08-06-durable-job-replay.md new file mode 100644 index 00000000..937e3f0f --- /dev/null +++ b/docs/superpowers/plans/2026-08-06-durable-job-replay.md @@ -0,0 +1,135 @@ +# Durable ETL Job Replay Implementation Plan + +> **Execution rule:** implement every state, lineage, and admission boundary test-first and preserve the terminal source as immutable evidence. + +**Goal:** Create a new owner-scoped durable job from a failed or cancelled source only after the client resupplies the exact original payload. + +**Architecture:** Store replay lineage on the new job row, bind immediate-source and root references to the same principal through composite owner-scoped foreign keys, enforce exact source/root/generation continuity and lineage-column immutability through a PostgreSQL database trigger, use a versioned principal-scoped replay-key hash in the existing submission identity column, serialize creation with the existing transaction-lock boundary, verify payload digest against the terminal source, and return the existing accepted-job wire model. + +**Tech Stack:** Java 25, Spring MVC, Spring transactions, JdbcTemplate, PostgreSQL 18, Flyway, H2 integration tests, JUnit 5, Mockito, JaCoCo, Maven. + +## Global constraints + +- Never update a terminal source back to `PENDING`. +- Allow only `FAILED` and `CANCELLED` sources. +- Validate identifier, replay key, principal, and complete payload before lock or table access. +- Persist no raw principal or raw replay key. +- Require PostgreSQL to reject source or root lineage whose `principal_scope_hash` differs from the new row. +- Require the database trigger to reject nonterminal sources, derived roots, generation skips, generation-one source/root divergence, and post-insert lineage mutation. +- Preserve zero-missed configured production instruction, line, method, and branch coverage. +- Preserve no-skipped project tests and beginner-readable public Javadoc. +- Use descriptive multi-word `snake_case` database objects. +- Keep all existing review-agent credentials and workflows unchanged. + +## Task 1 — Lock the V7 lineage schema first + +**Files** +- Modify: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobMigrationDocumentationTest.java` +- Create: `etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql` +- Create: `etl-service/src/test/postgresql/replay_lineage_migration.sql` + +- [ ] Require all three lineage columns, bounded generation, complete-null-or-complete-non-null lifecycle, self-reference rejection, named composite owner-scoped foreign keys, their named `(job_record_id, principal_scope_hash)` unique support constraint, and `ON DELETE RESTRICT`. +- [ ] Reject legacy one-column source or root foreign keys because they permit cross-owner lineage at the database layer. +- [ ] Require a database trigger and PL/pgSQL trigger function that validate the terminal source, first root, exact generation successor, initial pending lifecycle, and lineage-column immutability. +- [ ] Run focused migration and PostgreSQL-rehearsal contract tests and observe failure because V7, tenant-integrity constraints, or continuity enforcement is absent. +- [ ] Implement the additive transactional migration and disposable PostgreSQL 18 rehearsal. +- [ ] Rerun the focused tests and commit. + +## Task 2 — Define immutable replay models and errors + +**Files** +- Create: `etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplay.java` +- Modify: `etl-service/src/main/java/com/xtrmetl/etl/service/EtlRequestError.java` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayTest.java` +- Modify: `etl-service/src/test/java/com/xtrmetl/etl/service/EtlRequestExceptionTest.java` + +- [ ] Add fail-first model validation and stable RFC 9457 metadata tests. +- [ ] Add the immutable replay result with new job ID, `PENDING`, and replay flag only. +- [ ] Add required, mismatch, reused, in-progress, active, succeeded, and generation-exhausted errors. +- [ ] Run focused tests and commit. + +## Task 3 — Implement the service transaction test-first + +**Files** +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayServiceIntegrationTest.java` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayServiceBoundaryTest.java` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLineageIntegrityTest.java` +- Modify: `etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobService.java` +- Create: `etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplayService.java` + +**Public interface** + +```java +EtlJobReplay replayOwned( + UUID sourceJobRecordId, + String requestPayload, + String replayKey, + String principalScope +) +``` + +- [ ] Add failing tests for failed source, cancelled source, source immutability, payload mismatch, key replay/reuse, owner isolation, active/succeeded rejection, lineage root/generation, generation exhaustion, and validation before JDBC. +- [ ] Add a fail-first regression proving an inherited root that is itself a replay row is rejected. +- [ ] Run focused tests and observe compile/assertion failure. +- [ ] Add the versioned replay domain and transaction-lock identity. +- [ ] Add existing-replay lookup and source-lineage lookup. +- [ ] Require the inherited root to exist in the owner namespace and to have all lineage fields null before insertion. +- [ ] Insert one ordinary `PENDING` job with verified payload and lineage. +- [ ] Run focused tests and commit. + +## Task 4 — Add the HTTP resource + +**Files** +- Modify: `etl-service/src/main/java/com/xtrmetl/etl/controller/EtlJobController.java` +- Modify: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobControllerTest.java` +- Modify: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobControllerFailureTest.java` + +- [ ] Add fail-first tests for first acceptance, replay, authentication, missing key, malformed source, typed error, database failure, and unexpected failure. +- [ ] Implement `POST /api/etl/jobs/{sourceJobRecordId}/replays` with `202`, `Location`, no-store, and replay header. +- [ ] Run focused tests and commit. + +## Task 5 — Prove lifecycle and concurrency compatibility + +**Files** +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayClaimIntegrationTest.java` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayConcurrencyIntegrationTest.java` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLineageIntegrationTest.java` + +- [ ] Prove a new replay row can be claimed by the ordinary worker. +- [ ] Prove an unavailable transaction lock returns replay-in-progress without insertion. +- [ ] Prove retry after a committed first replay returns the same new row. +- [ ] Prove replay-of-replay preserves the root and increments generation by exactly one. +- [ ] Prove generation 100 fails before insertion. +- [ ] Prove PostgreSQL rejects a replay source or root from another `principal_scope_hash`, independent of application owner predicates. +- [ ] Prove PostgreSQL rejects nonterminal sources, derived roots, skipped generations, and lineage mutation. +- [ ] Run focused and full tests and commit. + +## Task 6 — Finish operations, provenance, and exact-head verification + +**Files** +- Create: `docs/operations/durable-job-replay.md` +- Create: `docs/doctoring/durable-job-replay-key-domain-separation.md` +- Create: `docs/doctoring/durable-job-replay-standards-evidence.md` +- Create: `docs/adr/2026-08-07-immutable-durable-job-replay.md` +- Modify: `docs/etl/durable-job-intake.md` +- Modify: `CHANGELOG.md` +- Create: `etl-service/src/test/java/com/xtrmetl/etl/documentation/DurableJobReplayDocumentationTest.java` + +- [ ] Require source immutability, payload digest proof, composite owner-scoped foreign keys, database-trigger continuity, immutable lineage fields, cross-owner lineage rejection, concurrency, generation limit, connector limitation, rollout, and rollback documentation first. +- [ ] Document W3C PROV mapping as an export contract, not a database-authority substitute. +- [ ] Record APA 7th primary references for PostgreSQL constraints, `CREATE TRIGGER`, and PL/pgSQL trigger functions together with the versioned replay-key compatibility boundary. +- [ ] Rehearse the migration on PostgreSQL 18 and verify owner-scoped source/root failures, exact generation progression, derived-root rejection, lineage immutability, deletion restriction, and rollback before production rollout. +- [ ] Run all verification through exact-head CI: `./mvnw -B test`, configured coverage gates, and `git diff --check`. +- [ ] Keep the PR draft until every stacked-target gate succeeds. + +## Plan self-review + +- Every issue #134 acceptance requirement maps to a task. +- The source is never mutated. +- Replay-key and payload conflicts are distinguished without disclosing source existence across principals. +- New jobs enter the existing worker lifecycle rather than creating a second execution engine. +- PostgreSQL and application owner predicates independently reject cross-owner lineage. +- A database trigger owns exact source/root/generation continuity even for maintenance or import writers that bypass the service. +- Lineage fields are immutable after insertion, so descendants cannot be silently reparented. +- No placeholder, ambiguous public signature, or unbounded database object name remains. +- Verification requires that no project test is skipped. diff --git a/docs/superpowers/specs/2026-08-06-durable-job-replay-design.md b/docs/superpowers/specs/2026-08-06-durable-job-replay-design.md new file mode 100644 index 00000000..a372463a --- /dev/null +++ b/docs/superpowers/specs/2026-08-06-durable-job-replay-design.md @@ -0,0 +1,221 @@ +# Durable ETL Job Replay Design + +## Purpose + +mightyETL terminalizes failed and cancelled jobs by clearing their retained request payload. Operators still need a safe way to retry the same intended work without mutating terminal history, trusting an unverified replacement payload, or losing the relationship between the original and the new attempt. + +This design adds one authenticated owner-scoped replay action. The client resupplies the complete bounded JSON payload; mightyETL validates it through the existing intake contract and requires its SHA-256 digest to equal the immutable terminal source digest before creating a new `PENDING` resource. + +## API contract + +```http +POST /api/etl/jobs/{source_job_record_id}/replays +Authorization: +Idempotency-Key: "new-replay-key" +Content-Type: application/json + +[{"id":"record_alpha","name":"accepted"}] +``` + +A first accepted replay returns RFC 9110 `202 Accepted`, `Location` for the new job, `Cache-Control: no-store`, and `Idempotency-Replayed: false`. The same principal, replay key, source job, and byte-identical payload returns the same new job with `Idempotency-Replayed: true`. + +The source remains terminal and unchanged. Only `FAILED` and `CANCELLED` are replayable. Active sources conflict because they still own or may own execution. `SUCCEEDED` conflicts because a first-slice replay could duplicate committed target effects. + +## Immutable relational lineage + +Migration `V7__add_etl_job_replay_lineage.sql` adds: + +```text +replay_source_job_record_id +replay_root_job_record_id +replay_generation_count +``` + +The root job has all three fields null. Every replay row has all three fields non-null, an immediate source different from itself, a root different from itself, and a generation from 1 through 100. The composite owner-scoped foreign keys use `ON DELETE RESTRICT` so terminal history cannot disappear through cascade deletion and one tenant cannot reference another tenant's source or root. + +The referenced key is the named support constraint: + +```text +etl_job_owner_identity_unique +UNIQUE (job_record_id, principal_scope_hash) +``` + +Each source and root relationship includes the new row's `principal_scope_hash`: + +```text +FOREIGN KEY (replay_source_job_record_id, principal_scope_hash) + REFERENCES etl_job_records (job_record_id, principal_scope_hash) + +FOREIGN KEY (replay_root_job_record_id, principal_scope_hash) + REFERENCES etl_job_records (job_record_id, principal_scope_hash) +``` + +Those declarative constraints establish owner scope, existence, self-reference rejection, completeness, deletion restriction, and the generation bound. They do not by themselves prove that a source is terminal, that the selected root is the first lineage row, that the new row preserves the immediate source digest, or that each replay hop advances exactly one generation. The migration therefore adds a database trigger and PL/pgSQL trigger function as the relational authority for continuity: + +```text +validate_etl_job_replay_lineage() +etl_job_replay_lineage_guard_trigger +``` + +On replay insertion, the database trigger requires all of the following: + +- the new derived row starts as `PENDING`, attempt zero, with a retained payload; +- the immediate source exists in the same principal scope and is `FAILED` or `CANCELLED`; +- the new row's `request_digest` exactly equals the immediate source `request_digest`; +- the root exists in the same principal scope, is terminal, and has all lineage fields null; +- generation 1 uses the same row for immediate source and root; +- later generations inherit the exact first root and equal the source generation plus one. + +On updates that name any lineage column, the trigger rejects every changed value. Lineage fields are immutable after insertion, so a maintenance script, import path, or future service cannot silently reparent a job after descendants exist. + +A terminal row becomes durable evidence when a descendant names it as an immediate source or lineage root. Child insertion locks the immediate source and root with PostgreSQL `FOR UPDATE` before it can validate and commit. A parent update already owns the parent row lock before its trigger executes; when the update would alter terminal replay evidence, the trigger performs an indexed descendant existence lookup without taking child row locks and then returns before the INSERT-only source/root locking path. This establishes one lock direction and avoids child-to-ancestor lock inversion: + +```text +child insertion → lock source/root → validate → commit or reject +parent mutation → parent row already locked → read descendant existence → commit or reject +``` + +If a parent update commits first, a later child validates the resulting evidence. If child insertion obtains the parent lock first, a later conflicting parent update waits, observes the committed descendant, and fails closed. Ordinary lifecycle updates remain possible before the first descendant exists; referenced replay evidence is immutable after that point, so an already-created descendant cannot silently acquire a different historical meaning. + +For the first replay: + +```text +source = terminal root job +root = terminal root job +generation = 1 +``` + +For replay of a replay: + +```text +source = immediate terminal replay +root = inherited first job +generation = source generation + 1 +``` + +The application independently validates the owner-scoped source, source digest, inherited root, and root-row identity before insertion. PostgreSQL independently rejects cross-owner, nonterminal, digest-divergent, discontinuous, derived-root, mutable-lineage, and referenced-evidence mutation. Relational rows remain authoritative even when lineage is later exported as W3C PROV. + +## Replay lookup indexes + +The trigger's immutable-evidence check and PostgreSQL's self-referencing foreign keys need bounded source/root lookup paths as durable history grows. The design therefore separates schema authority from online index construction: + +- `V7__add_etl_job_replay_lineage.sql` remains transactional and contains columns, constraints, trigger, and function only; +- `V8__add_etl_job_replay_source_lookup_index.sql` owns one partial `CREATE INDEX CONCURRENTLY` for `(replay_source_job_record_id, principal_scope_hash)`; +- `V9__add_etl_job_replay_root_lookup_index.sql` owns one partial `CREATE INDEX CONCURRENTLY` for `(replay_root_job_record_id, principal_scope_hash)`; +- each concurrent migration has its own `.sql.conf` with `executeInTransaction=false`; +- migration verification requires both indexes to be present, ready, and valid. + +One concurrent index per nontransactional migration gives each failure one auditable Flyway repair boundary. A cancelled build can leave an invalid index, so rollout must inspect PostgreSQL catalog state, remove only the failed artifact with `DROP INDEX CONCURRENTLY`, repair the exact migration record under the approved deployment procedure, and rerun without editing an applied migration. + +## Replay-key authority + +The replay key is normalized through the same bounded quoted-or-legacy safe profile as other idempotency keys. The new job stores a versioned principal-scoped replay identity in the existing `submission_key_hash` field: + +```text +SHA-256( + "mightyetl:durable-job-replay:v1:" + || principal_scope_hash + || ":" + || normalized_replay_key +) +``` + +This isolates replay keys from ordinary submission keys and from another tenant. Within one principal namespace, the same replay key can identify only one new job. Reusing it with another source or payload fails with `etl_job_replay_key_reused`. + +A transaction-level lock derived from the replay identity serializes concurrent creation. The table's existing principal-plus-submission-hash unique constraint remains the second integrity boundary. A concurrent request that cannot acquire the lock returns `etl_job_replay_in_progress`; retrying after the first transaction completes replays the committed new job. + +## Database transaction + +One transaction performs the following sequence: + +1. validate source identifier, replay key, principal, and complete bounded payload before lock or table access; +2. compute principal, replay-key, and payload digests; +3. acquire the replay-key transaction lock; +4. find and classify an existing job using that replay identity; +5. select the owner-scoped source and immutable lineage; +6. require terminal `FAILED` or `CANCELLED`; +7. require the supplied payload digest to equal the source `request_digest`; +8. derive root and bounded generation; +9. require the inherited root to exist in the owner namespace and have null lineage fields; +10. insert one new `PENDING` row with the verified payload and lineage; +11. let PostgreSQL validate same-owner references, immediate-source digest equality, terminal source, first root, exact generation continuity, initial lifecycle, and the transition from mutable lifecycle state to referenced immutable evidence; +12. return only the new operator-safe job identity. + +The source is never updated. Read-then-write state resurrection is prohibited. + +## Error taxonomy + +| HTTP | Stable code | Meaning | +| ---: | --- | --- | +| 400 | `etl_job_replay_key_required` | Replay key is missing or outside the bounded profile. | +| 404 | `etl_job_not_found` | Source is malformed, missing, or foreign-owned. | +| 409 | `etl_job_replay_in_progress` | Another transaction owns the principal-scoped replay identity. | +| 409 | `etl_job_replay_source_active` | Source is `PENDING` or `RUNNING`. | +| 409 | `etl_job_replay_source_succeeded` | Source already committed successful effects. | +| 409 | `etl_job_replay_generation_exhausted` | Generation 100 cannot create generation 101. | +| 422 | `etl_job_replay_payload_mismatch` | Resupplied JSON does not match the immutable source digest. | +| 422 | `etl_job_replay_key_reused` | Replay key already identifies another source or payload. | + +All covered request failures use the existing RFC 9457 problem model without payload, principal, key, hash, lineage internals, SQL, or exception text. A trigger rejection indicates internally inconsistent repository state or an unauthorized writer and is treated as an operator-visible integrity incident rather than reflected with raw database text. + +## Worker compatibility + +The new row is an ordinary `PENDING` job. Existing worker claim, lease fencing, retry, success, failure, cancellation, pagination, polling, and conditional-status contracts apply unchanged. No replay-specific worker or scheduler exists. Only lineage and admission differ. + +## Provenance export + +A later JSON-LD adapter may map the relational evidence as: + +```text +source job → prov:Entity +replay action → prov:Activity +new job → prov:Entity +new job → prov:wasDerivedFrom → source job +replay action → prov:used → source job +new job → prov:wasGeneratedBy → replay action +``` + +The export must not weaken owner authorization or replace database constraints and trigger enforcement. + +## Verification + +The exact-head suite must prove: + +1. failed and cancelled sources each create a distinct pending job; +2. source status, terminal evidence, timestamps, and cleared payload remain unchanged; +3. payload mismatch fails before insertion; +4. same source, key, and payload replay one new job; +5. same key with another source or payload fails closed; +6. foreign and missing sources remain indistinguishable; +7. pending, running, and succeeded sources are rejected; +8. replay of replay preserves the first root and increments generation exactly once; +9. generation 100 cannot create generation 101; +10. concurrent creation produces one row and an in-progress or later replay outcome; +11. the new job can be claimed and follows normal lifecycle contracts; +12. service defense in depth rejects an inherited root that is itself a replay row; +13. PostgreSQL 18 rejects cross-owner references, nonterminal sources, immediate-source digest mismatch, generation-one root divergence, derived roots, skipped generations, lineage mutation, and mutation of referenced root or immediate-source evidence; +14. PostgreSQL 18 applies V8 and V9 independently and requires both replay lookup indexes to be ready and valid; +15. migration completeness, source and root constraints, trigger/function presence, self-reference prohibition, descriptive naming, deletion restriction, concurrent-index recovery, rollout, and rollback are documented and tested; +16. all added production statements and branches retain zero-missed configured coverage and no project test is skipped. + +## Operational limitation + +Replay verifies that the resupplied payload matches the immutable source digest. It does not prove that replaying a connector is economically or externally safe. A target that cannot provide transactional or idempotent effects requires connector-specific policy before replay is enabled for that connector. + +## References — APA 7th + +Fielding, R., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://www.rfc-editor.org/rfc/rfc9110 + +Nottingham, M., Wilde, E., & Dalal, S. (2023). *Problem details for HTTP APIs* (RFC 9457). RFC Editor. https://www.rfc-editor.org/rfc/rfc9457 + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Constraints*. https://www.postgresql.org/docs/18/ddl-constraints.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE INDEX*. https://www.postgresql.org/docs/18/sql-createindex.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: CREATE TRIGGER*. https://www.postgresql.org/docs/18/sql-createtrigger.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: INSERT*. https://www.postgresql.org/docs/18/sql-insert.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: PL/pgSQL trigger functions*. https://www.postgresql.org/docs/18/plpgsql-trigger.html + +World Wide Web Consortium. (2013). *PROV-O: The PROV ontology*. https://www.w3.org/TR/prov-o/ diff --git a/etl-service/pom.xml b/etl-service/pom.xml index 9e0e56b5..2ebc1d80 100644 --- a/etl-service/pom.xml +++ b/etl-service/pom.xml @@ -116,6 +116,7 @@ com/xtrmetl/etl/job/*.class com/xtrmetl/etl/controller/EtlJobController*.class + com/xtrmetl/etl/controller/EtlJobReplayController*.class com/xtrmetl/etl/service/Sha256Digest*.class @@ -130,6 +131,7 @@ com/xtrmetl/etl/job/*.class com/xtrmetl/etl/controller/EtlJobController*.class + com/xtrmetl/etl/controller/EtlJobReplayController*.class com/xtrmetl/etl/service/Sha256Digest*.class diff --git a/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlApiProblemHandler.java b/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlApiProblemHandler.java index 6ab4ff3e..cef3f117 100644 --- a/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlApiProblemHandler.java +++ b/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlApiProblemHandler.java @@ -30,7 +30,11 @@ * captured by a broad exception handler and therefore retain their framework-owned status * semantics.

*/ -@RestControllerAdvice(assignableTypes = {EtlController.class, EtlJobController.class}) +@RestControllerAdvice(assignableTypes = { + EtlController.class, + EtlJobController.class, + EtlJobReplayController.class +}) public class EtlApiProblemHandler { private static final Logger log = LoggerFactory.getLogger(EtlApiProblemHandler.class); diff --git a/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlJobReplayController.java b/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlJobReplayController.java new file mode 100644 index 00000000..da889715 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/controller/EtlJobReplayController.java @@ -0,0 +1,126 @@ +package com.xtrmetl.etl.controller; + +import com.xtrmetl.etl.job.EtlJobAcceptedResponse; +import com.xtrmetl.etl.job.EtlJobReplay; +import com.xtrmetl.etl.job.EtlJobReplayService; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import io.micrometer.observation.annotation.Observed; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty; +import org.springframework.dao.DataAccessException; +import org.springframework.http.CacheControl; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.lang.Nullable; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.net.URI; +import java.security.Principal; +import java.util.Objects; +import java.util.UUID; + +/** + * Exposes immutable-lineage replay admission for owner-scoped terminal durable ETL jobs. + * + *

The source is never resurrected. A replay request supplies the complete candidate payload, + * which the service verifies against the source digest before creating one new ordinary pending + * job. The response is RFC 9110 noncommittal acceptance of that new resource, not evidence that its + * ETL effects have completed.

+ */ +@ConditionalOnBooleanProperty( + prefix = "xtrmetl.etl.jobs", + name = "intake-enabled", + havingValue = true, + matchIfMissing = false +) +@RestController +@RequestMapping("/api/etl/jobs") +public class EtlJobReplayController { + + private final EtlJobReplayService replayService; + + /** + * Creates the replay HTTP adapter. + * + * @param replayService immutable owner-scoped replay admission service + */ + public EtlJobReplayController(EtlJobReplayService replayService) { + this.replayService = Objects.requireNonNull( + replayService, + "replayService must not be null" + ); + } + + /** + * Accepts one verified replay of a failed or cancelled owner-scoped source. + * + * @param sourceJobRecordIdText opaque terminal source identifier text + * @param requestPayload exact bounded JSON array text to verify against the source digest + * @param replayKey required replay idempotency key + * @param principal authenticated principal namespace + * @return accepted new-job identity, status monitor, and replay evidence + */ + @PostMapping("/{sourceJobRecordId}/replays") + @Observed(name = "etl.jobs.replay", contextualName = "etl-job-replay") + public ResponseEntity replay( + @PathVariable("sourceJobRecordId") String sourceJobRecordIdText, + @RequestBody String requestPayload, + @RequestHeader(value = "Idempotency-Key", required = false) + @Nullable String replayKey, + @Nullable Principal principal + ) { + if (principal == null) { + throw new EtlRequestException(EtlRequestError.IDEMPOTENCY_PRINCIPAL_REQUIRED); + } + if (replayKey == null) { + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_KEY_REQUIRED); + } + UUID sourceJobRecordId = parseJobRecordId(sourceJobRecordIdText); + + final EtlJobReplay replay; + try { + replay = replayService.replayOwned( + sourceJobRecordId, + requestPayload, + replayKey, + principal.getName() + ); + } catch (EtlRequestException | DataAccessException exception) { + throw exception; + } catch (RuntimeException exception) { + throw new EtlUnexpectedException(exception); + } + + String statusUrl = "/api/etl/jobs/" + replay.jobRecordId(); + EtlJobAcceptedResponse responseBody = new EtlJobAcceptedResponse( + replay.jobRecordId(), + replay.jobStatus(), + statusUrl + ); + return ResponseEntity.accepted() + .cacheControl(CacheControl.noStore()) + .location(URI.create(statusUrl)) + .header( + EtlJobController.IDEMPOTENCY_REPLAYED_HEADER, + Boolean.toString(replay.replayed()) + ) + .contentType(MediaType.APPLICATION_JSON) + .body(responseBody); + } + + private static UUID parseJobRecordId(String jobRecordIdText) { + try { + return UUID.fromString(Objects.requireNonNull( + jobRecordIdText, + "sourceJobRecordIdText must not be null" + )); + } catch (IllegalArgumentException exception) { + throw new EtlRequestException(EtlRequestError.JOB_NOT_FOUND, exception); + } + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplay.java b/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplay.java new file mode 100644 index 00000000..65caf105 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplay.java @@ -0,0 +1,29 @@ +package com.xtrmetl.etl.job; + +import java.util.Objects; +import java.util.UUID; + +/** + * Reports one newly accepted or replayed immutable-lineage durable ETL job. + * + *

The source terminal resource and lineage remain internal persistence evidence. This result + * exposes only the new opaque job identifier, its current stable lifecycle state, and whether the + * same principal-scoped replay request had already created it. A first creation is pending; a later + * idempotent retry may correctly report that the same created job has since progressed.

+ * + * @param jobRecordId replay-created durable job identifier + * @param jobStatus current stable lifecycle state of that created job + * @param replayed {@code true} when this response reuses an already-created replay job + */ +public record EtlJobReplay( + UUID jobRecordId, + EtlJobStatus jobStatus, + boolean replayed +) { + + /** Validates the immutable replay result. */ + public EtlJobReplay { + Objects.requireNonNull(jobRecordId, "jobRecordId must not be null"); + Objects.requireNonNull(jobStatus, "jobStatus must not be null"); + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplayService.java b/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplayService.java new file mode 100644 index 00000000..0f3a15e4 --- /dev/null +++ b/etl-service/src/main/java/com/xtrmetl/etl/job/EtlJobReplayService.java @@ -0,0 +1,409 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import com.xtrmetl.etl.service.EtlRequestLock; +import com.xtrmetl.etl.service.PostgresEtlRequestLock; +import com.xtrmetl.etl.service.Sha256Digest; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Objects; +import java.util.UUID; +import java.util.regex.Pattern; + +/** + * Creates new durable jobs from immutable failed or cancelled owner-scoped source evidence. + * + *

The client resupplies the complete bounded JSON payload because terminal rows deliberately + * clear it. Replay validates the payload through the same record contract as ordinary intake and + * requires its SHA-256 digest to equal the source digest before insertion. The terminal source is + * never updated.

+ * + *

A versioned principal-scoped replay-key hash is stored in the existing submission identity + * column. The transaction-level request lock serializes one key within a principal namespace, and + * the table unique constraint remains a second integrity boundary. Replay-created rows enter the + * ordinary pending/worker lifecycle with immutable source, root, and bounded generation lineage.

+ */ +@Service +public class EtlJobReplayService { + + /** Maximum number of replay generations retained by the first lineage contract. */ + public static final int MAXIMUM_REPLAY_GENERATION = 100; + + private static final String REPLAY_KEY_DOMAIN = "mightyetl:durable-job-replay:v1:"; + private static final String REPLAY_LOCK_DOMAIN = "mightyetl:durable-job-replay-lock:v1:"; + private static final int MAX_PRINCIPAL_SCOPE_CODE_POINTS = 512; + private static final String KEY_VALUE_EXPRESSION = "[A-Za-z0-9._:-]{16,128}"; + private static final Pattern KEY_VALUE_PROFILE = Pattern.compile(KEY_VALUE_EXPRESSION); + private static final Pattern KEY_STRUCTURED_FIELD_PROFILE = Pattern.compile( + "\"(" + KEY_VALUE_EXPRESSION + ")\"" + ); + + private static final String SELECT_EXISTING_REPLAY_SQL = """ + SELECT job_record_id, request_digest, job_status, + replay_source_job_record_id + FROM etl_job_records + WHERE principal_scope_hash = ? + AND submission_key_hash = ? + """; + private static final String SELECT_REPLAY_SOURCE_SQL = """ + SELECT job_record_id, request_digest, job_status, + replay_root_job_record_id, replay_generation_count + FROM etl_job_records + WHERE job_record_id = ? + AND principal_scope_hash = ? + FOR UPDATE + """; + private static final String SELECT_OWNED_ROOT_COUNT_SQL = """ + SELECT COUNT(*) + FROM etl_job_records + WHERE job_record_id = ? + AND principal_scope_hash = ? + """; + private static final String SELECT_LINEAGE_ROOT_COUNT_SQL = """ + SELECT COUNT(*) + FROM etl_job_records + WHERE job_record_id = ? + AND principal_scope_hash = ? + AND replay_source_job_record_id IS NULL + AND replay_root_job_record_id IS NULL + AND replay_generation_count IS NULL + """; + private static final String INSERT_REPLAY_JOB_SQL = """ + INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + attempt_count, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count + ) VALUES (?, ?, ?, ?, ?, 'PENDING', 0, ?, ?, ?) + """; + + private final JdbcTemplate jdbcTemplate; + private final ObjectMapper objectMapper; + private final EtlBatchProperties batchProperties; + private final EtlRequestLock requestLock; + + /** + * Creates replay admission with the PostgreSQL transaction-lock implementation. + * + * @param jdbcTemplate parameterized durable job persistence + * @param objectMapper JSON parser configuration to copy + * @param batchProperties bounded request limits + */ + public EtlJobReplayService( + JdbcTemplate jdbcTemplate, + ObjectMapper objectMapper, + EtlBatchProperties batchProperties + ) { + this( + jdbcTemplate, + objectMapper, + batchProperties, + new PostgresEtlRequestLock(jdbcTemplate) + ); + } + + /** + * Creates replay admission with an explicit transaction-lifetime request lock. + * + * @param jdbcTemplate parameterized durable job persistence + * @param objectMapper JSON parser configuration to copy + * @param batchProperties bounded request limits + * @param requestLock transaction-lifetime replay-key lock + */ + @Autowired + public EtlJobReplayService( + JdbcTemplate jdbcTemplate, + ObjectMapper objectMapper, + EtlBatchProperties batchProperties, + EtlRequestLock requestLock + ) { + this.jdbcTemplate = Objects.requireNonNull(jdbcTemplate, "jdbcTemplate must not be null"); + ObjectMapper sourceMapper = Objects.requireNonNull( + objectMapper, + "objectMapper must not be null" + ); + this.objectMapper = sourceMapper.copy(); + this.objectMapper.enable(JsonParser.Feature.STRICT_DUPLICATE_DETECTION); + this.batchProperties = Objects.requireNonNull( + batchProperties, + "batchProperties must not be null" + ); + this.requestLock = Objects.requireNonNull(requestLock, "requestLock must not be null"); + } + + /** + * Creates or replays one new durable job from an immutable terminal source. + * + * @param sourceJobRecordId owner-scoped failed or cancelled source identifier + * @param requestPayload resupplied exact bounded JSON array text + * @param replayKey quoted Structured Field String or supported legacy raw safe value + * @param principalScope authenticated principal namespace + * @return new or previously-created replay job identity and current state + * @throws NullPointerException when the source identifier is {@code null} + * @throws EtlRequestException when validation, ownership, state, digest, key, or generation + * contracts fail + * @throws IllegalStateException when no actual transaction is active or stored lineage is + * internally inconsistent + */ + @Transactional + public EtlJobReplay replayOwned( + UUID sourceJobRecordId, + @Nullable String requestPayload, + @Nullable String replayKey, + @Nullable String principalScope + ) { + UUID validatedSourceId = Objects.requireNonNull( + sourceJobRecordId, + "sourceJobRecordId must not be null" + ); + String validatedKey = validateReplayKey(replayKey); + String validatedScope = validatePrincipalScope(principalScope); + String validatedPayload = validatePayload(requestPayload); + requireActiveTransaction(); + + String principalScopeHash = Sha256Digest.digest(validatedScope); + String replayKeyHash = Sha256Digest.digest( + REPLAY_KEY_DOMAIN + principalScopeHash + ':' + validatedKey + ); + String replayLockHash = Sha256Digest.digest( + REPLAY_LOCK_DOMAIN + principalScopeHash + ':' + replayKeyHash + ); + String requestDigest = Sha256Digest.digest(validatedPayload); + + if (!requestLock.tryLock(replayLockHash)) { + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_IN_PROGRESS); + } + + ExistingReplay existingReplay = findExistingReplay(principalScopeHash, replayKeyHash); + if (existingReplay != null) { + if (!validatedSourceId.equals(existingReplay.sourceJobRecordId()) + || !requestDigest.equals(existingReplay.requestDigest())) { + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_KEY_REUSED); + } + return new EtlJobReplay( + existingReplay.jobRecordId(), + existingReplay.jobStatus(), + true + ); + } + + ReplaySource source = findSource(validatedSourceId, principalScopeHash); + if (source == null) { + throw new EtlRequestException(EtlRequestError.JOB_NOT_FOUND); + } + switch (source.jobStatus()) { + case PENDING, RUNNING -> throw new EtlRequestException( + EtlRequestError.JOB_REPLAY_SOURCE_ACTIVE + ); + case SUCCEEDED -> throw new EtlRequestException( + EtlRequestError.JOB_REPLAY_SOURCE_SUCCEEDED + ); + case FAILED, CANCELLED -> { + // These terminal outcomes are the only first-slice replay sources. + } + } + if (!source.requestDigest().equals(requestDigest)) { + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_PAYLOAD_MISMATCH); + } + + UUID rootJobRecordId; + int replayGeneration; + if (source.replayRootJobRecordId() == null) { + if (source.replayGenerationCount() != null) { + throw new IllegalStateException("Replay source has incomplete root lineage"); + } + rootJobRecordId = source.jobRecordId(); + replayGeneration = 1; + } else { + Integer sourceGeneration = Objects.requireNonNull( + source.replayGenerationCount(), + "Replay source generation must accompany its root" + ); + if (sourceGeneration >= MAXIMUM_REPLAY_GENERATION) { + throw new EtlRequestException( + EtlRequestError.JOB_REPLAY_GENERATION_EXHAUSTED + ); + } + rootJobRecordId = source.replayRootJobRecordId(); + requireOwnedRoot(rootJobRecordId, principalScopeHash); + replayGeneration = sourceGeneration + 1; + } + + UUID newJobRecordId = UUID.randomUUID(); + jdbcTemplate.update( + INSERT_REPLAY_JOB_SQL, + newJobRecordId, + principalScopeHash, + replayKeyHash, + requestDigest, + validatedPayload, + source.jobRecordId(), + rootJobRecordId, + replayGeneration + ); + return new EtlJobReplay(newJobRecordId, EtlJobStatus.PENDING, false); + } + + @Nullable + private ExistingReplay findExistingReplay( + String principalScopeHash, + String replayKeyHash + ) { + List rows = jdbcTemplate.query( + SELECT_EXISTING_REPLAY_SQL, + (resultSet, rowNumber) -> new ExistingReplay( + resultSet.getObject("job_record_id", UUID.class), + resultSet.getString("request_digest"), + EtlJobStatus.valueOf(resultSet.getString("job_status")), + resultSet.getObject("replay_source_job_record_id", UUID.class) + ), + principalScopeHash, + replayKeyHash + ); + return rows.isEmpty() ? null : rows.getFirst(); + } + + @Nullable + private ReplaySource findSource(UUID sourceJobRecordId, String principalScopeHash) { + List rows = jdbcTemplate.query( + SELECT_REPLAY_SOURCE_SQL, + (resultSet, rowNumber) -> new ReplaySource( + resultSet.getObject("job_record_id", UUID.class), + resultSet.getString("request_digest"), + EtlJobStatus.valueOf(resultSet.getString("job_status")), + resultSet.getObject("replay_root_job_record_id", UUID.class), + resultSet.getObject("replay_generation_count", Integer.class) + ), + sourceJobRecordId, + principalScopeHash + ); + return rows.isEmpty() ? null : rows.getFirst(); + } + + private void requireOwnedRoot(UUID rootJobRecordId, String principalScopeHash) { + Integer ownedRootCount = jdbcTemplate.queryForObject( + SELECT_OWNED_ROOT_COUNT_SQL, + Integer.class, + rootJobRecordId, + principalScopeHash + ); + if (!Integer.valueOf(1).equals(ownedRootCount)) { + throw new IllegalStateException("Replay root is absent from the owner namespace"); + } + + Integer lineageRootCount = jdbcTemplate.queryForObject( + SELECT_LINEAGE_ROOT_COUNT_SQL, + Integer.class, + rootJobRecordId, + principalScopeHash + ); + if (!Integer.valueOf(1).equals(lineageRootCount)) { + throw new IllegalStateException("Replay root is not a lineage root"); + } + } + + private String validatePayload(@Nullable String requestPayload) { + if (requestPayload == null) { + throw new EtlRequestException(EtlRequestError.INVALID_JSON); + } + if (requestPayload.getBytes(StandardCharsets.UTF_8).length + > batchProperties.getMaxPayloadBytes()) { + throw new EtlRequestException(EtlRequestError.PAYLOAD_TOO_LARGE); + } + + final JsonNode root; + try { + root = objectMapper.readTree(requestPayload); + } catch (JsonProcessingException exception) { + throw new EtlRequestException(EtlRequestError.INVALID_JSON, exception); + } + if (root == null || root.isNull() || !root.isArray()) { + throw new EtlRequestException(EtlRequestError.INVALID_JSON); + } + if (root.size() > batchProperties.getMaxBatchRecords()) { + throw new EtlRequestException(EtlRequestError.BATCH_TOO_LARGE); + } + for (JsonNode record : root) { + EtlJobService.validateRecord(record); + } + return requestPayload; + } + + private static String validateReplayKey(@Nullable String replayKey) { + if (replayKey == null) { + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_KEY_REQUIRED); + } + var structuredFieldMatcher = KEY_STRUCTURED_FIELD_PROFILE.matcher(replayKey); + if (structuredFieldMatcher.matches()) { + return structuredFieldMatcher.group(1); + } + if (KEY_VALUE_PROFILE.matcher(replayKey).matches()) { + return replayKey; + } + throw new EtlRequestException(EtlRequestError.JOB_REPLAY_KEY_REQUIRED); + } + + private static String validatePrincipalScope(@Nullable String principalScope) { + if (principalScope == null + || principalScope.isBlank() + || principalScope.codePointCount(0, principalScope.length()) + > MAX_PRINCIPAL_SCOPE_CODE_POINTS) { + throw new EtlRequestException(EtlRequestError.IDEMPOTENCY_PRINCIPAL_REQUIRED); + } + return principalScope; + } + + private static void requireActiveTransaction() { + if (!TransactionSynchronizationManager.isActualTransactionActive()) { + throw new IllegalStateException( + "Durable ETL job replay requires an active transaction" + ); + } + } + + private record ExistingReplay( + UUID jobRecordId, + String requestDigest, + EtlJobStatus jobStatus, + @Nullable UUID sourceJobRecordId + ) { + private ExistingReplay { + Objects.requireNonNull(jobRecordId, "jobRecordId must not be null"); + Objects.requireNonNull(requestDigest, "requestDigest must not be null"); + Objects.requireNonNull(jobStatus, "jobStatus must not be null"); + } + } + + private record ReplaySource( + UUID jobRecordId, + String requestDigest, + EtlJobStatus jobStatus, + @Nullable UUID replayRootJobRecordId, + @Nullable Integer replayGenerationCount + ) { + private ReplaySource { + Objects.requireNonNull(jobRecordId, "jobRecordId must not be null"); + Objects.requireNonNull(requestDigest, "requestDigest must not be null"); + Objects.requireNonNull(jobStatus, "jobStatus must not be null"); + } + } +} diff --git a/etl-service/src/main/java/com/xtrmetl/etl/service/EtlRequestError.java b/etl-service/src/main/java/com/xtrmetl/etl/service/EtlRequestError.java index 332f13f6..e2c6c611 100644 --- a/etl-service/src/main/java/com/xtrmetl/etl/service/EtlRequestError.java +++ b/etl-service/src/main/java/com/xtrmetl/etl/service/EtlRequestError.java @@ -167,6 +167,69 @@ public enum EtlRequestError { "The durable job failed before cancellation could commit." ), + /** The replay key is absent or outside the bounded safe idempotency profile. */ + JOB_REPLAY_KEY_REQUIRED( + HttpStatus.BAD_REQUEST, + "etl_job_replay_key_required", + "urn:mightyetl:problem:etl-job-replay-key-required", + "ETL job replay key required", + "Replay requires a supported principal-scoped Idempotency-Key." + ), + + /** The resupplied payload does not match the immutable terminal source digest. */ + JOB_REPLAY_PAYLOAD_MISMATCH( + HttpStatus.UNPROCESSABLE_ENTITY, + "etl_job_replay_payload_mismatch", + "urn:mightyetl:problem:etl-job-replay-payload-mismatch", + "ETL job replay payload mismatch", + "The replay payload does not match the immutable source job payload digest." + ), + + /** The replay identity already belongs to another source or payload. */ + JOB_REPLAY_KEY_REUSED( + HttpStatus.UNPROCESSABLE_ENTITY, + "etl_job_replay_key_reused", + "urn:mightyetl:problem:etl-job-replay-key-reused", + "ETL job replay key reused", + "The Idempotency-Key already identifies a different durable job replay." + ), + + /** Another transaction owns the same principal-scoped replay identity. */ + JOB_REPLAY_IN_PROGRESS( + HttpStatus.CONFLICT, + "etl_job_replay_in_progress", + "urn:mightyetl:problem:etl-job-replay-in-progress", + "ETL job replay in progress", + "A durable job replay with the same principal-scoped Idempotency-Key is being created." + ), + + /** A pending or running source is still active and cannot be replayed. */ + JOB_REPLAY_SOURCE_ACTIVE( + HttpStatus.CONFLICT, + "etl_job_replay_source_active", + "urn:mightyetl:problem:etl-job-replay-source-active", + "ETL job replay source active", + "Pending or running durable jobs cannot be replayed." + ), + + /** A succeeded source is excluded to prevent silent duplicate target effects. */ + JOB_REPLAY_SOURCE_SUCCEEDED( + HttpStatus.CONFLICT, + "etl_job_replay_source_succeeded", + "urn:mightyetl:problem:etl-job-replay-source-succeeded", + "ETL job replay source succeeded", + "A succeeded durable job cannot be replayed through this endpoint." + ), + + /** The bounded immutable replay lineage cannot create another generation. */ + JOB_REPLAY_GENERATION_EXHAUSTED( + HttpStatus.CONFLICT, + "etl_job_replay_generation_exhausted", + "urn:mightyetl:problem:etl-job-replay-generation-exhausted", + "ETL job replay generation exhausted", + "The durable job replay lineage reached its maximum generation." + ), + /** The requested job does not exist in the authenticated principal's namespace. */ JOB_NOT_FOUND( HttpStatus.NOT_FOUND, @@ -196,47 +259,27 @@ public enum EtlRequestError { this.detail = Objects.requireNonNull(detail, "detail must not be null"); } - /** - * Returns the HTTP status for this deterministic request failure. - * - * @return immutable HTTP status - */ + /** @return HTTP status for this deterministic request failure */ public HttpStatus status() { return status; } - /** - * Returns the stable snake_case machine code. - * - * @return compatibility-safe error code - */ + /** @return stable snake_case compatibility-safe machine code */ public String errorCode() { return errorCode; } - /** - * Returns the stable RFC 9457 problem type URI. - * - * @return problem type URI - */ + /** @return stable RFC 9457 problem type URI */ public URI type() { return type; } - /** - * Returns the fixed human-readable category title. - * - * @return problem title - */ + /** @return fixed human-readable problem title */ public String title() { return title; } - /** - * Returns the fixed non-sensitive client guidance. - * - * @return problem detail - */ + /** @return fixed non-sensitive client guidance */ public String detail() { return detail; } diff --git a/etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql b/etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql new file mode 100644 index 00000000..db21a654 --- /dev/null +++ b/etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql @@ -0,0 +1,187 @@ +-- Add immutable source/root/generation lineage to replay-created durable job rows. +ALTER TABLE etl_job_records + ADD COLUMN replay_source_job_record_id UUID, + ADD COLUMN replay_root_job_record_id UUID, + ADD COLUMN replay_generation_count INTEGER; + +ALTER TABLE etl_job_records + ADD CONSTRAINT etl_job_owner_identity_unique + UNIQUE (job_record_id, principal_scope_hash), + ADD CONSTRAINT etl_job_replay_source_reference + FOREIGN KEY (replay_source_job_record_id, principal_scope_hash) + REFERENCES etl_job_records (job_record_id, principal_scope_hash) + ON DELETE RESTRICT, + ADD CONSTRAINT etl_job_replay_root_reference + FOREIGN KEY (replay_root_job_record_id, principal_scope_hash) + REFERENCES etl_job_records (job_record_id, principal_scope_hash) + ON DELETE RESTRICT, + ADD CONSTRAINT etl_job_replay_lineage_complete_check CHECK ( + ( + replay_source_job_record_id IS NULL + AND replay_root_job_record_id IS NULL + AND replay_generation_count IS NULL + ) + OR + ( + replay_source_job_record_id IS NOT NULL + AND replay_root_job_record_id IS NOT NULL + AND replay_generation_count BETWEEN 1 AND 100 + AND replay_source_job_record_id <> job_record_id + AND replay_root_job_record_id <> job_record_id + ) + ); + +-- Keep the relational lineage authoritative even when rows are imported outside the service. +-- Each derived row must point to an exact terminal predecessor, preserve the first root, and +-- advance the generation by exactly one. Lineage columns become immutable after insertion. +-- Once a row is referenced as an immediate source or root, its terminal replay evidence also +-- becomes immutable so later writes cannot change the meaning of already-created descendants. +CREATE FUNCTION validate_etl_job_replay_lineage() RETURNS trigger +LANGUAGE plpgsql +AS $etl_job_replay_lineage$ +DECLARE + source_job_status VARCHAR(32); + source_request_digest CHAR(64); + source_source_job_record_id UUID; + source_root_job_record_id UUID; + source_generation_count INTEGER; + root_job_status VARCHAR(32); + root_source_job_record_id UUID; + root_root_job_record_id UUID; + root_generation_count INTEGER; +BEGIN + IF TG_OP = 'UPDATE' THEN + IF OLD.replay_source_job_record_id + IS DISTINCT FROM NEW.replay_source_job_record_id + OR OLD.replay_root_job_record_id + IS DISTINCT FROM NEW.replay_root_job_record_id + OR OLD.replay_generation_count + IS DISTINCT FROM NEW.replay_generation_count THEN + RAISE EXCEPTION 'Replay lineage fields are immutable' + USING ERRCODE = '23514'; + END IF; + + IF OLD.job_status IN ('FAILED', 'CANCELLED') AND ( + OLD.job_status IS DISTINCT FROM NEW.job_status + OR OLD.request_digest IS DISTINCT FROM NEW.request_digest + OR OLD.request_payload IS DISTINCT FROM NEW.request_payload + OR OLD.attempt_count IS DISTINCT FROM NEW.attempt_count + OR OLD.failure_code IS DISTINCT FROM NEW.failure_code + OR OLD.cancellation_key_hash IS DISTINCT FROM NEW.cancellation_key_hash + OR OLD.cancellation_code IS DISTINCT FROM NEW.cancellation_code + OR OLD.job_cancelled_at IS DISTINCT FROM NEW.job_cancelled_at + OR OLD.created_at IS DISTINCT FROM NEW.created_at + OR OLD.updated_at IS DISTINCT FROM NEW.updated_at + ) THEN + -- The row being updated is already locked by PostgreSQL. Every child insertion + -- locks its source and root before it can commit, so an existence lookup is enough + -- to serialize this mutation without taking child locks in the reverse direction. + PERFORM 1 + FROM etl_job_records AS child_record + WHERE child_record.replay_source_job_record_id = OLD.job_record_id + OR child_record.replay_root_job_record_id = OLD.job_record_id + LIMIT 1; + + IF FOUND THEN + RAISE EXCEPTION 'Referenced replay evidence is immutable' + USING ERRCODE = '23514'; + END IF; + END IF; + + RETURN NEW; + END IF; + + IF NEW.replay_generation_count IS NULL THEN + RETURN NEW; + END IF; + + IF NEW.job_status <> 'PENDING' + OR NEW.attempt_count <> 0 + OR NEW.request_payload IS NULL THEN + RAISE EXCEPTION 'Replay rows must start as pending jobs' + USING ERRCODE = '23514'; + END IF; + + SELECT job_status, + request_digest, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count + INTO source_job_status, + source_request_digest, + source_source_job_record_id, + source_root_job_record_id, + source_generation_count + FROM etl_job_records + WHERE job_record_id = NEW.replay_source_job_record_id + AND principal_scope_hash = NEW.principal_scope_hash + FOR UPDATE; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Replay source is missing or belongs to another owner' + USING ERRCODE = '23514'; + END IF; + + IF source_job_status NOT IN ('FAILED', 'CANCELLED') THEN + RAISE EXCEPTION 'Replay source must be failed or cancelled' + USING ERRCODE = '23514'; + END IF; + + IF source_request_digest IS DISTINCT FROM NEW.request_digest THEN + RAISE EXCEPTION 'Replay request digest must match the immediate source' + USING ERRCODE = '23514'; + END IF; + + SELECT job_status, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count + INTO root_job_status, + root_source_job_record_id, + root_root_job_record_id, + root_generation_count + FROM etl_job_records + WHERE job_record_id = NEW.replay_root_job_record_id + AND principal_scope_hash = NEW.principal_scope_hash + FOR UPDATE; + + IF NOT FOUND THEN + RAISE EXCEPTION 'Replay root is missing or belongs to another owner' + USING ERRCODE = '23514'; + END IF; + + IF root_job_status NOT IN ('FAILED', 'CANCELLED') + OR root_source_job_record_id IS NOT NULL + OR root_root_job_record_id IS NOT NULL + OR root_generation_count IS NOT NULL THEN + RAISE EXCEPTION 'Replay root is not a lineage root' + USING ERRCODE = '23514'; + END IF; + + IF NEW.replay_generation_count = 1 THEN + IF NEW.replay_source_job_record_id <> NEW.replay_root_job_record_id THEN + RAISE EXCEPTION 'Generation one must reference the same source and root' + USING ERRCODE = '23514'; + END IF; + ELSIF source_source_job_record_id IS NULL + OR source_root_job_record_id + IS DISTINCT FROM NEW.replay_root_job_record_id + OR source_generation_count + IS DISTINCT FROM NEW.replay_generation_count - 1 THEN + RAISE EXCEPTION 'Replay generation does not follow the immediate source' + USING ERRCODE = '23514'; + END IF; + + RETURN NEW; +END; +$etl_job_replay_lineage$; + +CREATE TRIGGER etl_job_replay_lineage_guard_trigger +BEFORE INSERT OR UPDATE OF replay_source_job_record_id, + replay_root_job_record_id, replay_generation_count, job_status, + request_digest, request_payload, attempt_count, failure_code, + cancellation_key_hash, cancellation_code, job_cancelled_at, + created_at, updated_at +ON etl_job_records +FOR EACH ROW +EXECUTE FUNCTION validate_etl_job_replay_lineage(); diff --git a/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql b/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql new file mode 100644 index 00000000..8e1c175b --- /dev/null +++ b/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql @@ -0,0 +1,10 @@ +-- Support replay-source foreign-key enforcement and immutable-evidence descendant lookup. +-- CONCURRENTLY preserves durable-job inserts, lifecycle updates, and deletes during rollout. +-- The companion .sql.conf disables Flyway's per-migration transaction because PostgreSQL +-- rejects CREATE INDEX CONCURRENTLY inside a transaction block. +CREATE INDEX CONCURRENTLY etl_job_replay_source_lookup_index + ON etl_job_records ( + replay_source_job_record_id, + principal_scope_hash + ) + WHERE replay_source_job_record_id IS NOT NULL; diff --git a/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql.conf b/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql.conf new file mode 100644 index 00000000..73bd53a1 --- /dev/null +++ b/etl-service/src/main/resources/db/migration/V8__add_etl_job_replay_source_lookup_index.sql.conf @@ -0,0 +1 @@ +executeInTransaction=false diff --git a/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql b/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql new file mode 100644 index 00000000..1df3bc28 --- /dev/null +++ b/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql @@ -0,0 +1,10 @@ +-- Support replay-root foreign-key enforcement and immutable-evidence descendant lookup. +-- CONCURRENTLY preserves durable-job inserts, lifecycle updates, and deletes during rollout. +-- The companion .sql.conf disables Flyway's per-migration transaction because PostgreSQL +-- rejects CREATE INDEX CONCURRENTLY inside a transaction block. +CREATE INDEX CONCURRENTLY etl_job_replay_root_lookup_index + ON etl_job_records ( + replay_root_job_record_id, + principal_scope_hash + ) + WHERE replay_root_job_record_id IS NOT NULL; diff --git a/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql.conf b/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql.conf new file mode 100644 index 00000000..73bd53a1 --- /dev/null +++ b/etl-service/src/main/resources/db/migration/V9__add_etl_job_replay_root_lookup_index.sql.conf @@ -0,0 +1 @@ +executeInTransaction=false diff --git a/etl-service/src/test/java/com/xtrmetl/etl/documentation/CiCoverageDiagnosticsWorkflowTest.java b/etl-service/src/test/java/com/xtrmetl/etl/documentation/CiCoverageDiagnosticsWorkflowTest.java index 67148b3d..5963f963 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/documentation/CiCoverageDiagnosticsWorkflowTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/documentation/CiCoverageDiagnosticsWorkflowTest.java @@ -44,10 +44,18 @@ void diagnosesEveryStrictCoverageTarget() { assertTrue(workflow.contains( "\"com/xtrmetl/etl/job/EtlJobService\": \"EtlJobService.java\"" )); + assertTrue(workflow.contains( + "\"com/xtrmetl/etl/job/EtlJobReplayService\": " + + "\"EtlJobReplayService.java\"" + )); assertTrue(workflow.contains( "\"com/xtrmetl/etl/controller/EtlJobController\": " + "\"EtlJobController.java\"" )); + assertTrue(workflow.contains( + "\"com/xtrmetl/etl/controller/EtlJobReplayController\": " + + "\"EtlJobReplayController.java\"" + )); assertTrue(workflow.contains( "\"com/xtrmetl/etl/service/Sha256Digest\": \"Sha256Digest.java\"" )); diff --git a/etl-service/src/test/java/com/xtrmetl/etl/documentation/DurableJobReplayDocumentationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/documentation/DurableJobReplayDocumentationTest.java new file mode 100644 index 00000000..8cb34b05 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/documentation/DurableJobReplayDocumentationTest.java @@ -0,0 +1,159 @@ +package com.xtrmetl.etl.documentation; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Keeps replay source immutability, payload proof, lineage, compatibility, and operations aligned. + */ +class DurableJobReplayDocumentationTest { + + @Test + void operationsRunbookDocumentsAdmissionLineageAndRollback() throws IOException { + String runbook = read("docs/operations/durable-job-replay.md") + .replaceAll("\\s+", " "); + + assertTrue(runbook.contains("POST /api/etl/jobs/{source_job_record_id}/replays")); + assertTrue(runbook.contains("Replay never changes a terminal source back to `PENDING`")); + assertTrue(runbook.contains("Idempotency-Replayed: false")); + assertTrue(runbook.contains("Idempotency-Replayed: true")); + assertTrue(runbook.contains("etl_job_replay_payload_mismatch")); + assertTrue(runbook.contains("etl_job_replay_key_reused")); + assertTrue(runbook.contains("replay_source_job_record_id")); + assertTrue(runbook.contains("replay_root_job_record_id")); + assertTrue(runbook.contains("replay_generation_count")); + assertTrue(runbook.contains("etl_job_owner_identity_unique")); + assertTrue(runbook.contains("etl_job_replay_lineage_guard_trigger")); + assertTrue(runbook.contains("validate_etl_job_replay_lineage")); + assertTrue(runbook.contains("source generation plus one")); + assertTrue(runbook.contains("lineage fields are immutable")); + assertTrue(runbook.contains("(job_record_id, principal_scope_hash)")); + assertTrue(runbook.contains("cross-owner lineage")); + assertTrue(runbook.contains("ON DELETE RESTRICT")); + assertTrue(runbook.contains("generation 1 through 100")); + assertTrue(runbook.contains("prov:wasDerivedFrom")); + assertTrue(runbook.contains("Do not drop V7 while replay rows exist")); + assertTrue(runbook.contains("does not prove that replaying a connector")); + assertTrue(runbook.contains("RFC 9110")); + assertTrue(runbook.contains("RFC 9457")); + assertTrue(runbook.contains("PostgreSQL 18 documentation: CREATE TRIGGER")); + assertTrue(runbook.contains("PostgreSQL 18 documentation: Constraints")); + assertTrue(runbook.contains("PROV-O")); + } + + @Test + void designAndPlanPreserveTheSingleExecutionEngine() throws IOException { + String design = read( + "docs/superpowers/specs/2026-08-06-durable-job-replay-design.md" + ).replaceAll("\\s+", " "); + String plan = read( + "docs/superpowers/plans/2026-08-06-durable-job-replay.md" + ).replaceAll("\\s+", " "); + + assertTrue(design.contains("The source is never updated")); + assertTrue(design.contains("Only `FAILED` and `CANCELLED`")); + assertTrue(design.contains("ordinary `PENDING` job")); + assertTrue(design.contains("No replay-specific worker or scheduler exists")); + assertTrue(design.contains("replay_generation_count")); + assertTrue(design.contains("composite owner-scoped foreign keys")); + assertTrue(design.contains("database trigger")); + assertTrue(design.contains("exactly one generation")); + assertTrue(design.contains("immutable after insertion")); + assertTrue(plan.contains("Never update a terminal source back to `PENDING`")); + assertTrue(plan.contains("composite owner-scoped foreign keys")); + assertTrue(plan.contains("database trigger")); + assertTrue(plan.contains("Run all verification")); + assertTrue(plan.contains("no project test is skipped")); + } + + @Test + void changelogRecordsReplayAdmissionLineageAndSafety() throws IOException { + String changelog = read("CHANGELOG.md").replaceAll("\\s+", " "); + + assertTrue(changelog.contains("immutable failed or cancelled source")); + assertTrue(changelog.contains("byte-identical bounded JSON payload")); + assertTrue(changelog.contains("replay_source_job_record_id")); + assertTrue(changelog.contains("replay_root_job_record_id")); + assertTrue(changelog.contains("replay_generation_count")); + assertTrue(changelog.contains("etl_job_owner_identity_unique")); + assertTrue(changelog.contains("composite owner-scoped foreign keys")); + assertTrue(changelog.contains("exact source/root/generation continuity")); + assertTrue(changelog.contains("immutable lineage fields")); + assertTrue(changelog.contains("V7__add_etl_job_replay_lineage.sql")); + assertTrue(changelog.contains("does not prove external connector safety")); + } + + @Test + void doctoringPinsReplayStandardsAndVersionedKeyDomains() throws IOException { + String domainEvidence = read( + "docs/doctoring/durable-job-replay-key-domain-separation.md" + ).replaceAll("\\s+", " "); + String standardsEvidence = read( + "docs/doctoring/durable-job-replay-standards-evidence.md" + ).replaceAll("\\s+", " "); + + assertTrue(domainEvidence.contains("mightyetl:durable-job-replay:v1:")); + assertTrue(domainEvidence.contains("mightyetl:durable-job-replay-lock:v1:")); + assertTrue(domainEvidence.contains("isolated from ordinary submission-key hashing")); + assertTrue(domainEvidence.contains("exact strings are persisted behavior")); + assertTrue(domainEvidence.contains("does not claim cSHAKE")); + assertTrue(domainEvidence.contains("NIST Special Publication 800-185")); + + assertTrue(standardsEvidence.contains("CREATE TRIGGER")); + assertTrue(standardsEvidence.contains("PL/pgSQL trigger functions")); + assertTrue(standardsEvidence.contains("exact source/root/generation continuity")); + assertTrue(standardsEvidence.contains("lineage-column immutability")); + } + + @Test + void verificationDocsRequireExactReplayIndexCatalogDefinitions() throws IOException { + String runbook = read("docs/operations/durable-job-replay.md") + .replaceAll("\\s+", " "); + String standardsEvidence = read( + "docs/doctoring/durable-job-replay-standards-evidence.md" + ).replaceAll("\\s+", " "); + String changelog = read("CHANGELOG.md").replaceAll("\\s+", " "); + + assertTrue(runbook.contains( + "exact indexed column, one-key/one-attribute nonunique shape, " + + "and `IS NOT NULL` partial predicate" + )); + assertTrue(standardsEvidence.contains( + "`pg_get_indexdef` reconstructs each indexed column and `pg_get_expr` " + + "reconstructs each stored partial predicate" + )); + assertTrue(changelog.contains( + "exact replay-index column, predicate, and one-column nonunique shape" + )); + } + + private static String read(String relativePath) throws IOException { + return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8); + } + + /** @return repository root from reactor-root or module-local execution */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlJobReplayPostgresWorkflowTest.java b/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlJobReplayPostgresWorkflowTest.java new file mode 100644 index 00000000..0fe4e7e3 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/documentation/EtlJobReplayPostgresWorkflowTest.java @@ -0,0 +1,136 @@ +package com.xtrmetl.etl.documentation; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Keeps the production-PostgreSQL replay migration and rollback rehearsal executable in CI. + * + *

The ordinary cross-platform Java build validates these repository contracts without + * requiring PostgreSQL on every runner. The separate direct-{@code develop} integration + * workflow then executes the same verifier against PostgreSQL 18.

+ */ +class EtlJobReplayPostgresWorkflowTest { + + @Test + void directDevelopWorkflowRunsCompleteReplayMigrationChainOnPostgres18() throws IOException { + String workflow = read( + ".github/workflows/postgresql-migration-integration.yml" + ).replaceAll("\\s+", " "); + String verifier = read( + "scripts/verify-postgresql-migrations.sh" + ).replaceAll("\\s+", " "); + + assertTrue(workflow.contains("replay_lineage_migration:")); + assertTrue(workflow.contains("image: postgres:18-alpine")); + assertTrue(workflow.contains("--health-cmd \"pg_isready")); + assertTrue(workflow.contains("etl-service/src/test/postgresql/**")); + assertTrue(workflow.contains("bash scripts/verify-postgresql-migrations.sh")); + + assertTrue(verifier.contains("find \"${migration_directory}\"")); + assertTrue(verifier.contains("-name 'V*__*.sql'")); + assertTrue(verifier.contains("sort -zV")); + assertTrue(verifier.contains("psql --no-psqlrc --set ON_ERROR_STOP=1")); + assertTrue(verifier.contains( + "etl-service/src/test/postgresql/replay_lineage_migration.sql" + )); + assertTrue(verifier.contains( + "pg_get_indexdef(index_record.indexrelid, 1, true)" + )); + assertTrue(verifier.contains( + "pg_get_expr(index_record.indpred, index_record.indrelid)" + )); + assertTrue(verifier.contains("index_record.indnkeyatts = 1")); + assertTrue(verifier.contains("index_record.indnatts = 1")); + assertTrue(verifier.contains("NOT index_record.indisunique")); + assertTrue(verifier.contains( + "pg_get_indexdef(index_record.indexrelid, 1, true) = " + + "'replay_source_job_record_id'" + )); + assertTrue(verifier.contains( + "pg_get_expr(index_record.indpred, index_record.indrelid) = " + + "'(replay_source_job_record_id IS NOT NULL)'" + )); + assertTrue(verifier.contains( + "pg_get_indexdef(index_record.indexrelid, 1, true) = " + + "'replay_root_job_record_id'" + )); + assertTrue(verifier.contains( + "pg_get_expr(index_record.indpred, index_record.indrelid) = " + + "'(replay_root_job_record_id IS NOT NULL)'" + )); + } + + @Test + void postgresRehearsalCoversLineageContinuityTenantIntegrityAndRollback() + throws IOException { + String rehearsal = read( + "etl-service/src/test/postgresql/replay_lineage_migration.sql" + ).replaceAll("\\s+", " "); + + assertTrue(rehearsal.contains("replay lineage trigger or function is missing")); + assertTrue(rehearsal.contains("nonterminal replay source was accepted")); + assertTrue(rehearsal.contains("generation-one replay accepted a different root")); + assertTrue(rehearsal.contains("a derived replay row was accepted as lineage root")); + assertTrue(rehearsal.contains("a skipped replay generation was accepted")); + assertTrue(rehearsal.contains("cross-owner source lineage was accepted")); + assertTrue(rehearsal.contains("cross-owner root lineage was accepted")); + assertTrue(rehearsal.contains("replay lineage fields were mutable")); + assertTrue(rehearsal.contains("referenced replay root evidence was mutable")); + assertTrue(rehearsal.contains("referenced immediate-source evidence was mutable")); + assertTrue(rehearsal.contains( + "ON DELETE RESTRICT did not protect immediate replay history" + )); + assertTrue(rehearsal.contains( + "ON DELETE RESTRICT did not protect replay root history" + )); + assertTrue(rehearsal.contains( + "DROP TRIGGER etl_job_replay_lineage_guard_trigger ON etl_job_records" + )); + assertTrue(rehearsal.contains( + "DROP FUNCTION validate_etl_job_replay_lineage()" + )); + assertTrue(rehearsal.contains( + "ALTER TABLE etl_job_records DROP CONSTRAINT etl_job_replay_source_reference" + )); + assertTrue(rehearsal.contains( + "ALTER TABLE etl_job_records DROP COLUMN replay_source_job_record_id" + )); + assertTrue(rehearsal.contains("ROLLBACK")); + assertTrue(rehearsal.contains("rollback rehearsal did not restore V7")); + } + + private static String read(String relativePath) throws IOException { + return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8); + } + + /** + * Finds the repository root from reactor-root or module-local Maven execution. + * + * @return repository root that contains workflows, scripts, and test fixtures + */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/documentation/PostgresqlMigrationIntegrationWorkflowTest.java b/etl-service/src/test/java/com/xtrmetl/etl/documentation/PostgresqlMigrationIntegrationWorkflowTest.java new file mode 100644 index 00000000..f032d114 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/documentation/PostgresqlMigrationIntegrationWorkflowTest.java @@ -0,0 +1,113 @@ +package com.xtrmetl.etl.documentation; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Keeps the real PostgreSQL migration gate aligned with replay-lineage safety requirements. + */ +class PostgresqlMigrationIntegrationWorkflowTest { + + @Test + void workflowUsesLeastPrivilegeExactHeadCheckoutAndPostgresqlEighteen() throws IOException { + String workflow = read(".github/workflows/postgresql-migration-integration.yml"); + + assertTrue(workflow.contains("name: PostgreSQL Migration Integration")); + assertTrue(workflow.contains("branches:\n - develop")); + assertTrue(workflow.contains("permissions:\n contents: read")); + assertTrue(workflow.contains("timeout-minutes: 15")); + assertTrue(workflow.contains("image: postgres:18-alpine")); + assertTrue(workflow.contains( + "if: github.event_name != 'workflow_dispatch' || " + + "github.ref_name == github.event.repository.default_branch" + )); + assertTrue(workflow.contains( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" + )); + assertTrue(workflow.contains( + "repository: ${{ github.event_name == 'pull_request' " + + "&& github.event.pull_request.head.repo.full_name || github.repository }}" + )); + assertTrue(workflow.contains( + "ref: ${{ github.event_name == 'pull_request' " + + "&& github.event.pull_request.head.sha " + + "|| github.event.repository.default_branch }}" + )); + assertTrue(workflow.contains("persist-credentials: false")); + assertTrue(workflow.contains("bash scripts/verify-postgresql-migrations.sh")); + assertFalse(workflow.contains("refs/pull/")); + assertFalse(workflow.contains("github.event.pull_request.merge_commit_sha")); + assertFalse(workflow.contains("pull_request_target:")); + assertFalse(workflow.contains("COPILOT_GITHUB_TOKEN")); + assertFalse(workflow.contains("NVIDIA_NIM_API_KEY")); + assertFalse(workflow.contains("contents: write")); + } + + @Test + void verificationScriptAppliesEveryMigrationAndChecksReplayConstraints() throws IOException { + String script = read("scripts/verify-postgresql-migrations.sh"); + + assertTrue(script.contains("set -Eeuo pipefail")); + assertTrue(script.contains("pg_isready")); + assertTrue(script.contains("sort -zV")); + assertTrue(script.contains("--set ON_ERROR_STOP=1")); + assertTrue(script.contains("replay_source_job_record_id")); + assertTrue(script.contains("replay_root_job_record_id")); + assertTrue(script.contains("replay_generation_count")); + assertTrue(script.contains("constraint_record.confdeltype = 'r'")); + assertTrue(script.contains("replay_check_definition NOT ILIKE '%100%'")); + assertTrue(script.contains("cancellation_key_hash")); + assertTrue(script.contains("job_cancelled_at")); + assertTrue(script.contains("pg_dump --schema-only --no-owner --no-privileges")); + assertFalse(script.contains("set +e")); + assertFalse(script.contains("|| true")); + } + + /** + * Reads one repository contract with platform-independent line endings. + * + *

Git may materialize text files with CRLF on Windows runners. Normalizing both CRLF + * and lone carriage returns keeps semantic workflow assertions identical across the CI + * operating-system matrix without weakening their exact content requirements.

+ * + * @param relativePath repository-relative file path + * @return UTF-8 content using LF line endings + * @throws IOException when the repository contract cannot be read + */ + private static String read(String relativePath) throws IOException { + return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + } + + /** + * Finds the Maven reactor root from repository-root or module-local execution. + * + * @return repository root containing the workflow and migration verifier + */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobCoveragePolicyTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobCoveragePolicyTest.java index 74fa5e0a..9f53a5b6 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobCoveragePolicyTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobCoveragePolicyTest.java @@ -36,6 +36,7 @@ class EtlJobCoveragePolicyTest { private static final Set DURABLE_JOB_CLASS_FILES = Set.of( "com/xtrmetl/etl/job/*.class", "com/xtrmetl/etl/controller/EtlJobController*.class", + "com/xtrmetl/etl/controller/EtlJobReplayController*.class", "com/xtrmetl/etl/service/Sha256Digest*.class" ); diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayBoundaryTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayBoundaryTest.java new file mode 100644 index 00000000..6985c256 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayBoundaryTest.java @@ -0,0 +1,228 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * Covers immutable replay-result construction and fail-closed admission before persistence. + */ +class EtlJobReplayBoundaryTest { + + private static final UUID SOURCE_ID = UUID.fromString( + "cf4f083f-8c90-4f34-a8b6-b53761de44ef" + ); + private static final String PAYLOAD = "[{\"id\":\"record_alpha\"}]"; + private static final String REPLAY_KEY = "1e05bdca-447c-4ad3-882c-e33963ce517c"; + + @AfterEach + void clearSyntheticTransactionState() { + TransactionSynchronizationManager.clear(); + } + + @Test + void replayResultRequiresIdentityAndStatusButPreservesCurrentState() { + EtlJobReplay pending = new EtlJobReplay(SOURCE_ID, EtlJobStatus.PENDING, false); + EtlJobReplay terminalReplay = new EtlJobReplay( + SOURCE_ID, + EtlJobStatus.SUCCEEDED, + true + ); + + assertEquals(EtlJobStatus.PENDING, pending.jobStatus()); + assertEquals(EtlJobStatus.SUCCEEDED, terminalReplay.jobStatus()); + assertThrows( + NullPointerException.class, + () -> new EtlJobReplay(null, EtlJobStatus.PENDING, false) + ); + assertThrows( + NullPointerException.class, + () -> new EtlJobReplay(SOURCE_ID, null, false) + ); + } + + @Test + void validatesIdentityKeyPrincipalAndPayloadBeforeDatabaseWork() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + EtlJobReplayService service = service(jdbcTemplate, lockHash -> true); + + assertThrows( + NullPointerException.class, + () -> service.replayOwned(null, PAYLOAD, REPLAY_KEY, "tenant_alpha") + ); + assertError( + EtlRequestError.JOB_REPLAY_KEY_REQUIRED, + () -> service.replayOwned(SOURCE_ID, PAYLOAD, null, "tenant_alpha") + ); + assertError( + EtlRequestError.JOB_REPLAY_KEY_REQUIRED, + () -> service.replayOwned(SOURCE_ID, PAYLOAD, "unsafe key", "tenant_alpha") + ); + assertError( + EtlRequestError.IDEMPOTENCY_PRINCIPAL_REQUIRED, + () -> service.replayOwned(SOURCE_ID, PAYLOAD, REPLAY_KEY, null) + ); + assertError( + EtlRequestError.IDEMPOTENCY_PRINCIPAL_REQUIRED, + () -> service.replayOwned(SOURCE_ID, PAYLOAD, REPLAY_KEY, " ".repeat(513)) + ); + assertError( + EtlRequestError.IDEMPOTENCY_PRINCIPAL_REQUIRED, + () -> service.replayOwned(SOURCE_ID, PAYLOAD, REPLAY_KEY, "a".repeat(513)) + ); + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, null, REPLAY_KEY, "tenant_alpha") + ); + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, "", REPLAY_KEY, "tenant_alpha") + ); + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, "null", REPLAY_KEY, "tenant_alpha") + ); + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, "not-json", REPLAY_KEY, "tenant_alpha") + ); + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, "{}", REPLAY_KEY, "tenant_alpha") + ); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void rejectsAnAbsentParsedRootBeforeDatabaseWork() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + ObjectMapper absentRootMapper = new ObjectMapper() { + @Override + public ObjectMapper copy() { + return this; + } + + @Override + public JsonNode readTree(String content) { + return null; + } + }; + EtlJobReplayService service = new EtlJobReplayService( + jdbcTemplate, + absentRootMapper, + new EtlBatchProperties(), + lockHash -> true + ); + + assertError( + EtlRequestError.INVALID_JSON, + () -> service.replayOwned(SOURCE_ID, "[]", REPLAY_KEY, "tenant_alpha") + ); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void requiresAnActualTransactionBeforeLockOrTableAccess() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + EtlJobReplayService service = service(jdbcTemplate, lockHash -> true); + + IllegalStateException nonEmptyBatch = assertThrows( + IllegalStateException.class, + () -> service.replayOwned( + SOURCE_ID, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + IllegalStateException emptyBatch = assertThrows( + IllegalStateException.class, + () -> service.replayOwned( + SOURCE_ID, + "[]", + REPLAY_KEY, + "tenant_alpha" + ) + ); + + assertEquals( + "Durable ETL job replay requires an active transaction", + nonEmptyBatch.getMessage() + ); + assertEquals(nonEmptyBatch.getMessage(), emptyBatch.getMessage()); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void reportsAnUnavailableReplayLockWithoutTableAccess() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + EtlJobReplayService service = service(jdbcTemplate, lockHash -> false); + TransactionSynchronizationManager.setActualTransactionActive(true); + + assertError( + EtlRequestError.JOB_REPLAY_IN_PROGRESS, + () -> service.replayOwned( + SOURCE_ID, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void constructorsRejectMissingCollaborators() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + ObjectMapper mapper = new ObjectMapper(); + EtlBatchProperties properties = new EtlBatchProperties(); + + assertThrows( + NullPointerException.class, + () -> new EtlJobReplayService(null, mapper, properties, hash -> true) + ); + assertThrows( + NullPointerException.class, + () -> new EtlJobReplayService(jdbcTemplate, null, properties, hash -> true) + ); + assertThrows( + NullPointerException.class, + () -> new EtlJobReplayService(jdbcTemplate, mapper, null, hash -> true) + ); + assertThrows( + NullPointerException.class, + () -> new EtlJobReplayService(jdbcTemplate, mapper, properties, null) + ); + new EtlJobReplayService(jdbcTemplate, mapper, properties); + } + + private static EtlJobReplayService service( + JdbcTemplate jdbcTemplate, + com.xtrmetl.etl.service.EtlRequestLock requestLock + ) { + return new EtlJobReplayService( + jdbcTemplate, + new ObjectMapper(), + new EtlBatchProperties(), + requestLock + ); + } + + private static void assertError(EtlRequestError expected, Runnable invocation) { + EtlRequestException exception = assertThrows(EtlRequestException.class, invocation::run); + assertEquals(expected, exception.error()); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayClaimIntegrationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayClaimIntegrationTest.java new file mode 100644 index 00000000..b5fb617e --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayClaimIntegrationTest.java @@ -0,0 +1,180 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestLock; +import com.xtrmetl.etl.service.Sha256Digest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; +import java.time.Duration; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Proves replay admission creates an ordinary pending job that the existing worker can claim. + */ +@SpringJUnitConfig(EtlJobReplayClaimIntegrationTest.TestConfiguration.class) +class EtlJobReplayClaimIntegrationTest { + + private static final String PAYLOAD = "[{\"id\":\"record_alpha\"}]"; + private static final String REPLAY_KEY = "1e05bdca-447c-4ad3-882c-e33963ce517c"; + + private final EtlJobReplayService replayService; + private final EtlJobLeaseRepository leaseRepository; + private final JdbcTemplate jdbcTemplate; + + @Autowired + EtlJobReplayClaimIntegrationTest( + EtlJobReplayService replayService, + EtlJobLeaseRepository leaseRepository, + JdbcTemplate jdbcTemplate + ) { + this.replayService = replayService; + this.leaseRepository = leaseRepository; + this.jdbcTemplate = jdbcTemplate; + } + + @BeforeEach + void createJobTable() { + jdbcTemplate.execute("DROP TABLE IF EXISTS etl_job_records"); + jdbcTemplate.execute(""" + CREATE TABLE etl_job_records ( + job_record_id UUID PRIMARY KEY, + principal_scope_hash CHAR(64) NOT NULL, + submission_key_hash CHAR(64) NOT NULL, + request_digest CHAR(64) NOT NULL, + request_payload CLOB, + job_status VARCHAR(32) NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + failure_code VARCHAR(128), + lease_claim_id UUID, + lease_owner_id VARCHAR(128), + lease_expires_at TIMESTAMP WITH TIME ZONE, + cancellation_key_hash CHAR(64), + cancellation_code VARCHAR(128), + job_cancelled_at TIMESTAMP WITH TIME ZONE, + replay_source_job_record_id UUID, + replay_root_job_record_id UUID, + replay_generation_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT etl_job_submission_scope_unique + UNIQUE (principal_scope_hash, submission_key_hash) + ) + """); + } + + @Test + void replayCreatedPendingJobIsClaimedWithoutAReplaySpecificWorkerPath() { + UUID sourceId = UUID.fromString("c0b2860a-fd63-431a-96cb-48f3f4d7b19d"); + jdbcTemplate.update( + """ + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, attempt_count, + failure_code + ) VALUES (?, ?, ?, ?, NULL, 'FAILED', 1, 'etl_target_failure') + """, + sourceId, + Sha256Digest.digest("tenant_alpha"), + Sha256Digest.digest("source-key"), + Sha256Digest.digest(PAYLOAD) + ); + + EtlJobReplay replay = replayService.replayOwned( + sourceId, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + EtlJobLease lease = leaseRepository.claimNext( + "worker-alpha", + Duration.ofMinutes(5), + 3 + ).orElseThrow(); + + assertEquals(replay.jobRecordId(), lease.jobRecordId()); + assertEquals(PAYLOAD, lease.requestPayload()); + assertEquals(1, lease.attemptCount()); + assertEquals(sourceId, jdbcTemplate.queryForObject( + "SELECT replay_source_job_record_id FROM etl_job_records WHERE job_record_id=?", + UUID.class, + replay.jobRecordId() + )); + } + + /** Minimal transaction-enabled context for replay and worker claim integration. */ + @Configuration + @EnableTransactionManagement + static class TestConfiguration { + + @Bean + DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .generateUniqueName(true) + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + EtlBatchProperties etlBatchProperties() { + return new EtlBatchProperties(); + } + + @Bean + EtlRequestLock etlRequestLock() { + return lockHash -> true; + } + + @Bean + EtlJobReplayService replayService( + JdbcTemplate jdbcTemplate, + ObjectMapper objectMapper, + EtlBatchProperties batchProperties, + EtlRequestLock requestLock + ) { + return new EtlJobReplayService( + jdbcTemplate, + objectMapper, + batchProperties, + requestLock + ); + } + + @Bean + EtlJobLeaseRepository leaseRepository( + JdbcTemplate jdbcTemplate, + PlatformTransactionManager transactionManager + ) { + return new EtlJobLeaseRepository(jdbcTemplate, transactionManager); + } + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayControllerTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayControllerTest.java new file mode 100644 index 00000000..b4271785 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayControllerTest.java @@ -0,0 +1,143 @@ +package com.xtrmetl.etl.job; + +import com.xtrmetl.etl.controller.EtlApiProblemHandler; +import com.xtrmetl.etl.controller.EtlJobReplayController; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.dao.DataAccessException; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.security.Principal; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +/** + * Defines success, replay, authentication, identifier, and failure behavior for replay admission. + */ +class EtlJobReplayControllerTest { + + private static final UUID SOURCE_ID = UUID.fromString( + "cf4f083f-8c90-4f34-a8b6-b53761de44ef" + ); + private static final UUID NEW_JOB_ID = UUID.fromString( + "86e4d474-dabf-4d6a-9de4-4e8230589363" + ); + private static final String PAYLOAD = "[{\"id\":\"record_alpha\"}]"; + private static final String REPLAY_KEY = "\"1e05bdca-447c-4ad3-882c-e33963ce517c\""; + private static final Principal PRINCIPAL = () -> "tenant_alpha"; + + private EtlJobReplayService replayService; + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + replayService = mock(EtlJobReplayService.class); + mockMvc = MockMvcBuilders + .standaloneSetup(new EtlJobReplayController(replayService)) + .setControllerAdvice(new EtlApiProblemHandler()) + .build(); + } + + @Test + void acceptsANewReplayAndReturnsItsStatusMonitor() throws Exception { + when(replayService.replayOwned(SOURCE_ID, PAYLOAD, REPLAY_KEY, "tenant_alpha")) + .thenReturn(new EtlJobReplay(NEW_JOB_ID, EtlJobStatus.PENDING, false)); + + mockMvc.perform(request(SOURCE_ID).principal(PRINCIPAL)) + .andExpect(status().isAccepted()) + .andExpect(header().string("Cache-Control", "no-store")) + .andExpect(header().string("Location", "/api/etl/jobs/" + NEW_JOB_ID)) + .andExpect(header().string("Idempotency-Replayed", "false")) + .andExpect(jsonPath("$.jobRecordId").value(NEW_JOB_ID.toString())) + .andExpect(jsonPath("$.jobStatus").value("PENDING")) + .andExpect(jsonPath("$.statusUrl").value("/api/etl/jobs/" + NEW_JOB_ID)); + + verify(replayService).replayOwned( + SOURCE_ID, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + } + + @Test + void returnsTheCurrentStatusWhenAReplayRequestIsRepeatedLater() throws Exception { + when(replayService.replayOwned(SOURCE_ID, PAYLOAD, REPLAY_KEY, "tenant_alpha")) + .thenReturn(new EtlJobReplay(NEW_JOB_ID, EtlJobStatus.SUCCEEDED, true)); + + mockMvc.perform(request(SOURCE_ID).principal(PRINCIPAL)) + .andExpect(status().isAccepted()) + .andExpect(header().string("Idempotency-Replayed", "true")) + .andExpect(jsonPath("$.jobStatus").value("SUCCEEDED")); + } + + @Test + void rejectsAuthenticationKeyAndMalformedIdentifierBeforeServiceAccess() throws Exception { + mockMvc.perform(request(SOURCE_ID)) + .andExpect(status().isUnauthorized()) + .andExpect(jsonPath("$.errorCode").value( + "etl_idempotency_principal_required" + )); + mockMvc.perform(post("/api/etl/jobs/" + SOURCE_ID + "/replays") + .principal(PRINCIPAL) + .contentType(MediaType.APPLICATION_JSON) + .content(PAYLOAD)) + .andExpect(status().isBadRequest()) + .andExpect(jsonPath("$.errorCode").value("etl_job_replay_key_required")); + mockMvc.perform(post("/api/etl/jobs/not-a-uuid/replays") + .principal(PRINCIPAL) + .header("Idempotency-Key", REPLAY_KEY) + .contentType(MediaType.APPLICATION_JSON) + .content(PAYLOAD)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.errorCode").value("etl_job_not_found")); + + verifyNoInteractions(replayService); + } + + @Test + void preservesTypedFailuresAndSanitizesDatabaseAndUnexpectedFailures() throws Exception { + when(replayService.replayOwned(any(UUID.class), anyString(), anyString(), anyString())) + .thenThrow(new EtlRequestException(EtlRequestError.JOB_REPLAY_PAYLOAD_MISMATCH)) + .thenThrow(new DataAccessException("secret database detail") { }) + .thenThrow(new IllegalStateException("secret runtime detail")); + + mockMvc.perform(request(SOURCE_ID).principal(PRINCIPAL)) + .andExpect(status().isUnprocessableEntity()) + .andExpect(jsonPath("$.errorCode").value("etl_job_replay_payload_mismatch")); + mockMvc.perform(request(SOURCE_ID).principal(PRINCIPAL)) + .andExpect(status().isInternalServerError()) + .andExpect(jsonPath("$.errorCode").value("etl_target_failure")) + .andExpect(jsonPath("$.detail").value( + "The ETL target could not process the request." + )); + mockMvc.perform(request(SOURCE_ID).principal(PRINCIPAL)) + .andExpect(status().isInternalServerError()) + .andExpect(jsonPath("$.errorCode").value("etl_internal_error")) + .andExpect(jsonPath("$.detail").value( + "The ETL request could not be processed." + )); + } + + private static org.springframework.test.web.servlet.request.MockHttpServletRequestBuilder + request(UUID sourceId) { + return post("/api/etl/jobs/" + sourceId + "/replays") + .header("Idempotency-Key", REPLAY_KEY) + .contentType(MediaType.APPLICATION_JSON) + .content(PAYLOAD); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayDigestContinuityMigrationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayDigestContinuityMigrationTest.java new file mode 100644 index 00000000..2c6b53fd --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayDigestContinuityMigrationTest.java @@ -0,0 +1,76 @@ +package com.xtrmetl.etl.job; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Guards database-level digest continuity for immutable durable-job replay lineage. + * + *

Application admission already compares the resupplied payload digest with the terminal + * source. The database trigger is an independent integrity boundary for imports and other direct + * writers, so every derived row must retain the exact immediate source request digest.

+ */ +class EtlJobReplayDigestContinuityMigrationTest { + + @Test + void lineageTriggerRequiresImmediateSourceDigestEquality() throws IOException { + String migration = normalize(read( + "etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql" + )); + + assertTrue(migration.contains("source_request_digest")); + assertTrue(migration.contains("request_digest")); + assertTrue(migration.contains( + "source_request_digest IS DISTINCT FROM NEW.request_digest" + )); + assertTrue(migration.contains("Replay request digest must match the immediate source")); + } + + @Test + void postgresqlRehearsalRejectsMismatchedReplayDigest() throws IOException { + String rehearsal = normalize(read( + "etl-service/src/test/postgresql/replay_lineage_migration.sql" + )); + + assertTrue(rehearsal.contains("digest_continuity_check")); + assertTrue(rehearsal.contains("replay digest mismatch was accepted")); + } + + private static String read(String relativePath) throws IOException { + return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8); + } + + private static String normalize(String value) { + return value.replaceAll("\\s+", " ").trim(); + } + + /** + * Finds the reactor root from repository-root or module-local Maven execution. + * + * @return repository root containing migrations and integration fixtures + */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLineageIntegrityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLineageIntegrityTest.java new file mode 100644 index 00000000..c9afdc7e --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLineageIntegrityTest.java @@ -0,0 +1,188 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestLock; +import com.xtrmetl.etl.service.Sha256Digest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Proves replay admission rejects inherited roots that are themselves replay rows. + * + *

This is a defense-in-depth contract for repositories or migrations that import historical + * rows before PostgreSQL's lineage trigger is available. A replay root must be the immutable first + * job in the lineage, never another derived replay row.

+ */ +@SpringJUnitConfig(EtlJobReplayLineageIntegrityTest.TestConfiguration.class) +class EtlJobReplayLineageIntegrityTest { + + private static final String PAYLOAD = "[{\"id\":\"record_alpha\"}]"; + private static final String REPLAY_KEY = "94ccf28c-9649-4a06-b06f-11e70c57c5d2"; + private static final String PRINCIPAL_SCOPE = "tenant_alpha"; + + private final EtlJobReplayService replayService; + private final JdbcTemplate jdbcTemplate; + + @Autowired + EtlJobReplayLineageIntegrityTest( + EtlJobReplayService replayService, + JdbcTemplate jdbcTemplate + ) { + this.replayService = replayService; + this.jdbcTemplate = jdbcTemplate; + } + + @BeforeEach + void createJobTable() { + jdbcTemplate.execute("DROP TABLE IF EXISTS etl_job_records"); + jdbcTemplate.execute(""" + CREATE TABLE etl_job_records ( + job_record_id UUID PRIMARY KEY, + principal_scope_hash CHAR(64) NOT NULL, + submission_key_hash CHAR(64) NOT NULL, + request_digest CHAR(64) NOT NULL, + request_payload CLOB, + job_status VARCHAR(32) NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + failure_code VARCHAR(128), + replay_source_job_record_id UUID, + replay_root_job_record_id UUID, + replay_generation_count INTEGER, + CONSTRAINT etl_job_submission_scope_unique + UNIQUE (principal_scope_hash, submission_key_hash) + ) + """); + } + + @Test + void rejectsAnInheritedRootThatIsItselfAReplay() { + UUID rootJobRecordId = insertFailedJob(null, null, null); + UUID generationOneJobRecordId = insertFailedJob( + rootJobRecordId, + rootJobRecordId, + 1 + ); + UUID malformedGenerationTwoJobRecordId = insertFailedJob( + generationOneJobRecordId, + generationOneJobRecordId, + 2 + ); + + IllegalStateException exception = assertThrows( + IllegalStateException.class, + () -> replayService.replayOwned( + malformedGenerationTwoJobRecordId, + PAYLOAD, + REPLAY_KEY, + PRINCIPAL_SCOPE + ) + ); + + assertEquals("Replay root is not a lineage root", exception.getMessage()); + } + + private UUID insertFailedJob( + UUID replaySourceJobRecordId, + UUID replayRootJobRecordId, + Integer replayGenerationCount + ) { + UUID jobRecordId = UUID.randomUUID(); + jdbcTemplate.update( + """ + INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + attempt_count, + failure_code, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count + ) VALUES (?, ?, ?, ?, NULL, 'FAILED', 0, ?, ?, ?, ?) + """, + jobRecordId, + Sha256Digest.digest(PRINCIPAL_SCOPE), + Sha256Digest.digest(UUID.randomUUID().toString()), + Sha256Digest.digest(PAYLOAD), + "etl_target_failure", + replaySourceJobRecordId, + replayRootJobRecordId, + replayGenerationCount + ); + return jobRecordId; + } + + /** Minimal transaction-enabled context for replay-lineage integrity verification. */ + @Configuration + @EnableTransactionManagement + static class TestConfiguration { + + @Bean + DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .generateUniqueName(true) + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + EtlBatchProperties etlBatchProperties() { + return new EtlBatchProperties(); + } + + @Bean + EtlRequestLock etlRequestLock() { + return lockHash -> true; + } + + @Bean + EtlJobReplayService replayService( + JdbcTemplate jdbcTemplate, + ObjectMapper objectMapper, + EtlBatchProperties batchProperties, + EtlRequestLock requestLock + ) { + return new EtlJobReplayService( + jdbcTemplate, + objectMapper, + batchProperties, + requestLock + ); + } + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLookupIndexMigrationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLookupIndexMigrationTest.java new file mode 100644 index 00000000..dd4dae28 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayLookupIndexMigrationTest.java @@ -0,0 +1,155 @@ +package com.xtrmetl.etl.job; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Guards nonblocking PostgreSQL indexes for replay descendant and foreign-key lookups. + * + *

The lineage trigger checks descendants whenever durable terminal evidence changes. Without + * indexes beginning with the source and root identifiers, ordinary lifecycle updates can degrade + * into full-table scans as durable-job history grows. Each index is isolated from the transactional + * lineage schema and from the other index so a concurrent-build failure has one auditable Flyway + * repair boundary.

+ */ +class EtlJobReplayLookupIndexMigrationTest { + + private static final String V7_MIGRATION = + "etl-service/src/main/resources/db/migration/V7__add_etl_job_replay_lineage.sql"; + private static final String V8_MIGRATION = + "etl-service/src/main/resources/db/migration/" + + "V8__add_etl_job_replay_source_lookup_index.sql"; + private static final String V9_MIGRATION = + "etl-service/src/main/resources/db/migration/" + + "V9__add_etl_job_replay_root_lookup_index.sql"; + private static final String V8_CONFIGURATION = V8_MIGRATION + ".conf"; + private static final String V9_CONFIGURATION = V9_MIGRATION + ".conf"; + + @Test + void separatesTransactionalLineageFromConcurrentLookupIndexes() throws IOException { + String lineageMigration = normalize(read(V7_MIGRATION)); + String sourceIndexMigration = normalize(read(V8_MIGRATION)); + String rootIndexMigration = normalize(read(V9_MIGRATION)); + + assertFalse( + lineageMigration.contains("CREATE INDEX"), + "the transactional lineage migration must not contain a production index build" + ); + assertTrue(sourceIndexMigration.contains( + "CREATE INDEX CONCURRENTLY etl_job_replay_source_lookup_index" + )); + assertTrue(sourceIndexMigration.contains( + "ON etl_job_records ( replay_source_job_record_id, principal_scope_hash )" + )); + assertTrue(sourceIndexMigration.contains( + "WHERE replay_source_job_record_id IS NOT NULL" + )); + assertFalse( + sourceIndexMigration.contains("etl_job_replay_root_lookup_index"), + "one nontransactional migration must own only one concurrent index build" + ); + + assertTrue(rootIndexMigration.contains( + "CREATE INDEX CONCURRENTLY etl_job_replay_root_lookup_index" + )); + assertTrue(rootIndexMigration.contains( + "ON etl_job_records ( replay_root_job_record_id, principal_scope_hash )" + )); + assertTrue(rootIndexMigration.contains( + "WHERE replay_root_job_record_id IS NOT NULL" + )); + assertFalse( + rootIndexMigration.contains("etl_job_replay_source_lookup_index"), + "one nontransactional migration must own only one concurrent index build" + ); + } + + @Test + void disablesFlywayTransactionForEachConcurrentReplayIndex() throws IOException { + String applicationProperties = read( + "etl-service/src/main/resources/application.properties" + ); + + for (String configuration : new String[]{V8_CONFIGURATION, V9_CONFIGURATION}) { + Path configurationPath = projectRoot().resolve(configuration); + assertTrue( + Files.exists(configurationPath), + "each concurrent replay-index migration requires a Flyway script configuration" + ); + assertTrue( + Files.readString(configurationPath, StandardCharsets.UTF_8) + .contains("executeInTransaction=false") + ); + } + assertTrue(applicationProperties.contains( + "spring.flyway.postgresql.transactional-lock=false" + )); + } + + @Test + void verifierRequiresReadyAndValidReplayLookupIndexes() throws IOException { + String verifier = normalize(read("scripts/verify-postgresql-migrations.sh")); + + assertTrue(verifier.contains("etl_job_replay_source_lookup_index")); + assertTrue(verifier.contains("etl_job_replay_root_lookup_index")); + assertTrue(verifier.contains("index_record.indisready")); + assertTrue(verifier.contains("index_record.indisvalid")); + assertTrue(verifier.contains("replay lookup indexes are missing or invalid")); + } + + @Test + void runbookDocumentsConcurrentFailureRecoveryAndRollback() throws IOException { + String runbook = normalize(read("docs/operations/durable-job-replay.md")); + + assertTrue(runbook.contains("V8__add_etl_job_replay_source_lookup_index.sql")); + assertTrue(runbook.contains("V9__add_etl_job_replay_root_lookup_index.sql")); + assertTrue(runbook.contains("CREATE INDEX CONCURRENTLY")); + assertTrue(runbook.contains("invalid index")); + assertTrue(runbook.contains( + "DROP INDEX CONCURRENTLY etl_job_replay_source_lookup_index" + )); + assertTrue(runbook.contains( + "DROP INDEX CONCURRENTLY etl_job_replay_root_lookup_index" + )); + assertTrue(runbook.contains("executeInTransaction=false")); + } + + private static String read(String relativePath) throws IOException { + return Files.readString(projectRoot().resolve(relativePath), StandardCharsets.UTF_8); + } + + private static String normalize(String value) { + return value.replaceAll("\\s+", " ").trim(); + } + + /** + * Finds the reactor root from repository-root or module-local Maven execution. + * + * @return repository root containing migrations, scripts, and documentation + */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayMigrationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayMigrationTest.java new file mode 100644 index 00000000..d8b4ea9e --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayMigrationTest.java @@ -0,0 +1,159 @@ +package com.xtrmetl.etl.job; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Guards the complete, bounded, owner-scoped, non-cascading durable job replay lineage schema. + */ +class EtlJobReplayMigrationTest { + + @Test + void replayMigrationAddsCompleteRestrictedOwnerScopedLineage() throws IOException { + String migration = normalizedMigration(); + + assertTrue(migration.contains("ADD COLUMN replay_source_job_record_id UUID")); + assertTrue(migration.contains("ADD COLUMN replay_root_job_record_id UUID")); + assertTrue(migration.contains("ADD COLUMN replay_generation_count INTEGER")); + assertTrue(migration.contains("CONSTRAINT etl_job_owner_identity_unique")); + assertTrue(migration.contains( + "UNIQUE (job_record_id, principal_scope_hash)" + )); + assertTrue(migration.contains("CONSTRAINT etl_job_replay_source_reference")); + assertTrue(migration.contains( + "FOREIGN KEY (replay_source_job_record_id, principal_scope_hash) " + + "REFERENCES etl_job_records (job_record_id, principal_scope_hash) " + + "ON DELETE RESTRICT" + )); + assertTrue(migration.contains("CONSTRAINT etl_job_replay_root_reference")); + assertTrue(migration.contains( + "FOREIGN KEY (replay_root_job_record_id, principal_scope_hash) " + + "REFERENCES etl_job_records (job_record_id, principal_scope_hash) " + + "ON DELETE RESTRICT" + )); + assertTrue(migration.contains("CONSTRAINT etl_job_replay_lineage_complete_check")); + assertTrue(migration.contains("replay_source_job_record_id IS NULL")); + assertTrue(migration.contains("replay_root_job_record_id IS NULL")); + assertTrue(migration.contains("replay_generation_count IS NULL")); + assertTrue(migration.contains("replay_source_job_record_id IS NOT NULL")); + assertTrue(migration.contains("replay_root_job_record_id IS NOT NULL")); + assertTrue(migration.contains("replay_generation_count BETWEEN 1 AND 100")); + assertTrue(migration.contains("replay_source_job_record_id <> job_record_id")); + assertTrue(migration.contains("replay_root_job_record_id <> job_record_id")); + assertTrue(migration.contains( + "CREATE FUNCTION validate_etl_job_replay_lineage() RETURNS trigger" + )); + assertTrue(migration.contains( + "CREATE TRIGGER etl_job_replay_lineage_guard_trigger" + )); + assertTrue(migration.contains( + "BEFORE INSERT OR UPDATE OF replay_source_job_record_id, " + + "replay_root_job_record_id, replay_generation_count, job_status, " + + "request_digest, request_payload, attempt_count, failure_code, " + + "cancellation_key_hash, cancellation_code, job_cancelled_at, " + + "created_at, updated_at" + )); + assertTrue(migration.contains( + "NEW.replay_source_job_record_id <> NEW.replay_root_job_record_id" + )); + assertTrue(migration.contains( + "source_generation_count IS DISTINCT FROM NEW.replay_generation_count - 1" + )); + assertTrue(migration.contains("Replay lineage fields are immutable")); + assertTrue(migration.contains("Referenced replay evidence is immutable")); + assertTrue(migration.contains("FOR UPDATE")); + assertTrue(migration.contains( + "child_record.replay_source_job_record_id = OLD.job_record_id" + )); + assertTrue(migration.contains( + "child_record.replay_root_job_record_id = OLD.job_record_id" + )); + assertFalse(migration.contains("FOR KEY SHARE")); + assertFalse(migration.contains( + "FOREIGN KEY (replay_source_job_record_id) " + + "REFERENCES etl_job_records (job_record_id)" + )); + assertFalse(migration.contains( + "FOREIGN KEY (replay_root_job_record_id) " + + "REFERENCES etl_job_records (job_record_id)" + )); + assertFalse(migration.contains("ON DELETE CASCADE")); + assertFalse(migration.contains("replay_payload")); + assertFalse(migration.contains("principal_name")); + } + + @Test + void updateGuardAvoidsChildToAncestorLockInversion() throws IOException { + String migration = normalizedMigration(); + Pattern updateReturnsBeforeInsertValidation = Pattern.compile( + "IF TG_OP = 'UPDATE' THEN .*Referenced replay evidence is immutable.*" + + "RETURN NEW; END IF; IF NEW\\.replay_generation_count IS NULL" + ); + Pattern descendantLookupTakesRowLock = Pattern.compile( + "FROM etl_job_records AS child_record .*FOR UPDATE;.*IF FOUND THEN" + ); + + assertTrue( + updateReturnsBeforeInsertValidation.matcher(migration).find(), + "UPDATE validation must return before INSERT-only source/root locking" + ); + assertFalse( + descendantLookupTakesRowLock.matcher(migration).find(), + "Referenced-child existence checks must not lock child rows in reverse order" + ); + } + + @Test + void referencedEvidenceLookupRunsOnlyForReplayEligibleTerminalRows() throws IOException { + String migration = normalizedMigration(); + Pattern terminalEvidenceGuard = Pattern.compile( + "IF OLD\\.job_status IN \\('FAILED', 'CANCELLED'\\) AND \\( " + + "OLD\\.job_status IS DISTINCT FROM NEW\\.job_status .*" + + "OLD\\.updated_at IS DISTINCT FROM NEW\\.updated_at \\) THEN .*?" + + "PERFORM 1 FROM etl_job_records AS child_record" + ); + + assertTrue( + terminalEvidenceGuard.matcher(migration).find(), + "Only replay-eligible terminal rows should pay the descendant lookup cost" + ); + } + + private static String normalizedMigration() throws IOException { + return Files.readString( + projectRoot().resolve( + "etl-service/src/main/resources/db/migration/" + + "V7__add_etl_job_replay_lineage.sql" + ), + StandardCharsets.UTF_8 + ).replaceAll("\\s+", " "); + } + + /** @return reactor root from repository-root or module-local execution */ + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayPayloadBoundaryTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayPayloadBoundaryTest.java new file mode 100644 index 00000000..944416d1 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayPayloadBoundaryTest.java @@ -0,0 +1,97 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.transaction.support.TransactionSynchronizationManager; + +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyNoInteractions; + +/** + * Covers replay payload byte, record-count, and ordinary record-contract rejection before JDBC. + */ +class EtlJobReplayPayloadBoundaryTest { + + private static final UUID SOURCE_ID = UUID.fromString( + "cf4f083f-8c90-4f34-a8b6-b53761de44ef" + ); + private static final String REPLAY_KEY = "1e05bdca-447c-4ad3-882c-e33963ce517c"; + + @AfterEach + void clearSyntheticTransactionState() { + TransactionSynchronizationManager.clear(); + } + + @Test + void rejectsOversizedPayloadBeforeLockOrTableAccess() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + EtlBatchProperties properties = new EtlBatchProperties(); + properties.setMaxPayloadBytes(8); + EtlJobReplayService service = service(jdbcTemplate, properties); + + assertError( + EtlRequestError.PAYLOAD_TOO_LARGE, + () -> service.replayOwned( + SOURCE_ID, + "[{\"id\":\"record_alpha\"}]", + REPLAY_KEY, + "tenant_alpha" + ) + ); + verifyNoInteractions(jdbcTemplate); + } + + @Test + void rejectsOversizedBatchAndInvalidRecordsBeforeLockOrTableAccess() { + JdbcTemplate jdbcTemplate = mock(JdbcTemplate.class); + EtlBatchProperties properties = new EtlBatchProperties(); + properties.setMaxBatchRecords(1); + EtlJobReplayService service = service(jdbcTemplate, properties); + + assertError( + EtlRequestError.BATCH_TOO_LARGE, + () -> service.replayOwned( + SOURCE_ID, + "[{\"id\":\"a\"},{\"id\":\"b\"}]", + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertError( + EtlRequestError.INVALID_RECORD, + () -> service.replayOwned( + SOURCE_ID, + "[{}]", + REPLAY_KEY, + "tenant_alpha" + ) + ); + verifyNoInteractions(jdbcTemplate); + } + + private static EtlJobReplayService service( + JdbcTemplate jdbcTemplate, + EtlBatchProperties properties + ) { + return new EtlJobReplayService( + jdbcTemplate, + new ObjectMapper(), + properties, + lockHash -> true + ); + } + + private static void assertError(EtlRequestError expected, Runnable invocation) { + EtlRequestException exception = assertThrows(EtlRequestException.class, invocation::run); + assertEquals(expected, exception.error()); + } +} diff --git a/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayServiceIntegrationTest.java b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayServiceIntegrationTest.java new file mode 100644 index 00000000..010898ab --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/job/EtlJobReplayServiceIntegrationTest.java @@ -0,0 +1,474 @@ +package com.xtrmetl.etl.job; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.xtrmetl.etl.service.EtlBatchProperties; +import com.xtrmetl.etl.service.EtlRequestError; +import com.xtrmetl.etl.service.EtlRequestException; +import com.xtrmetl.etl.service.EtlRequestLock; +import com.xtrmetl.etl.service.Sha256Digest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseBuilder; +import org.springframework.jdbc.datasource.embedded.EmbeddedDatabaseType; +import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; +import java.time.Instant; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Covers immutable replay admission, owner isolation, payload proof, and lineage generation. + */ +@SpringJUnitConfig(EtlJobReplayServiceIntegrationTest.TestConfiguration.class) +class EtlJobReplayServiceIntegrationTest { + + private static final String PAYLOAD = "[{\"id\":\"record_alpha\"}]"; + private static final String OTHER_PAYLOAD = "[{\"id\":\"record_beta\"}]"; + private static final String REPLAY_KEY = "1e05bdca-447c-4ad3-882c-e33963ce517c"; + private static final String OTHER_REPLAY_KEY = "519bc126-1398-4b4e-a4e3-1fb18a00f19b"; + + private final EtlJobReplayService replayService; + private final JdbcTemplate jdbcTemplate; + + @Autowired + EtlJobReplayServiceIntegrationTest( + EtlJobReplayService replayService, + JdbcTemplate jdbcTemplate + ) { + this.replayService = replayService; + this.jdbcTemplate = jdbcTemplate; + } + + @BeforeEach + void createJobTable() { + jdbcTemplate.execute("DROP TABLE IF EXISTS etl_job_records"); + jdbcTemplate.execute(""" + CREATE TABLE etl_job_records ( + job_record_id UUID PRIMARY KEY, + principal_scope_hash CHAR(64) NOT NULL, + submission_key_hash CHAR(64) NOT NULL, + request_digest CHAR(64) NOT NULL, + request_payload CLOB, + job_status VARCHAR(32) NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + failure_code VARCHAR(128), + lease_claim_id UUID, + lease_owner_id VARCHAR(128), + lease_expires_at TIMESTAMP WITH TIME ZONE, + cancellation_key_hash CHAR(64), + cancellation_code VARCHAR(128), + job_cancelled_at TIMESTAMP WITH TIME ZONE, + replay_source_job_record_id UUID, + replay_root_job_record_id UUID, + replay_generation_count INTEGER, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT etl_job_submission_scope_unique + UNIQUE (principal_scope_hash, submission_key_hash) + ) + """); + } + + @Test + void createsOnePendingReplayFromAFailedSourceAndReplaysIt() { + UUID sourceId = insertTerminalSource(EtlJobStatus.FAILED, "tenant_alpha", PAYLOAD); + Instant sourceUpdated = instantColumn(sourceId, "updated_at"); + + EtlJobReplay first = replayService.replayOwned( + sourceId, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + EtlJobReplay replay = replayService.replayOwned( + sourceId, + PAYLOAD, + "\"" + REPLAY_KEY + "\"", + "tenant_alpha" + ); + + assertFalse(first.replayed()); + assertTrue(replay.replayed()); + assertEquals(first.jobRecordId(), replay.jobRecordId()); + assertEquals(EtlJobStatus.PENDING, first.jobStatus()); + assertEquals(sourceId, uuidColumn(first.jobRecordId(), "replay_source_job_record_id")); + assertEquals(sourceId, uuidColumn(first.jobRecordId(), "replay_root_job_record_id")); + assertEquals(1, integerColumn(first.jobRecordId(), "replay_generation_count")); + assertEquals(PAYLOAD, textColumn(first.jobRecordId(), "request_payload")); + assertEquals("FAILED", textColumn(sourceId, "job_status")); + assertNull(textColumn(sourceId, "request_payload")); + assertEquals(sourceUpdated, instantColumn(sourceId, "updated_at")); + } + + @Test + void createsAReplayFromCancelledSourceAndKeepsCancellationEvidence() { + UUID sourceId = insertTerminalSource(EtlJobStatus.CANCELLED, "tenant_alpha", PAYLOAD); + jdbcTemplate.update( + "UPDATE etl_job_records SET cancellation_key_hash=?, cancellation_code=?, " + + "job_cancelled_at=CURRENT_TIMESTAMP WHERE job_record_id=?", + "d".repeat(64), + EtlJobService.CANCELLED_BY_OWNER_CODE, + sourceId + ); + + EtlJobReplay replay = replayService.replayOwned( + sourceId, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + + assertEquals(EtlJobStatus.PENDING, replay.jobStatus()); + assertEquals("CANCELLED", textColumn(sourceId, "job_status")); + assertEquals( + EtlJobService.CANCELLED_BY_OWNER_CODE, + textColumn(sourceId, "cancellation_code") + ); + assertNull(textColumn(sourceId, "request_payload")); + } + + @Test + void rejectsMismatchedPayloadAndReplayKeyReuse() { + UUID sourceId = insertTerminalSource(EtlJobStatus.FAILED, "tenant_alpha", PAYLOAD); + EtlRequestException mismatch = assertThrows( + EtlRequestException.class, + () -> replayService.replayOwned( + sourceId, + OTHER_PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertEquals(EtlRequestError.JOB_REPLAY_PAYLOAD_MISMATCH, mismatch.error()); + + EtlJobReplay first = replayService.replayOwned( + sourceId, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + assertError( + EtlRequestError.JOB_REPLAY_KEY_REUSED, + () -> replayService.replayOwned( + sourceId, + OTHER_PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + + UUID otherSource = insertTerminalSource( + EtlJobStatus.FAILED, + "tenant_alpha", + PAYLOAD + ); + EtlRequestException reused = assertThrows( + EtlRequestException.class, + () -> replayService.replayOwned( + otherSource, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + + assertEquals(EtlRequestError.JOB_REPLAY_KEY_REUSED, reused.error()); + assertEquals(1, replayRowCount()); + assertNotEquals(sourceId, first.jobRecordId()); + } + + @Test + void hidesForeignAndMissingSourcesAndRejectsUnsupportedStates() { + UUID failed = insertTerminalSource(EtlJobStatus.FAILED, "tenant_alpha", PAYLOAD); + assertError( + EtlRequestError.JOB_NOT_FOUND, + () -> replayService.replayOwned(failed, PAYLOAD, REPLAY_KEY, "tenant_beta") + ); + assertError( + EtlRequestError.JOB_NOT_FOUND, + () -> replayService.replayOwned( + UUID.randomUUID(), + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertError( + EtlRequestError.JOB_REPLAY_SOURCE_ACTIVE, + () -> replayService.replayOwned( + insertSource(EtlJobStatus.PENDING, "tenant_alpha", PAYLOAD, null, null), + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertError( + EtlRequestError.JOB_REPLAY_SOURCE_ACTIVE, + () -> replayService.replayOwned( + insertSource(EtlJobStatus.RUNNING, "tenant_alpha", PAYLOAD, null, null), + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertError( + EtlRequestError.JOB_REPLAY_SOURCE_SUCCEEDED, + () -> replayService.replayOwned( + insertTerminalSource(EtlJobStatus.SUCCEEDED, "tenant_alpha", PAYLOAD), + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + } + + @Test + void replayOfReplayPreservesRootAndBoundsGeneration() { + UUID root = insertTerminalSource(EtlJobStatus.FAILED, "tenant_alpha", PAYLOAD); + UUID generationOne = insertSource( + EtlJobStatus.FAILED, + "tenant_alpha", + PAYLOAD, + root, + 1 + ); + EtlJobReplay generationTwo = replayService.replayOwned( + generationOne, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ); + + assertEquals(generationOne, uuidColumn( + generationTwo.jobRecordId(), + "replay_source_job_record_id" + )); + assertEquals(root, uuidColumn( + generationTwo.jobRecordId(), + "replay_root_job_record_id" + )); + assertEquals(2, integerColumn( + generationTwo.jobRecordId(), + "replay_generation_count" + )); + + UUID generationHundred = insertSource( + EtlJobStatus.CANCELLED, + "tenant_alpha", + PAYLOAD, + root, + EtlJobReplayService.MAXIMUM_REPLAY_GENERATION + ); + assertError( + EtlRequestError.JOB_REPLAY_GENERATION_EXHAUSTED, + () -> replayService.replayOwned( + generationHundred, + PAYLOAD, + OTHER_REPLAY_KEY, + "tenant_alpha" + ) + ); + } + + @Test + void rejectsIncompleteOrForeignRootLineage() { + UUID incompleteRoot = insertSource( + EtlJobStatus.FAILED, + "tenant_alpha", + PAYLOAD, + null, + 1 + ); + IllegalStateException incomplete = assertThrows( + IllegalStateException.class, + () -> replayService.replayOwned( + incompleteRoot, + PAYLOAD, + REPLAY_KEY, + "tenant_alpha" + ) + ); + assertEquals("Replay source has incomplete root lineage", incomplete.getMessage()); + + UUID absentRoot = insertSource( + EtlJobStatus.FAILED, + "tenant_alpha", + PAYLOAD, + UUID.randomUUID(), + 1 + ); + IllegalStateException absent = assertThrows( + IllegalStateException.class, + () -> replayService.replayOwned( + absentRoot, + PAYLOAD, + OTHER_REPLAY_KEY, + "tenant_alpha" + ) + ); + assertEquals("Replay root is absent from the owner namespace", absent.getMessage()); + assertEquals( + 0, + jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM etl_job_records " + + "WHERE job_status='PENDING' " + + "AND replay_generation_count IS NOT NULL", + Integer.class + ) + ); + } + + private UUID insertTerminalSource( + EtlJobStatus status, + String principal, + String payload + ) { + return insertSource(status, principal, payload, null, null); + } + + private UUID insertSource( + EtlJobStatus status, + String principal, + String payload, + UUID replayRoot, + Integer replayGeneration + ) { + UUID id = UUID.randomUUID(); + UUID replaySource = replayRoot == null ? null : replayRoot; + jdbcTemplate.update( + """ + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, attempt_count, + failure_code, replay_source_job_record_id, + replay_root_job_record_id, replay_generation_count + ) VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?) + """, + id, + Sha256Digest.digest(principal), + Sha256Digest.digest(UUID.randomUUID().toString()), + Sha256Digest.digest(payload), + status == EtlJobStatus.PENDING || status == EtlJobStatus.RUNNING + ? payload : null, + status.name(), + status == EtlJobStatus.FAILED ? "etl_target_failure" : null, + replaySource, + replayRoot, + replayGeneration + ); + return id; + } + + private static void assertError(EtlRequestError expected, Runnable invocation) { + EtlRequestException exception = assertThrows(EtlRequestException.class, invocation::run); + assertEquals(expected, exception.error()); + } + + private int replayRowCount() { + Integer count = jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM etl_job_records WHERE replay_generation_count IS NOT NULL", + Integer.class + ); + return count == null ? 0 : count; + } + + private String textColumn(UUID id, String column) { + return jdbcTemplate.queryForObject( + "SELECT " + column + " FROM etl_job_records WHERE job_record_id=?", + String.class, + id + ); + } + + private UUID uuidColumn(UUID id, String column) { + return jdbcTemplate.queryForObject( + "SELECT " + column + " FROM etl_job_records WHERE job_record_id=?", + UUID.class, + id + ); + } + + private Integer integerColumn(UUID id, String column) { + return jdbcTemplate.queryForObject( + "SELECT " + column + " FROM etl_job_records WHERE job_record_id=?", + Integer.class, + id + ); + } + + private Instant instantColumn(UUID id, String column) { + return jdbcTemplate.queryForObject( + "SELECT " + column + " FROM etl_job_records WHERE job_record_id=?", + (resultSet, rowNumber) -> resultSet.getTimestamp(column).toInstant(), + id + ); + } + + /** Minimal transaction-enabled context for replay service integration. */ + @Configuration + @EnableTransactionManagement + static class TestConfiguration { + + @Bean + DataSource dataSource() { + return new EmbeddedDatabaseBuilder() + .generateUniqueName(true) + .setType(EmbeddedDatabaseType.H2) + .build(); + } + + @Bean + JdbcTemplate jdbcTemplate(DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Bean + PlatformTransactionManager transactionManager(DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Bean + ObjectMapper objectMapper() { + return new ObjectMapper(); + } + + @Bean + EtlBatchProperties etlBatchProperties() { + return new EtlBatchProperties(); + } + + @Bean + EtlRequestLock etlRequestLock() { + return lockHash -> true; + } + + @Bean + EtlJobReplayService replayService( + JdbcTemplate jdbcTemplate, + ObjectMapper objectMapper, + EtlBatchProperties batchProperties, + EtlRequestLock requestLock + ) { + return new EtlJobReplayService( + jdbcTemplate, + objectMapper, + batchProperties, + requestLock + ); + } + } +} diff --git a/etl-service/src/test/postgresql/replay_lineage_migration.sql b/etl-service/src/test/postgresql/replay_lineage_migration.sql new file mode 100644 index 00000000..60b5893c --- /dev/null +++ b/etl-service/src/test/postgresql/replay_lineage_migration.sql @@ -0,0 +1,436 @@ +-- Rehearse V7 lineage authority plus V8/V9 replay lookup indexes on PostgreSQL 18. +-- This script runs only against the disposable PostgreSQL integration-test database. + +DO $migration_object_check$ +DECLARE + trigger_count integer; + function_count integer; + lookup_index_count integer; +BEGIN + SELECT count(*) + INTO trigger_count + FROM pg_trigger AS trigger_record + JOIN pg_class AS table_record + ON table_record.oid = trigger_record.tgrelid + WHERE table_record.relname = 'etl_job_records' + AND trigger_record.tgname = 'etl_job_replay_lineage_guard_trigger' + AND NOT trigger_record.tgisinternal; + + SELECT count(*) + INTO function_count + FROM pg_proc AS function_record + WHERE function_record.proname = 'validate_etl_job_replay_lineage'; + + SELECT count(*) + INTO lookup_index_count + FROM pg_class AS index_class + JOIN pg_index AS index_record + ON index_record.indexrelid = index_class.oid + WHERE index_class.relname IN ( + 'etl_job_replay_source_lookup_index', + 'etl_job_replay_root_lookup_index' + ) + AND index_record.indisready + AND index_record.indisvalid; + + IF trigger_count <> 1 OR function_count <> 1 OR lookup_index_count <> 2 THEN + RAISE EXCEPTION 'replay lineage trigger or function is missing'; + END IF; +END +$migration_object_check$; + +BEGIN; + +-- Pending controls for owner A and owner B. +INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status +) VALUES + ( + '00000000-0000-4000-8000-000000000001', + repeat('a', 64), + repeat('1', 64), + repeat('a', 64), + '{}', + 'PENDING' + ), + ( + '00000000-0000-4000-8000-000000000003', + repeat('b', 64), + repeat('3', 64), + repeat('c', 64), + '{}', + 'PENDING' + ); + +-- Two independent same-owner terminal roots plus one terminal root for owner B. +INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + failure_code +) VALUES + ( + '00000000-0000-4000-8000-000000000002', + repeat('a', 64), + repeat('2', 64), + repeat('b', 64), + NULL, + 'FAILED', + 'etl_replay_source_failed' + ), + ( + '00000000-0000-4000-8000-000000000014', + repeat('a', 64), + repeat('e', 64), + repeat('b', 64), + NULL, + 'FAILED', + 'etl_alternate_root_failed' + ); + +INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + cancellation_key_hash, + cancellation_code, + job_cancelled_at +) VALUES ( + '00000000-0000-4000-8000-000000000004', + repeat('b', 64), + repeat('4', 64), + repeat('d', 64), + NULL, + 'CANCELLED', + repeat('e', 64), + 'etl_job_cancelled_by_owner', + CURRENT_TIMESTAMP +); + +-- Create a valid first-generation replay with the exact source digest, then terminalize it. +INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count +) VALUES ( + '00000000-0000-4000-8000-000000000005', + repeat('a', 64), + repeat('5', 64), + repeat('b', 64), + '{}', + 'PENDING', + '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000002', + 1 +); + +UPDATE etl_job_records + SET job_status = 'FAILED', + request_payload = NULL, + failure_code = 'etl_replay_generation_failed' + WHERE job_record_id = '00000000-0000-4000-8000-000000000005'; + +-- A valid second generation retains the first root, source digest, and one-step succession. +INSERT INTO etl_job_records ( + job_record_id, + principal_scope_hash, + submission_key_hash, + request_digest, + request_payload, + job_status, + replay_source_job_record_id, + replay_root_job_record_id, + replay_generation_count +) VALUES ( + '00000000-0000-4000-8000-000000000008', + repeat('a', 64), + repeat('8', 64), + repeat('b', 64), + '{}', + 'PENDING', + '00000000-0000-4000-8000-000000000005', + '00000000-0000-4000-8000-000000000002', + 2 +); + +DO $lineage_rejection_checks$ +BEGIN + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000009', + repeat('a', 64), repeat('9', 64), repeat('a', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000001', + '00000000-0000-4000-8000-000000000001', 1 + ); + RAISE EXCEPTION 'nonterminal replay source was accepted'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; + + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000010', + repeat('a', 64), repeat('a', 64), repeat('b', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000014', 1 + ); + RAISE EXCEPTION 'generation-one replay accepted a different root'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; + + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000011', + repeat('a', 64), repeat('b', 64), repeat('b', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000005', + '00000000-0000-4000-8000-000000000005', 2 + ); + RAISE EXCEPTION 'a derived replay row was accepted as lineage root'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; + + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000012', + repeat('a', 64), repeat('c', 64), repeat('b', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000005', + '00000000-0000-4000-8000-000000000002', 3 + ); + RAISE EXCEPTION 'a skipped replay generation was accepted'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; + + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000006', + repeat('b', 64), repeat('6', 64), repeat('d', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000004', 1 + ); + RAISE EXCEPTION 'cross-owner source lineage was accepted'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; + + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000007', + repeat('b', 64), repeat('7', 64), repeat('d', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000004', + '00000000-0000-4000-8000-000000000002', 1 + ); + RAISE EXCEPTION 'cross-owner root lineage was accepted'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; +END +$lineage_rejection_checks$; + +DO $digest_continuity_check$ +BEGIN + BEGIN + INSERT INTO etl_job_records ( + job_record_id, principal_scope_hash, submission_key_hash, + request_digest, request_payload, job_status, + replay_source_job_record_id, replay_root_job_record_id, + replay_generation_count + ) VALUES ( + '00000000-0000-4000-8000-000000000013', + repeat('a', 64), repeat('d', 64), repeat('c', 64), '{}', 'PENDING', + '00000000-0000-4000-8000-000000000002', + '00000000-0000-4000-8000-000000000002', 1 + ); + RAISE EXCEPTION 'replay digest mismatch was accepted'; + EXCEPTION + WHEN check_violation OR foreign_key_violation THEN NULL; + END; +END +$digest_continuity_check$; + +DO $immutability_checks$ +BEGIN + BEGIN + UPDATE etl_job_records + SET replay_root_job_record_id = '00000000-0000-4000-8000-000000000014' + WHERE job_record_id = '00000000-0000-4000-8000-000000000005'; + RAISE EXCEPTION 'replay lineage fields were mutable'; + EXCEPTION + WHEN check_violation THEN NULL; + END; + + BEGIN + UPDATE etl_job_records + SET request_digest = repeat('0', 64) + WHERE job_record_id = '00000000-0000-4000-8000-000000000002'; + RAISE EXCEPTION 'referenced replay root evidence was mutable'; + EXCEPTION + WHEN check_violation THEN NULL; + END; + + BEGIN + UPDATE etl_job_records + SET failure_code = 'etl_replay_generation_changed' + WHERE job_record_id = '00000000-0000-4000-8000-000000000005'; + RAISE EXCEPTION 'referenced immediate-source evidence was mutable'; + EXCEPTION + WHEN check_violation THEN NULL; + END; +END +$immutability_checks$; + +DO $delete_restrict_check$ +BEGIN + BEGIN + DELETE FROM etl_job_records + WHERE job_record_id = '00000000-0000-4000-8000-000000000005'; + RAISE EXCEPTION 'ON DELETE RESTRICT did not protect immediate replay history'; + EXCEPTION + WHEN foreign_key_violation THEN NULL; + END; + + BEGIN + DELETE FROM etl_job_records + WHERE job_record_id = '00000000-0000-4000-8000-000000000002'; + RAISE EXCEPTION 'ON DELETE RESTRICT did not protect replay root history'; + EXCEPTION + WHEN foreign_key_violation THEN NULL; + END; +END +$delete_restrict_check$; + +ROLLBACK; + +-- Rehearse ordered rollback without changing the migrated database. +BEGIN; + +DROP INDEX etl_job_replay_source_lookup_index; +DROP INDEX etl_job_replay_root_lookup_index; +DROP TRIGGER etl_job_replay_lineage_guard_trigger ON etl_job_records; +DROP FUNCTION validate_etl_job_replay_lineage(); +ALTER TABLE etl_job_records DROP CONSTRAINT etl_job_replay_source_reference; +ALTER TABLE etl_job_records DROP CONSTRAINT etl_job_replay_root_reference; +ALTER TABLE etl_job_records DROP CONSTRAINT etl_job_replay_lineage_complete_check; +ALTER TABLE etl_job_records DROP CONSTRAINT etl_job_owner_identity_unique; +ALTER TABLE etl_job_records DROP COLUMN replay_source_job_record_id; +ALTER TABLE etl_job_records DROP COLUMN replay_root_job_record_id; +ALTER TABLE etl_job_records DROP COLUMN replay_generation_count; + +ROLLBACK; + +DO $rollback_restoration_check$ +DECLARE + restored_column_count integer; + restored_constraint_count integer; + restored_trigger_count integer; + restored_function_count integer; + restored_index_count integer; +BEGIN + SELECT count(*) + INTO restored_column_count + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'etl_job_records' + AND column_name IN ( + 'replay_source_job_record_id', + 'replay_root_job_record_id', + 'replay_generation_count' + ); + + SELECT count(*) + INTO restored_constraint_count + FROM pg_constraint AS constraint_record + JOIN pg_class AS table_record + ON table_record.oid = constraint_record.conrelid + WHERE table_record.relname = 'etl_job_records' + AND constraint_record.conname IN ( + 'etl_job_replay_source_reference', + 'etl_job_replay_root_reference', + 'etl_job_replay_lineage_complete_check', + 'etl_job_owner_identity_unique' + ); + + SELECT count(*) + INTO restored_trigger_count + FROM pg_trigger AS trigger_record + JOIN pg_class AS table_record + ON table_record.oid = trigger_record.tgrelid + WHERE table_record.relname = 'etl_job_records' + AND trigger_record.tgname = 'etl_job_replay_lineage_guard_trigger' + AND NOT trigger_record.tgisinternal; + + SELECT count(*) + INTO restored_function_count + FROM pg_proc AS function_record + WHERE function_record.proname = 'validate_etl_job_replay_lineage'; + + SELECT count(*) + INTO restored_index_count + FROM pg_class AS index_class + JOIN pg_index AS index_record + ON index_record.indexrelid = index_class.oid + WHERE index_class.relname IN ( + 'etl_job_replay_source_lookup_index', + 'etl_job_replay_root_lookup_index' + ) + AND index_record.indisready + AND index_record.indisvalid; + + IF restored_column_count <> 3 + OR restored_constraint_count <> 4 + OR restored_trigger_count <> 1 + OR restored_function_count <> 1 + OR restored_index_count <> 2 THEN + RAISE EXCEPTION 'rollback rehearsal did not restore V7'; + END IF; +END +$rollback_restoration_check$; diff --git a/scripts/verify-postgresql-migrations.sh b/scripts/verify-postgresql-migrations.sh new file mode 100644 index 00000000..c61c756c --- /dev/null +++ b/scripts/verify-postgresql-migrations.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +: "${PGHOST:=127.0.0.1}" +: "${PGPORT:=5432}" +: "${PGDATABASE:=mightyetl_replay_test}" +: "${PGUSER:=mightyetl_test}" +: "${PGPASSWORD:=mightyetl_test_password}" +export PGHOST PGPORT PGDATABASE PGUSER PGPASSWORD + +migration_directory="etl-service/src/main/resources/db/migration" +if [[ ! -d "${migration_directory}" ]]; then + printf 'Migration directory not found: %s\n' "${migration_directory}" >&2 + exit 1 +fi + +for attempt_number in $(seq 1 30); do + if pg_isready --host "${PGHOST}" --port "${PGPORT}" --dbname "${PGDATABASE}" --username "${PGUSER}" >/dev/null 2>&1; then + break + fi + if [[ "${attempt_number}" -eq 30 ]]; then + printf 'PostgreSQL did not become ready after 30 attempts.\n' >&2 + exit 1 + fi + sleep 2 +done + +mapfile -d '' migration_files < <( + find "${migration_directory}" -maxdepth 1 -type f -name 'V*__*.sql' -print0 | sort -zV +) +if [[ "${#migration_files[@]}" -eq 0 ]]; then + printf 'No versioned SQL migrations found.\n' >&2 + exit 1 +fi + +for migration_file in "${migration_files[@]}"; do + printf 'Applying %s\n' "${migration_file}" + psql --no-psqlrc --set ON_ERROR_STOP=1 --file "${migration_file}" >/dev/null +done + +psql --no-psqlrc --set ON_ERROR_STOP=1 <<'SQL' +DO $verification_block$ +DECLARE + missing_column_count integer; + restrict_foreign_key_count integer; + replay_lookup_index_count integer; + replay_check_definition text; + cancellation_check_definition text; +BEGIN + IF to_regclass('public.etl_job_records') IS NULL THEN + RAISE EXCEPTION 'etl_job_records was not created'; + END IF; + + SELECT count(*) + INTO missing_column_count + FROM ( + VALUES + ('replay_source_job_record_id', 'uuid'), + ('replay_root_job_record_id', 'uuid'), + ('replay_generation_count', 'integer'), + ('cancellation_key_hash', 'character'), + ('cancellation_code', 'character varying'), + ('job_cancelled_at', 'timestamp with time zone') + ) AS expected_columns(column_name, data_type) + WHERE NOT EXISTS ( + SELECT 1 + FROM information_schema.columns AS actual_columns + WHERE actual_columns.table_schema = 'public' + AND actual_columns.table_name = 'etl_job_records' + AND actual_columns.column_name = expected_columns.column_name + AND actual_columns.data_type = expected_columns.data_type + ); + + IF missing_column_count <> 0 THEN + RAISE EXCEPTION 'one or more cancellation/replay columns are missing or have the wrong type'; + END IF; + + SELECT count(*) + INTO restrict_foreign_key_count + FROM pg_constraint AS constraint_record + JOIN pg_class AS table_record + ON table_record.oid = constraint_record.conrelid + JOIN unnest(constraint_record.conkey) AS constrained_attribute(attribute_number) + ON true + JOIN pg_attribute AS attribute_record + ON attribute_record.attrelid = table_record.oid + AND attribute_record.attnum = constrained_attribute.attribute_number + WHERE table_record.relname = 'etl_job_records' + AND constraint_record.contype = 'f' + AND constraint_record.confrelid = table_record.oid + AND constraint_record.confdeltype = 'r' + AND attribute_record.attname IN ( + 'replay_source_job_record_id', + 'replay_root_job_record_id' + ); + + IF restrict_foreign_key_count <> 2 THEN + RAISE EXCEPTION 'replay source and root must each use a self-reference with ON DELETE RESTRICT'; + END IF; + + -- Names and ready/valid flags are insufficient: CREATE INDEX CONCURRENTLY IF NOT EXISTS + -- would retain a valid but incorrectly shaped index with the expected name. Reconstruct the + -- indexed column and stored predicate from PostgreSQL catalogs and require the exact bounded + -- one-column, nonunique partial-index contracts used by the lineage trigger. + SELECT count(*) + INTO replay_lookup_index_count + FROM pg_class AS index_class + JOIN pg_index AS index_record + ON index_record.indexrelid = index_class.oid + JOIN pg_class AS table_record + ON table_record.oid = index_record.indrelid + WHERE table_record.relname = 'etl_job_records' + AND index_record.indisready + AND index_record.indisvalid + AND index_record.indnkeyatts = 1 + AND index_record.indnatts = 1 + AND NOT index_record.indisunique + AND ( + ( + index_class.relname = 'etl_job_replay_source_lookup_index' + AND pg_get_indexdef(index_record.indexrelid, 1, true) + = 'replay_source_job_record_id' + AND pg_get_expr(index_record.indpred, index_record.indrelid) + = '(replay_source_job_record_id IS NOT NULL)' + ) + OR + ( + index_class.relname = 'etl_job_replay_root_lookup_index' + AND pg_get_indexdef(index_record.indexrelid, 1, true) + = 'replay_root_job_record_id' + AND pg_get_expr(index_record.indpred, index_record.indrelid) + = '(replay_root_job_record_id IS NOT NULL)' + ) + ); + + IF replay_lookup_index_count <> 2 THEN + RAISE EXCEPTION 'replay lookup indexes are missing or invalid, or have unexpected definitions'; + END IF; + + SELECT string_agg(pg_get_constraintdef(constraint_record.oid), ' ') + INTO replay_check_definition + FROM pg_constraint AS constraint_record + JOIN pg_class AS table_record + ON table_record.oid = constraint_record.conrelid + WHERE table_record.relname = 'etl_job_records' + AND constraint_record.contype = 'c' + AND pg_get_constraintdef(constraint_record.oid) ILIKE '%replay_generation_count%'; + + IF replay_check_definition IS NULL + OR replay_check_definition NOT ILIKE '%replay_source_job_record_id%' + OR replay_check_definition NOT ILIKE '%replay_root_job_record_id%' + OR replay_check_definition NOT ILIKE '%100%' THEN + RAISE EXCEPTION 'replay lineage checks do not bind source, root, and the bounded generation'; + END IF; + + SELECT string_agg(pg_get_constraintdef(constraint_record.oid), ' ') + INTO cancellation_check_definition + FROM pg_constraint AS constraint_record + JOIN pg_class AS table_record + ON table_record.oid = constraint_record.conrelid + WHERE table_record.relname = 'etl_job_records' + AND constraint_record.contype = 'c' + AND pg_get_constraintdef(constraint_record.oid) ILIKE '%cancellation_key_hash%'; + + IF cancellation_check_definition IS NULL + OR cancellation_check_definition NOT ILIKE '%job_cancelled_at%' + OR cancellation_check_definition NOT ILIKE '%CANCELLED%' THEN + RAISE EXCEPTION 'cancellation lifecycle checks are incomplete'; + END IF; +END +$verification_block$; +SQL + +rehearsal_file="etl-service/src/test/postgresql/replay_lineage_migration.sql" +if [[ ! -f "${rehearsal_file}" ]]; then + printf 'Replay-lineage rehearsal not found: %s\n' "${rehearsal_file}" >&2 + exit 1 +fi + +printf 'Running %s\n' "${rehearsal_file}" +psql --no-psqlrc --set ON_ERROR_STOP=1 --file "${rehearsal_file}" >/dev/null + +pg_dump --schema-only --no-owner --no-privileges > /tmp/mightyetl-postgresql-schema.sql +if [[ ! -s /tmp/mightyetl-postgresql-schema.sql ]]; then + printf 'Schema-only dump was empty.\n' >&2 + exit 1 +fi + +printf 'PostgreSQL migration and replay-lineage verification succeeded.\n'