From eb7671a1cad16464eb15b199a64fc99b2e29e963 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 12:44:01 +0900 Subject: [PATCH 01/30] test(ops): require provenance-bound PostgreSQL backup artifact --- .../PostgresLogicalBackupContractTest.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java new file mode 100644 index 00000000..fde4a2c1 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -0,0 +1,61 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Defines the first executable recovery contract for a local PostgreSQL logical backup. + */ +class PostgresLogicalBackupContractTest { + + @Test + void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOException { + Path scriptPath = projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"); + assertTrue( + Files.isRegularFile(scriptPath), + "A supported logical-backup command must exist before named volumes can be called recoverable" + ); + + String script = Files.readString(scriptPath, StandardCharsets.UTF_8); + assertTrue(script.contains("set -euo pipefail"), "backup failures must fail closed"); + assertTrue(script.contains("umask 077"), "backup artifacts must be private by default"); + assertTrue(script.contains("${BACKUP_DIRECTORY:?"), "operators must choose the backup destination explicitly"); + assertTrue(script.contains("${APPLICATION_SOURCE_SHA:?"), "backup provenance must bind the exact application source"); + assertTrue(script.contains("pg_dump"), "the local PostgreSQL profile must use a database-consistent logical dump"); + assertTrue(script.contains("--format=custom"), "the archive must use PostgreSQL custom format for pg_restore validation"); + assertTrue(script.contains("pg_restore --list"), "the completed archive must be structurally verified before publication"); + assertTrue(script.contains("server_version_num"), "the manifest must record the PostgreSQL server version"); + assertTrue(script.contains("flyway_schema_history"), "the manifest must bind the Flyway migration level"); + assertTrue(script.contains("backup_sha256"), "the manifest must bind an integrity digest"); + assertTrue(script.contains("application_source_sha"), "the manifest must record exact source identity"); + assertTrue(script.contains("mv --"), "temporary backup artifacts must be atomically published only after verification"); + assertFalse(script.contains("docker cp /var/lib/postgresql/data"), "copying a live PostgreSQL data directory is not backup"); + assertFalse(script.contains("echo \"$PGPASSWORD\""), "database credentials must never be printed"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From c2abe450b97e9ce0d6b0dc4e15587eed37f24deb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 12:53:17 +0900 Subject: [PATCH 02/30] feat(ops): create verified PostgreSQL logical backup bundle --- scripts/ops/postgres-logical-backup.sh | 99 ++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100755 scripts/ops/postgres-logical-backup.sh diff --git a/scripts/ops/postgres-logical-backup.sh b/scripts/ops/postgres-logical-backup.sh new file mode 100755 index 00000000..f84a11c4 --- /dev/null +++ b/scripts/ops/postgres-logical-backup.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +: "${BACKUP_DIRECTORY:?Set BACKUP_DIRECTORY to an existing or creatable backup directory}" +: "${APPLICATION_SOURCE_SHA:?Set APPLICATION_SOURCE_SHA to the exact mightyETL source commit}" +: "${PGHOST:?Set PGHOST for the PostgreSQL server to back up}" +: "${PGPORT:?Set PGPORT for the PostgreSQL server to back up}" +: "${PGDATABASE:?Set PGDATABASE for the PostgreSQL database to back up}" +: "${PGUSER:?Set PGUSER for the PostgreSQL role used by backup tooling}" + +if [[ ! "$APPLICATION_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + printf 'APPLICATION_SOURCE_SHA must be a 40-character lowercase Git commit SHA\n' >&2 + exit 2 +fi + +for command_name in pg_dump pg_restore psql mktemp mv; do + if ! command -v "$command_name" >/dev/null 2>&1; then + printf 'Required command is unavailable: %s\n' "$command_name" >&2 + exit 3 + fi +done + +sha256_file() { + local file_path=$1 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file_path" | awk '{print $1}' + return + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file_path" | awk '{print $1}' + return + fi + printf 'Neither sha256sum nor shasum is available\n' >&2 + return 1 +} + +mkdir -p -- "$BACKUP_DIRECTORY" +backup_directory=$(cd -- "$BACKUP_DIRECTORY" && pwd -P) +created_at_utc=$(date -u '+%Y-%m-%dT%H:%M:%SZ') +backup_identity="mightyetl-postgres-${created_at_utc//[:]/}-${APPLICATION_SOURCE_SHA}" +final_bundle="${backup_directory}/${backup_identity}" + +if [[ -e "$final_bundle" ]]; then + printf 'Refusing to replace an existing backup bundle: %s\n' "$final_bundle" >&2 + exit 4 +fi + +temporary_bundle=$(mktemp -d "${backup_directory}/.mightyetl-postgres-backup.XXXXXX") +cleanup() { + rm -rf -- "$temporary_bundle" +} +trap cleanup EXIT + +archive_path="${temporary_bundle}/database.dump" +manifest_path="${temporary_bundle}/manifest.txt" + +pg_dump \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --format=custom \ + --file="$archive_path" + +# A completed custom archive must be structurally readable before it is published. +pg_restore --list "$archive_path" >/dev/null + +server_version_num=$(psql \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --command='SHOW server_version_num') + +flyway_schema_version=$(psql \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --command="SELECT COALESCE((SELECT version FROM flyway_schema_history WHERE success = true ORDER BY installed_rank DESC LIMIT 1), 'none')") + +backup_sha256=$(sha256_file "$archive_path") + +printf '%s\n' \ + "manifest_version=1" \ + "application_source_sha=${APPLICATION_SOURCE_SHA}" \ + "created_at_utc=${created_at_utc}" \ + "server_version_num=${server_version_num}" \ + "flyway_schema_version=${flyway_schema_version}" \ + "backup_sha256=${backup_sha256}" \ + > "$manifest_path" + +# Publish archive and manifest together through one same-filesystem directory rename. +mv -- "$temporary_bundle" "$final_bundle" +trap - EXIT +printf '%s\n' "$final_bundle" From 6e4239bc70b3d967081b23a7a33910088bfe9b4c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 12:56:14 +0900 Subject: [PATCH 03/30] test(ops): require truthful PostgreSQL recovery guidance --- .../PostgresLogicalBackupContractTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index fde4a2c1..0d865593 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -41,6 +41,24 @@ void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOExc assertFalse(script.contains("echo \"$PGPASSWORD\""), "database credentials must never be printed"); } + @Test + void recoveryRunbookSeparatesVerifiedBackupFromUnprovenRestoreAndRecoveryObjectives() throws IOException { + Path runbookPath = projectRoot().resolve("docs/ops/postgres-recovery.md"); + assertTrue(Files.isRegularFile(runbookPath), "the executable backup boundary needs an operator recovery runbook"); + + String runbook = Files.readString(runbookPath, StandardCharsets.UTF_8); + assertTrue(runbook.contains("APPLICATION_SOURCE_SHA"), "operators need exact application-source provenance"); + assertTrue(runbook.contains("pg_restore --list"), "runbook must explain archive structural verification"); + assertTrue(runbook.contains("flyway_schema_history"), "runbook must explain migration-level provenance"); + assertTrue(runbook.contains("Backup is not restore"), "a produced archive must not be represented as restore proof"); + assertTrue(runbook.contains("RPO: not measured"), "RPO must remain evidence-based rather than invented"); + assertTrue(runbook.contains("RTO: not measured"), "RTO must remain evidence-based rather than invented"); + assertTrue(runbook.contains("Kafka"), "database recovery scope must distinguish Kafka side effects"); + assertTrue(runbook.contains("Debezium"), "database recovery scope must distinguish Debezium state"); + assertTrue(runbook.contains("DLT"), "database recovery scope must distinguish dead-letter state"); + assertTrue(runbook.contains("external target"), "database recovery scope must distinguish external target effects"); + } + private static Path projectRoot() { Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); Path lastPomParent = null; From 58780585e6e89173db3a2f4573eff0bcc27fd7ed Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:01:40 +0900 Subject: [PATCH 04/30] docs(ops): define PostgreSQL backup and recovery boundary --- docs/ops/postgres-recovery.md | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 docs/ops/postgres-recovery.md diff --git a/docs/ops/postgres-recovery.md b/docs/ops/postgres-recovery.md new file mode 100644 index 00000000..d7d7b6df --- /dev/null +++ b/docs/ops/postgres-recovery.md @@ -0,0 +1,102 @@ +# PostgreSQL logical backup and recovery boundary + +Status: `active_pr` (#208). This document describes the bounded logical-backup capability on this branch. It is not `implemented_on_develop` until protected integration. + +## Purpose and scope + +The local mightyETL PostgreSQL profile uses persistent storage, but a named Docker volume is not a backup. `scripts/ops/postgres-logical-backup.sh` creates a separate PostgreSQL logical archive plus a provenance manifest so an operator can preserve database state independently of the live volume. + +This first recovery increment proves **backup artifact creation and verification only**. It does not yet prove a clean restore, destructive-loss recovery, service restart, end-to-end CDC recovery, or a measured recovery objective. + +## Prerequisites + +Use PostgreSQL client tooling compatible with the database server and provide ordinary libpq connection settings without writing credentials into source control: + +- `PGHOST` +- `PGPORT` +- `PGDATABASE` +- `PGUSER` +- `PGPASSWORD` or another deployment-approved libpq authentication mechanism +- `BACKUP_DIRECTORY`, an operator-controlled destination +- `APPLICATION_SOURCE_SHA`, the exact 40-character mightyETL Git commit represented by the running deployment + +The script creates files under `umask 077`. The operator remains responsible for storage encryption, host and object-store access control, retention, replication, deletion, and custody of database credentials. Do not place `PGPASSWORD` in command histories, logs, manifests, or committed configuration. + +Example for a controlled local environment: + +```bash +export BACKUP_DIRECTORY="$HOME/mightyetl-backups" +export APPLICATION_SOURCE_SHA="$(git rev-parse HEAD)" +export PGHOST="127.0.0.1" +export PGPORT="5432" +export PGDATABASE="postgres" +export PGUSER="postgres" +# Supply PGPASSWORD through the deployment's approved secret mechanism. +./scripts/ops/postgres-logical-backup.sh +``` + +Do not substitute a guessed SHA. In a packaged or remote deployment, obtain `APPLICATION_SOURCE_SHA` from the exact deployed release/provenance record rather than whichever checkout happens to be on an operator workstation. + +## Artifact and verification contract + +The command creates a private temporary directory inside `BACKUP_DIRECTORY`, writes a PostgreSQL custom-format archive with `pg_dump --format=custom`, and requires `pg_restore --list` to read the completed archive before publication. PostgreSQL documents the custom archive as a format intended for `pg_restore` and suitable for selective/reordered restore operations. The structural check proves that the artifact is readable as a PostgreSQL archive; it does **not** prove that restoring it into a clean server will satisfy mightyETL recovery invariants. + +The companion `manifest.txt` records: + +- manifest version; +- `application_source_sha`; +- UTC creation time; +- PostgreSQL `server_version_num`; +- latest successful `flyway_schema_history` migration version; +- `backup_sha256` for `database.dump`. + +Archive and manifest are published together only after the archive check and metadata queries succeed. The final bundle is never intentionally replaced by the command. Before moving or restoring an archive, recompute SHA-256 and require equality with `backup_sha256`. + +## Backup is not restore + +**Backup is not restore.** A successful `pg_dump`, `pg_restore --list`, and digest check establishes a verified backup artifact, not disaster-recovery success. + +Current evidence state: + +- RPO: not measured +- RTO: not measured +- clean-target restore: not yet proven +- destructive-loss replacement: not yet proven +- application restart after restore: not yet proven +- restored durable-job/idempotency invariants: not yet proven + +Do not advertise an RPO or RTO until a repeatable recovery rehearsal measures it from an explicitly defined failure point and workload. + +## Recovery-domain boundaries + +A PostgreSQL logical restore cannot by itself rewind or reconcile every mightyETL side effect. A full #188 recovery rehearsal must treat these as separate authorities: + +- **Kafka:** broker topics, consumer groups, offsets, retained records, and acknowledged publications are outside a PostgreSQL dump. +- **Debezium:** connector offset and schema-history state may live outside the restored application database and must be reconciled against the chosen recovery point. +- **DLT:** dead-letter records, retention, deletion, and redrive authorization are broker/data-governance state, not PostgreSQL backup contents. +- **external target:** warehouse, BI, JDBC, or other external target writes are not rolled back by restoring PostgreSQL. Recovery must use proven idempotency, reconciliation, or compensation for each target boundary. + +A database-only restore must therefore never be described as end-to-end exactly-once recovery. + +## Next recovery acceptance increment + +The next bounded #188 increment should create a disposable clean PostgreSQL target and prove, without touching a production database: + +1. manifest and SHA-256 verification before restore; +2. restore from the custom archive using `pg_restore`; +3. PostgreSQL and Flyway schema/migration identity after restore; +4. representative durable ETL/idempotency data invariants; +5. application startup/readiness against the restored database; +6. deliberate destructive-loss and replacement procedure; +7. documented treatment of Kafka, Debezium, DLT, and external-target divergence; +8. measured elapsed recovery evidence before any RTO claim. + +Rollback of this backup feature means disabling/removing the operator command and its documentation, not deleting previously created recovery evidence. Existing archives remain sensitive operational artifacts and must follow the operator's retention/destruction policy. + +## References + +PostgreSQL Global Development Group. (2026). *pg_dump (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/app-pgdump.html + +PostgreSQL Global Development Group. (2026). *pg_restore (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/app-pgrestore.html + +PostgreSQL Global Development Group. (2026). *Backup and restore (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/backup.html From 34ef1f3c87705757d057505f556b40ca7f380424 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:04:16 +0900 Subject: [PATCH 05/30] test(ops): require truthful backup changelog contract --- .../PostgresLogicalBackupContractTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index 0d865593..5cd7af3f 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -59,6 +59,19 @@ void recoveryRunbookSeparatesVerifiedBackupFromUnprovenRestoreAndRecoveryObjecti assertTrue(runbook.contains("external target"), "database recovery scope must distinguish external target effects"); } + @Test + void changelogRecordsTheNewBackupCapabilityWithoutClaimingRestoreReadiness() throws IOException { + String changelog = Files.readString(projectRoot().resolve("CHANGELOG.md"), StandardCharsets.UTF_8); + assertTrue( + changelog.contains("verified PostgreSQL logical backup"), + "the operator-visible backup capability must be discoverable in the changelog" + ); + assertTrue( + changelog.contains("does not prove restore or disaster-recovery readiness"), + "the changelog must not promote a backup artifact into an untested recovery claim" + ); + } + private static Path projectRoot() { Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); Path lastPomParent = null; From 37d1680f10e186b470275e1f5e909d345a34bd32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:07:58 +0900 Subject: [PATCH 06/30] docs(changelog): record verified PostgreSQL backup boundary --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b08b8c..3a41a0c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a private, verified PostgreSQL logical backup bundle with exact application-source, PostgreSQL/Flyway migration, and SHA-256 provenance; this does not prove restore or disaster-recovery readiness, which remains tracked by #188. - Principal-scoped durable asynchronous ETL job intake and owner-scoped status resources, Flyway `etl_job_records` migration, deterministic replay/conflict coverage, and the explicit worker boundary in `docs/etl/durable-job-intake.md`. - Durable idempotency ledger migration, PostgreSQL transaction advisory-lock adapter, deterministic concurrency/rollback coverage, and the operator/client contract `docs/etl/idempotent-retries.md`. - ETL problem-details client and operator contract: `docs/api/problem-details.md`. From 46f9ef3a3ffd3d179c3826b17bfed7a1d4346a30 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:10:54 +0900 Subject: [PATCH 07/30] test(ops): fail closed on concurrent backup identity collision --- .../PostgresLogicalBackupContractTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index 5cd7af3f..a768bef3 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -41,6 +41,25 @@ void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOExc assertFalse(script.contains("echo \"$PGPASSWORD\""), "database credentials must never be printed"); } + @Test + void backupToolReservesTheFinalIdentityBeforeWritingToPreventSameSecondCollisions() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"), + StandardCharsets.UTF_8 + ); + String reservation = "reservation_directory=\"${backup_directory}/.${backup_identity}.reservation\""; + String atomicReservation = "if ! mkdir -- \"$reservation_directory\""; + String temporaryBundle = "temporary_bundle=$(mktemp -d"; + + assertTrue(script.contains(reservation), "a timestamp/source backup identity needs a same-filesystem reservation"); + assertTrue(script.contains(atomicReservation), "concurrent creation of the same identity must fail closed"); + assertTrue(script.contains("rm -rf -- \"$reservation_directory\""), "the reservation must be released on exit"); + assertTrue( + script.indexOf(atomicReservation) < script.indexOf(temporaryBundle), + "identity reservation must happen before backup work starts" + ); + } + @Test void recoveryRunbookSeparatesVerifiedBackupFromUnprovenRestoreAndRecoveryObjectives() throws IOException { Path runbookPath = projectRoot().resolve("docs/ops/postgres-recovery.md"); From 8c248b02ba14caeb90eb91260eb1cdf611d241fa Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:12:44 +0900 Subject: [PATCH 08/30] fix(ops): reserve backup identities before dump work --- scripts/ops/postgres-logical-backup.sh | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/scripts/ops/postgres-logical-backup.sh b/scripts/ops/postgres-logical-backup.sh index f84a11c4..b47e7a96 100755 --- a/scripts/ops/postgres-logical-backup.sh +++ b/scripts/ops/postgres-logical-backup.sh @@ -40,18 +40,30 @@ backup_directory=$(cd -- "$BACKUP_DIRECTORY" && pwd -P) created_at_utc=$(date -u '+%Y-%m-%dT%H:%M:%SZ') backup_identity="mightyetl-postgres-${created_at_utc//[:]/}-${APPLICATION_SOURCE_SHA}" final_bundle="${backup_directory}/${backup_identity}" +reservation_directory="${backup_directory}/.${backup_identity}.reservation" if [[ -e "$final_bundle" ]]; then printf 'Refusing to replace an existing backup bundle: %s\n' "$final_bundle" >&2 exit 4 fi -temporary_bundle=$(mktemp -d "${backup_directory}/.mightyetl-postgres-backup.XXXXXX") +# mkdir is the same-filesystem compare-and-set for a second-resolution backup identity. +# A concurrent invocation with the same source and timestamp must fail before dump work begins. +if ! mkdir -- "$reservation_directory" 2>/dev/null; then + printf 'Backup identity is already reserved by another invocation: %s\n' "$backup_identity" >&2 + exit 4 +fi + +temporary_bundle="" cleanup() { - rm -rf -- "$temporary_bundle" + if [[ -n "$temporary_bundle" ]]; then + rm -rf -- "$temporary_bundle" + fi + rm -rf -- "$reservation_directory" } trap cleanup EXIT +temporary_bundle=$(mktemp -d "${backup_directory}/.mightyetl-postgres-backup.XXXXXX") archive_path="${temporary_bundle}/database.dump" manifest_path="${temporary_bundle}/manifest.txt" @@ -95,5 +107,7 @@ printf '%s\n' \ # Publish archive and manifest together through one same-filesystem directory rename. mv -- "$temporary_bundle" "$final_bundle" +temporary_bundle="" +rm -rf -- "$reservation_directory" trap - EXIT printf '%s\n' "$final_bundle" From e65f9bdab0f7f59685a7adabb89471b88c59da95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:20:34 +0900 Subject: [PATCH 09/30] test(ops): require disposable-target PostgreSQL restore rehearsal --- .../PostgresLogicalBackupContractTest.java | 55 +++++++++++-------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index a768bef3..34253335 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -12,17 +12,14 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Defines the first executable recovery contract for a local PostgreSQL logical backup. + * Defines executable recovery contracts for PostgreSQL backup and disposable-target restore rehearsal. */ class PostgresLogicalBackupContractTest { @Test void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOException { Path scriptPath = projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"); - assertTrue( - Files.isRegularFile(scriptPath), - "A supported logical-backup command must exist before named volumes can be called recoverable" - ); + assertTrue(Files.isRegularFile(scriptPath), "a supported logical-backup command must exist"); String script = Files.readString(scriptPath, StandardCharsets.UTF_8); assertTrue(script.contains("set -euo pipefail"), "backup failures must fail closed"); @@ -30,7 +27,7 @@ void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOExc assertTrue(script.contains("${BACKUP_DIRECTORY:?"), "operators must choose the backup destination explicitly"); assertTrue(script.contains("${APPLICATION_SOURCE_SHA:?"), "backup provenance must bind the exact application source"); assertTrue(script.contains("pg_dump"), "the local PostgreSQL profile must use a database-consistent logical dump"); - assertTrue(script.contains("--format=custom"), "the archive must use PostgreSQL custom format for pg_restore validation"); + assertTrue(script.contains("--format=custom"), "the archive must use PostgreSQL custom format"); assertTrue(script.contains("pg_restore --list"), "the completed archive must be structurally verified before publication"); assertTrue(script.contains("server_version_num"), "the manifest must record the PostgreSQL server version"); assertTrue(script.contains("flyway_schema_history"), "the manifest must bind the Flyway migration level"); @@ -43,10 +40,7 @@ void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOExc @Test void backupToolReservesTheFinalIdentityBeforeWritingToPreventSameSecondCollisions() throws IOException { - String script = Files.readString( - projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"), - StandardCharsets.UTF_8 - ); + String script = Files.readString(projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"), StandardCharsets.UTF_8); String reservation = "reservation_directory=\"${backup_directory}/.${backup_identity}.reservation\""; String atomicReservation = "if ! mkdir -- \"$reservation_directory\""; String temporaryBundle = "temporary_bundle=$(mktemp -d"; @@ -54,10 +48,33 @@ void backupToolReservesTheFinalIdentityBeforeWritingToPreventSameSecondCollision assertTrue(script.contains(reservation), "a timestamp/source backup identity needs a same-filesystem reservation"); assertTrue(script.contains(atomicReservation), "concurrent creation of the same identity must fail closed"); assertTrue(script.contains("rm -rf -- \"$reservation_directory\""), "the reservation must be released on exit"); - assertTrue( - script.indexOf(atomicReservation) < script.indexOf(temporaryBundle), - "identity reservation must happen before backup work starts" - ); + assertTrue(script.indexOf(atomicReservation) < script.indexOf(temporaryBundle), "reserve identity before backup work starts"); + } + + @Test + void restoreRehearsalVerifiesProvenanceAndRequiresAnEmptyExplicitTarget() throws IOException { + Path scriptPath = projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"); + assertTrue(Files.isRegularFile(scriptPath), "a verified backup needs a bounded disposable-target restore rehearsal command"); + + String script = Files.readString(scriptPath, StandardCharsets.UTF_8); + assertTrue(script.contains("set -euo pipefail"), "restore rehearsal failures must fail closed"); + assertTrue(script.contains("${BACKUP_BUNDLE:?"), "the archive bundle must be explicit"); + assertTrue(script.contains("${EXPECTED_APPLICATION_SOURCE_SHA:?"), "restore must bind the expected application source"); + assertTrue(script.contains("${RECOVERY_PGHOST:?"), "restore target host must be explicit"); + assertTrue(script.contains("${RECOVERY_PGDATABASE:?"), "restore target database must be explicit"); + assertTrue(script.contains("backup_sha256"), "restore must verify the manifest digest"); + assertTrue(script.contains("application_source_sha"), "restore must verify application-source provenance"); + assertTrue(script.contains("pg_restore --list"), "archive structure must be verified before restore"); + assertTrue(script.contains("user_table_count"), "restore must prove the target is empty before writing"); + assertTrue(script.contains("pg_restore"), "rehearsal must use PostgreSQL restore tooling"); + assertTrue(script.contains("--exit-on-error"), "restore must fail at the first PostgreSQL restore error"); + assertTrue(script.contains("--no-owner"), "rehearsal must not require archived ownership to exist"); + assertTrue(script.contains("--no-privileges"), "rehearsal must not import archived grants into the target"); + assertTrue(script.contains("restored_flyway_schema_version"), "post-restore migration identity must be verified"); + assertFalse(script.contains("source \"$manifest_path\""), "an untrusted manifest must never be executed as shell code"); + assertFalse(script.contains("--clean"), "the rehearsal must not make an arbitrary target destructive"); + assertFalse(script.contains("dropdb"), "the rehearsal must not drop a database"); + assertFalse(script.contains("createdb"), "the operator must provision the disposable target explicitly"); } @Test @@ -81,14 +98,8 @@ void recoveryRunbookSeparatesVerifiedBackupFromUnprovenRestoreAndRecoveryObjecti @Test void changelogRecordsTheNewBackupCapabilityWithoutClaimingRestoreReadiness() throws IOException { String changelog = Files.readString(projectRoot().resolve("CHANGELOG.md"), StandardCharsets.UTF_8); - assertTrue( - changelog.contains("verified PostgreSQL logical backup"), - "the operator-visible backup capability must be discoverable in the changelog" - ); - assertTrue( - changelog.contains("does not prove restore or disaster-recovery readiness"), - "the changelog must not promote a backup artifact into an untested recovery claim" - ); + assertTrue(changelog.contains("verified PostgreSQL logical backup"), "the backup capability must be discoverable"); + assertTrue(changelog.contains("does not prove restore or disaster-recovery readiness"), "backup must not inflate readiness"); } private static Path projectRoot() { From fb6f7d3b1685108fd636f74d035b4497592d1500 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 13:25:08 +0900 Subject: [PATCH 10/30] feat(ops): rehearse PostgreSQL restore on an explicit empty target --- .../ops/postgres-logical-restore-rehearsal.sh | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100755 scripts/ops/postgres-logical-restore-rehearsal.sh diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh new file mode 100755 index 00000000..de2fe441 --- /dev/null +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +: "${BACKUP_BUNDLE:?Set BACKUP_BUNDLE to one verified mightyETL PostgreSQL backup bundle}" +: "${EXPECTED_APPLICATION_SOURCE_SHA:?Set EXPECTED_APPLICATION_SOURCE_SHA to the exact source revision expected in the backup}" +: "${RECOVERY_PGHOST:?Set RECOVERY_PGHOST to the disposable PostgreSQL restore target}" +: "${RECOVERY_PGPORT:?Set RECOVERY_PGPORT to the disposable PostgreSQL restore target port}" +: "${RECOVERY_PGDATABASE:?Set RECOVERY_PGDATABASE to the explicitly provisioned empty restore database}" +: "${RECOVERY_PGUSER:?Set RECOVERY_PGUSER to the restore role}" + +if [[ ! "$EXPECTED_APPLICATION_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + printf 'EXPECTED_APPLICATION_SOURCE_SHA must be a 40-character lowercase Git commit SHA\n' >&2 + exit 2 +fi + +for command_name in pg_restore psql; do + if ! command -v "$command_name" >/dev/null 2>&1; then + printf 'Required command is unavailable: %s\n' "$command_name" >&2 + exit 3 + fi +done + +sha256_file() { + local file_path=$1 + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$file_path" | awk '{print $1}' + return + fi + if command -v shasum >/dev/null 2>&1; then + shasum -a 256 "$file_path" | awk '{print $1}' + return + fi + printf 'Neither sha256sum nor shasum is available\n' >&2 + return 1 +} + +backup_bundle=$(cd -- "$BACKUP_BUNDLE" 2>/dev/null && pwd -P) || { + printf 'BACKUP_BUNDLE must be an existing directory\n' >&2 + exit 4 +} +archive_path="${backup_bundle}/database.dump" +manifest_path="${backup_bundle}/manifest.txt" + +if [[ -L "$BACKUP_BUNDLE" || ! -f "$archive_path" || -L "$archive_path" || ! -f "$manifest_path" || -L "$manifest_path" ]]; then + printf 'Backup bundle must contain regular, non-symlink database.dump and manifest.txt files\n' >&2 + exit 4 +fi + +manifest_value() { + local wanted_key=$1 + local matched_value="" + local match_count=0 + local line key value + + while IFS= read -r line || [[ -n "$line" ]]; do + if [[ "$line" != *=* ]]; then + printf 'Invalid backup manifest line\n' >&2 + return 1 + fi + key=${line%%=*} + value=${line#*=} + if [[ "$key" == "$wanted_key" ]]; then + match_count=$((match_count + 1)) + matched_value=$value + fi + done < "$manifest_path" + + if [[ "$match_count" -ne 1 ]]; then + printf 'Backup manifest must contain exactly one %s entry\n' "$wanted_key" >&2 + return 1 + fi + printf '%s' "$matched_value" +} + +manifest_version=$(manifest_value manifest_version) +application_source_sha=$(manifest_value application_source_sha) +expected_backup_sha256=$(manifest_value backup_sha256) +expected_flyway_schema_version=$(manifest_value flyway_schema_version) + +if [[ "$manifest_version" != "1" ]]; then + printf 'Unsupported backup manifest version\n' >&2 + exit 4 +fi +if [[ ! "$application_source_sha" =~ ^[0-9a-f]{40}$ || "$application_source_sha" != "$EXPECTED_APPLICATION_SOURCE_SHA" ]]; then + printf 'Backup application source does not match the expected source revision\n' >&2 + exit 4 +fi +if [[ ! "$expected_backup_sha256" =~ ^[0-9a-f]{64}$ ]]; then + printf 'Backup manifest SHA-256 is invalid\n' >&2 + exit 4 +fi + +actual_backup_sha256=$(sha256_file "$archive_path") +if [[ "$actual_backup_sha256" != "$expected_backup_sha256" ]]; then + printf 'Backup archive SHA-256 verification failed\n' >&2 + exit 4 +fi + +# Verify that the archive can be parsed before any write reaches the recovery target. +pg_restore --list "$archive_path" >/dev/null + +psql_recovery=( + psql + --host="$RECOVERY_PGHOST" + --port="$RECOVERY_PGPORT" + --username="$RECOVERY_PGUSER" + --dbname="$RECOVERY_PGDATABASE" + --no-psqlrc + --tuples-only + --no-align + --set=ON_ERROR_STOP=1 +) + +user_table_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')") +user_table_count=${user_table_count//[[:space:]]/} +if [[ ! "$user_table_count" =~ ^[0-9]+$ || "$user_table_count" != "0" ]]; then + printf 'Recovery target must be explicitly provisioned and empty before rehearsal\n' >&2 + exit 5 +fi + +pg_restore \ + --host="$RECOVERY_PGHOST" \ + --port="$RECOVERY_PGPORT" \ + --username="$RECOVERY_PGUSER" \ + --dbname="$RECOVERY_PGDATABASE" \ + --exit-on-error \ + --no-owner \ + --no-privileges \ + "$archive_path" + +restored_flyway_schema_version=$("${psql_recovery[@]}" --command="SELECT COALESCE((SELECT version FROM flyway_schema_history WHERE success = true ORDER BY installed_rank DESC LIMIT 1), 'none')") +restored_flyway_schema_version=${restored_flyway_schema_version//$'\r'/} +restored_flyway_schema_version=${restored_flyway_schema_version//$'\n'/} + +if [[ "$restored_flyway_schema_version" != "$expected_flyway_schema_version" ]]; then + printf 'Restored Flyway schema version does not match backup provenance\n' >&2 + exit 6 +fi + +printf '%s\n' 'PostgreSQL restore rehearsal completed on the explicit disposable target' From 5ca064072000e769ac14c6d8ed653a6b77c2bd9b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 14:08:11 +0900 Subject: [PATCH 11/30] test(ops): fail on cross-major PostgreSQL restore rehearsal --- ...ogicalRestoreVersionCompatibilityTest.java | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java new file mode 100644 index 00000000..23d48960 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java @@ -0,0 +1,62 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Defines the PostgreSQL major-version compatibility boundary for restore rehearsals. + */ +class PostgresLogicalRestoreVersionCompatibilityTest { + + @Test + void restoreRehearsalRejectsMajorVersionMismatchBeforeDatabaseWrites() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8); + + String backupVersionRead = "backup_server_version_num=$(manifest_value server_version_num)"; + String targetVersionRead = "target_server_version_num=$(\"${psql_recovery[@]}\" --command='SHOW server_version_num')"; + String backupMajor = "backup_server_major=$((backup_server_version_num / 10000))"; + String targetMajor = "target_server_major=$((target_server_version_num / 10000))"; + String majorMismatch = "if [[ \"$backup_server_major\" != \"$target_server_major\" ]]"; + String restoreWrite = "pg_restore \\\n --host=\"$RECOVERY_PGHOST\""; + + assertTrue(script.contains(backupVersionRead), + "restore must read the PostgreSQL server version captured by backup provenance"); + assertTrue(script.contains(targetVersionRead), + "restore must resolve the disposable target PostgreSQL server version before writing"); + assertTrue(script.contains(backupMajor), + "restore must derive the source PostgreSQL major version from server_version_num"); + assertTrue(script.contains(targetMajor), + "restore must derive the target PostgreSQL major version from server_version_num"); + assertTrue(script.contains(majorMismatch), + "the bounded rehearsal must fail closed on a cross-major restore target"); + assertTrue(script.indexOf(majorMismatch) < script.indexOf(restoreWrite), + "major-version compatibility must be checked before pg_restore writes to the target"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From 927578ceb98480563c88896f2eb3fec02c8c5a72 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 14:11:00 +0900 Subject: [PATCH 12/30] fix(ops): reject cross-major PostgreSQL restore rehearsals --- .../ops/postgres-logical-restore-rehearsal.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index de2fe441..5dd0be49 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -77,6 +77,7 @@ manifest_version=$(manifest_value manifest_version) application_source_sha=$(manifest_value application_source_sha) expected_backup_sha256=$(manifest_value backup_sha256) expected_flyway_schema_version=$(manifest_value flyway_schema_version) +backup_server_version_num=$(manifest_value server_version_num) if [[ "$manifest_version" != "1" ]]; then printf 'Unsupported backup manifest version\n' >&2 @@ -90,6 +91,10 @@ if [[ ! "$expected_backup_sha256" =~ ^[0-9a-f]{64}$ ]]; then printf 'Backup manifest SHA-256 is invalid\n' >&2 exit 4 fi +if [[ ! "$backup_server_version_num" =~ ^[0-9]{6,9}$ ]]; then + printf 'Backup PostgreSQL server version provenance is invalid\n' >&2 + exit 4 +fi actual_backup_sha256=$(sha256_file "$archive_path") if [[ "$actual_backup_sha256" != "$expected_backup_sha256" ]]; then @@ -112,6 +117,20 @@ psql_recovery=( --set=ON_ERROR_STOP=1 ) +target_server_version_num=$("${psql_recovery[@]}" --command='SHOW server_version_num') +target_server_version_num=${target_server_version_num//[[:space:]]/} +if [[ ! "$target_server_version_num" =~ ^[0-9]{6,9}$ ]]; then + printf 'Recovery target PostgreSQL server version is invalid\n' >&2 + exit 5 +fi + +backup_server_major=$((backup_server_version_num / 10000)) +target_server_major=$((target_server_version_num / 10000)) +if [[ "$backup_server_major" != "$target_server_major" ]]; then + printf 'Recovery target PostgreSQL major version does not match backup provenance\n' >&2 + exit 5 +fi + user_table_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') AND n.nspname NOT LIKE 'pg_toast%' AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')") user_table_count=${user_table_count//[[:space:]]/} if [[ ! "$user_table_count" =~ ^[0-9]+$ || "$user_table_count" != "0" ]]; then From 3761d7f020700a526e05f9f8fe6d6857f9025308 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 14:13:10 +0900 Subject: [PATCH 13/30] test(ops): normalize restore script line endings --- .../PostgresLogicalRestoreVersionCompatibilityTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java index 23d48960..5ecf8bdc 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreVersionCompatibilityTest.java @@ -19,7 +19,9 @@ class PostgresLogicalRestoreVersionCompatibilityTest { void restoreRehearsalRejectsMajorVersionMismatchBeforeDatabaseWrites() throws IOException { String script = Files.readString( projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), - StandardCharsets.UTF_8); + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); String backupVersionRead = "backup_server_version_num=$(manifest_value server_version_num)"; String targetVersionRead = "target_server_version_num=$(\"${psql_recovery[@]}\" --command='SHOW server_version_num')"; From 70741b0a28e1972a289822524f1f129766614a85 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 14:15:42 +0900 Subject: [PATCH 14/30] test(ops): require independent restore archive digest --- ...ostgresLogicalRestoreAuthenticityTest.java | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java new file mode 100644 index 00000000..7a4cfc4c --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java @@ -0,0 +1,61 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Defines the independent archive-digest boundary for PostgreSQL restore rehearsals. + */ +class PostgresLogicalRestoreAuthenticityTest { + + @Test + void restoreRequiresAnOutOfBandExpectedArchiveDigestBeforeArchiveInspection() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + + String expectedDigestInput = + ": \"${EXPECTED_BACKUP_SHA256:?Set EXPECTED_BACKUP_SHA256 to the independently recorded backup archive digest}\""; + String expectedDigestValidation = + "if [[ ! \"$EXPECTED_BACKUP_SHA256\" =~ ^[0-9a-f]{64}$ ]]"; + String manifestDigestMatch = + "if [[ \"$expected_backup_sha256\" != \"$EXPECTED_BACKUP_SHA256\" ]]"; + String archiveInspection = "pg_restore --list \"$archive_path\""; + + assertTrue(script.contains(expectedDigestInput), + "restore must require an expected archive digest supplied outside the mutable backup bundle"); + assertTrue(script.contains(expectedDigestValidation), + "restore must reject malformed out-of-band digest evidence"); + assertTrue(script.contains(manifestDigestMatch), + "the bundle manifest digest must match independently supplied digest evidence"); + assertTrue(script.indexOf(manifestDigestMatch) < script.indexOf(archiveInspection), + "independent digest evidence must be checked before pg_restore parses the archive"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From 9c5f248490ee6fcaf350200131e2e01198a18e7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 14:21:53 +0900 Subject: [PATCH 15/30] fix(ops): bind restore to independent archive digest --- scripts/ops/postgres-logical-restore-rehearsal.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index 5dd0be49..4c9a936c 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -4,6 +4,7 @@ umask 077 : "${BACKUP_BUNDLE:?Set BACKUP_BUNDLE to one verified mightyETL PostgreSQL backup bundle}" : "${EXPECTED_APPLICATION_SOURCE_SHA:?Set EXPECTED_APPLICATION_SOURCE_SHA to the exact source revision expected in the backup}" +: "${EXPECTED_BACKUP_SHA256:?Set EXPECTED_BACKUP_SHA256 to the independently recorded backup archive digest}" : "${RECOVERY_PGHOST:?Set RECOVERY_PGHOST to the disposable PostgreSQL restore target}" : "${RECOVERY_PGPORT:?Set RECOVERY_PGPORT to the disposable PostgreSQL restore target port}" : "${RECOVERY_PGDATABASE:?Set RECOVERY_PGDATABASE to the explicitly provisioned empty restore database}" @@ -13,6 +14,10 @@ if [[ ! "$EXPECTED_APPLICATION_SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then printf 'EXPECTED_APPLICATION_SOURCE_SHA must be a 40-character lowercase Git commit SHA\n' >&2 exit 2 fi +if [[ ! "$EXPECTED_BACKUP_SHA256" =~ ^[0-9a-f]{64}$ ]]; then + printf 'EXPECTED_BACKUP_SHA256 must be a 64-character lowercase SHA-256 digest\n' >&2 + exit 2 +fi for command_name in pg_restore psql; do if ! command -v "$command_name" >/dev/null 2>&1; then @@ -91,6 +96,10 @@ if [[ ! "$expected_backup_sha256" =~ ^[0-9a-f]{64}$ ]]; then printf 'Backup manifest SHA-256 is invalid\n' >&2 exit 4 fi +if [[ "$expected_backup_sha256" != "$EXPECTED_BACKUP_SHA256" ]]; then + printf 'Backup manifest digest does not match independently recorded archive evidence\n' >&2 + exit 4 +fi if [[ ! "$backup_server_version_num" =~ ^[0-9]{6,9}$ ]]; then printf 'Backup PostgreSQL server version provenance is invalid\n' >&2 exit 4 From 399984b8a46c68793271746fb34e33d497f69d9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 15:05:21 +0900 Subject: [PATCH 16/30] test(ops): require independent restore manifest integrity --- ...esLogicalRestoreManifestIntegrityTest.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreManifestIntegrityTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreManifestIntegrityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreManifestIntegrityTest.java new file mode 100644 index 00000000..3ac66a61 --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreManifestIntegrityTest.java @@ -0,0 +1,64 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Defines the independent manifest-integrity boundary for PostgreSQL restore rehearsals. + */ +class PostgresLogicalRestoreManifestIntegrityTest { + + @Test + void restoreAuthenticatesManifestBeforeTrustingProvenanceFields() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + + String expectedManifestDigestInput = + ": \"${EXPECTED_MANIFEST_SHA256:?Set EXPECTED_MANIFEST_SHA256 to the independently recorded backup manifest digest}\""; + String expectedManifestDigestValidation = + "if [[ ! \"$EXPECTED_MANIFEST_SHA256\" =~ ^[0-9a-f]{64}$ ]]"; + String actualManifestDigest = "actual_manifest_sha256=$(sha256_file \"$manifest_path\")"; + String manifestDigestMatch = + "if [[ \"$actual_manifest_sha256\" != \"$EXPECTED_MANIFEST_SHA256\" ]]"; + String firstManifestFieldRead = "manifest_version=$(manifest_value manifest_version)"; + + assertTrue(script.contains(expectedManifestDigestInput), + "restore must require manifest integrity evidence supplied outside the mutable backup bundle"); + assertTrue(script.contains(expectedManifestDigestValidation), + "restore must reject malformed out-of-band manifest digest evidence"); + assertTrue(script.contains(actualManifestDigest), + "restore must hash the manifest before trusting any provenance field"); + assertTrue(script.contains(manifestDigestMatch), + "restore must compare the manifest to independently supplied integrity evidence"); + assertTrue(script.indexOf(manifestDigestMatch) < script.indexOf(firstManifestFieldRead), + "manifest integrity must be established before source/version/Flyway provenance is parsed"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From 227000aa9fa919865bfcd04d8424363050e39f78 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 15:09:12 +0900 Subject: [PATCH 17/30] fix(ops): verify restore manifest integrity before provenance --- scripts/ops/postgres-logical-restore-rehearsal.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index 4c9a936c..6fd2d6eb 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -5,6 +5,7 @@ umask 077 : "${BACKUP_BUNDLE:?Set BACKUP_BUNDLE to one verified mightyETL PostgreSQL backup bundle}" : "${EXPECTED_APPLICATION_SOURCE_SHA:?Set EXPECTED_APPLICATION_SOURCE_SHA to the exact source revision expected in the backup}" : "${EXPECTED_BACKUP_SHA256:?Set EXPECTED_BACKUP_SHA256 to the independently recorded backup archive digest}" +: "${EXPECTED_MANIFEST_SHA256:?Set EXPECTED_MANIFEST_SHA256 to the independently recorded backup manifest digest}" : "${RECOVERY_PGHOST:?Set RECOVERY_PGHOST to the disposable PostgreSQL restore target}" : "${RECOVERY_PGPORT:?Set RECOVERY_PGPORT to the disposable PostgreSQL restore target port}" : "${RECOVERY_PGDATABASE:?Set RECOVERY_PGDATABASE to the explicitly provisioned empty restore database}" @@ -18,6 +19,10 @@ if [[ ! "$EXPECTED_BACKUP_SHA256" =~ ^[0-9a-f]{64}$ ]]; then printf 'EXPECTED_BACKUP_SHA256 must be a 64-character lowercase SHA-256 digest\n' >&2 exit 2 fi +if [[ ! "$EXPECTED_MANIFEST_SHA256" =~ ^[0-9a-f]{64}$ ]]; then + printf 'EXPECTED_MANIFEST_SHA256 must be a 64-character lowercase SHA-256 digest\n' >&2 + exit 2 +fi for command_name in pg_restore psql; do if ! command -v "$command_name" >/dev/null 2>&1; then @@ -52,6 +57,12 @@ if [[ -L "$BACKUP_BUNDLE" || ! -f "$archive_path" || -L "$archive_path" || ! -f exit 4 fi +actual_manifest_sha256=$(sha256_file "$manifest_path") +if [[ "$actual_manifest_sha256" != "$EXPECTED_MANIFEST_SHA256" ]]; then + printf 'Backup manifest SHA-256 verification failed\n' >&2 + exit 4 +fi + manifest_value() { local wanted_key=$1 local matched_value="" From 2c8716e39900fbf8992c09d6a4d80360fa8ad453 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 15:21:26 +0900 Subject: [PATCH 18/30] test(ops): require restored application relation invariants --- ...ogicalRestoreApplicationInvariantTest.java | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreApplicationInvariantTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreApplicationInvariantTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreApplicationInvariantTest.java new file mode 100644 index 00000000..2e3fb81b --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreApplicationInvariantTest.java @@ -0,0 +1,70 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Binds PostgreSQL restore rehearsal success to the minimum application relations owned by the + * current protected mightyETL database contract. + */ +class PostgresLogicalRestoreApplicationInvariantTest { + + @Test + void restoreVerifiesCriticalApplicationRelationsBeforeReportingSuccess() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + + String requiredRelationsQuery = + "SELECT count(*) FROM (VALUES (to_regclass('public.processed_data')), " + + "(to_regclass('public.etl_idempotency_records')), " + + "(to_regclass('public.etl_job_records'))) AS required(relation_oid) " + + "WHERE relation_oid IS NOT NULL"; + String invariantFailure = + "Restored database is missing one or more required mightyETL application relations"; + String flywayVerification = + "if [[ \"$restored_flyway_schema_version\" != \"$expected_flyway_schema_version\" ]]"; + String success = + "PostgreSQL restore rehearsal completed on the explicit disposable target"; + + assertTrue(script.contains(requiredRelationsQuery), + "restore must verify the current processed-data, idempotency, and durable-job relations"); + assertTrue(script.contains("required_application_relation_count"), + "restore must bind the relation check to an explicit finite result"); + assertTrue(script.contains("!= \"3\""), + "restore must fail closed unless all three required relations exist"); + assertTrue(script.contains(invariantFailure), + "restore must emit a stable operator classification when application relations are missing"); + assertTrue(script.indexOf(flywayVerification) < script.indexOf(requiredRelationsQuery), + "application relation verification must occur after Flyway provenance verification"); + assertTrue(script.indexOf(requiredRelationsQuery) < script.indexOf(success), + "restore must not report success before application relations are verified"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From f95ef877c366550c24172d4ed840bd17344391ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 15:23:41 +0900 Subject: [PATCH 19/30] fix(ops): verify restored application relations --- scripts/ops/postgres-logical-restore-rehearsal.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index 6fd2d6eb..e3d22e68 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -177,4 +177,11 @@ if [[ "$restored_flyway_schema_version" != "$expected_flyway_schema_version" ]]; exit 6 fi +required_application_relation_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM (VALUES (to_regclass('public.processed_data')), (to_regclass('public.etl_idempotency_records')), (to_regclass('public.etl_job_records'))) AS required(relation_oid) WHERE relation_oid IS NOT NULL") +required_application_relation_count=${required_application_relation_count//[[:space:]]/} +if [[ ! "$required_application_relation_count" =~ ^[0-9]+$ || "$required_application_relation_count" != "3" ]]; then + printf 'Restored database is missing one or more required mightyETL application relations\n' >&2 + exit 6 +fi + printf '%s\n' 'PostgreSQL restore rehearsal completed on the explicit disposable target' From b125e0693298de16e2634f4dc8a08d069c81bbd5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 18:10:51 +0900 Subject: [PATCH 20/30] test(ops): require atomic restore transaction --- ...ostgresLogicalRestoreAuthenticityTest.java | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java index 7a4cfc4c..bf3089ca 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreAuthenticityTest.java @@ -8,20 +8,17 @@ import java.nio.file.Path; import java.nio.file.Paths; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * Defines the independent archive-digest boundary for PostgreSQL restore rehearsals. + * Defines integrity and atomic-write boundaries for PostgreSQL restore rehearsals. */ class PostgresLogicalRestoreAuthenticityTest { @Test void restoreRequiresAnOutOfBandExpectedArchiveDigestBeforeArchiveInspection() throws IOException { - String script = Files.readString( - projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), - StandardCharsets.UTF_8) - .replace("\r\n", "\n") - .replace('\r', '\n'); + String script = restoreScript(); String expectedDigestInput = ": \"${EXPECTED_BACKUP_SHA256:?Set EXPECTED_BACKUP_SHA256 to the independently recorded backup archive digest}\""; @@ -41,6 +38,30 @@ void restoreRequiresAnOutOfBandExpectedArchiveDigestBeforeArchiveInspection() th "independent digest evidence must be checked before pg_restore parses the archive"); } + @Test + void restoreUsesOneTransactionSoACommandFailureCannotLeavePartialApplicationState() throws IOException { + String script = restoreScript(); + + int restoreWrite = script.lastIndexOf("pg_restore \\\n"); + assertTrue(restoreWrite >= 0, "restore script must contain a database-writing pg_restore invocation"); + String restoreCommand = script.substring(restoreWrite, script.indexOf("\n\n", restoreWrite)); + + assertTrue(restoreCommand.contains("--single-transaction"), + "disposable-target restore must be atomic so any command failure rolls back the complete restore"); + assertFalse(restoreCommand.contains("--transaction-size"), + "single-transaction recovery must not be weakened into a partially committed transaction batch"); + assertTrue(restoreCommand.contains("--exit-on-error"), + "restore must keep explicit fail-fast semantics even though single-transaction also implies it"); + } + + private static String restoreScript() throws IOException { + return Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + } + private static Path projectRoot() { Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); Path lastPomParent = null; From 579bcb11aca336aa33dbd7721032a94fbe4c81cd Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 18:12:48 +0900 Subject: [PATCH 21/30] fix(ops): make restore rehearsal atomic --- scripts/ops/postgres-logical-restore-rehearsal.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index e3d22e68..336fc7d7 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -164,6 +164,7 @@ pg_restore \ --username="$RECOVERY_PGUSER" \ --dbname="$RECOVERY_PGDATABASE" \ --exit-on-error \ + --single-transaction \ --no-owner \ --no-privileges \ "$archive_path" From 7fe5d201725b16739bd67e27732c997db91e8761 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 18:16:15 +0900 Subject: [PATCH 22/30] test(ops): require current restore runbook truth --- .../PostgresLogicalBackupContractTest.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index 34253335..76af9608 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -95,6 +95,26 @@ void recoveryRunbookSeparatesVerifiedBackupFromUnprovenRestoreAndRecoveryObjecti assertTrue(runbook.contains("external target"), "database recovery scope must distinguish external target effects"); } + @Test + void recoveryRunbookDescribesTheCurrentVerifiedRestoreBoundaryWithoutInflatingDisasterRecovery() throws IOException { + String runbook = Files.readString(projectRoot().resolve("docs/ops/postgres-recovery.md"), StandardCharsets.UTF_8); + + assertTrue(runbook.contains("postgres-logical-restore-rehearsal.sh"), + "operators must be able to find the implemented restore rehearsal command"); + assertTrue(runbook.contains("EXPECTED_MANIFEST_SHA256") && runbook.contains("EXPECTED_BACKUP_SHA256"), + "runbook must explain the out-of-band manifest and archive integrity evidence"); + assertTrue(runbook.contains("--single-transaction"), + "runbook must document that the disposable restore is all-or-nothing on PostgreSQL command failure"); + assertTrue(runbook.contains("processed_data") + && runbook.contains("etl_idempotency_records") + && runbook.contains("etl_job_records"), + "runbook must state the current post-restore application relation invariants"); + assertFalse(runbook.contains("clean-target restore: not yet proven"), + "runbook must not describe an implemented and verified branch capability as still absent"); + assertTrue(runbook.contains("application startup/readiness") && runbook.contains("not yet proven"), + "runbook must keep the still-unproven application startup/readiness boundary explicit"); + } + @Test void changelogRecordsTheNewBackupCapabilityWithoutClaimingRestoreReadiness() throws IOException { String changelog = Files.readString(projectRoot().resolve("CHANGELOG.md"), StandardCharsets.UTF_8); From 02dd83bdfad3141366da7008b77e4c0ce7b81ab0 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 18:22:51 +0900 Subject: [PATCH 23/30] docs(ops): align runbook with verified restore rehearsal --- docs/ops/postgres-recovery.md | 112 +++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 28 deletions(-) diff --git a/docs/ops/postgres-recovery.md b/docs/ops/postgres-recovery.md index d7d7b6df..874a29c3 100644 --- a/docs/ops/postgres-recovery.md +++ b/docs/ops/postgres-recovery.md @@ -1,14 +1,14 @@ # PostgreSQL logical backup and recovery boundary -Status: `active_pr` (#208). This document describes the bounded logical-backup capability on this branch. It is not `implemented_on_develop` until protected integration. +Status: `active_pr` (#208). This document describes the bounded PostgreSQL logical-backup and disposable-target restore-rehearsal capability on this branch. It is not `implemented_on_develop` until protected integration. ## Purpose and scope -The local mightyETL PostgreSQL profile uses persistent storage, but a named Docker volume is not a backup. `scripts/ops/postgres-logical-backup.sh` creates a separate PostgreSQL logical archive plus a provenance manifest so an operator can preserve database state independently of the live volume. +The local mightyETL PostgreSQL profile uses persistent storage, but a named Docker volume is not a backup. `scripts/ops/postgres-logical-backup.sh` creates a separate PostgreSQL custom-format logical archive plus a provenance manifest. `scripts/ops/postgres-logical-restore-rehearsal.sh` then provides a distinct fail-closed rehearsal that restores a verified bundle into an explicitly provisioned empty PostgreSQL database and checks bounded application invariants. -This first recovery increment proves **backup artifact creation and verification only**. It does not yet prove a clean restore, destructive-loss recovery, service restart, end-to-end CDC recovery, or a measured recovery objective. +The branch therefore proves more than archive creation, but it still does **not** prove disaster recovery. Application startup/readiness, destructive-loss replacement, representative durable-row recovery, Kafka/Debezium/DLT reconciliation, external-target compensation, PITR, and measured RPO/RTO remain outside the current evidence boundary. -## Prerequisites +## Backup prerequisites Use PostgreSQL client tooling compatible with the database server and provide ordinary libpq connection settings without writing credentials into source control: @@ -20,7 +20,7 @@ Use PostgreSQL client tooling compatible with the database server and provide or - `BACKUP_DIRECTORY`, an operator-controlled destination - `APPLICATION_SOURCE_SHA`, the exact 40-character mightyETL Git commit represented by the running deployment -The script creates files under `umask 077`. The operator remains responsible for storage encryption, host and object-store access control, retention, replication, deletion, and custody of database credentials. Do not place `PGPASSWORD` in command histories, logs, manifests, or committed configuration. +The backup script creates files under `umask 077`. The operator remains responsible for storage encryption, host and object-store access control, retention, replication, deletion, and custody of database credentials. Do not place `PGPASSWORD` in command histories, logs, manifests, or committed configuration. Example for a controlled local environment: @@ -37,11 +37,9 @@ export PGUSER="postgres" Do not substitute a guessed SHA. In a packaged or remote deployment, obtain `APPLICATION_SOURCE_SHA` from the exact deployed release/provenance record rather than whichever checkout happens to be on an operator workstation. -## Artifact and verification contract +## Backup artifact and provenance contract -The command creates a private temporary directory inside `BACKUP_DIRECTORY`, writes a PostgreSQL custom-format archive with `pg_dump --format=custom`, and requires `pg_restore --list` to read the completed archive before publication. PostgreSQL documents the custom archive as a format intended for `pg_restore` and suitable for selective/reordered restore operations. The structural check proves that the artifact is readable as a PostgreSQL archive; it does **not** prove that restoring it into a clean server will satisfy mightyETL recovery invariants. - -The companion `manifest.txt` records: +The backup command creates a private temporary directory inside `BACKUP_DIRECTORY`, writes a PostgreSQL custom-format archive with `pg_dump --format=custom`, and requires `pg_restore --list` to read the completed archive before publication. The companion `manifest.txt` records: - manifest version; - `application_source_sha`; @@ -50,20 +48,78 @@ The companion `manifest.txt` records: - latest successful `flyway_schema_history` migration version; - `backup_sha256` for `database.dump`. -Archive and manifest are published together only after the archive check and metadata queries succeed. The final bundle is never intentionally replaced by the command. Before moving or restoring an archive, recompute SHA-256 and require equality with `backup_sha256`. +Archive and manifest are published together only after archive verification and metadata queries succeed. The command reserves the final bundle identity before dump work so concurrent invocations cannot silently replace the same timestamp/source identity. Publication is a same-filesystem directory rename and never intentionally replaces an existing final bundle. + +The manifest is part of the evidence and must not authenticate itself. Before a restore rehearsal, independently record and protect both: + +1. the SHA-256 of `database.dump` as `EXPECTED_BACKUP_SHA256`; and +2. the SHA-256 of `manifest.txt` as `EXPECTED_MANIFEST_SHA256`. + +Those digests must be kept outside the mutable backup bundle in the deployment's approved recovery-evidence channel. Computing a digest only after accepting an untrusted transferred bundle is not independent provenance. + +## Disposable-target restore prerequisites + +`postgres-logical-restore-rehearsal.sh` requires all recovery authority to be explicit: + +- `BACKUP_BUNDLE`, the verified bundle directory; +- `EXPECTED_APPLICATION_SOURCE_SHA`, the expected 40-character source revision; +- `EXPECTED_BACKUP_SHA256`, the independently recorded archive digest; +- `EXPECTED_MANIFEST_SHA256`, the independently recorded manifest digest; +- `RECOVERY_PGHOST`; +- `RECOVERY_PGPORT`; +- `RECOVERY_PGDATABASE`, an explicitly provisioned empty database; +- `RECOVERY_PGUSER`; +- a deployment-approved libpq authentication mechanism such as `PGPASSWORD`, without printing or committing the credential. + +The rehearsal rejects symlinked archive/manifest files. It authenticates `manifest.txt` against `EXPECTED_MANIFEST_SHA256` **before** parsing provenance fields, then requires the manifest source SHA and archive digest to match the independently supplied expected values. It recomputes the archive SHA-256 and requires `pg_restore --list` to parse the custom archive before any target write. + +## Restore safety and compatibility boundary + +Before restore, the command: + +1. reads the recovery server's `server_version_num` and requires the same PostgreSQL major version as the backup provenance; +2. queries the target catalog and requires zero non-system relations of the governed relation kinds; +3. never uses `--clean`, `dropdb`, or `createdb`; the operator must deliberately provision the disposable empty target. + +The database-writing restore uses: + +```text +pg_restore --exit-on-error --single-transaction --no-owner --no-privileges +``` + +`--single-transaction` makes the restore all-or-nothing for PostgreSQL command failures: an error rolls back the complete restore instead of accepting earlier committed objects as a partially restored application state. `--no-owner` and `--no-privileges` avoid importing archived ownership/grants into the disposable target; they do not establish the final production authorization model. + +PostgreSQL's current version 18 guidance explicitly documents `-1` / `--single-transaction` for whole-dump restore and notes that even a small error rolls back the entire restore. This branch intentionally chooses that behavior for the bounded rehearsal because accepting a partial application restore would be a false recovery success. + +## Post-restore application invariants + +A successful `pg_restore` is not by itself accepted. The rehearsal then requires: + +- the latest successful `flyway_schema_history` version to equal the version recorded in the authenticated manifest; +- `public.processed_data` to exist; +- `public.etl_idempotency_records` to exist; +- `public.etl_job_records` to exist. + +These are structural application invariants only. They do not yet prove representative row contents, durable-job ownership/lineage, idempotency replay behavior, service startup, health/readiness, or application transactions against the restored database. ## Backup is not restore -**Backup is not restore.** A successful `pg_dump`, `pg_restore --list`, and digest check establishes a verified backup artifact, not disaster-recovery success. +**Backup is not restore.** A successful `pg_dump`, `pg_restore --list`, and digest check establishes a verified backup artifact. The separate restore-rehearsal command adds bounded clean-target restore evidence, but neither artifact alone establishes complete disaster recovery. -Current evidence state: +Current evidence state on this active PR: -- RPO: not measured -- RTO: not measured -- clean-target restore: not yet proven -- destructive-loss replacement: not yet proven -- application restart after restore: not yet proven -- restored durable-job/idempotency invariants: not yet proven +- verified logical backup artifact: proven by repository contracts; +- independently bound archive and manifest integrity inputs: proven by repository contracts; +- disposable clean-target restore: proven by the bounded restore-rehearsal contract; +- restore command-failure atomicity via `--single-transaction`: proven by the bounded restore-rehearsal contract; +- Flyway migration identity after restore: proven by the bounded restore-rehearsal contract; +- required application relation presence after restore: proven by the bounded restore-rehearsal contract; +- representative durable-job/idempotency row invariants: not yet proven; +- application startup/readiness against the restored database: not yet proven; +- destructive-loss replacement: not yet proven; +- end-to-end CDC/external-target recovery: not yet proven; +- RPO: not measured; +- RTO: not measured. Do not advertise an RPO or RTO until a repeatable recovery rehearsal measures it from an explicitly defined failure point and workload. @@ -80,18 +136,16 @@ A database-only restore must therefore never be described as end-to-end exactly- ## Next recovery acceptance increment -The next bounded #188 increment should create a disposable clean PostgreSQL target and prove, without touching a production database: +The next bounded #188 work should extend the disposable rehearsal without touching production data and prove, in order: -1. manifest and SHA-256 verification before restore; -2. restore from the custom archive using `pg_restore`; -3. PostgreSQL and Flyway schema/migration identity after restore; -4. representative durable ETL/idempotency data invariants; -5. application startup/readiness against the restored database; -6. deliberate destructive-loss and replacement procedure; -7. documented treatment of Kafka, Debezium, DLT, and external-target divergence; -8. measured elapsed recovery evidence before any RTO claim. +1. realistic PostgreSQL backup and restore execution against an ephemeral supported server rather than source-contract inspection alone; +2. representative durable ETL/idempotency data invariants and immutable lifecycle evidence after restore; +3. application startup/readiness and a bounded read/write smoke path against the restored database; +4. deliberate destructive-loss/replacement procedure with retry/incident handling that preserves the last known good backup; +5. documented and tested Kafka, Debezium, DLT, and external-target divergence/reconciliation boundaries; +6. elapsed recovery measurements before any profile-specific RTO/RPO claim. -Rollback of this backup feature means disabling/removing the operator command and its documentation, not deleting previously created recovery evidence. Existing archives remain sensitive operational artifacts and must follow the operator's retention/destruction policy. +Rollback of this repository recovery tooling means disabling/removing the operator commands and documentation, not deleting previously created recovery evidence. Existing archives remain sensitive operational artifacts and must follow the operator's retention/destruction policy. ## References @@ -99,4 +153,6 @@ PostgreSQL Global Development Group. (2026). *pg_dump (PostgreSQL 18 documentati PostgreSQL Global Development Group. (2026). *pg_restore (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/app-pgrestore.html +PostgreSQL Global Development Group. (2026). *Populating a database (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/populate.html + PostgreSQL Global Development Group. (2026). *Backup and restore (PostgreSQL 18 documentation)*. https://www.postgresql.org/docs/18/backup.html From 5ea1d328a40544240504d271ebf8a5f01292d723 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:32:28 +0900 Subject: [PATCH 24/30] test(ops): require restored durable data invariants --- ...ogicalRestoreDurableDataInvariantTest.java | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java new file mode 100644 index 00000000..52f03d5c --- /dev/null +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java @@ -0,0 +1,82 @@ +package com.xtrmetl.etl.operations; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Requires PostgreSQL recovery rehearsal to verify representative persisted idempotency and durable + * job invariants before reporting a successful restore. + */ +class PostgresLogicalRestoreDurableDataInvariantTest { + + @Test + void restoreRejectsPersistedRowsThatViolateDurableApplicationInvariants() throws IOException { + String script = Files.readString( + projectRoot().resolve("scripts/ops/postgres-logical-restore-rehearsal.sh"), + StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + + assertTrue(script.contains("idempotency_integrity_violation_count="), + "restore must explicitly evaluate persisted idempotency ledger rows"); + assertTrue(script.contains("FROM etl_idempotency_records"), + "restore must read the restored idempotency ledger rather than infer row integrity from table existence"); + assertTrue(script.contains("idempotency_key_hash !~ '^[0-9a-f]{64}$'"), + "restore must reject malformed persisted idempotency key hashes"); + assertTrue(script.contains("request_digest !~ '^[0-9a-f]{64}$'"), + "restore must reject malformed persisted request digests"); + assertTrue(script.contains("Restored idempotency ledger violates mightyETL integrity invariants"), + "restore must provide a stable operator classification for idempotency integrity failure"); + + assertTrue(script.contains("durable_job_integrity_violation_count="), + "restore must explicitly evaluate persisted durable-job rows"); + assertTrue(script.contains("FROM etl_job_records"), + "restore must read restored durable-job state rather than infer row integrity from table existence"); + assertTrue(script.contains("job_status NOT IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')"), + "restore must reject durable jobs outside the protected lifecycle state domain"); + assertTrue(script.contains("attempt_count < 0"), + "restore must reject negative persisted durable-job attempt counts"); + assertTrue(script.contains("job_status IN ('PENDING', 'RUNNING') AND request_payload IS NULL"), + "restore must reject active durable jobs whose recoverable payload is missing"); + assertTrue(script.contains("job_status IN ('SUCCEEDED', 'FAILED') AND request_payload IS NOT NULL"), + "restore must reject terminal durable jobs that still retain request payloads"); + assertTrue(script.contains("Restored durable-job ledger violates mightyETL lifecycle invariants"), + "restore must provide a stable operator classification for durable-job integrity failure"); + + String relationCheck = "required_application_relation_count="; + String idempotencyCheck = "idempotency_integrity_violation_count="; + String durableJobCheck = "durable_job_integrity_violation_count="; + String success = "PostgreSQL restore rehearsal completed on the explicit disposable target"; + assertTrue(script.indexOf(relationCheck) < script.indexOf(idempotencyCheck), + "row-level invariants must run only after required application relations are proven present"); + assertTrue(script.indexOf(idempotencyCheck) < script.indexOf(durableJobCheck), + "idempotency and durable-job invariants must run in a deterministic order"); + assertTrue(script.indexOf(durableJobCheck) < script.indexOf(success), + "restore must not report success before durable row invariants are verified"); + } + + private static Path projectRoot() { + Path current = Paths.get(System.getProperty("user.dir")).toAbsolutePath(); + Path lastPomParent = null; + while (current != null) { + if (Files.exists(current.resolve(".git"))) { + return current; + } + if (Files.exists(current.resolve("pom.xml"))) { + lastPomParent = current; + } + current = current.getParent(); + } + if (lastPomParent != null) { + return lastPomParent; + } + throw new IllegalStateException("Could not find project root"); + } +} From 41f840d49f170496bf60cf2fd0d2f210712b88a4 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:36:38 +0900 Subject: [PATCH 25/30] fix(ops): verify restored durable data invariants --- scripts/ops/postgres-logical-restore-rehearsal.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index 336fc7d7..27dbe8d8 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -185,4 +185,18 @@ if [[ ! "$required_application_relation_count" =~ ^[0-9]+$ || "$required_applica exit 6 fi +idempotency_integrity_violation_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM etl_idempotency_records WHERE idempotency_key_hash !~ '^[0-9a-f]{64}$' OR request_digest !~ '^[0-9a-f]{64}$'") +idempotency_integrity_violation_count=${idempotency_integrity_violation_count//[[:space:]]/} +if [[ ! "$idempotency_integrity_violation_count" =~ ^[0-9]+$ || "$idempotency_integrity_violation_count" != "0" ]]; then + printf 'Restored idempotency ledger violates mightyETL integrity invariants\n' >&2 + exit 6 +fi + +durable_job_integrity_violation_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM etl_job_records WHERE principal_scope_hash !~ '^[0-9a-f]{64}$' OR submission_key_hash !~ '^[0-9a-f]{64}$' OR request_digest !~ '^[0-9a-f]{64}$' OR job_status NOT IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED') OR attempt_count < 0 OR (job_status IN ('PENDING', 'RUNNING') AND request_payload IS NULL) OR (job_status IN ('SUCCEEDED', 'FAILED') AND request_payload IS NOT NULL) OR (failure_code IS NOT NULL AND failure_code !~ '^[a-z0-9_]{1,64}$')") +durable_job_integrity_violation_count=${durable_job_integrity_violation_count//[[:space:]]/} +if [[ ! "$durable_job_integrity_violation_count" =~ ^[0-9]+$ || "$durable_job_integrity_violation_count" != "0" ]]; then + printf 'Restored durable-job ledger violates mightyETL lifecycle invariants\n' >&2 + exit 6 +fi + printf '%s\n' 'PostgreSQL restore rehearsal completed on the explicit disposable target' From 510d033982f935aa9d933ea90dcb52824f2c6b7a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:39:28 +0900 Subject: [PATCH 26/30] test(ops): bind restored failure-code grammar --- .../PostgresLogicalRestoreDurableDataInvariantTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java index 52f03d5c..a18b33e1 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalRestoreDurableDataInvariantTest.java @@ -47,6 +47,8 @@ void restoreRejectsPersistedRowsThatViolateDurableApplicationInvariants() throws "restore must reject active durable jobs whose recoverable payload is missing"); assertTrue(script.contains("job_status IN ('SUCCEEDED', 'FAILED') AND request_payload IS NOT NULL"), "restore must reject terminal durable jobs that still retain request payloads"); + assertTrue(script.contains("failure_code !~ '^[a-z][a-z0-9_]{2,127}$'"), + "restore must enforce the exact protected durable-job failure-code grammar"); assertTrue(script.contains("Restored durable-job ledger violates mightyETL lifecycle invariants"), "restore must provide a stable operator classification for durable-job integrity failure"); From f33de0a8a6709947c0b19a94a931f1870afa386c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:42:14 +0900 Subject: [PATCH 27/30] fix(ops): match durable failure-code constraint --- scripts/ops/postgres-logical-restore-rehearsal.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ops/postgres-logical-restore-rehearsal.sh b/scripts/ops/postgres-logical-restore-rehearsal.sh index 27dbe8d8..37003185 100755 --- a/scripts/ops/postgres-logical-restore-rehearsal.sh +++ b/scripts/ops/postgres-logical-restore-rehearsal.sh @@ -192,7 +192,7 @@ if [[ ! "$idempotency_integrity_violation_count" =~ ^[0-9]+$ || "$idempotency_in exit 6 fi -durable_job_integrity_violation_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM etl_job_records WHERE principal_scope_hash !~ '^[0-9a-f]{64}$' OR submission_key_hash !~ '^[0-9a-f]{64}$' OR request_digest !~ '^[0-9a-f]{64}$' OR job_status NOT IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED') OR attempt_count < 0 OR (job_status IN ('PENDING', 'RUNNING') AND request_payload IS NULL) OR (job_status IN ('SUCCEEDED', 'FAILED') AND request_payload IS NOT NULL) OR (failure_code IS NOT NULL AND failure_code !~ '^[a-z0-9_]{1,64}$')") +durable_job_integrity_violation_count=$("${psql_recovery[@]}" --command="SELECT count(*) FROM etl_job_records WHERE principal_scope_hash !~ '^[0-9a-f]{64}$' OR submission_key_hash !~ '^[0-9a-f]{64}$' OR request_digest !~ '^[0-9a-f]{64}$' OR job_status NOT IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED') OR attempt_count < 0 OR (job_status IN ('PENDING', 'RUNNING') AND request_payload IS NULL) OR (job_status IN ('SUCCEEDED', 'FAILED') AND request_payload IS NOT NULL) OR (failure_code IS NOT NULL AND failure_code !~ '^[a-z][a-z0-9_]{2,127}$')") durable_job_integrity_violation_count=${durable_job_integrity_violation_count//[[:space:]]/} if [[ ! "$durable_job_integrity_violation_count" =~ ^[0-9]+$ || "$durable_job_integrity_violation_count" != "0" ]]; then printf 'Restored durable-job ledger violates mightyETL lifecycle invariants\n' >&2 From 2f001bf5dc48c814a580bbc7d2baf53fd195dd32 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:45:52 +0900 Subject: [PATCH 28/30] test(ops): reject migration drift during backup --- .../PostgresLogicalBackupContractTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index 76af9608..941abaa6 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -38,6 +38,34 @@ void backupToolCreatesAPrivateVerifiedCustomArchiveWithProvenance() throws IOExc assertFalse(script.contains("echo \"$PGPASSWORD\""), "database credentials must never be printed"); } + @Test + void backupToolRejectsMigrationLevelDriftAcrossTheDumpWindow() throws IOException { + String script = Files.readString(projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"), StandardCharsets.UTF_8) + .replace("\r\n", "\n") + .replace('\r', '\n'); + + String before = "flyway_schema_version_before_dump="; + String dump = "pg_dump \\\"; + String after = "flyway_schema_version_after_dump="; + String driftGuard = "if [[ \"$flyway_schema_version_before_dump\" != \"$flyway_schema_version_after_dump\" ]]"; + String manifest = "\"flyway_schema_version=${flyway_schema_version_before_dump}\""; + + assertTrue(script.contains(before), + "backup must observe the migration level before taking the archive snapshot"); + assertTrue(script.contains(after), + "backup must observe the migration level again after the archive is complete"); + assertTrue(script.contains(driftGuard), + "backup must fail closed if migration authority changes while the archive is being captured"); + assertTrue(script.contains("Database migration level changed while backup was being captured"), + "migration drift needs a stable operator failure classification"); + assertTrue(script.indexOf(before) < script.indexOf(dump), + "the first migration observation must happen before pg_dump"); + assertTrue(script.indexOf(dump) < script.indexOf(after), + "the second migration observation must happen after pg_dump"); + assertTrue(script.indexOf(after) < script.indexOf(manifest), + "the manifest may record a migration level only after the dump window is proven stable"); + } + @Test void backupToolReservesTheFinalIdentityBeforeWritingToPreventSameSecondCollisions() throws IOException { String script = Files.readString(projectRoot().resolve("scripts/ops/postgres-logical-backup.sh"), StandardCharsets.UTF_8); From ec4b919969467831019e3874a11d96a72ac58545 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:47:34 +0900 Subject: [PATCH 29/30] test(ops): fix migration-drift test marker --- .../etl/operations/PostgresLogicalBackupContractTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java index 941abaa6..4b788e45 100644 --- a/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java +++ b/etl-service/src/test/java/com/xtrmetl/etl/operations/PostgresLogicalBackupContractTest.java @@ -45,7 +45,7 @@ void backupToolRejectsMigrationLevelDriftAcrossTheDumpWindow() throws IOExceptio .replace('\r', '\n'); String before = "flyway_schema_version_before_dump="; - String dump = "pg_dump \\\"; + String dump = "--format=custom"; String after = "flyway_schema_version_after_dump="; String driftGuard = "if [[ \"$flyway_schema_version_before_dump\" != \"$flyway_schema_version_after_dump\" ]]"; String manifest = "\"flyway_schema_version=${flyway_schema_version_before_dump}\""; From 871153c2294cd56cf59f15517cd538ae0a8530d6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 10 Aug 2026 19:49:51 +0900 Subject: [PATCH 30/30] fix(ops): reject migration drift across backup window --- scripts/ops/postgres-logical-backup.sh | 30 ++++++++++++++++++-------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/scripts/ops/postgres-logical-backup.sh b/scripts/ops/postgres-logical-backup.sh index b47e7a96..628d6b68 100755 --- a/scripts/ops/postgres-logical-backup.sh +++ b/scripts/ops/postgres-logical-backup.sh @@ -35,6 +35,16 @@ sha256_file() { return 1 } +query_flyway_schema_version() { + psql \ + --host="$PGHOST" \ + --port="$PGPORT" \ + --username="$PGUSER" \ + --dbname="$PGDATABASE" \ + --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ + --command="SELECT COALESCE((SELECT version FROM flyway_schema_history WHERE success = true ORDER BY installed_rank DESC LIMIT 1), 'none')" +} + mkdir -p -- "$BACKUP_DIRECTORY" backup_directory=$(cd -- "$BACKUP_DIRECTORY" && pwd -P) created_at_utc=$(date -u '+%Y-%m-%dT%H:%M:%SZ') @@ -67,6 +77,10 @@ temporary_bundle=$(mktemp -d "${backup_directory}/.mightyetl-postgres-backup.XXX archive_path="${temporary_bundle}/database.dump" manifest_path="${temporary_bundle}/manifest.txt" +# Bind migration provenance on both sides of the dump window. A concurrent Flyway migration may +# otherwise make a post-dump manifest claim a schema level newer than the archive snapshot. +flyway_schema_version_before_dump=$(query_flyway_schema_version) + pg_dump \ --host="$PGHOST" \ --port="$PGPORT" \ @@ -78,6 +92,12 @@ pg_dump \ # A completed custom archive must be structurally readable before it is published. pg_restore --list "$archive_path" >/dev/null +flyway_schema_version_after_dump=$(query_flyway_schema_version) +if [[ "$flyway_schema_version_before_dump" != "$flyway_schema_version_after_dump" ]]; then + printf 'Database migration level changed while backup was being captured\n' >&2 + exit 5 +fi + server_version_num=$(psql \ --host="$PGHOST" \ --port="$PGPORT" \ @@ -86,14 +106,6 @@ server_version_num=$(psql \ --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ --command='SHOW server_version_num') -flyway_schema_version=$(psql \ - --host="$PGHOST" \ - --port="$PGPORT" \ - --username="$PGUSER" \ - --dbname="$PGDATABASE" \ - --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 \ - --command="SELECT COALESCE((SELECT version FROM flyway_schema_history WHERE success = true ORDER BY installed_rank DESC LIMIT 1), 'none')") - backup_sha256=$(sha256_file "$archive_path") printf '%s\n' \ @@ -101,7 +113,7 @@ printf '%s\n' \ "application_source_sha=${APPLICATION_SOURCE_SHA}" \ "created_at_utc=${created_at_utc}" \ "server_version_num=${server_version_num}" \ - "flyway_schema_version=${flyway_schema_version}" \ + "flyway_schema_version=${flyway_schema_version_before_dump}" \ "backup_sha256=${backup_sha256}" \ > "$manifest_path"